Browse code

Remove always-on watch for networks and endpoints

Always on watching of networks and endpoints can
affect scalability of the cluster beyond a few nodes.
Remove pro active watching and watch only the objects
you are interested in.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

Jana Radhakrishnan authored on 2015/10/05 20:21:15
Showing 18 changed files
... ...
@@ -14,6 +14,7 @@ import (
14 14
 
15 15
 	"github.com/docker/docker/pkg/reexec"
16 16
 	"github.com/docker/libnetwork"
17
+	"github.com/docker/libnetwork/datastore"
17 18
 	"github.com/docker/libnetwork/netlabel"
18 19
 	"github.com/docker/libnetwork/options"
19 20
 	"github.com/docker/libnetwork/testutils"
... ...
@@ -88,11 +89,13 @@ func i2sbL(i interface{}) []*sandboxResource {
88 88
 }
89 89
 
90 90
 func createTestNetwork(t *testing.T, network string) (libnetwork.NetworkController, libnetwork.Network) {
91
+	// Cleanup local datastore file
92
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
93
+
91 94
 	c, err := libnetwork.New()
92 95
 	if err != nil {
93 96
 		t.Fatal(err)
94 97
 	}
95
-	defer c.Stop()
96 98
 
97 99
 	netOption := options.Generic{
98 100
 		netlabel.GenericData: options.Generic{
... ...
@@ -175,6 +178,9 @@ func TestJson(t *testing.T) {
175 175
 func TestCreateDeleteNetwork(t *testing.T) {
176 176
 	defer testutils.SetupTestOSContext(t)()
177 177
 
178
+	// Cleanup local datastore file
179
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
180
+
178 181
 	c, err := libnetwork.New()
179 182
 	if err != nil {
180 183
 		t.Fatal(err)
... ...
@@ -249,6 +255,9 @@ func TestCreateDeleteNetwork(t *testing.T) {
249 249
 func TestGetNetworksAndEndpoints(t *testing.T) {
250 250
 	defer testutils.SetupTestOSContext(t)()
251 251
 
252
+	// Cleanup local datastore file
253
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
254
+
252 255
 	c, err := libnetwork.New()
253 256
 	if err != nil {
254 257
 		t.Fatal(err)
... ...
@@ -518,6 +527,9 @@ func TestGetNetworksAndEndpoints(t *testing.T) {
518 518
 func TestProcGetServices(t *testing.T) {
519 519
 	defer testutils.SetupTestOSContext(t)()
520 520
 
521
+	// Cleanup local datastore file
522
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
523
+
521 524
 	c, err := libnetwork.New()
522 525
 	if err != nil {
523 526
 		t.Fatal(err)
... ...
@@ -686,6 +698,7 @@ func TestProcGetService(t *testing.T) {
686 686
 	defer testutils.SetupTestOSContext(t)()
687 687
 
688 688
 	c, nw := createTestNetwork(t, "network")
689
+	defer c.Stop()
689 690
 	ep1, err := nw.CreateEndpoint("db")
690 691
 	if err != nil {
691 692
 		t.Fatal(err)
... ...
@@ -738,6 +751,8 @@ func TestProcPublishUnpublishService(t *testing.T) {
738 738
 	defer testutils.SetupTestOSContext(t)()
739 739
 
740 740
 	c, _ := createTestNetwork(t, "network")
741
+	defer c.Stop()
742
+
741 743
 	vars := make(map[string]string)
742 744
 
743 745
 	vbad, err := json.Marshal("bad service create data")
... ...
@@ -870,6 +885,7 @@ func TestAttachDetachBackend(t *testing.T) {
870 870
 	defer testutils.SetupTestOSContext(t)()
871 871
 
872 872
 	c, nw := createTestNetwork(t, "network")
873
+	defer c.Stop()
873 874
 	ep1, err := nw.CreateEndpoint("db")
874 875
 	if err != nil {
875 876
 		t.Fatal(err)
... ...
@@ -994,6 +1010,9 @@ func TestAttachDetachBackend(t *testing.T) {
994 994
 }
995 995
 
996 996
 func TestDetectGetNetworksInvalidQueryComposition(t *testing.T) {
997
+	// Cleanup local datastore file
998
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
999
+
997 1000
 	c, err := libnetwork.New()
998 1001
 	if err != nil {
999 1002
 		t.Fatal(err)
... ...
@@ -1011,6 +1030,7 @@ func TestDetectGetEndpointsInvalidQueryComposition(t *testing.T) {
1011 1011
 	defer testutils.SetupTestOSContext(t)()
1012 1012
 
1013 1013
 	c, _ := createTestNetwork(t, "network")
1014
+	defer c.Stop()
1014 1015
 
1015 1016
 	vars := map[string]string{urlNwName: "network", urlEpName: "x", urlEpPID: "y"}
1016 1017
 	_, errRsp := procGetEndpoints(c, vars, nil)
... ...
@@ -1023,6 +1043,7 @@ func TestDetectGetServicesInvalidQueryComposition(t *testing.T) {
1023 1023
 	defer testutils.SetupTestOSContext(t)()
1024 1024
 
1025 1025
 	c, _ := createTestNetwork(t, "network")
1026
+	defer c.Stop()
1026 1027
 
1027 1028
 	vars := map[string]string{urlNwName: "network", urlEpName: "x", urlEpPID: "y"}
1028 1029
 	_, errRsp := procGetServices(c, vars, nil)
... ...
@@ -1040,6 +1061,8 @@ func TestFindNetworkUtil(t *testing.T) {
1040 1040
 	defer testutils.SetupTestOSContext(t)()
1041 1041
 
1042 1042
 	c, nw := createTestNetwork(t, "network")
1043
+	defer c.Stop()
1044
+
1043 1045
 	nid := nw.ID()
1044 1046
 
1045 1047
 	_, errRsp := findNetwork(c, "", byName)
... ...
@@ -1102,6 +1125,9 @@ func TestFindNetworkUtil(t *testing.T) {
1102 1102
 func TestCreateDeleteEndpoints(t *testing.T) {
1103 1103
 	defer testutils.SetupTestOSContext(t)()
1104 1104
 
1105
+	// Cleanup local datastore file
1106
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
1107
+
1105 1108
 	c, err := libnetwork.New()
1106 1109
 	if err != nil {
1107 1110
 		t.Fatal(err)
... ...
@@ -1225,6 +1251,9 @@ func TestCreateDeleteEndpoints(t *testing.T) {
1225 1225
 func TestJoinLeave(t *testing.T) {
1226 1226
 	defer testutils.SetupTestOSContext(t)()
1227 1227
 
1228
+	// Cleanup local datastore file
1229
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
1230
+
1228 1231
 	c, err := libnetwork.New()
1229 1232
 	if err != nil {
1230 1233
 		t.Fatal(err)
... ...
@@ -1382,6 +1411,8 @@ func TestFindEndpointUtilPanic(t *testing.T) {
1382 1382
 	defer testutils.SetupTestOSContext(t)()
1383 1383
 	defer checkPanic(t)
1384 1384
 	c, nw := createTestNetwork(t, "network")
1385
+	defer c.Stop()
1386
+
1385 1387
 	nid := nw.ID()
1386 1388
 	findEndpoint(c, nid, "", byID, -1)
1387 1389
 }
... ...
@@ -1390,6 +1421,8 @@ func TestFindServiceUtilPanic(t *testing.T) {
1390 1390
 	defer testutils.SetupTestOSContext(t)()
1391 1391
 	defer checkPanic(t)
1392 1392
 	c, _ := createTestNetwork(t, "network")
1393
+	defer c.Stop()
1394
+
1393 1395
 	findService(c, "random_service", -1)
1394 1396
 }
1395 1397
 
... ...
@@ -1397,6 +1430,8 @@ func TestFindEndpointUtil(t *testing.T) {
1397 1397
 	defer testutils.SetupTestOSContext(t)()
1398 1398
 
1399 1399
 	c, nw := createTestNetwork(t, "network")
1400
+	defer c.Stop()
1401
+
1400 1402
 	nid := nw.ID()
1401 1403
 
1402 1404
 	ep, err := nw.CreateEndpoint("secondEp", nil)
... ...
@@ -1443,7 +1478,8 @@ func TestFindEndpointUtil(t *testing.T) {
1443 1443
 		t.Fatalf("Unexepected failure: %v", errRsp)
1444 1444
 	}
1445 1445
 
1446
-	if ep0 != ep1 || ep0 != ep2 || ep0 != ep3 || ep0 != ep4 || ep0 != ep5 {
1446
+	if ep0.ID() != ep1.ID() || ep0.ID() != ep2.ID() ||
1447
+		ep0.ID() != ep3.ID() || ep0.ID() != ep4.ID() || ep0.ID() != ep5.ID() {
1447 1448
 		t.Fatalf("Diffenrent queries returned different endpoints")
1448 1449
 	}
1449 1450
 
... ...
@@ -1665,6 +1701,9 @@ func TestwriteJSON(t *testing.T) {
1665 1665
 func TestHttpHandlerUninit(t *testing.T) {
1666 1666
 	defer testutils.SetupTestOSContext(t)()
1667 1667
 
1668
+	// Cleanup local datastore file
1669
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
1670
+
1668 1671
 	c, err := libnetwork.New()
1669 1672
 	if err != nil {
1670 1673
 		t.Fatal(err)
... ...
@@ -1732,6 +1771,9 @@ func TestHttpHandlerBadBody(t *testing.T) {
1732 1732
 
1733 1733
 	rsp := newWriter()
1734 1734
 
1735
+	// Cleanup local datastore file
1736
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
1737
+
1735 1738
 	c, err := libnetwork.New()
1736 1739
 	if err != nil {
1737 1740
 		t.Fatal(err)
... ...
@@ -1765,6 +1807,9 @@ func TestEndToEnd(t *testing.T) {
1765 1765
 
1766 1766
 	rsp := newWriter()
1767 1767
 
1768
+	// Cleanup local datastore file
1769
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
1770
+
1768 1771
 	c, err := libnetwork.New()
1769 1772
 	if err != nil {
1770 1773
 		t.Fatal(err)
... ...
@@ -2213,6 +2258,9 @@ func TestEndToEndErrorMessage(t *testing.T) {
2213 2213
 
2214 2214
 	rsp := newWriter()
2215 2215
 
2216
+	// Cleanup local datastore file
2217
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
2218
+
2216 2219
 	c, err := libnetwork.New()
2217 2220
 	if err != nil {
2218 2221
 		t.Fatal(err)
... ...
@@ -22,10 +22,12 @@ import (
22 22
 	"github.com/docker/docker/pkg/reexec"
23 23
 
24 24
 	"github.com/Sirupsen/logrus"
25
+	psignal "github.com/docker/docker/pkg/signal"
25 26
 	"github.com/docker/docker/pkg/term"
26 27
 	"github.com/docker/libnetwork"
27 28
 	"github.com/docker/libnetwork/api"
28 29
 	"github.com/docker/libnetwork/config"
30
+	"github.com/docker/libnetwork/datastore"
29 31
 	"github.com/docker/libnetwork/driverapi"
30 32
 	"github.com/docker/libnetwork/netlabel"
31 33
 	"github.com/docker/libnetwork/options"
... ...
@@ -76,6 +78,7 @@ func processConfig(cfg *config.Config) []config.Option {
76 76
 	if cfg == nil {
77 77
 		return options
78 78
 	}
79
+
79 80
 	dn := "bridge"
80 81
 	if strings.TrimSpace(cfg.Daemon.DefaultNetwork) != "" {
81 82
 		dn = cfg.Daemon.DefaultNetwork
... ...
@@ -91,12 +94,12 @@ func processConfig(cfg *config.Config) []config.Option {
91 91
 	if cfg.Daemon.Labels != nil {
92 92
 		options = append(options, config.OptionLabels(cfg.Daemon.Labels))
93 93
 	}
94
-	if strings.TrimSpace(cfg.GlobalStore.Client.Provider) != "" {
95
-		options = append(options, config.OptionKVProvider(cfg.GlobalStore.Client.Provider))
96
-	}
97
-	if strings.TrimSpace(cfg.GlobalStore.Client.Address) != "" {
98
-		options = append(options, config.OptionKVProviderURL(cfg.GlobalStore.Client.Address))
94
+
95
+	if dcfg, ok := cfg.Scopes[datastore.GlobalScope]; ok && dcfg.IsValid() {
96
+		options = append(options, config.OptionKVProvider(dcfg.Client.Provider))
97
+		options = append(options, config.OptionKVProviderURL(dcfg.Client.Address))
99 98
 	}
99
+
100 100
 	dOptions, err := startDiscovery(&cfg.Cluster)
101 101
 	if err != nil {
102 102
 		logrus.Infof("Skipping discovery : %s", err.Error())
... ...
@@ -182,8 +185,9 @@ func createDefaultNetwork(c libnetwork.NetworkController) {
182 182
 			genericOption[netlabel.GenericData] = map[string]interface{}{
183 183
 				"BridgeName": nw,
184 184
 			}
185
-			networkOption := libnetwork.NetworkOptionGeneric(genericOption)
186
-			createOptions = append(createOptions, networkOption)
185
+			createOptions = append(createOptions,
186
+				libnetwork.NetworkOptionGeneric(genericOption),
187
+				libnetwork.NetworkOptionPersist(false))
187 188
 		}
188 189
 		_, err := c.NewNetwork(d, nw, createOptions...)
189 190
 		if err != nil {
... ...
@@ -214,6 +218,7 @@ func (d *dnetConnection) dnetDaemon(cfgFile string) error {
214 214
 		fmt.Println("Error starting dnetDaemon :", err)
215 215
 		return err
216 216
 	}
217
+
217 218
 	createDefaultNetwork(controller)
218 219
 	httpHandler := api.NewHTTPHandler(controller)
219 220
 	r := mux.NewRouter().StrictSlash(false)
... ...
@@ -231,10 +236,21 @@ func (d *dnetConnection) dnetDaemon(cfgFile string) error {
231 231
 	post.Methods("GET", "PUT", "POST", "DELETE").HandlerFunc(httpHandler)
232 232
 
233 233
 	handleSignals(controller)
234
+	setupDumpStackTrap()
234 235
 
235 236
 	return http.ListenAndServe(d.addr, r)
236 237
 }
237 238
 
239
+func setupDumpStackTrap() {
240
+	c := make(chan os.Signal, 1)
241
+	signal.Notify(c, syscall.SIGUSR1)
242
+	go func() {
243
+		for range c {
244
+			psignal.DumpStacks()
245
+		}
246
+	}()
247
+}
248
+
238 249
 func handleSignals(controller libnetwork.NetworkController) {
239 250
 	c := make(chan os.Signal, 1)
240 251
 	signals := []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT}
... ...
@@ -7,19 +7,21 @@ import (
7 7
 	log "github.com/Sirupsen/logrus"
8 8
 	"github.com/docker/docker/pkg/discovery"
9 9
 	"github.com/docker/libkv/store"
10
+	"github.com/docker/libnetwork/datastore"
10 11
 	"github.com/docker/libnetwork/netlabel"
11 12
 )
12 13
 
13 14
 // Config encapsulates configurations of various Libnetwork components
14 15
 type Config struct {
15
-	Daemon                  DaemonCfg
16
-	Cluster                 ClusterCfg
17
-	GlobalStore, LocalStore DatastoreCfg
16
+	Daemon  DaemonCfg
17
+	Cluster ClusterCfg
18
+	Scopes  map[string]*datastore.ScopeCfg
18 19
 }
19 20
 
20 21
 // DaemonCfg represents libnetwork core configuration
21 22
 type DaemonCfg struct {
22 23
 	Debug          bool
24
+	DataDir        string
23 25
 	DefaultNetwork string
24 26
 	DefaultDriver  string
25 27
 	Labels         []string
... ...
@@ -34,26 +36,28 @@ type ClusterCfg struct {
34 34
 	Heartbeat uint64
35 35
 }
36 36
 
37
-// DatastoreCfg represents Datastore configuration.
38
-type DatastoreCfg struct {
39
-	Embedded bool
40
-	Client   DatastoreClientCfg
41
-}
42
-
43
-// DatastoreClientCfg represents Datastore Client-only mode configuration
44
-type DatastoreClientCfg struct {
45
-	Provider string
46
-	Address  string
47
-	Config   *store.Config
37
+// LoadDefaultScopes loads default scope configs for scopes which
38
+// doesn't have explicit user specified configs.
39
+func (c *Config) LoadDefaultScopes(dataDir string) {
40
+	for k, v := range datastore.DefaultScopes(dataDir) {
41
+		if _, ok := c.Scopes[k]; !ok {
42
+			c.Scopes[k] = v
43
+		}
44
+	}
48 45
 }
49 46
 
50 47
 // ParseConfig parses the libnetwork configuration file
51 48
 func ParseConfig(tomlCfgFile string) (*Config, error) {
52
-	var cfg Config
53
-	if _, err := toml.DecodeFile(tomlCfgFile, &cfg); err != nil {
49
+	cfg := &Config{
50
+		Scopes: map[string]*datastore.ScopeCfg{},
51
+	}
52
+
53
+	if _, err := toml.DecodeFile(tomlCfgFile, cfg); err != nil {
54 54
 		return nil, err
55 55
 	}
56
-	return &cfg, nil
56
+
57
+	cfg.LoadDefaultScopes(cfg.Daemon.DataDir)
58
+	return cfg, nil
57 59
 }
58 60
 
59 61
 // Option is a option setter function type used to pass varios configurations
... ...
@@ -98,7 +102,10 @@ func OptionLabels(labels []string) Option {
98 98
 func OptionKVProvider(provider string) Option {
99 99
 	return func(c *Config) {
100 100
 		log.Infof("Option OptionKVProvider: %s", provider)
101
-		c.GlobalStore.Client.Provider = strings.TrimSpace(provider)
101
+		if _, ok := c.Scopes[datastore.GlobalScope]; !ok {
102
+			c.Scopes[datastore.GlobalScope] = &datastore.ScopeCfg{}
103
+		}
104
+		c.Scopes[datastore.GlobalScope].Client.Provider = strings.TrimSpace(provider)
102 105
 	}
103 106
 }
104 107
 
... ...
@@ -106,7 +113,10 @@ func OptionKVProvider(provider string) Option {
106 106
 func OptionKVProviderURL(url string) Option {
107 107
 	return func(c *Config) {
108 108
 		log.Infof("Option OptionKVProviderURL: %s", url)
109
-		c.GlobalStore.Client.Address = strings.TrimSpace(url)
109
+		if _, ok := c.Scopes[datastore.GlobalScope]; !ok {
110
+			c.Scopes[datastore.GlobalScope] = &datastore.ScopeCfg{}
111
+		}
112
+		c.Scopes[datastore.GlobalScope].Client.Address = strings.TrimSpace(url)
110 113
 	}
111 114
 }
112 115
 
... ...
@@ -124,6 +134,13 @@ func OptionDiscoveryAddress(address string) Option {
124 124
 	}
125 125
 }
126 126
 
127
+// OptionDataDir function returns an option setter for data folder
128
+func OptionDataDir(dataDir string) Option {
129
+	return func(c *Config) {
130
+		c.Daemon.DataDir = dataDir
131
+	}
132
+}
133
+
127 134
 // ProcessOptions processes options and stores it in config
128 135
 func (c *Config) ProcessOptions(options ...Option) {
129 136
 	for _, opt := range options {
... ...
@@ -145,7 +162,10 @@ func IsValidName(name string) bool {
145 145
 func OptionLocalKVProvider(provider string) Option {
146 146
 	return func(c *Config) {
147 147
 		log.Infof("Option OptionLocalKVProvider: %s", provider)
148
-		c.LocalStore.Client.Provider = strings.TrimSpace(provider)
148
+		if _, ok := c.Scopes[datastore.LocalScope]; !ok {
149
+			c.Scopes[datastore.LocalScope] = &datastore.ScopeCfg{}
150
+		}
151
+		c.Scopes[datastore.LocalScope].Client.Provider = strings.TrimSpace(provider)
149 152
 	}
150 153
 }
151 154
 
... ...
@@ -153,7 +173,10 @@ func OptionLocalKVProvider(provider string) Option {
153 153
 func OptionLocalKVProviderURL(url string) Option {
154 154
 	return func(c *Config) {
155 155
 		log.Infof("Option OptionLocalKVProviderURL: %s", url)
156
-		c.LocalStore.Client.Address = strings.TrimSpace(url)
156
+		if _, ok := c.Scopes[datastore.LocalScope]; !ok {
157
+			c.Scopes[datastore.LocalScope] = &datastore.ScopeCfg{}
158
+		}
159
+		c.Scopes[datastore.LocalScope].Client.Address = strings.TrimSpace(url)
157 160
 	}
158 161
 }
159 162
 
... ...
@@ -161,6 +184,9 @@ func OptionLocalKVProviderURL(url string) Option {
161 161
 func OptionLocalKVProviderConfig(config *store.Config) Option {
162 162
 	return func(c *Config) {
163 163
 		log.Infof("Option OptionLocalKVProviderConfig: %v", config)
164
-		c.LocalStore.Client.Config = config
164
+		if _, ok := c.Scopes[datastore.LocalScope]; !ok {
165
+			c.Scopes[datastore.LocalScope] = &datastore.ScopeCfg{}
166
+		}
167
+		c.Scopes[datastore.LocalScope].Client.Config = config
165 168
 	}
166 169
 }
... ...
@@ -124,73 +124,71 @@ type ipamData struct {
124 124
 }
125 125
 
126 126
 type driverTable map[string]*driverData
127
+
128
+//type networkTable map[string]*network
129
+//type endpointTable map[string]*endpoint
127 130
 type ipamTable map[string]*ipamData
128
-type networkTable map[string]*network
129
-type endpointTable map[string]*endpoint
130 131
 type sandboxTable map[string]*sandbox
131 132
 
132 133
 type controller struct {
133
-	id                      string
134
-	networks                networkTable
135
-	drivers                 driverTable
136
-	ipamDrivers             ipamTable
137
-	sandboxes               sandboxTable
138
-	cfg                     *config.Config
139
-	globalStore, localStore datastore.DataStore
140
-	discovery               hostdiscovery.HostDiscovery
141
-	extKeyListener          net.Listener
134
+	id string
135
+	//networks       networkTable
136
+	drivers        driverTable
137
+	ipamDrivers    ipamTable
138
+	sandboxes      sandboxTable
139
+	cfg            *config.Config
140
+	stores         []datastore.DataStore
141
+	discovery      hostdiscovery.HostDiscovery
142
+	extKeyListener net.Listener
143
+	watchCh        chan *endpoint
144
+	unWatchCh      chan *endpoint
145
+	svcDb          map[string]svcMap
142 146
 	sync.Mutex
143 147
 }
144 148
 
145 149
 // New creates a new instance of network controller.
146 150
 func New(cfgOptions ...config.Option) (NetworkController, error) {
147 151
 	var cfg *config.Config
152
+	cfg = &config.Config{
153
+		Daemon: config.DaemonCfg{
154
+			DriverCfg: make(map[string]interface{}),
155
+		},
156
+		Scopes: make(map[string]*datastore.ScopeCfg),
157
+	}
158
+
148 159
 	if len(cfgOptions) > 0 {
149
-		cfg = &config.Config{
150
-			Daemon: config.DaemonCfg{
151
-				DriverCfg: make(map[string]interface{}),
152
-			},
153
-		}
154 160
 		cfg.ProcessOptions(cfgOptions...)
155 161
 	}
162
+	cfg.LoadDefaultScopes(cfg.Daemon.DataDir)
163
+
156 164
 	c := &controller{
157 165
 		id:          stringid.GenerateRandomID(),
158 166
 		cfg:         cfg,
159
-		networks:    networkTable{},
160 167
 		sandboxes:   sandboxTable{},
161 168
 		drivers:     driverTable{},
162
-		ipamDrivers: ipamTable{}}
163
-	if err := initDrivers(c); err != nil {
164
-		return nil, err
169
+		ipamDrivers: ipamTable{},
170
+		svcDb:       make(map[string]svcMap),
165 171
 	}
166 172
 
167
-	if cfg != nil {
168
-		if err := c.initGlobalStore(); err != nil {
169
-			// Failing to initalize datastore is a bad situation to be in.
170
-			// But it cannot fail creating the Controller
171
-			log.Debugf("Failed to Initialize Datastore due to %v. Operating in non-clustered mode", err)
172
-		}
173
-		if err := c.initLocalStore(); err != nil {
174
-			log.Debugf("Failed to Initialize LocalDatastore due to %v.", err)
175
-		}
176
-	}
177
-
178
-	if err := initIpams(c, c.localStore, c.globalStore); err != nil {
173
+	if err := c.initStores(); err != nil {
179 174
 		return nil, err
180 175
 	}
181 176
 
182
-	if cfg != nil {
183
-		if err := c.restoreFromGlobalStore(); err != nil {
184
-			log.Debugf("Failed to restore from global Datastore due to %v", err)
185
-		}
177
+	if cfg != nil && cfg.Cluster.Watcher != nil {
186 178
 		if err := c.initDiscovery(cfg.Cluster.Watcher); err != nil {
187 179
 			// Failing to initalize discovery is a bad situation to be in.
188 180
 			// But it cannot fail creating the Controller
189 181
 			log.Debugf("Failed to Initialize Discovery : %v", err)
190 182
 		}
191
-		if err := c.restoreFromLocalStore(); err != nil {
192
-			log.Debugf("Failed to restore from local Datastore due to %v", err)
193
-		}
183
+	}
184
+
185
+	if err := initDrivers(c); err != nil {
186
+		return nil, err
187
+	}
188
+
189
+	if err := initIpams(c, c.getStore(datastore.LocalScope),
190
+		c.getStore(datastore.GlobalScope)); err != nil {
191
+		return nil, err
194 192
 	}
195 193
 
196 194
 	if err := c.startExternalKeyListener(); err != nil {
... ...
@@ -325,15 +323,6 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
325 325
 	if !config.IsValidName(name) {
326 326
 		return nil, ErrInvalidName(name)
327 327
 	}
328
-	// Check if a network already exists with the specified network name
329
-	c.Lock()
330
-	for _, n := range c.networks {
331
-		if n.name == name {
332
-			c.Unlock()
333
-			return nil, NetworkNameError(name)
334
-		}
335
-	}
336
-	c.Unlock()
337 328
 
338 329
 	// Construct the network object
339 330
 	network := &network{
... ...
@@ -342,13 +331,15 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
342 342
 		ipamType:    ipamapi.DefaultIPAM,
343 343
 		id:          stringid.GenerateRandomID(),
344 344
 		ctrlr:       c,
345
-		endpoints:   endpointTable{},
346 345
 		persist:     true,
346
+		drvOnce:     &sync.Once{},
347 347
 	}
348 348
 
349 349
 	network.processOptions(options...)
350 350
 
351
-	if _, err := c.loadNetworkDriver(network); err != nil {
351
+	// Make sure we have a driver available for this network type
352
+	// before we allocate anything.
353
+	if _, err := network.driver(); err != nil {
352 354
 		return nil, err
353 355
 	}
354 356
 
... ...
@@ -364,7 +355,16 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
364 364
 		}
365 365
 	}()
366 366
 
367
-	if err = c.addNetwork(network); err != nil {
367
+	// addNetwork can be called for local scope network lazily when
368
+	// an endpoint is created after a restart and the network was
369
+	// created in previous life. Make sure you wrap around the driver
370
+	// notification of network creation in once call so that the driver
371
+	// invoked only once in case both the network and endpoint creation
372
+	// happens in the same lifetime.
373
+	network.drvOnce.Do(func() {
374
+		err = c.addNetwork(network)
375
+	})
376
+	if err != nil {
368 377
 		return nil, err
369 378
 	}
370 379
 
... ...
@@ -380,35 +380,28 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
380 380
 }
381 381
 
382 382
 func (c *controller) addNetwork(n *network) error {
383
-	if _, err := c.loadNetworkDriver(n); err != nil {
383
+	d, err := n.driver()
384
+	if err != nil {
384 385
 		return err
385 386
 	}
386
-	n.Lock()
387
-	d := n.driver
388
-	n.Unlock()
389 387
 
390 388
 	// Create the network
391 389
 	if err := d.CreateNetwork(n.id, n.generic, n.getIPv4Data(), n.getIPv6Data()); err != nil {
392 390
 		return err
393 391
 	}
394
-	if n.isGlobalScoped() {
395
-		if err := n.watchEndpoints(); err != nil {
396
-			return err
397
-		}
398
-	}
399
-	c.Lock()
400
-	c.networks[n.id] = n
401
-	c.Unlock()
402 392
 
403 393
 	return nil
404 394
 }
405 395
 
406 396
 func (c *controller) Networks() []Network {
407
-	c.Lock()
408
-	defer c.Unlock()
397
+	var list []Network
409 398
 
410
-	list := make([]Network, 0, len(c.networks))
411
-	for _, n := range c.networks {
399
+	networks, err := c.getNetworksFromStore()
400
+	if err != nil {
401
+		log.Error(err)
402
+	}
403
+
404
+	for _, n := range networks {
412 405
 		list = append(list, n)
413 406
 	}
414 407
 
... ...
@@ -450,12 +443,13 @@ func (c *controller) NetworkByID(id string) (Network, error) {
450 450
 	if id == "" {
451 451
 		return nil, ErrInvalidID(id)
452 452
 	}
453
-	c.Lock()
454
-	defer c.Unlock()
455
-	if n, ok := c.networks[id]; ok {
456
-		return n, nil
453
+
454
+	n, err := c.getNetworkFromStore(id)
455
+	if err != nil {
456
+		return nil, ErrNoSuchNetwork(id)
457 457
 	}
458
-	return nil, ErrNoSuchNetwork(id)
458
+
459
+	return n, nil
459 460
 }
460 461
 
461 462
 // NewSandbox creates a new sandbox for the passed container id
... ...
@@ -620,30 +614,7 @@ func (c *controller) getIpamDriver(name string) (ipamapi.Ipam, error) {
620 620
 }
621 621
 
622 622
 func (c *controller) Stop() {
623
-	if c.localStore != nil {
624
-		c.localStore.KVStore().Close()
625
-	}
623
+	c.closeStores()
626 624
 	c.stopExternalKeyListener()
627 625
 	osl.GC()
628 626
 }
629
-
630
-func (c *controller) loadNetworkDriver(n *network) (driverapi.Driver, error) {
631
-	// Check if a driver for the specified network type is available
632
-	c.Lock()
633
-	dd, ok := c.drivers[n.networkType]
634
-	c.Unlock()
635
-	if !ok {
636
-		var err error
637
-		dd, err = c.loadDriver(n.networkType)
638
-		if err != nil {
639
-			return nil, err
640
-		}
641
-	}
642
-
643
-	n.Lock()
644
-	n.svcRecords = svcMap{}
645
-	n.driver = dd.driver
646
-	n.dataScope = dd.capability.DataScope
647
-	n.Unlock()
648
-	return dd.driver, nil
649
-}
... ...
@@ -1,10 +1,6 @@
1 1
 package driverapi
2 2
 
3
-import (
4
-	"net"
5
-
6
-	"github.com/docker/libnetwork/datastore"
7
-)
3
+import "net"
8 4
 
9 5
 // NetworkPluginEndpointType represents the Endpoint Type used by Plugin system
10 6
 const NetworkPluginEndpointType = "NetworkDriver"
... ...
@@ -105,7 +101,7 @@ type DriverCallback interface {
105 105
 
106 106
 // Capability represents the high level capabilities of the drivers which libnetwork can make use of
107 107
 type Capability struct {
108
-	DataScope datastore.DataScope
108
+	DataScope string
109 109
 }
110 110
 
111 111
 // DiscoveryType represents the type of discovery element the DiscoverNew function is invoked on
... ...
@@ -3,6 +3,7 @@ package libnetwork
3 3
 import (
4 4
 	"strings"
5 5
 
6
+	"github.com/docker/libnetwork/datastore"
6 7
 	"github.com/docker/libnetwork/driverapi"
7 8
 	"github.com/docker/libnetwork/ipamapi"
8 9
 	builtinIpam "github.com/docker/libnetwork/ipams/builtin"
... ...
@@ -32,9 +33,9 @@ func makeDriverConfig(c *controller, ntype string) map[string]interface{} {
32 32
 
33 33
 	config := make(map[string]interface{})
34 34
 
35
-	if c.validateGlobalStoreConfig() {
36
-		config[netlabel.KVProvider] = c.cfg.GlobalStore.Client.Provider
37
-		config[netlabel.KVProviderURL] = c.cfg.GlobalStore.Client.Address
35
+	if dcfg, ok := c.cfg.Scopes[datastore.GlobalScope]; ok && dcfg.IsValid() {
36
+		config[netlabel.KVProvider] = dcfg.Client.Provider
37
+		config[netlabel.KVProviderURL] = dcfg.Client.Address
38 38
 	}
39 39
 
40 40
 	for _, label := range c.cfg.Daemon.Labels {
... ...
@@ -43,12 +43,16 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,
43 43
 		return err
44 44
 	}
45 45
 
46
+	// Since we perform lazy configuration make sure we try
47
+	// configuring the driver when we enter CreateEndpoint since
48
+	// CreateNetwork may not be called in every node.
49
+	if err := d.configure(); err != nil {
50
+		return err
51
+	}
52
+
46 53
 	n := d.network(nid)
47 54
 	if n == nil {
48
-		n, err = d.createNetworkfromStore(nid)
49
-		if err != nil {
50
-			return fmt.Errorf("network id %q not found", nid)
51
-		}
55
+		return fmt.Errorf("network id %q not found", nid)
52 56
 	}
53 57
 
54 58
 	ep := &endpoint{
... ...
@@ -45,12 +45,13 @@ type network struct {
45 45
 }
46 46
 
47 47
 func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error {
48
-	var err error
49 48
 	if id == "" {
50 49
 		return fmt.Errorf("invalid network id")
51 50
 	}
52 51
 
53
-	if err = d.configure(); err != nil {
52
+	// Since we perform lazy configuration make sure we try
53
+	// configuring the driver when we enter CreateNetwork
54
+	if err := d.configure(); err != nil {
54 55
 		return err
55 56
 	}
56 57
 
... ...
@@ -71,29 +72,16 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
71 71
 		n.subnets = append(n.subnets, s)
72 72
 	}
73 73
 
74
-	for {
75
-		// If the datastore has the network object already
76
-		// there is no need to do a write.
77
-		err = d.store.GetObject(datastore.Key(n.Key()...), n)
78
-		if err == nil || err != datastore.ErrKeyNotFound {
79
-			break
80
-		}
81
-
82
-		err = n.writeToStore()
83
-		if err == nil || err != datastore.ErrKeyModified {
84
-			break
85
-		}
86
-	}
87
-
88
-	if err != nil {
74
+	if err := n.writeToStore(); err != nil {
89 75
 		return fmt.Errorf("failed to update data store for network %v: %v", n.id, err)
90 76
 	}
77
+
91 78
 	d.addNetwork(n)
92 79
 
93 80
 	return nil
94 81
 }
95 82
 
96
-func (d *driver) createNetworkfromStore(nid string) (*network, error) {
83
+/* func (d *driver) createNetworkfromStore(nid string) (*network, error) {
97 84
 	n := &network{
98 85
 		id:        nid,
99 86
 		driver:    d,
... ...
@@ -107,7 +95,7 @@ func (d *driver) createNetworkfromStore(nid string) (*network, error) {
107 107
 		return nil, fmt.Errorf("unable to get network %q from data store, %v", nid, err)
108 108
 	}
109 109
 	return n, nil
110
-}
110
+}*/
111 111
 
112 112
 func (d *driver) DeleteNetwork(nid string) error {
113 113
 	if nid == "" {
... ...
@@ -313,9 +301,34 @@ func (d *driver) deleteNetwork(nid string) {
313 313
 
314 314
 func (d *driver) network(nid string) *network {
315 315
 	d.Lock()
316
-	defer d.Unlock()
316
+	networks := d.networks
317
+	d.Unlock()
317 318
 
318
-	return d.networks[nid]
319
+	n, ok := networks[nid]
320
+	if !ok {
321
+		n = d.getNetworkFromStore(nid)
322
+		if n != nil {
323
+			n.driver = d
324
+			n.endpoints = endpointTable{}
325
+			n.once = &sync.Once{}
326
+			networks[nid] = n
327
+		}
328
+	}
329
+
330
+	return n
331
+}
332
+
333
+func (d *driver) getNetworkFromStore(nid string) *network {
334
+	if d.store == nil {
335
+		return nil
336
+	}
337
+
338
+	n := &network{id: nid}
339
+	if err := d.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
340
+		return nil
341
+	}
342
+
343
+	return n
319 344
 }
320 345
 
321 346
 func (n *network) sandbox() osl.Sandbox {
... ...
@@ -408,30 +421,23 @@ func (n *network) SetValue(value []byte) error {
408 408
 	subnetIP, _ := types.ParseCIDR(subnetIPstr)
409 409
 	gwIP, _ := types.ParseCIDR(gwIPstr)
410 410
 
411
-	// If the network is being created by reading from the
412
-	// datastore subnets have to created. If the network
413
-	// already exists update only the subnets' vni field
414
-	if len(n.subnets) == 0 {
415
-		s := &subnet{
416
-			subnetIP: subnetIP,
417
-			gwIP:     gwIP,
418
-			vni:      vni,
419
-			once:     &sync.Once{},
420
-		}
421
-		n.subnets = append(n.subnets, s)
422
-		return nil
411
+	s := &subnet{
412
+		subnetIP: subnetIP,
413
+		gwIP:     gwIP,
414
+		vni:      vni,
415
+		once:     &sync.Once{},
423 416
 	}
417
+	n.subnets = append(n.subnets, s)
424 418
 
425 419
 	sNet := n.getMatchingSubnet(subnetIP)
426 420
 	if sNet != nil {
427
-		if vni != 0 {
428
-			sNet.vni = vni
429
-		}
421
+		sNet.vni = vni
430 422
 	}
423
+
431 424
 	return nil
432 425
 }
433 426
 
434
-func (n *network) DataScope() datastore.DataScope {
427
+func (n *network) DataScope() string {
435 428
 	return datastore.GlobalScope
436 429
 }
437 430
 
... ...
@@ -6,7 +6,6 @@ import (
6 6
 
7 7
 	"github.com/Sirupsen/logrus"
8 8
 	"github.com/docker/libkv/store"
9
-	"github.com/docker/libnetwork/config"
10 9
 	"github.com/docker/libnetwork/datastore"
11 10
 	"github.com/docker/libnetwork/driverapi"
12 11
 	"github.com/docker/libnetwork/idm"
... ...
@@ -84,8 +83,8 @@ func (d *driver) configure() error {
84 84
 		provURL, urlOk := d.config[netlabel.KVProviderURL]
85 85
 
86 86
 		if provOk && urlOk {
87
-			cfg := &config.DatastoreCfg{
88
-				Client: config.DatastoreClientCfg{
87
+			cfg := &datastore.ScopeCfg{
88
+				Client: datastore.ScopeClientCfg{
89 89
 					Provider: provider.(string),
90 90
 					Address:  provURL.(string),
91 91
 				},
... ...
@@ -94,7 +93,7 @@ func (d *driver) configure() error {
94 94
 			if confOk {
95 95
 				cfg.Client.Config = provConfig.(*store.Config)
96 96
 			}
97
-			d.store, err = datastore.NewDataStore(cfg)
97
+			d.store, err = datastore.NewDataStore(datastore.GlobalScope, cfg)
98 98
 			if err != nil {
99 99
 				err = fmt.Errorf("failed to initialize data store: %v", err)
100 100
 				return
... ...
@@ -12,6 +12,7 @@ import (
12 12
 	"github.com/docker/libnetwork/datastore"
13 13
 	"github.com/docker/libnetwork/ipamapi"
14 14
 	"github.com/docker/libnetwork/netlabel"
15
+	"github.com/docker/libnetwork/options"
15 16
 	"github.com/docker/libnetwork/types"
16 17
 )
17 18
 
... ...
@@ -107,6 +108,37 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
107 107
 	return nil
108 108
 }
109 109
 
110
+func (ep *endpoint) New() datastore.KVObject {
111
+	return &endpoint{network: ep.getNetwork()}
112
+}
113
+
114
+func (ep *endpoint) CopyTo(o datastore.KVObject) error {
115
+	ep.Lock()
116
+	defer ep.Unlock()
117
+
118
+	dstEp := o.(*endpoint)
119
+	dstEp.name = ep.name
120
+	dstEp.id = ep.id
121
+	dstEp.sandboxID = ep.sandboxID
122
+	dstEp.dbIndex = ep.dbIndex
123
+	dstEp.dbExists = ep.dbExists
124
+
125
+	if ep.iface != nil {
126
+		dstEp.iface = &endpointInterface{}
127
+		ep.iface.CopyTo(dstEp.iface)
128
+	}
129
+
130
+	dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
131
+	copy(dstEp.exposedPorts, ep.exposedPorts)
132
+
133
+	dstEp.generic = options.Generic{}
134
+	for k, v := range ep.generic {
135
+		dstEp.generic[k] = v
136
+	}
137
+
138
+	return nil
139
+}
140
+
110 141
 func (ep *endpoint) ID() string {
111 142
 	ep.Lock()
112 143
 	defer ep.Unlock()
... ...
@@ -122,16 +154,28 @@ func (ep *endpoint) Name() string {
122 122
 }
123 123
 
124 124
 func (ep *endpoint) Network() string {
125
-	return ep.getNetwork().name
125
+	if ep.network == nil {
126
+		return ""
127
+	}
128
+
129
+	return ep.network.name
126 130
 }
127 131
 
128 132
 // endpoint Key structure : endpoint/network-id/endpoint-id
129 133
 func (ep *endpoint) Key() []string {
130
-	return []string{datastore.EndpointKeyPrefix, ep.getNetwork().id, ep.id}
134
+	if ep.network == nil {
135
+		return nil
136
+	}
137
+
138
+	return []string{datastore.EndpointKeyPrefix, ep.network.id, ep.id}
131 139
 }
132 140
 
133 141
 func (ep *endpoint) KeyPrefix() []string {
134
-	return []string{datastore.EndpointKeyPrefix, ep.getNetwork().id}
142
+	if ep.network == nil {
143
+		return nil
144
+	}
145
+
146
+	return []string{datastore.EndpointKeyPrefix, ep.network.id}
135 147
 }
136 148
 
137 149
 func (ep *endpoint) networkIDFromKey(key string) (string, error) {
... ...
@@ -177,7 +221,7 @@ func (ep *endpoint) Exists() bool {
177 177
 }
178 178
 
179 179
 func (ep *endpoint) Skip() bool {
180
-	return ep.getNetwork().Skip()
180
+	return ep.getNetwork().Skip() || ep.DataScope() == datastore.LocalScope
181 181
 }
182 182
 
183 183
 func (ep *endpoint) processOptions(options ...EndpointOption) {
... ...
@@ -191,8 +235,22 @@ func (ep *endpoint) processOptions(options ...EndpointOption) {
191 191
 	}
192 192
 }
193 193
 
194
-func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error {
194
+func (ep *endpoint) getNetwork() *network {
195
+	ep.Lock()
196
+	defer ep.Unlock()
197
+
198
+	return ep.network
199
+}
200
+
201
+func (ep *endpoint) getNetworkFromStore() (*network, error) {
202
+	if ep.network == nil {
203
+		return nil, fmt.Errorf("invalid network object in endpoint %s", ep.Name())
204
+	}
205
+
206
+	return ep.network.ctrlr.getNetworkFromStore(ep.network.id)
207
+}
195 208
 
209
+func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error {
196 210
 	if sbox == nil {
197 211
 		return types.BadRequestErrorf("endpoint cannot be joined by nil container")
198 212
 	}
... ...
@@ -215,15 +273,27 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
215 215
 		return types.BadRequestErrorf("not a valid Sandbox interface")
216 216
 	}
217 217
 
218
+	network, err := ep.getNetworkFromStore()
219
+	if err != nil {
220
+		return fmt.Errorf("failed to get network from store during join: %v", err)
221
+	}
222
+
223
+	ep, err = network.getEndpointFromStore(ep.ID())
224
+	if err != nil {
225
+		return fmt.Errorf("failed to get endpoint from store during join: %v", err)
226
+	}
227
+
218 228
 	ep.Lock()
219 229
 	if ep.sandboxID != "" {
220 230
 		ep.Unlock()
221 231
 		return types.ForbiddenErrorf("a sandbox has already joined the endpoint")
222 232
 	}
233
+	ep.Unlock()
223 234
 
235
+	ep.Lock()
236
+	ep.network = network
224 237
 	ep.sandboxID = sbox.ID()
225 238
 	ep.joinInfo = &endpointJoinInfo{}
226
-	network := ep.network
227 239
 	epid := ep.id
228 240
 	ep.Unlock()
229 241
 	defer func() {
... ...
@@ -235,12 +305,16 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
235 235
 	}()
236 236
 
237 237
 	network.Lock()
238
-	driver := network.driver
239 238
 	nid := network.id
240 239
 	network.Unlock()
241 240
 
242 241
 	ep.processOptions(options...)
243 242
 
243
+	driver, err := network.driver()
244
+	if err != nil {
245
+		return fmt.Errorf("failed to join endpoint: %v", err)
246
+	}
247
+
244 248
 	err = driver.Join(nid, epid, sbox.Key(), ep, sbox.Labels())
245 249
 	if err != nil {
246 250
 		return err
... ...
@@ -262,14 +336,15 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
262 262
 		return err
263 263
 	}
264 264
 
265
-	if err = sb.updateDNS(ep.getNetwork().enableIPv6); err != nil {
265
+	// Watch for service records
266
+	network.getController().watchSvcRecord(ep)
267
+
268
+	if err = sb.updateDNS(network.enableIPv6); err != nil {
266 269
 		return err
267 270
 	}
268 271
 
269
-	if !ep.isLocalScoped() {
270
-		if err = network.ctrlr.updateToStore(ep); err != nil {
271
-			return err
272
-		}
272
+	if err = network.getController().updateToStore(ep); err != nil {
273
+		return err
273 274
 	}
274 275
 
275 276
 	sb.Lock()
... ...
@@ -327,6 +402,16 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
327 327
 		return types.BadRequestErrorf("not a valid Sandbox interface")
328 328
 	}
329 329
 
330
+	n, err := ep.getNetworkFromStore()
331
+	if err != nil {
332
+		return fmt.Errorf("failed to get network from store during leave: %v", err)
333
+	}
334
+
335
+	ep, err = n.getEndpointFromStore(ep.ID())
336
+	if err != nil {
337
+		return fmt.Errorf("failed to get endpoint from store during leave: %v", err)
338
+	}
339
+
330 340
 	ep.Lock()
331 341
 	sid := ep.sandboxID
332 342
 	ep.Unlock()
... ...
@@ -342,21 +427,19 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
342 342
 
343 343
 	ep.Lock()
344 344
 	ep.sandboxID = ""
345
-	n := ep.network
345
+	ep.network = n
346 346
 	ep.Unlock()
347 347
 
348
-	n.Lock()
349
-	c := n.ctrlr
350
-	d := n.driver
351
-	n.Unlock()
348
+	if err := n.getController().updateToStore(ep); err != nil {
349
+		ep.Lock()
350
+		ep.sandboxID = sid
351
+		ep.Unlock()
352
+		return err
353
+	}
352 354
 
353
-	if !ep.isLocalScoped() {
354
-		if err := c.updateToStore(ep); err != nil {
355
-			ep.Lock()
356
-			ep.sandboxID = sid
357
-			ep.Unlock()
358
-			return err
359
-		}
355
+	d, err := n.driver()
356
+	if err != nil {
357
+		return fmt.Errorf("failed to leave endpoint: %v", err)
360 358
 	}
361 359
 
362 360
 	if err := d.Leave(n.id, ep.id); err != nil {
... ...
@@ -367,6 +450,9 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
367 367
 		return err
368 368
 	}
369 369
 
370
+	// unwatch for service records
371
+	n.getController().unWatchSvcRecord(ep)
372
+
370 373
 	if sb.needDefaultGW() {
371 374
 		ep := sb.getEPwithoutGateway()
372 375
 		if ep == nil {
... ...
@@ -379,45 +465,44 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
379 379
 
380 380
 func (ep *endpoint) Delete() error {
381 381
 	var err error
382
+	n, err := ep.getNetworkFromStore()
383
+	if err != nil {
384
+		return fmt.Errorf("failed to get network during Delete: %v", err)
385
+	}
386
+
387
+	ep, err = n.getEndpointFromStore(ep.ID())
388
+	if err != nil {
389
+		return fmt.Errorf("failed to get endpoint from store during Delete: %v", err)
390
+	}
391
+
382 392
 	ep.Lock()
383 393
 	epid := ep.id
384 394
 	name := ep.name
385
-	n := ep.network
386 395
 	if ep.sandboxID != "" {
387 396
 		ep.Unlock()
388 397
 		return &ActiveContainerError{name: name, id: epid}
389 398
 	}
390
-	n.Lock()
391
-	ctrlr := n.ctrlr
392
-	n.Unlock()
393 399
 	ep.Unlock()
394 400
 
395
-	if !ep.isLocalScoped() {
396
-		if err = ctrlr.deleteFromStore(ep); err != nil {
397
-			return err
398
-		}
401
+	if err = n.DecEndpointCnt(); err != nil {
402
+		return err
399 403
 	}
400 404
 	defer func() {
401 405
 		if err != nil {
402
-			ep.dbExists = false
403
-			if !ep.isLocalScoped() {
404
-				if e := ctrlr.updateToStore(ep); e != nil {
405
-					log.Warnf("failed to recreate endpoint in store %s : %v", name, e)
406
-				}
406
+			if e := n.IncEndpointCnt(); e != nil {
407
+				log.Warnf("failed to update network %s : %v", n.name, e)
407 408
 			}
408 409
 		}
409 410
 	}()
410 411
 
411
-	// Update the endpoint count in network and update it in the datastore
412
-	n.DecEndpointCnt()
413
-	if err = ctrlr.updateToStore(n); err != nil {
412
+	if err = n.getController().deleteFromStore(ep); err != nil {
414 413
 		return err
415 414
 	}
416 415
 	defer func() {
417 416
 		if err != nil {
418
-			n.IncEndpointCnt()
419
-			if e := ctrlr.updateToStore(n); e != nil {
420
-				log.Warnf("failed to update network %s : %v", n.name, e)
417
+			ep.dbExists = false
418
+			if e := n.getController().updateToStore(ep); e != nil {
419
+				log.Warnf("failed to recreate endpoint in store %s : %v", name, e)
421 420
 			}
422 421
 		}
423 422
 	}()
... ...
@@ -438,38 +523,21 @@ func (ep *endpoint) deleteEndpoint() error {
438 438
 	epid := ep.id
439 439
 	ep.Unlock()
440 440
 
441
-	n.Lock()
442
-	_, ok := n.endpoints[epid]
443
-	if !ok {
444
-		n.Unlock()
445
-		return nil
441
+	driver, err := n.driver()
442
+	if err != nil {
443
+		return fmt.Errorf("failed to delete endpoint: %v", err)
446 444
 	}
447 445
 
448
-	nid := n.id
449
-	driver := n.driver
450
-	delete(n.endpoints, epid)
451
-	n.Unlock()
452
-
453
-	if err := driver.DeleteEndpoint(nid, epid); err != nil {
446
+	if err := driver.DeleteEndpoint(n.id, epid); err != nil {
454 447
 		if _, ok := err.(types.ForbiddenError); ok {
455
-			n.Lock()
456
-			n.endpoints[epid] = ep
457
-			n.Unlock()
458 448
 			return err
459 449
 		}
460 450
 		log.Warnf("driver error deleting endpoint %s : %v", name, err)
461 451
 	}
462 452
 
463
-	n.updateSvcRecord(ep, false)
464 453
 	return nil
465 454
 }
466 455
 
467
-func (ep *endpoint) getNetwork() *network {
468
-	ep.Lock()
469
-	defer ep.Unlock()
470
-	return ep.network
471
-}
472
-
473 456
 func (ep *endpoint) getSandbox() (*sandbox, bool) {
474 457
 	ep.Lock()
475 458
 	c := ep.network.getController()
... ...
@@ -545,14 +613,8 @@ func JoinOptionPriority(ep Endpoint, prio int) EndpointOption {
545 545
 	}
546 546
 }
547 547
 
548
-func (ep *endpoint) DataScope() datastore.DataScope {
549
-	ep.Lock()
550
-	defer ep.Unlock()
551
-	return ep.network.dataScope
552
-}
553
-
554
-func (ep *endpoint) isLocalScoped() bool {
555
-	return ep.DataScope() == datastore.LocalScope
548
+func (ep *endpoint) DataScope() string {
549
+	return ep.getNetwork().DataScope()
556 550
 }
557 551
 
558 552
 func (ep *endpoint) assignAddress() error {
... ...
@@ -2,6 +2,7 @@ package libnetwork
2 2
 
3 3
 import (
4 4
 	"encoding/json"
5
+	"fmt"
5 6
 	"net"
6 7
 
7 8
 	"github.com/docker/libnetwork/driverapi"
... ...
@@ -115,6 +116,21 @@ func (epi *endpointInterface) UnmarshalJSON(b []byte) error {
115 115
 	return nil
116 116
 }
117 117
 
118
+func (epi *endpointInterface) CopyTo(dstEpi *endpointInterface) error {
119
+	dstEpi.mac = types.GetMacCopy(epi.mac)
120
+	dstEpi.addr = types.GetIPNetCopy(epi.addr)
121
+	dstEpi.addrv6 = types.GetIPNetCopy(epi.addrv6)
122
+	dstEpi.srcName = epi.srcName
123
+	dstEpi.dstPrefix = epi.dstPrefix
124
+	dstEpi.poolID = epi.poolID
125
+
126
+	for _, route := range epi.routes {
127
+		dstEpi.routes = append(dstEpi.routes, types.GetIPNetCopy(route))
128
+	}
129
+
130
+	return nil
131
+}
132
+
118 133
 type endpointJoinInfo struct {
119 134
 	gw           net.IP
120 135
 	gw6          net.IP
... ...
@@ -122,21 +138,38 @@ type endpointJoinInfo struct {
122 122
 }
123 123
 
124 124
 func (ep *endpoint) Info() EndpointInfo {
125
-	return ep
125
+	n, err := ep.getNetworkFromStore()
126
+	if err != nil {
127
+		return nil
128
+	}
129
+
130
+	ep, err = n.getEndpointFromStore(ep.ID())
131
+	if err != nil {
132
+		return nil
133
+	}
134
+
135
+	sb, ok := ep.getSandbox()
136
+	if !ok {
137
+		// endpoint hasn't joined any sandbox.
138
+		// Just return the endpoint
139
+		return ep
140
+	}
141
+
142
+	return sb.getEndpoint(ep.ID())
126 143
 }
127 144
 
128 145
 func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
129
-	ep.Lock()
130
-	network := ep.network
131
-	epid := ep.id
132
-	ep.Unlock()
146
+	n, err := ep.getNetworkFromStore()
147
+	if err != nil {
148
+		return nil, fmt.Errorf("could not find network in store for driver info: %v", err)
149
+	}
133 150
 
134
-	network.Lock()
135
-	driver := network.driver
136
-	nid := network.id
137
-	network.Unlock()
151
+	driver, err := n.driver()
152
+	if err != nil {
153
+		return nil, fmt.Errorf("failed to get driver info: %v", err)
154
+	}
138 155
 
139
-	return driver.EndpointOperInfo(nid, epid)
156
+	return driver.EndpointOperInfo(n.ID(), ep.ID())
140 157
 }
141 158
 
142 159
 func (ep *endpoint) Iface() InterfaceInfo {
... ...
@@ -6,7 +6,6 @@ import (
6 6
 	"net"
7 7
 	"testing"
8 8
 
9
-	"github.com/docker/libnetwork/datastore"
10 9
 	"github.com/docker/libnetwork/driverapi"
11 10
 	"github.com/docker/libnetwork/netlabel"
12 11
 	"github.com/docker/libnetwork/types"
... ...
@@ -32,11 +31,6 @@ func TestDriverRegistration(t *testing.T) {
32 32
 	}
33 33
 }
34 34
 
35
-func SetTestDataStore(c NetworkController, custom datastore.DataStore) {
36
-	con := c.(*controller)
37
-	con.globalStore = custom
38
-}
39
-
40 35
 func TestNetworkMarshalling(t *testing.T) {
41 36
 	n := &network{
42 37
 		name:        "Miao",
... ...
@@ -50,7 +50,7 @@ func TestMain(m *testing.M) {
50 50
 		os.Exit(1)
51 51
 	}
52 52
 
53
-	libnetwork.SetTestDataStore(controller, datastore.NewCustomDataStore(datastore.NewMockStore()))
53
+	//libnetwork.SetTestDataStore(controller, datastore.NewCustomDataStore(datastore.NewMockStore()))
54 54
 
55 55
 	x := m.Run()
56 56
 	controller.Stop()
... ...
@@ -60,6 +60,9 @@ func TestMain(m *testing.M) {
60 60
 func createController() error {
61 61
 	var err error
62 62
 
63
+	// Cleanup local datastore file
64
+	os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
65
+
63 66
 	option := options.Generic{
64 67
 		"EnableIPForwarding": true,
65 68
 	}
... ...
@@ -358,27 +361,6 @@ func TestNilRemoteDriver(t *testing.T) {
358 358
 	}
359 359
 }
360 360
 
361
-func TestDuplicateNetwork(t *testing.T) {
362
-	if !testutils.IsRunningInContainer() {
363
-		defer testutils.SetupTestOSContext(t)()
364
-	}
365
-
366
-	// Creating a default bridge name network (can't be removed)
367
-	_, err := controller.NewNetwork(bridgeNetType, "testdup")
368
-	if err != nil {
369
-		t.Fatal(err)
370
-	}
371
-
372
-	_, err = controller.NewNetwork(bridgeNetType, "testdup")
373
-	if err == nil {
374
-		t.Fatal("Expected to fail. But instead succeeded")
375
-	}
376
-
377
-	if _, ok := err.(libnetwork.NetworkNameError); !ok {
378
-		t.Fatalf("Did not fail with expected error. Actual error: %v", err)
379
-	}
380
-}
381
-
382 361
 func TestNetworkName(t *testing.T) {
383 362
 	if !testutils.IsRunningInContainer() {
384 363
 		defer testutils.SetupTestOSContext(t)()
... ...
@@ -703,7 +685,7 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
703 703
 	if netWanted == nil {
704 704
 		t.Fatal(err)
705 705
 	}
706
-	if net1 != netWanted {
706
+	if net1.ID() != netWanted.ID() {
707 707
 		t.Fatal(err)
708 708
 	}
709 709
 
... ...
@@ -712,7 +694,7 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
712 712
 	if netWanted == nil {
713 713
 		t.Fatal(err)
714 714
 	}
715
-	if net2 != netWanted {
715
+	if net2.ID() != netWanted.ID() {
716 716
 		t.Fatal(err)
717 717
 	}
718 718
 }
... ...
@@ -843,7 +825,7 @@ func TestControllerQuery(t *testing.T) {
843 843
 	if err != nil {
844 844
 		t.Fatalf("Unexpected failure for NetworkByID(): %v", err)
845 845
 	}
846
-	if net1 != g {
846
+	if net1.ID() != g.ID() {
847 847
 		t.Fatalf("NetworkByID() returned unexpected element: %v", g)
848 848
 	}
849 849
 
... ...
@@ -863,7 +845,7 @@ func TestControllerQuery(t *testing.T) {
863 863
 	if err != nil {
864 864
 		t.Fatalf("Unexpected failure for NetworkByID(): %v", err)
865 865
 	}
866
-	if net2 != g {
866
+	if net2.ID() != g.ID() {
867 867
 		t.Fatalf("NetworkByID() returned unexpected element: %v", g)
868 868
 	}
869 869
 }
... ...
@@ -940,7 +922,7 @@ func TestNetworkQuery(t *testing.T) {
940 940
 	if err != nil {
941 941
 		t.Fatal(err)
942 942
 	}
943
-	if ep12 != e {
943
+	if ep12.ID() != e.ID() {
944 944
 		t.Fatalf("EndpointByID() returned %v instead of %v", e, ep12)
945 945
 	}
946 946
 
... ...
@@ -2,6 +2,7 @@ package libnetwork
2 2
 
3 3
 import (
4 4
 	"encoding/json"
5
+	"fmt"
5 6
 	"net"
6 7
 	"sync"
7 8
 
... ...
@@ -127,7 +128,6 @@ type network struct {
127 127
 	networkType  string
128 128
 	id           string
129 129
 	ipamType     string
130
-	driver       driverapi.Driver
131 130
 	addrSpace    string
132 131
 	ipamV4Config []*IpamConf
133 132
 	ipamV6Config []*IpamConf
... ...
@@ -135,14 +135,14 @@ type network struct {
135 135
 	ipamV6Info   []*IpamInfo
136 136
 	enableIPv6   bool
137 137
 	endpointCnt  uint64
138
-	endpoints    endpointTable
139 138
 	generic      options.Generic
140 139
 	dbIndex      uint64
141 140
 	svcRecords   svcMap
142 141
 	dbExists     bool
143 142
 	persist      bool
144 143
 	stopWatchCh  chan struct{}
145
-	dataScope    datastore.DataScope
144
+	scope        string
145
+	drvOnce      *sync.Once
146 146
 	sync.Mutex
147 147
 }
148 148
 
... ...
@@ -164,11 +164,7 @@ func (n *network) Type() string {
164 164
 	n.Lock()
165 165
 	defer n.Unlock()
166 166
 
167
-	if n.driver == nil {
168
-		return ""
169
-	}
170
-
171
-	return n.driver.Type()
167
+	return n.networkType
172 168
 }
173 169
 
174 170
 func (n *network) Key() []string {
... ...
@@ -220,10 +216,72 @@ func (n *network) Skip() bool {
220 220
 	return !n.persist
221 221
 }
222 222
 
223
-func (n *network) DataScope() datastore.DataScope {
223
+func (n *network) New() datastore.KVObject {
224
+	n.Lock()
225
+	defer n.Unlock()
226
+
227
+	return &network{
228
+		ctrlr:   n.ctrlr,
229
+		drvOnce: &sync.Once{},
230
+	}
231
+}
232
+
233
+// CopyTo deep copies to the destination IpamInfo
234
+func (i *IpamInfo) CopyTo(dstI *IpamInfo) error {
235
+	dstI.PoolID = i.PoolID
236
+	if i.Meta != nil {
237
+		dstI.Meta = make(map[string]string)
238
+		for k, v := range i.Meta {
239
+			dstI.Meta[k] = v
240
+		}
241
+	}
242
+
243
+	dstI.AddressSpace = i.AddressSpace
244
+	dstI.Pool = types.GetIPNetCopy(i.Pool)
245
+	dstI.Gateway = types.GetIPNetCopy(i.Gateway)
246
+
247
+	if i.AuxAddresses != nil {
248
+		dstI.AuxAddresses = make(map[string]*net.IPNet)
249
+		for k, v := range i.AuxAddresses {
250
+			dstI.AuxAddresses[k] = types.GetIPNetCopy(v)
251
+		}
252
+	}
253
+
254
+	return nil
255
+}
256
+
257
+func (n *network) CopyTo(o datastore.KVObject) error {
224 258
 	n.Lock()
225 259
 	defer n.Unlock()
226
-	return n.dataScope
260
+
261
+	dstN := o.(*network)
262
+	dstN.name = n.name
263
+	dstN.id = n.id
264
+	dstN.networkType = n.networkType
265
+	dstN.ipamType = n.ipamType
266
+	dstN.endpointCnt = n.endpointCnt
267
+	dstN.enableIPv6 = n.enableIPv6
268
+	dstN.persist = n.persist
269
+	dstN.dbIndex = n.dbIndex
270
+	dstN.dbExists = n.dbExists
271
+	dstN.drvOnce = n.drvOnce
272
+
273
+	for _, v4info := range n.ipamV4Info {
274
+		dstV4Info := &IpamInfo{}
275
+		v4info.CopyTo(dstV4Info)
276
+		dstN.ipamV4Info = append(dstN.ipamV4Info, dstV4Info)
277
+	}
278
+
279
+	dstN.generic = options.Generic{}
280
+	for k, v := range n.generic {
281
+		dstN.generic[k] = v
282
+	}
283
+
284
+	return nil
285
+}
286
+
287
+func (n *network) DataScope() string {
288
+	return n.driverScope()
227 289
 }
228 290
 
229 291
 func (n *network) EndpointCnt() uint64 {
... ...
@@ -232,16 +290,20 @@ func (n *network) EndpointCnt() uint64 {
232 232
 	return n.endpointCnt
233 233
 }
234 234
 
235
-func (n *network) IncEndpointCnt() {
235
+func (n *network) IncEndpointCnt() error {
236 236
 	n.Lock()
237 237
 	n.endpointCnt++
238 238
 	n.Unlock()
239
+
240
+	return n.getController().updateToStore(n)
239 241
 }
240 242
 
241
-func (n *network) DecEndpointCnt() {
243
+func (n *network) DecEndpointCnt() error {
242 244
 	n.Lock()
243 245
 	n.endpointCnt--
244 246
 	n.Unlock()
247
+
248
+	return n.getController().updateToStore(n)
245 249
 }
246 250
 
247 251
 // TODO : Can be made much more generic with the help of reflection (but has some golang limitations)
... ...
@@ -372,17 +434,55 @@ func (n *network) processOptions(options ...NetworkOption) {
372 372
 	}
373 373
 }
374 374
 
375
-func (n *network) Delete() error {
376
-	var err error
375
+func (n *network) driverScope() string {
376
+	c := n.getController()
377
+
378
+	c.Lock()
379
+	// Check if a driver for the specified network type is available
380
+	dd, ok := c.drivers[n.networkType]
381
+	c.Unlock()
382
+
383
+	if !ok {
384
+		var err error
385
+		dd, err = c.loadDriver(n.networkType)
386
+		if err != nil {
387
+			// If driver could not be resolved simply return an empty string
388
+			return ""
389
+		}
390
+	}
391
+
392
+	return dd.capability.DataScope
393
+}
377 394
 
378
-	ctrlr := n.getController()
395
+func (n *network) driver() (driverapi.Driver, error) {
396
+	c := n.getController()
379 397
 
380
-	ctrlr.Lock()
381
-	_, ok := ctrlr.networks[n.id]
382
-	ctrlr.Unlock()
398
+	c.Lock()
399
+	// Check if a driver for the specified network type is available
400
+	dd, ok := c.drivers[n.networkType]
401
+	c.Unlock()
383 402
 
384 403
 	if !ok {
385
-		return &UnknownNetworkError{name: n.name, id: n.id}
404
+		var err error
405
+		dd, err = c.loadDriver(n.networkType)
406
+		if err != nil {
407
+			return nil, err
408
+		}
409
+	}
410
+
411
+	return dd.driver, nil
412
+}
413
+
414
+func (n *network) Delete() error {
415
+	n.Lock()
416
+	c := n.ctrlr
417
+	name := n.name
418
+	id := n.id
419
+	n.Unlock()
420
+
421
+	n, err := c.getNetworkFromStore(id)
422
+	if err != nil {
423
+		return &UnknownNetworkError{name: name, id: id}
386 424
 	}
387 425
 
388 426
 	numEps := n.EndpointCnt()
... ...
@@ -390,9 +490,22 @@ func (n *network) Delete() error {
390 390
 		return &ActiveEndpointsError{name: n.name, id: n.id}
391 391
 	}
392 392
 
393
-	// deleteNetworkFromStore performs an atomic delete operation and the network.endpointCnt field will help
394
-	// prevent any possible race between endpoint join and network delete
395
-	if err = ctrlr.deleteFromStore(n); err != nil {
393
+	if err = n.deleteNetwork(); err != nil {
394
+		return err
395
+	}
396
+	defer func() {
397
+		if err != nil {
398
+			if e := c.addNetwork(n); e != nil {
399
+				log.Warnf("failed to rollback deleteNetwork for network %s: %v",
400
+					n.Name(), err)
401
+			}
402
+		}
403
+	}()
404
+
405
+	// deleteFromStore performs an atomic delete operation and the
406
+	// network.endpointCnt field will help prevent any possible
407
+	// race between endpoint join and network delete
408
+	if err = n.getController().deleteFromStore(n); err != nil {
396 409
 		if err == datastore.ErrKeyModified {
397 410
 			return types.InternalErrorf("operation in progress. delete failed for network %s. Please try again.")
398 411
 		}
... ...
@@ -402,65 +515,68 @@ func (n *network) Delete() error {
402 402
 	defer func() {
403 403
 		if err != nil {
404 404
 			n.dbExists = false
405
-			if e := ctrlr.updateToStore(n); e != nil {
405
+			if e := n.getController().updateToStore(n); e != nil {
406 406
 				log.Warnf("failed to recreate network in store %s : %v", n.name, e)
407 407
 			}
408 408
 		}
409 409
 	}()
410 410
 
411
-	if err = n.deleteNetwork(); err != nil {
412
-		return err
413
-	}
414
-
415 411
 	n.ipamRelease()
416 412
 
417 413
 	return nil
418 414
 }
419 415
 
420 416
 func (n *network) deleteNetwork() error {
421
-	n.Lock()
422
-	id := n.id
423
-	d := n.driver
424
-	n.ctrlr.Lock()
425
-	delete(n.ctrlr.networks, id)
426
-	n.ctrlr.Unlock()
427
-	n.Unlock()
417
+	d, err := n.driver()
418
+	if err != nil {
419
+		return fmt.Errorf("failed deleting network: %v", err)
420
+	}
421
+
422
+	// If it is bridge network type make sure we call the driver about the network
423
+	// because the network may have been created in some past life of libnetwork.
424
+	if n.Type() == "bridge" {
425
+		n.drvOnce.Do(func() {
426
+			err = n.getController().addNetwork(n)
427
+		})
428
+		if err != nil {
429
+			return err
430
+		}
431
+	}
428 432
 
429
-	if err := d.DeleteNetwork(n.id); err != nil {
433
+	if err := d.DeleteNetwork(n.ID()); err != nil {
430 434
 		// Forbidden Errors should be honored
431 435
 		if _, ok := err.(types.ForbiddenError); ok {
432
-			n.ctrlr.Lock()
433
-			n.ctrlr.networks[n.id] = n
434
-			n.ctrlr.Unlock()
435 436
 			return err
436 437
 		}
437 438
 		log.Warnf("driver error deleting network %s : %v", n.name, err)
438 439
 	}
439
-	n.stopWatch()
440
+
440 441
 	return nil
441 442
 }
442 443
 
443 444
 func (n *network) addEndpoint(ep *endpoint) error {
444
-	var err error
445
-	n.Lock()
446
-	n.endpoints[ep.id] = ep
447
-	d := n.driver
448
-	n.Unlock()
445
+	d, err := n.driver()
446
+	if err != nil {
447
+		return fmt.Errorf("failed to add endpoint: %v", err)
448
+	}
449 449
 
450
-	defer func() {
450
+	// If it is bridge network type make sure we call the driver about the network
451
+	// because the network may have been created in some past life of libnetwork.
452
+	if n.Type() == "bridge" {
453
+		n.drvOnce.Do(func() {
454
+			err = n.getController().addNetwork(n)
455
+		})
451 456
 		if err != nil {
452
-			n.Lock()
453
-			delete(n.endpoints, ep.id)
454
-			n.Unlock()
457
+			return err
455 458
 		}
456
-	}()
459
+	}
457 460
 
458 461
 	err = d.CreateEndpoint(n.id, ep.id, ep.Interface(), ep.generic)
459 462
 	if err != nil {
460
-		return types.InternalErrorf("failed to create endpoint %s on network %s: %v", ep.Name(), n.Name(), err)
463
+		return types.InternalErrorf("failed to create endpoint %s on network %s: %v",
464
+			ep.Name(), n.Name(), err)
461 465
 	}
462 466
 
463
-	n.updateSvcRecord(ep, true)
464 467
 	return nil
465 468
 }
466 469
 
... ...
@@ -476,7 +592,16 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
476 476
 
477 477
 	ep := &endpoint{name: name, generic: make(map[string]interface{}), iface: &endpointInterface{}}
478 478
 	ep.id = stringid.GenerateRandomID()
479
+
480
+	// Initialize ep.network with a possibly stale copy of n. We need this to get network from
481
+	// store. But once we get it from store we will have the most uptodate copy possible.
479 482
 	ep.network = n
483
+	ep.network, err = ep.getNetworkFromStore()
484
+	if err != nil {
485
+		return nil, fmt.Errorf("failed to get network during CreateEndpoint: %v", err)
486
+	}
487
+	n = ep.network
488
+
480 489
 	ep.processOptions(options...)
481 490
 
482 491
 	if err = ep.assignAddress(); err != nil {
... ...
@@ -488,46 +613,46 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
488 488
 		}
489 489
 	}()
490 490
 
491
-	ctrlr := n.getController()
492
-
493
-	n.IncEndpointCnt()
494
-	if err = ctrlr.updateToStore(n); err != nil {
491
+	if err = n.addEndpoint(ep); err != nil {
495 492
 		return nil, err
496 493
 	}
497 494
 	defer func() {
498 495
 		if err != nil {
499
-			n.DecEndpointCnt()
500
-			if err = ctrlr.updateToStore(n); err != nil {
501
-				log.Warnf("endpoint count cleanup failed when updating network for %s : %v", name, err)
496
+			if e := ep.deleteEndpoint(); e != nil {
497
+				log.Warnf("cleaning up endpoint failed %s : %v", name, e)
502 498
 			}
503 499
 		}
504 500
 	}()
505
-	if err = n.addEndpoint(ep); err != nil {
501
+
502
+	if err = n.getController().updateToStore(ep); err != nil {
506 503
 		return nil, err
507 504
 	}
508 505
 	defer func() {
509 506
 		if err != nil {
510
-			if e := ep.Delete(); ep != nil {
511
-				log.Warnf("cleaning up endpoint failed %s : %v", name, e)
507
+			if e := n.getController().deleteFromStore(ep); e != nil {
508
+				log.Warnf("error rolling back endpoint %s from store: %v", name, e)
512 509
 			}
513 510
 		}
514 511
 	}()
515 512
 
516
-	if !ep.isLocalScoped() {
517
-		if err = ctrlr.updateToStore(ep); err != nil {
518
-			return nil, err
519
-		}
513
+	// Increment endpoint count to indicate completion of endpoint addition
514
+	if err = n.IncEndpointCnt(); err != nil {
515
+		return nil, err
520 516
 	}
521 517
 
522 518
 	return ep, nil
523 519
 }
524 520
 
525 521
 func (n *network) Endpoints() []Endpoint {
526
-	n.Lock()
527
-	defer n.Unlock()
528
-	list := make([]Endpoint, 0, len(n.endpoints))
529
-	for _, e := range n.endpoints {
530
-		list = append(list, e)
522
+	var list []Endpoint
523
+
524
+	endpoints, err := n.getEndpointsFromStore()
525
+	if err != nil {
526
+		log.Error(err)
527
+	}
528
+
529
+	for _, ep := range endpoints {
530
+		list = append(list, ep)
531 531
 	}
532 532
 
533 533
 	return list
... ...
@@ -568,28 +693,32 @@ func (n *network) EndpointByID(id string) (Endpoint, error) {
568 568
 	if id == "" {
569 569
 		return nil, ErrInvalidID(id)
570 570
 	}
571
-	n.Lock()
572
-	defer n.Unlock()
573
-	if e, ok := n.endpoints[id]; ok {
574
-		return e, nil
571
+
572
+	ep, err := n.getEndpointFromStore(id)
573
+	if err != nil {
574
+		return nil, ErrNoSuchEndpoint(id)
575 575
 	}
576
-	return nil, ErrNoSuchEndpoint(id)
577
-}
578 576
 
579
-func (n *network) isGlobalScoped() bool {
580
-	return n.DataScope() == datastore.GlobalScope
577
+	return ep, nil
581 578
 }
582 579
 
583
-func (n *network) updateSvcRecord(ep *endpoint, isAdd bool) {
580
+func (n *network) updateSvcRecord(ep *endpoint, localEps []*endpoint, isAdd bool) {
581
+	c := n.getController()
582
+	sr, ok := c.svcDb[n.ID()]
583
+	if !ok {
584
+		c.svcDb[n.ID()] = svcMap{}
585
+		sr = c.svcDb[n.ID()]
586
+	}
587
+
584 588
 	n.Lock()
585 589
 	var recs []etchosts.Record
586 590
 	if iface := ep.Iface(); iface.Address() != nil {
587 591
 		if isAdd {
588
-			n.svcRecords[ep.Name()] = iface.Address().IP
589
-			n.svcRecords[ep.Name()+"."+n.name] = iface.Address().IP
592
+			sr[ep.Name()] = iface.Address().IP
593
+			sr[ep.Name()+"."+n.name] = iface.Address().IP
590 594
 		} else {
591
-			delete(n.svcRecords, ep.Name())
592
-			delete(n.svcRecords, ep.Name()+"."+n.name)
595
+			delete(sr, ep.Name())
596
+			delete(sr, ep.Name()+"."+n.name)
593 597
 		}
594 598
 
595 599
 		recs = append(recs, etchosts.Record{
... ...
@@ -610,12 +739,11 @@ func (n *network) updateSvcRecord(ep *endpoint, isAdd bool) {
610 610
 	}
611 611
 
612 612
 	var sbList []*sandbox
613
-	n.WalkEndpoints(func(e Endpoint) bool {
614
-		if sb, hasSandbox := e.(*endpoint).getSandbox(); hasSandbox {
613
+	for _, ep := range localEps {
614
+		if sb, hasSandbox := ep.getSandbox(); hasSandbox {
615 615
 			sbList = append(sbList, sb)
616 616
 		}
617
-		return false
618
-	})
617
+	}
619 618
 
620 619
 	for _, sb := range sbList {
621 620
 		if isAdd {
... ...
@@ -631,7 +759,9 @@ func (n *network) getSvcRecords() []etchosts.Record {
631 631
 	defer n.Unlock()
632 632
 
633 633
 	var recs []etchosts.Record
634
-	for h, ip := range n.svcRecords {
634
+	sr, _ := n.ctrlr.svcDb[n.id]
635
+
636
+	for h, ip := range sr {
635 637
 		recs = append(recs, etchosts.Record{
636 638
 			Hosts: h,
637 639
 			IP:    ip.String(),
... ...
@@ -799,7 +929,7 @@ func (n *network) deriveAddressSpace() (string, error) {
799 799
 	if !ok {
800 800
 		return "", types.NotFoundErrorf("could not find ipam driver %s to get default address space", n.ipamType)
801 801
 	}
802
-	if n.isGlobalScoped() {
802
+	if n.DataScope() == datastore.GlobalScope {
803 803
 		return ipd.defaultGlobalAddressSpace, nil
804 804
 	}
805 805
 	return ipd.defaultLocalAddressSpace, nil
... ...
@@ -247,6 +247,19 @@ func (sb *sandbox) getConnectedEndpoints() []*endpoint {
247 247
 	return eps
248 248
 }
249 249
 
250
+func (sb *sandbox) getEndpoint(id string) *endpoint {
251
+	sb.Lock()
252
+	defer sb.Unlock()
253
+
254
+	for _, ep := range sb.endpoints {
255
+		if ep.id == id {
256
+			return ep
257
+		}
258
+	}
259
+
260
+	return nil
261
+}
262
+
250 263
 func (sb *sandbox) updateGateway(ep *endpoint) error {
251 264
 	sb.Lock()
252 265
 	osSbox := sb.osSbox
... ...
@@ -359,7 +372,13 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
359 359
 	return nil
360 360
 }
361 361
 
362
-func (sb *sandbox) clearNetworkResources(ep *endpoint) error {
362
+func (sb *sandbox) clearNetworkResources(origEp *endpoint) error {
363
+	ep := sb.getEndpoint(origEp.id)
364
+	if ep == nil {
365
+		return fmt.Errorf("could not find the sandbox endpoint data for endpoint %s",
366
+			ep.name)
367
+	}
368
+
363 369
 	sb.Lock()
364 370
 	osSbox := sb.osSbox
365 371
 	sb.Unlock()
... ...
@@ -837,7 +856,7 @@ func (eh epHeap) Less(i, j int) bool {
837 837
 		cjp = 0
838 838
 	}
839 839
 	if cip == cjp {
840
-		return eh[i].getNetwork().Name() < eh[j].getNetwork().Name()
840
+		return eh[i].network.Name() < eh[j].network.Name()
841 841
 	}
842 842
 
843 843
 	return cip > cjp
... ...
@@ -115,21 +115,21 @@ func TestSandboxAddMultiPrio(t *testing.T) {
115 115
 		t.Fatal(err)
116 116
 	}
117 117
 
118
-	if ctrlr.sandboxes[sid].endpoints[0] != ep3 {
118
+	if ctrlr.sandboxes[sid].endpoints[0].ID() != ep3.ID() {
119 119
 		t.Fatal("Expected ep3 to be at the top of the heap. But did not find ep3 at the top of the heap")
120 120
 	}
121 121
 
122 122
 	if err := ep3.Leave(sbx); err != nil {
123 123
 		t.Fatal(err)
124 124
 	}
125
-	if ctrlr.sandboxes[sid].endpoints[0] != ep2 {
125
+	if ctrlr.sandboxes[sid].endpoints[0].ID() != ep2.ID() {
126 126
 		t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap")
127 127
 	}
128 128
 
129 129
 	if err := ep2.Leave(sbx); err != nil {
130 130
 		t.Fatal(err)
131 131
 	}
132
-	if ctrlr.sandboxes[sid].endpoints[0] != ep1 {
132
+	if ctrlr.sandboxes[sid].endpoints[0].ID() != ep1.ID() {
133 133
 		t.Fatal("Expected ep1 to be at the top of the heap after removing ep2. But did not find ep1 at the top of the heap")
134 134
 	}
135 135
 
... ...
@@ -138,7 +138,7 @@ func TestSandboxAddMultiPrio(t *testing.T) {
138 138
 		t.Fatal(err)
139 139
 	}
140 140
 
141
-	if ctrlr.sandboxes[sid].endpoints[0] != ep3 {
141
+	if ctrlr.sandboxes[sid].endpoints[0].ID() != ep3.ID() {
142 142
 		t.Fatal("Expected ep3 to be at the top of the heap after adding ep3 back. But did not find ep3 at the top of the heap")
143 143
 	}
144 144
 
... ...
@@ -185,7 +185,7 @@ func TestSandboxAddSamePrio(t *testing.T) {
185 185
 		t.Fatal(err)
186 186
 	}
187 187
 
188
-	if ctrlr.sandboxes[sid].endpoints[0] != ep1 {
188
+	if ctrlr.sandboxes[sid].endpoints[0].ID() != ep1.ID() {
189 189
 		t.Fatal("Expected ep1 to be at the top of the heap. But did not find ep1 at the top of the heap")
190 190
 	}
191 191
 
... ...
@@ -193,7 +193,7 @@ func TestSandboxAddSamePrio(t *testing.T) {
193 193
 		t.Fatal(err)
194 194
 	}
195 195
 
196
-	if ctrlr.sandboxes[sid].endpoints[0] != ep2 {
196
+	if ctrlr.sandboxes[sid].endpoints[0].ID() != ep2.ID() {
197 197
 		t.Fatal("Expected ep2 to be at the top of the heap after removing ep3. But did not find ep2 at the top of the heap")
198 198
 	}
199 199
 
... ...
@@ -1,408 +1,348 @@
1 1
 package libnetwork
2 2
 
3 3
 import (
4
-	"encoding/json"
5 4
 	"fmt"
6
-	"time"
7 5
 
8 6
 	log "github.com/Sirupsen/logrus"
9
-	"github.com/docker/libkv/store"
10
-	"github.com/docker/libnetwork/config"
11 7
 	"github.com/docker/libnetwork/datastore"
12 8
 )
13 9
 
14
-var (
15
-	defaultBoltTimeout      = 3 * time.Second
16
-	defaultLocalStoreConfig = config.DatastoreCfg{
17
-		Embedded: true,
18
-		Client: config.DatastoreClientCfg{
19
-			Provider: "boltdb",
20
-			Address:  defaultPrefix + "/boltdb.db",
21
-			Config: &store.Config{
22
-				Bucket:            "libnetwork",
23
-				ConnectionTimeout: defaultBoltTimeout,
24
-			},
25
-		},
26
-	}
27
-)
28
-
29
-func (c *controller) validateGlobalStoreConfig() bool {
30
-	return c.cfg != nil && c.cfg.GlobalStore.Client.Provider != "" && c.cfg.GlobalStore.Client.Address != ""
31
-}
32
-
33
-func (c *controller) initGlobalStore() error {
10
+func (c *controller) initStores() error {
34 11
 	c.Lock()
35
-	cfg := c.cfg
36
-	c.Unlock()
37
-	if !c.validateGlobalStoreConfig() {
38
-		return fmt.Errorf("globalstore initialization requires a valid configuration")
12
+	if c.cfg == nil {
13
+		c.Unlock()
14
+		return nil
39 15
 	}
16
+	scopeConfigs := c.cfg.Scopes
17
+	c.Unlock()
40 18
 
41
-	store, err := datastore.NewDataStore(&cfg.GlobalStore)
42
-	if err != nil {
43
-		return err
19
+	for scope, scfg := range scopeConfigs {
20
+		store, err := datastore.NewDataStore(scope, scfg)
21
+		if err != nil {
22
+			return err
23
+		}
24
+		c.Lock()
25
+		c.stores = append(c.stores, store)
26
+		c.Unlock()
44 27
 	}
45
-	c.Lock()
46
-	c.globalStore = store
47
-	c.Unlock()
28
+
29
+	c.startWatch()
48 30
 	return nil
49 31
 }
50 32
 
51
-func (c *controller) initLocalStore() error {
52
-	c.Lock()
53
-	cfg := c.cfg
54
-	c.Unlock()
55
-	localStore, err := datastore.NewDataStore(c.getLocalStoreConfig(cfg))
56
-	if err != nil {
57
-		return err
33
+func (c *controller) closeStores() {
34
+	for _, store := range c.getStores() {
35
+		store.Close()
58 36
 	}
59
-	c.Lock()
60
-	c.localStore = localStore
61
-	c.Unlock()
62
-	return nil
63 37
 }
64 38
 
65
-func (c *controller) restoreFromGlobalStore() error {
39
+func (c *controller) getStore(scope string) datastore.DataStore {
66 40
 	c.Lock()
67
-	s := c.globalStore
68
-	c.Unlock()
69
-	if s == nil {
70
-		return nil
41
+	defer c.Unlock()
42
+
43
+	for _, store := range c.stores {
44
+		if store.Scope() == scope {
45
+			return store
46
+		}
71 47
 	}
72
-	c.restore("global")
73
-	return c.watchNetworks()
48
+
49
+	return nil
74 50
 }
75 51
 
76
-func (c *controller) restoreFromLocalStore() error {
52
+func (c *controller) getStores() []datastore.DataStore {
77 53
 	c.Lock()
78
-	s := c.localStore
79
-	c.Unlock()
80
-	if s != nil {
81
-		c.restore("local")
82
-	}
83
-	return nil
54
+	defer c.Unlock()
55
+
56
+	return c.stores
84 57
 }
85 58
 
86
-func (c *controller) restore(store string) {
87
-	nws, err := c.getNetworksFromStore(store == "global")
88
-	if err == nil {
89
-		c.processNetworkUpdate(nws, nil)
90
-	} else if err != datastore.ErrKeyNotFound {
91
-		log.Warnf("failed to read networks from %s store during init : %v", store, err)
59
+func (c *controller) getNetworkFromStore(nid string) (*network, error) {
60
+	for _, store := range c.getStores() {
61
+		n := &network{id: nid, ctrlr: c}
62
+		err := store.GetObject(datastore.Key(n.Key()...), n)
63
+		if err != nil && err != datastore.ErrKeyNotFound {
64
+			return nil, fmt.Errorf("could not find network %s: %v", nid, err)
65
+		}
66
+
67
+		// Continue searching in the next store if the key is not found in this store
68
+		if err == datastore.ErrKeyNotFound {
69
+			continue
70
+		}
71
+
72
+		return n, nil
92 73
 	}
74
+
75
+	return nil, fmt.Errorf("network %s not found", nid)
93 76
 }
94 77
 
95
-func (c *controller) getNetworksFromStore(global bool) ([]*store.KVPair, error) {
96
-	var cs datastore.DataStore
97
-	c.Lock()
98
-	if global {
99
-		cs = c.globalStore
100
-	} else {
101
-		cs = c.localStore
78
+func (c *controller) getNetworksFromStore() ([]*network, error) {
79
+	var nl []*network
80
+
81
+	for _, store := range c.getStores() {
82
+		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
83
+			&network{ctrlr: c})
84
+		if err != nil && err != datastore.ErrKeyNotFound {
85
+			return nil, fmt.Errorf("failed to get networks for scope %s: %v",
86
+				store.Scope(), err)
87
+		}
88
+
89
+		// Continue searching in the next store if no keys found in this store
90
+		if err == datastore.ErrKeyNotFound {
91
+			continue
92
+		}
93
+
94
+		for _, kvo := range kvol {
95
+			n := kvo.(*network)
96
+			n.ctrlr = c
97
+			nl = append(nl, n)
98
+		}
102 99
 	}
103
-	c.Unlock()
104
-	return cs.KVStore().List(datastore.Key(datastore.NetworkKeyPrefix))
100
+
101
+	return nl, nil
105 102
 }
106 103
 
107
-func (c *controller) newNetworkFromStore(n *network) error {
108
-	n.Lock()
109
-	n.ctrlr = c
110
-	n.endpoints = endpointTable{}
111
-	n.Unlock()
104
+func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
105
+	for _, store := range n.ctrlr.getStores() {
106
+		ep := &endpoint{id: eid, network: n}
107
+		err := store.GetObject(datastore.Key(ep.Key()...), ep)
108
+		if err != nil && err != datastore.ErrKeyNotFound {
109
+			return nil, fmt.Errorf("could not find endpoint %s: %v", eid, err)
110
+		}
111
+
112
+		// Continue searching in the next store if the key is not found in this store
113
+		if err == datastore.ErrKeyNotFound {
114
+			continue
115
+		}
116
+
117
+		return ep, nil
118
+	}
112 119
 
113
-	return c.addNetwork(n)
120
+	return nil, fmt.Errorf("endpoint %s not found", eid)
114 121
 }
115 122
 
116
-func (c *controller) newEndpointFromStore(key string, ep *endpoint) error {
117
-	ep.Lock()
118
-	n := ep.network
119
-	id := ep.id
120
-	ep.Unlock()
123
+func (n *network) getEndpointsFromStore() ([]*endpoint, error) {
124
+	var epl []*endpoint
121 125
 
122
-	_, err := n.EndpointByID(id)
123
-	if err != nil {
124
-		if _, ok := err.(ErrNoSuchEndpoint); ok {
125
-			return n.addEndpoint(ep)
126
+	tmp := endpoint{network: n}
127
+	for _, store := range n.getController().getStores() {
128
+		kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &endpoint{network: n})
129
+		if err != nil && err != datastore.ErrKeyNotFound {
130
+			return nil,
131
+				fmt.Errorf("failed to get endpoints for network %s scope %s: %v",
132
+					n.Name(), store.Scope(), err)
133
+		}
134
+
135
+		// Continue searching in the next store if no keys found in this store
136
+		if err == datastore.ErrKeyNotFound {
137
+			continue
138
+		}
139
+
140
+		for _, kvo := range kvol {
141
+			ep := kvo.(*endpoint)
142
+			ep.network = n
143
+			epl = append(epl, ep)
126 144
 		}
127 145
 	}
128
-	return err
146
+
147
+	return epl, nil
129 148
 }
130 149
 
131
-func (c *controller) updateToStore(kvObject datastore.KV) error {
132
-	if kvObject.Skip() {
133
-		return nil
134
-	}
135
-	cs := c.getDataStore(kvObject.DataScope())
150
+func (c *controller) updateToStore(kvObject datastore.KVObject) error {
151
+	cs := c.getStore(kvObject.DataScope())
136 152
 	if cs == nil {
137
-		log.Debugf("datastore not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...))
153
+		log.Warnf("datastore for scope %s not initialized. kv object %s is not added to the store", kvObject.DataScope(), datastore.Key(kvObject.Key()...))
138 154
 		return nil
139 155
 	}
140 156
 
141
-	return cs.PutObjectAtomic(kvObject)
157
+	if err := cs.PutObjectAtomic(kvObject); err != nil {
158
+		return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err)
159
+	}
160
+
161
+	return nil
142 162
 }
143 163
 
144
-func (c *controller) deleteFromStore(kvObject datastore.KV) error {
145
-	if kvObject.Skip() {
146
-		return nil
147
-	}
148
-	cs := c.getDataStore(kvObject.DataScope())
164
+func (c *controller) deleteFromStore(kvObject datastore.KVObject) error {
165
+	cs := c.getStore(kvObject.DataScope())
149 166
 	if cs == nil {
150
-		log.Debugf("datastore not initialized. kv object %s is not deleted from datastore", datastore.Key(kvObject.Key()...))
167
+		log.Debugf("datastore for scope %s not initialized. kv object %s is not deleted from datastore", kvObject.DataScope(), datastore.Key(kvObject.Key()...))
151 168
 		return nil
152 169
 	}
153 170
 
171
+retry:
154 172
 	if err := cs.DeleteObjectAtomic(kvObject); err != nil {
173
+		if err == datastore.ErrKeyModified {
174
+			if err := cs.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
175
+				return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
176
+			}
177
+			goto retry
178
+		}
155 179
 		return err
156 180
 	}
157 181
 
158 182
 	return nil
159 183
 }
160 184
 
161
-func (c *controller) watchNetworks() error {
162
-	if !c.validateGlobalStoreConfig() {
163
-		return nil
164
-	}
185
+type netWatch struct {
186
+	localEps  map[string]*endpoint
187
+	remoteEps map[string]*endpoint
188
+	stopCh    chan struct{}
189
+}
165 190
 
191
+func (c *controller) getLocalEps(nw *netWatch) []*endpoint {
166 192
 	c.Lock()
167
-	cs := c.globalStore
168
-	c.Unlock()
193
+	defer c.Unlock()
169 194
 
170
-	networkKey := datastore.Key(datastore.NetworkKeyPrefix)
171
-	if err := ensureKeys(networkKey, cs); err != nil {
172
-		return fmt.Errorf("failed to ensure if the network keys are valid and present in store: %v", err)
195
+	var epl []*endpoint
196
+	for _, ep := range nw.localEps {
197
+		epl = append(epl, ep)
173 198
 	}
174
-	nwPairs, err := cs.KVStore().WatchTree(networkKey, nil)
175
-	if err != nil {
176
-		return err
177
-	}
178
-	go func() {
179
-		for {
180
-			select {
181
-			case nws := <-nwPairs:
182
-				c.Lock()
183
-				tmpview := networkTable{}
184
-				lview := c.networks
185
-				c.Unlock()
186
-				for k, v := range lview {
187
-					if v.isGlobalScoped() {
188
-						tmpview[k] = v
189
-					}
190
-				}
191
-				c.processNetworkUpdate(nws, &tmpview)
192
-
193
-				// Delete processing
194
-				for k := range tmpview {
195
-					c.Lock()
196
-					existing, ok := c.networks[k]
197
-					c.Unlock()
198
-					if !ok {
199
-						continue
200
-					}
201
-					tmp := network{}
202
-					if err := c.globalStore.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
203
-						continue
204
-					}
205
-					if err := existing.deleteNetwork(); err != nil {
206
-						log.Debugf("Delete failed %s: %s", existing.name, err)
207
-					}
208
-				}
209
-			}
210
-		}
211
-	}()
212
-	return nil
199
+
200
+	return epl
213 201
 }
214 202
 
215
-func (n *network) watchEndpoints() error {
216
-	if n.Skip() || !n.ctrlr.validateGlobalStoreConfig() {
217
-		return nil
218
-	}
203
+func (c *controller) watchSvcRecord(ep *endpoint) {
204
+	c.watchCh <- ep
205
+}
219 206
 
220
-	n.Lock()
221
-	cs := n.ctrlr.globalStore
222
-	tmp := endpoint{network: n}
223
-	n.stopWatchCh = make(chan struct{})
224
-	stopCh := n.stopWatchCh
225
-	n.Unlock()
207
+func (c *controller) unWatchSvcRecord(ep *endpoint) {
208
+	c.unWatchCh <- ep
209
+}
226 210
 
227
-	endpointKey := datastore.Key(tmp.KeyPrefix()...)
228
-	if err := ensureKeys(endpointKey, cs); err != nil {
229
-		return fmt.Errorf("failed to ensure if the endpoint keys are valid and present in store: %v", err)
230
-	}
231
-	epPairs, err := cs.KVStore().WatchTree(endpointKey, stopCh)
232
-	if err != nil {
233
-		return err
234
-	}
235
-	go func() {
236
-		for {
237
-			select {
238
-			case <-stopCh:
239
-				return
240
-			case eps := <-epPairs:
241
-				n.Lock()
242
-				tmpview := endpointTable{}
243
-				lview := n.endpoints
244
-				n.Unlock()
245
-				for k, v := range lview {
246
-					if v.network.isGlobalScoped() {
247
-						tmpview[k] = v
248
-					}
211
+func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, nCh <-chan datastore.KVObject) {
212
+	for {
213
+		select {
214
+		case <-nw.stopCh:
215
+			return
216
+		case o := <-nCh:
217
+			n := o.(*network)
218
+
219
+			epl, err := n.getEndpointsFromStore()
220
+			if err != nil {
221
+				break
222
+			}
223
+
224
+			c.Lock()
225
+			var addEp []*endpoint
226
+
227
+			delEpMap := make(map[string]*endpoint)
228
+			for k, v := range nw.remoteEps {
229
+				delEpMap[k] = v
230
+			}
231
+
232
+			for _, lEp := range epl {
233
+				if _, ok := nw.localEps[lEp.ID()]; ok {
234
+					continue
249 235
 				}
250
-				n.ctrlr.processEndpointsUpdate(eps, &tmpview)
251
-				// Delete processing
252
-				for k := range tmpview {
253
-					n.Lock()
254
-					existing, ok := n.endpoints[k]
255
-					n.Unlock()
256
-					if !ok {
257
-						continue
258
-					}
259
-					tmp := endpoint{}
260
-					if err := cs.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
261
-						continue
262
-					}
263
-					if err := existing.deleteEndpoint(); err != nil {
264
-						log.Debugf("Delete failed %s: %s", existing.name, err)
265
-					}
236
+
237
+				if _, ok := nw.remoteEps[lEp.ID()]; ok {
238
+					delete(delEpMap, lEp.ID())
239
+					continue
266 240
 				}
241
+
242
+				nw.remoteEps[lEp.ID()] = lEp
243
+				addEp = append(addEp, lEp)
244
+
245
+			}
246
+			c.Unlock()
247
+
248
+			for _, lEp := range addEp {
249
+				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), true)
267 250
 			}
268
-		}
269
-	}()
270
-	return nil
271
-}
272 251
 
273
-func (n *network) stopWatch() {
274
-	n.Lock()
275
-	if n.stopWatchCh != nil {
276
-		close(n.stopWatchCh)
277
-		n.stopWatchCh = nil
252
+			for _, lEp := range delEpMap {
253
+				ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), false)
254
+
255
+			}
256
+		}
278 257
 	}
279
-	n.Unlock()
280 258
 }
281 259
 
282
-func (c *controller) processNetworkUpdate(nws []*store.KVPair, prune *networkTable) {
283
-	for _, kve := range nws {
284
-		var n network
285
-		err := json.Unmarshal(kve.Value, &n)
286
-		if err != nil {
287
-			log.Error(err)
288
-			continue
289
-		}
290
-		if prune != nil {
291
-			delete(*prune, n.id)
292
-		}
293
-		n.SetIndex(kve.LastIndex)
260
+func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) {
261
+	c.Lock()
262
+	nw, ok := nmap[ep.getNetwork().ID()]
263
+	c.Unlock()
264
+
265
+	if ok {
266
+		// Update the svc db for the local endpoint join right away
267
+		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)
268
+
294 269
 		c.Lock()
295
-		existing, ok := c.networks[n.id]
270
+		nw.localEps[ep.ID()] = ep
296 271
 		c.Unlock()
297
-		if ok {
298
-			existing.Lock()
299
-			// Skip existing network update
300
-			if existing.dbIndex != n.Index() {
301
-				// Can't use SetIndex() since existing is locked.
302
-				existing.dbIndex = n.Index()
303
-				existing.dbExists = true
304
-				existing.endpointCnt = n.endpointCnt
305
-			}
306
-			existing.Unlock()
307
-			continue
308
-		}
309
-
310
-		if err = c.newNetworkFromStore(&n); err != nil {
311
-			log.Error(err)
312
-		}
272
+		return
313 273
 	}
314
-}
315 274
 
316
-func (c *controller) processEndpointUpdate(ep *endpoint) bool {
317
-	nw := ep.network
318
-	if nw == nil {
319
-		return true
275
+	nw = &netWatch{
276
+		localEps:  make(map[string]*endpoint),
277
+		remoteEps: make(map[string]*endpoint),
320 278
 	}
321
-	nw.Lock()
322
-	id := nw.id
323
-	nw.Unlock()
279
+
280
+	// Update the svc db for the local endpoint join right away
281
+	// Do this before adding this ep to localEps so that we don't
282
+	// try to update this ep's container's svc records
283
+	ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)
324 284
 
325 285
 	c.Lock()
326
-	n, ok := c.networks[id]
286
+	nw.localEps[ep.ID()] = ep
287
+	nmap[ep.getNetwork().ID()] = nw
288
+	nw.stopCh = make(chan struct{})
327 289
 	c.Unlock()
328
-	if !ok {
329
-		return true
330
-	}
331
-	existing, _ := n.EndpointByID(ep.id)
332
-	if existing == nil {
333
-		return true
334
-	}
335 290
 
336
-	ee := existing.(*endpoint)
337
-	ee.Lock()
338
-	if ee.dbIndex != ep.Index() {
339
-		// Can't use SetIndex() because ee is locked.
340
-		ee.dbIndex = ep.Index()
341
-		ee.dbExists = true
342
-		ee.sandboxID = ep.sandboxID
291
+	store := c.getStore(ep.getNetwork().DataScope())
292
+	if store == nil {
293
+		return
343 294
 	}
344
-	ee.Unlock()
345 295
 
346
-	return false
347
-}
296
+	if !store.Watchable() {
297
+		return
298
+	}
348 299
 
349
-func ensureKeys(key string, cs datastore.DataStore) error {
350
-	exists, err := cs.KVStore().Exists(key)
300
+	ch, err := store.Watch(ep.getNetwork(), nw.stopCh)
351 301
 	if err != nil {
352
-		return err
302
+		log.Warnf("Error creating watch for network: %v", err)
303
+		return
353 304
 	}
354
-	if exists {
355
-		return nil
356
-	}
357
-	return cs.KVStore().Put(key, []byte{}, nil)
358
-}
359 305
 
360
-func (c *controller) getLocalStoreConfig(cfg *config.Config) *config.DatastoreCfg {
361
-	if cfg != nil && cfg.LocalStore.Client.Provider != "" && cfg.LocalStore.Client.Address != "" {
362
-		return &cfg.LocalStore
363
-	}
364
-	return &defaultLocalStoreConfig
306
+	go c.networkWatchLoop(nw, ep, ch)
365 307
 }
366 308
 
367
-func (c *controller) getDataStore(dataScope datastore.DataScope) (dataStore datastore.DataStore) {
309
+func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) {
368 310
 	c.Lock()
369
-	if dataScope == datastore.GlobalScope {
370
-		dataStore = c.globalStore
371
-	} else if dataScope == datastore.LocalScope {
372
-		dataStore = c.localStore
311
+	nw, ok := nmap[ep.getNetwork().ID()]
312
+
313
+	if ok {
314
+		delete(nw.localEps, ep.ID())
315
+		c.Unlock()
316
+
317
+		// Update the svc db about local endpoint leave right away
318
+		// Do this after we remove this ep from localEps so that we
319
+		// don't try to remove this svc record from this ep's container.
320
+		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), false)
321
+
322
+		c.Lock()
323
+		if len(nw.localEps) == 0 {
324
+			close(nw.stopCh)
325
+			delete(nmap, ep.getNetwork().ID())
326
+		}
373 327
 	}
374 328
 	c.Unlock()
375
-	return
376 329
 }
377 330
 
378
-func (c *controller) processEndpointsUpdate(eps []*store.KVPair, prune *endpointTable) {
379
-	for _, epe := range eps {
380
-		var ep endpoint
381
-		err := json.Unmarshal(epe.Value, &ep)
382
-		if err != nil {
383
-			log.Error(err)
384
-			continue
385
-		}
386
-		if prune != nil {
387
-			delete(*prune, ep.id)
388
-		}
389
-		ep.SetIndex(epe.LastIndex)
390
-		if nid, err := ep.networkIDFromKey(epe.Key); err != nil {
391
-			log.Error(err)
392
-			continue
393
-		} else {
394
-			if n, err := c.NetworkByID(nid); err != nil {
395
-				log.Error(err)
396
-				continue
397
-			} else {
398
-				ep.network = n.(*network)
399
-			}
400
-		}
401
-		if c.processEndpointUpdate(&ep) {
402
-			err = c.newEndpointFromStore(epe.Key, &ep)
403
-			if err != nil {
404
-				log.Error(err)
405
-			}
331
+func (c *controller) watchLoop(nmap map[string]*netWatch) {
332
+	for {
333
+		select {
334
+		case ep := <-c.watchCh:
335
+			c.processEndpointCreate(nmap, ep)
336
+		case ep := <-c.unWatchCh:
337
+			c.processEndpointDelete(nmap, ep)
406 338
 		}
407 339
 	}
408 340
 }
341
+
342
+func (c *controller) startWatch() {
343
+	c.watchCh = make(chan *endpoint)
344
+	c.unWatchCh = make(chan *endpoint)
345
+	nmap := make(map[string]*netWatch)
346
+
347
+	go c.watchLoop(nmap)
348
+}
... ...
@@ -33,7 +33,7 @@ func testNewController(t *testing.T, provider, url string) (NetworkController, e
33 33
 }
34 34
 
35 35
 func TestBoltdbBackend(t *testing.T) {
36
-	defer os.Remove(defaultLocalStoreConfig.Client.Address)
36
+	defer os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address)
37 37
 	testLocalBackend(t, "", "", nil)
38 38
 	defer os.Remove("/tmp/boltdb.db")
39 39
 	config := &store.Config{Bucket: "testBackend", ConnectionTimeout: 3 * time.Second}
... ...
@@ -64,7 +64,7 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con
64 64
 	if err != nil {
65 65
 		t.Fatalf("Error creating endpoint: %v", err)
66 66
 	}
67
-	store := ctrl.(*controller).localStore.KVStore()
67
+	store := ctrl.(*controller).getStore(datastore.LocalScope).KVStore()
68 68
 	if exists, err := store.Exists(datastore.Key(datastore.NetworkKeyPrefix, string(nw.ID()))); !exists || err != nil {
69 69
 		t.Fatalf("Network key should have been created.")
70 70
 	}
... ...
@@ -100,7 +100,7 @@ func TestNoPersist(t *testing.T) {
100 100
 	if err != nil {
101 101
 		t.Fatalf("Error creating endpoint: %v", err)
102 102
 	}
103
-	store := ctrl.(*controller).localStore.KVStore()
103
+	store := ctrl.(*controller).getStore(datastore.LocalScope).KVStore()
104 104
 	if exists, _ := store.Exists(datastore.Key(datastore.NetworkKeyPrefix, string(nw.ID()))); exists {
105 105
 		t.Fatalf("Network with persist=false should not be stored in KV Store")
106 106
 	}
... ...
@@ -138,12 +138,8 @@ func TestLocalStoreLockTimeout(t *testing.T) {
138 138
 	}
139 139
 	defer ctrl1.Stop()
140 140
 	// Use the same boltdb file without closing the previous controller
141
-	ctrl2, err := New(cfgOptions...)
142
-	if err != nil {
143
-		t.Fatalf("Error new controller: %v", err)
144
-	}
145
-	store := ctrl2.(*controller).localStore
146
-	if store != nil {
147
-		t.Fatalf("localstore is expected to be nil")
141
+	_, err = New(cfgOptions...)
142
+	if err == nil {
143
+		t.Fatalf("Expected to fail but succeeded")
148 144
 	}
149 145
 }