Browse code

libnetwork: remove more datastore scope plumbing

Signed-off-by: Cory Snider <csnider@mirantis.com>

Cory Snider authored on 2023/01/14 09:04:43
Showing 9 changed files
... ...
@@ -360,7 +360,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
360 360
 
361 361
 	n, err := c.NewNetwork(driver, create.Name, id, nwOptions...)
362 362
 	if err != nil {
363
-		if _, ok := err.(libnetwork.ErrDataStoreNotInitialized); ok {
363
+		if errors.Is(err, libnetwork.ErrDataStoreNotInitialized) {
364 364
 			//nolint: revive
365 365
 			return nil, errors.New("This node is not a swarm manager. Use \"docker swarm init\" or \"docker swarm join\" to connect this node to swarm and try again.")
366 366
 		}
... ...
@@ -88,7 +88,7 @@ type Controller struct {
88 88
 	drvRegistry      *drvregistry.DrvRegistry
89 89
 	sandboxes        sandboxTable
90 90
 	cfg              *config.Config
91
-	stores           []datastore.DataStore
91
+	store            datastore.DataStore
92 92
 	extKeyListener   net.Listener
93 93
 	watchCh          chan *Endpoint
94 94
 	unWatchCh        chan *Endpoint
... ...
@@ -130,7 +130,7 @@ func New(cfgOptions ...config.Option) (*Controller, error) {
130 130
 		return nil, err
131 131
 	}
132 132
 
133
-	drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil, c.cfg.PluginGetter)
133
+	drvRegistry, err := drvregistry.New(c.getStore(), nil, c.RegisterDriver, nil, c.cfg.PluginGetter)
134 134
 	if err != nil {
135 135
 		return nil, err
136 136
 	}
... ...
@@ -149,7 +149,7 @@ func New(cfgOptions ...config.Option) (*Controller, error) {
149 149
 		}
150 150
 	}
151 151
 
152
-	if err = initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope), c.cfg.DefaultAddressPool); err != nil {
152
+	if err = initIPAMDrivers(drvRegistry, nil, nil, c.cfg.DefaultAddressPool); err != nil {
153 153
 		return nil, err
154 154
 	}
155 155
 
... ...
@@ -696,7 +696,7 @@ var joinCluster NetworkWalker = func(nw Network) bool {
696 696
 }
697 697
 
698 698
 func (c *Controller) reservePools() {
699
-	networks, err := c.getNetworksForScope(datastore.LocalScope)
699
+	networks, err := c.getNetworks()
700 700
 	if err != nil {
701 701
 		logrus.Warnf("Could not retrieve networks from local store during ipam allocation for existing networks: %v", err)
702 702
 		return
... ...
@@ -1151,7 +1151,7 @@ func (c *Controller) cleanupLocalEndpoints() {
1151 1151
 			eps[ep.id] = true
1152 1152
 		}
1153 1153
 	}
1154
-	nl, err := c.getNetworksForScope(datastore.LocalScope)
1154
+	nl, err := c.getNetworks()
1155 1155
 	if err != nil {
1156 1156
 		logrus.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
1157 1157
 		return
... ...
@@ -109,7 +109,7 @@ func (ec *endpointCnt) EndpointCnt() uint64 {
109 109
 }
110 110
 
111 111
 func (ec *endpointCnt) updateStore() error {
112
-	store := ec.n.getController().getStore(ec.DataScope())
112
+	store := ec.n.getController().getStore()
113 113
 	if store == nil {
114 114
 		return fmt.Errorf("store not found for scope %s on endpoint count update", ec.DataScope())
115 115
 	}
... ...
@@ -138,7 +138,7 @@ func (ec *endpointCnt) setCnt(cnt uint64) error {
138 138
 }
139 139
 
140 140
 func (ec *endpointCnt) atomicIncDecEpCnt(inc bool) error {
141
-	store := ec.n.getController().getStore(ec.DataScope())
141
+	store := ec.n.getController().getStore()
142 142
 	if store == nil {
143 143
 		return fmt.Errorf("store not found for scope %s", ec.DataScope())
144 144
 	}
... ...
@@ -1,6 +1,7 @@
1 1
 package libnetwork
2 2
 
3 3
 import (
4
+	"errors"
4 5
 	"fmt"
5 6
 )
6 7
 
... ...
@@ -186,8 +187,4 @@ func (mr ManagerRedirectError) Maskable() {}
186 186
 
187 187
 // ErrDataStoreNotInitialized is returned if an invalid data scope is passed
188 188
 // for getting data store
189
-type ErrDataStoreNotInitialized string
190
-
191
-func (dsni ErrDataStoreNotInitialized) Error() string {
192
-	return fmt.Sprintf("datastore for scope %q is not initialized", string(dsni))
193
-}
189
+var ErrDataStoreNotInitialized = errors.New("datastore is not initialized")
... ...
@@ -187,7 +187,7 @@ func (sb *Sandbox) storeDelete() error {
187 187
 }
188 188
 
189 189
 func (c *Controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
190
-	store := c.getStore(datastore.LocalScope)
190
+	store := c.getStore()
191 191
 	if store == nil {
192 192
 		logrus.Error("Could not find local scope store while trying to cleanup sandboxes")
193 193
 		return
... ...
@@ -22,40 +22,27 @@ func (c *Controller) initStores() error {
22 22
 	if c.cfg == nil {
23 23
 		return nil
24 24
 	}
25
-	store, err := datastore.NewDataStore(c.cfg.Scope)
25
+	var err error
26
+	c.store, err = datastore.NewDataStore(c.cfg.Scope)
26 27
 	if err != nil {
27 28
 		return err
28 29
 	}
29
-	c.stores = []datastore.DataStore{store}
30 30
 
31 31
 	c.startWatch()
32 32
 	return nil
33 33
 }
34 34
 
35 35
 func (c *Controller) closeStores() {
36
-	for _, store := range c.getStores() {
36
+	if store := c.store; store != nil {
37 37
 		store.Close()
38 38
 	}
39 39
 }
40 40
 
41
-func (c *Controller) getStore(scope string) datastore.DataStore {
41
+func (c *Controller) getStore() datastore.DataStore {
42 42
 	c.mu.Lock()
43 43
 	defer c.mu.Unlock()
44 44
 
45
-	for _, store := range c.stores {
46
-		if store.Scope() == scope {
47
-			return store
48
-		}
49
-	}
50
-
51
-	return nil
52
-}
53
-
54
-func (c *Controller) getStores() []datastore.DataStore {
55
-	c.mu.Lock()
56
-	defer c.mu.Unlock()
57
-
58
-	return c.stores
45
+	return c.store
59 46
 }
60 47
 
61 48
 func (c *Controller) getNetworkFromStore(nid string) (*network, error) {
... ...
@@ -67,10 +54,10 @@ func (c *Controller) getNetworkFromStore(nid string) (*network, error) {
67 67
 	return nil, ErrNoSuchNetwork(nid)
68 68
 }
69 69
 
70
-func (c *Controller) getNetworksForScope(scope string) ([]*network, error) {
70
+func (c *Controller) getNetworks() ([]*network, error) {
71 71
 	var nl []*network
72 72
 
73
-	store := c.getStore(scope)
73
+	store := c.getStore()
74 74
 	if store == nil {
75 75
 		return nil, nil
76 76
 	}
... ...
@@ -78,8 +65,7 @@ func (c *Controller) getNetworksForScope(scope string) ([]*network, error) {
78 78
 	kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
79 79
 		&network{ctrlr: c})
80 80
 	if err != nil && err != datastore.ErrKeyNotFound {
81
-		return nil, fmt.Errorf("failed to get networks for scope %s: %v",
82
-			scope, err)
81
+		return nil, fmt.Errorf("failed to get networks: %w", err)
83 82
 	}
84 83
 
85 84
 	for _, kvo := range kvol {
... ...
@@ -95,7 +81,7 @@ func (c *Controller) getNetworksForScope(scope string) ([]*network, error) {
95 95
 
96 96
 		n.epCnt = ec
97 97
 		if n.scope == "" {
98
-			n.scope = scope
98
+			n.scope = store.Scope()
99 99
 		}
100 100
 		nl = append(nl, n)
101 101
 	}
... ...
@@ -103,92 +89,80 @@ func (c *Controller) getNetworksForScope(scope string) ([]*network, error) {
103 103
 	return nl, nil
104 104
 }
105 105
 
106
-func (c *Controller) getNetworksFromStore() []*network {
106
+func (c *Controller) getNetworksFromStore() []*network { // FIXME: unify with c.getNetworks()
107 107
 	var nl []*network
108 108
 
109
-	for _, store := range c.getStores() {
110
-		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix), &network{ctrlr: c})
111
-		// Continue searching in the next store if no keys found in this store
112
-		if err != nil {
113
-			if err != datastore.ErrKeyNotFound {
114
-				logrus.Debugf("failed to get networks for scope %s: %v", store.Scope(), err)
115
-			}
116
-			continue
109
+	store := c.getStore()
110
+	kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix), &network{ctrlr: c})
111
+	if err != nil {
112
+		if err != datastore.ErrKeyNotFound {
113
+			logrus.Debugf("failed to get networks from store: %v", err)
117 114
 		}
115
+		return nil
116
+	}
118 117
 
119
-		kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{})
120
-		if err != nil && err != datastore.ErrKeyNotFound {
121
-			logrus.Warnf("failed to get endpoint_count map for scope %s: %v", store.Scope(), err)
122
-		}
118
+	kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{})
119
+	if err != nil && err != datastore.ErrKeyNotFound {
120
+		logrus.Warnf("failed to get endpoint_count map from store: %v", err)
121
+	}
123 122
 
124
-		for _, kvo := range kvol {
125
-			n := kvo.(*network)
126
-			n.mu.Lock()
127
-			n.ctrlr = c
128
-			ec := &endpointCnt{n: n}
129
-			// Trim the leading & trailing "/" to make it consistent across all stores
130
-			if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok {
131
-				ec = val.(*endpointCnt)
132
-				ec.n = n
133
-				n.epCnt = ec
134
-			}
135
-			if n.scope == "" {
136
-				n.scope = store.Scope()
137
-			}
138
-			n.mu.Unlock()
139
-			nl = append(nl, n)
123
+	for _, kvo := range kvol {
124
+		n := kvo.(*network)
125
+		n.mu.Lock()
126
+		n.ctrlr = c
127
+		ec := &endpointCnt{n: n}
128
+		// Trim the leading & trailing "/" to make it consistent across all stores
129
+		if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok {
130
+			ec = val.(*endpointCnt)
131
+			ec.n = n
132
+			n.epCnt = ec
133
+		}
134
+		if n.scope == "" {
135
+			n.scope = store.Scope()
140 136
 		}
137
+		n.mu.Unlock()
138
+		nl = append(nl, n)
141 139
 	}
142 140
 
143 141
 	return nl
144 142
 }
145 143
 
146 144
 func (n *network) getEndpointFromStore(eid string) (*Endpoint, error) {
147
-	var errors []string
148
-	for _, store := range n.ctrlr.getStores() {
149
-		ep := &Endpoint{id: eid, network: n}
150
-		err := store.GetObject(datastore.Key(ep.Key()...), ep)
151
-		// Continue searching in the next store if the key is not found in this store
152
-		if err != nil {
153
-			if err != datastore.ErrKeyNotFound {
154
-				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
155
-				logrus.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
156
-			}
157
-			continue
158
-		}
159
-		return ep, nil
145
+	store := n.ctrlr.getStore()
146
+	ep := &Endpoint{id: eid, network: n}
147
+	err := store.GetObject(datastore.Key(ep.Key()...), ep)
148
+	if err != nil {
149
+		return nil, fmt.Errorf("could not find endpoint %s: %w", eid, err)
160 150
 	}
161
-	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
151
+	return ep, nil
162 152
 }
163 153
 
164 154
 func (n *network) getEndpointsFromStore() ([]*Endpoint, error) {
165 155
 	var epl []*Endpoint
166 156
 
167 157
 	tmp := Endpoint{network: n}
168
-	for _, store := range n.getController().getStores() {
169
-		kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &Endpoint{network: n})
170
-		// Continue searching in the next store if no keys found in this store
171
-		if err != nil {
172
-			if err != datastore.ErrKeyNotFound {
173
-				logrus.Debugf("failed to get endpoints for network %s scope %s: %v",
174
-					n.Name(), store.Scope(), err)
175
-			}
176
-			continue
158
+	store := n.getController().getStore()
159
+	kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &Endpoint{network: n})
160
+	if err != nil {
161
+		if err != datastore.ErrKeyNotFound {
162
+			return nil, fmt.Errorf("failed to get endpoints for network %s scope %s: %w",
163
+				n.Name(), store.Scope(), err)
177 164
 		}
165
+		return nil, nil
166
+	}
178 167
 
179
-		for _, kvo := range kvol {
180
-			ep := kvo.(*Endpoint)
181
-			epl = append(epl, ep)
182
-		}
168
+	for _, kvo := range kvol {
169
+		ep := kvo.(*Endpoint)
170
+		epl = append(epl, ep)
183 171
 	}
184 172
 
185 173
 	return epl, nil
186 174
 }
187 175
 
188 176
 func (c *Controller) updateToStore(kvObject datastore.KVObject) error {
189
-	cs := c.getStore(kvObject.DataScope())
177
+	cs := c.getStore()
190 178
 	if cs == nil {
191
-		return ErrDataStoreNotInitialized(kvObject.DataScope())
179
+		return ErrDataStoreNotInitialized
192 180
 	}
193 181
 
194 182
 	if err := cs.PutObjectAtomic(kvObject); err != nil {
... ...
@@ -202,9 +176,9 @@ func (c *Controller) updateToStore(kvObject datastore.KVObject) error {
202 202
 }
203 203
 
204 204
 func (c *Controller) deleteFromStore(kvObject datastore.KVObject) error {
205
-	cs := c.getStore(kvObject.DataScope())
205
+	cs := c.getStore()
206 206
 	if cs == nil {
207
-		return ErrDataStoreNotInitialized(kvObject.DataScope())
207
+		return ErrDataStoreNotInitialized
208 208
 	}
209 209
 
210 210
 retry:
... ...
@@ -258,7 +232,8 @@ func (c *Controller) networkWatchLoop(nw *netWatch, ep *Endpoint, ecCh <-chan da
258 258
 
259 259
 			epl, err := ec.n.getEndpointsFromStore()
260 260
 			if err != nil {
261
-				break
261
+				logrus.WithError(err).Debug("error getting endpoints from store")
262
+				continue
262 263
 			}
263 264
 
264 265
 			c.mu.Lock()
... ...
@@ -356,7 +331,7 @@ func (c *Controller) processEndpointCreate(nmap map[string]*netWatch, ep *Endpoi
356 356
 	nw.stopCh = make(chan struct{})
357 357
 	c.mu.Unlock()
358 358
 
359
-	store := c.getStore(n.DataScope())
359
+	store := c.getStore()
360 360
 	if store == nil {
361 361
 		return
362 362
 	}
... ...
@@ -30,7 +30,7 @@ func TestNoPersist(t *testing.T) {
30 30
 	if err != nil {
31 31
 		t.Fatalf("Error creating endpoint: %v", err)
32 32
 	}
33
-	store := ctrl.getStore(datastore.LocalScope).KVStore()
33
+	store := ctrl.getStore().KVStore()
34 34
 	if exists, _ := store.Exists(datastore.Key(datastore.NetworkKeyPrefix, nw.ID())); exists {
35 35
 		t.Fatalf("Network with persist=false should not be stored in KV Store")
36 36
 	}
... ...
@@ -36,7 +36,7 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con
36 36
 	if err != nil {
37 37
 		t.Fatalf("Error creating endpoint: %v", err)
38 38
 	}
39
-	store := ctrl.getStore(datastore.LocalScope).KVStore()
39
+	store := ctrl.getStore().KVStore()
40 40
 	if exists, err := store.Exists(datastore.Key(datastore.NetworkKeyPrefix, nw.ID())); !exists || err != nil {
41 41
 		t.Fatalf("Network key should have been created.")
42 42
 	}