Browse code

replace individual endpoint_cnt read from store with 1 bulk read

getNetworksFromStore reads networks and endpoint_cnt from the kvstores.
endpoint_cnt especially is read in a for-loop for each network and that
causes a lot of stress in poorly performing KV-Stores.
This fix eases the load on the kvstore by fetching all the endpoint_cnt
in a single read and the operation is performed on it.

Signed-off-by: Madhu Venugopal <madhu@docker.com>

Madhu Venugopal authored on 2017/02/03 07:45:38
Showing 2 changed files
... ...
@@ -40,6 +40,8 @@ type DataStore interface {
40 40
 	// key. The caller must pass a KVObject of the same type as
41 41
 	// the objects that need to be listed
42 42
 	List(string, KVObject) ([]KVObject, error)
43
+	// Map returns a Map of KVObjects
44
+	Map(key string, kvObject KVObject) (map[string]KVObject, error)
43 45
 	// Scope returns the scope of the store
44 46
 	Scope() string
45 47
 	// KVStore returns access to the KV Store
... ...
@@ -512,23 +514,34 @@ func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) {
512 512
 		return ds.cache.list(kvObject)
513 513
 	}
514 514
 
515
+	var kvol []KVObject
516
+	cb := func(key string, val KVObject) {
517
+		kvol = append(kvol, val)
518
+	}
519
+	err := ds.iterateKVPairsFromStore(key, kvObject, cb)
520
+	if err != nil {
521
+		return nil, err
522
+	}
523
+	return kvol, nil
524
+}
525
+
526
+func (ds *datastore) iterateKVPairsFromStore(key string, kvObject KVObject, callback func(string, KVObject)) error {
515 527
 	// Bail out right away if the kvObject does not implement KVConstructor
516 528
 	ctor, ok := kvObject.(KVConstructor)
517 529
 	if !ok {
518
-		return nil, fmt.Errorf("error listing objects, object does not implement KVConstructor interface")
530
+		return fmt.Errorf("error listing objects, object does not implement KVConstructor interface")
519 531
 	}
520 532
 
521 533
 	// Make sure the parent key exists
522 534
 	if err := ds.ensureParent(key); err != nil {
523
-		return nil, err
535
+		return err
524 536
 	}
525 537
 
526 538
 	kvList, err := ds.store.List(key)
527 539
 	if err != nil {
528
-		return nil, err
540
+		return err
529 541
 	}
530 542
 
531
-	var kvol []KVObject
532 543
 	for _, kvPair := range kvList {
533 544
 		if len(kvPair.Value) == 0 {
534 545
 			continue
... ...
@@ -536,16 +549,33 @@ func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) {
536 536
 
537 537
 		dstO := ctor.New()
538 538
 		if err := dstO.SetValue(kvPair.Value); err != nil {
539
-			return nil, err
539
+			return err
540 540
 		}
541 541
 
542 542
 		// Make sure the object has a correct view of the DB index in
543 543
 		// case we need to modify it and update the DB.
544 544
 		dstO.SetIndex(kvPair.LastIndex)
545
+		callback(kvPair.Key, dstO)
546
+	}
547
+
548
+	return nil
549
+}
545 550
 
546
-		kvol = append(kvol, dstO)
551
+func (ds *datastore) Map(key string, kvObject KVObject) (map[string]KVObject, error) {
552
+	if ds.sequential {
553
+		ds.Lock()
554
+		defer ds.Unlock()
547 555
 	}
548 556
 
557
+	kvol := make(map[string]KVObject)
558
+	cb := func(key string, val KVObject) {
559
+		// Trim the leading & trailing "/" to make it consistent across all stores
560
+		kvol[strings.Trim(key, "/")] = val
561
+	}
562
+	err := ds.iterateKVPairsFromStore(key, kvObject, cb)
563
+	if err != nil {
564
+		return nil, err
565
+	}
549 566
 	return kvol, nil
550 567
 }
551 568
 
... ...
@@ -2,6 +2,7 @@ package libnetwork
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"strings"
5 6
 
6 7
 	"github.com/Sirupsen/logrus"
7 8
 	"github.com/docker/libkv/store/boltdb"
... ...
@@ -152,21 +153,24 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
152 152
 			continue
153 153
 		}
154 154
 
155
+		kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{})
156
+		if err != nil {
157
+			if err != datastore.ErrKeyNotFound {
158
+				logrus.Warnf("failed to get endpoint_count map for scope %s: %v", store.Scope(), err)
159
+			}
160
+		}
161
+
155 162
 		for _, kvo := range kvol {
156 163
 			n := kvo.(*network)
157 164
 			n.Lock()
158 165
 			n.ctrlr = c
159
-			n.Unlock()
160
-
161 166
 			ec := &endpointCnt{n: n}
162
-			err = store.GetObject(datastore.Key(ec.Key()...), ec)
163
-			if err != nil && !n.inDelete {
164
-				logrus.Warnf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
165
-				continue
167
+			// Trim the leading & trailing "/" to make it consistent across all stores
168
+			if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok {
169
+				ec = val.(*endpointCnt)
170
+				ec.n = n
171
+				n.epCnt = ec
166 172
 			}
167
-
168
-			n.Lock()
169
-			n.epCnt = ec
170 173
 			n.scope = store.Scope()
171 174
 			n.Unlock()
172 175
 			nl = append(nl, n)