@@ -3,7 +3,7 @@

 # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When

 # updating the binary version, consider updating github.com/docker/libnetwork

 # in vendor.conf accordingly

-LIBNETWORK_COMMIT=3ac297bc7fd0afec9051bbb47024c9bc1d75bf5b

+LIBNETWORK_COMMIT=f30a35b091cc2a431ef9856c75c343f75bb5f2e2

 install_proxy() {

 	case "$1" in

@@ -37,7 +37,7 @@ github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b

 #get libnetwork packages

 # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly

-github.com/docker/libnetwork d00ceed44cc447c77f25cdf5d59e83163bdcb4c9

+github.com/docker/libnetwork f30a35b091cc2a431ef9856c75c343f75bb5f2e2

 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9

 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80

 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

@@ -15,6 +15,17 @@ There are many networking solutions available to suit a broad range of use-cases

 ```go

+import (

+	"fmt"

+	"log"

+

+	"github.com/docker/docker/pkg/reexec"

+	"github.com/docker/libnetwork"

+	"github.com/docker/libnetwork/config"

+	"github.com/docker/libnetwork/netlabel"

+	"github.com/docker/libnetwork/options"

+)

+

 func main() {

 	if reexec.Init() {

 		return

@@ -194,7 +194,7 @@ func (c *controller) handleKeyChange(keys []*types.EncryptionKey) error {

 func (c *controller) agentSetup(clusterProvider cluster.Provider) error {

 	agent := c.getAgent()

-	// If the agent is already present there is no need to try to initilize it again

+	// If the agent is already present there is no need to try to initialize it again

 	if agent != nil {

 		return nil

 	}

@@ -372,7 +372,7 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial

 			h.Lock()

 		}

-		// Previous atomic push was succesfull. Save private copy to local copy

+		// Previous atomic push was successful. Save private copy to local copy

 		h.unselected = nh.unselected

 		h.head = nh.head

 		h.dbExists = nh.dbExists

deleted file mode 100644

@@ -1,29 +0,0 @@

-package common

-

-import (

-	"runtime"

-	"strings"

-)

-

-func callerInfo(i int) string {

-	ptr, _, _, ok := runtime.Caller(i)

-	fName := "unknown"

-	if ok {

-		f := runtime.FuncForPC(ptr)

-		if f != nil {

-			// f.Name() is like: github.com/docker/libnetwork/common.MethodName

-			tmp := strings.Split(f.Name(), ".")

-			if len(tmp) > 0 {

-				fName = tmp[len(tmp)-1]

-			}

-		}

-	}

-

-	return fName

-}

-

-// CallerName returns the name of the function at the specified level

-// level == 0 means current method name

-func CallerName(level int) string {

-	return callerInfo(2 + level)

-}

deleted file mode 100644

@@ -1,135 +0,0 @@

-package common

-

-import (

-	"sync"

-

-	mapset "github.com/deckarep/golang-set"

-)

-

-// SetMatrix is a map of Sets

-type SetMatrix interface {

-	// Get returns the members of the set for a specific key as a slice.

-	Get(key string) ([]interface{}, bool)

-	// Contains is used to verify if an element is in a set for a specific key

-	// returns true if the element is in the set

-	// returns true if there is a set for the key

-	Contains(key string, value interface{}) (bool, bool)

-	// Insert inserts the value in the set of a key

-	// returns true if the value is inserted (was not already in the set), false otherwise

-	// returns also the length of the set for the key

-	Insert(key string, value interface{}) (bool, int)

-	// Remove removes the value in the set for a specific key

-	// returns true if the value is deleted, false otherwise

-	// returns also the length of the set for the key

-	Remove(key string, value interface{}) (bool, int)

-	// Cardinality returns the number of elements in the set for a key

-	// returns false if the set is not present

-	Cardinality(key string) (int, bool)

-	// String returns the string version of the set, empty otherwise

-	// returns false if the set is not present

-	String(key string) (string, bool)

-	// Returns all the keys in the map

-	Keys() []string

-}

-

-type setMatrix struct {

-	matrix map[string]mapset.Set

-

-	sync.Mutex

-}

-

-// NewSetMatrix creates a new set matrix object

-func NewSetMatrix() SetMatrix {

-	s := &setMatrix{}

-	s.init()

-	return s

-}

-

-func (s *setMatrix) init() {

-	s.matrix = make(map[string]mapset.Set)

-}

-

-func (s *setMatrix) Get(key string) ([]interface{}, bool) {

-	s.Lock()

-	defer s.Unlock()

-	set, ok := s.matrix[key]

-	if !ok {

-		return nil, ok

-	}

-	return set.ToSlice(), ok

-}

-

-func (s *setMatrix) Contains(key string, value interface{}) (bool, bool) {

-	s.Lock()

-	defer s.Unlock()

-	set, ok := s.matrix[key]

-	if !ok {

-		return false, ok

-	}

-	return set.Contains(value), ok

-}

-

-func (s *setMatrix) Insert(key string, value interface{}) (bool, int) {

-	s.Lock()

-	defer s.Unlock()

-	set, ok := s.matrix[key]

-	if !ok {

-		s.matrix[key] = mapset.NewSet()

-		s.matrix[key].Add(value)

-		return true, 1

-	}

-

-	return set.Add(value), set.Cardinality()

-}

-

-func (s *setMatrix) Remove(key string, value interface{}) (bool, int) {

-	s.Lock()

-	defer s.Unlock()

-	set, ok := s.matrix[key]

-	if !ok {

-		return false, 0

-	}

-

-	var removed bool

-	if set.Contains(value) {

-		set.Remove(value)

-		removed = true

-		// If the set is empty remove it from the matrix

-		if set.Cardinality() == 0 {

-			delete(s.matrix, key)

-		}

-	}

-

-	return removed, set.Cardinality()

-}

-

-func (s *setMatrix) Cardinality(key string) (int, bool) {

-	s.Lock()

-	defer s.Unlock()

-	set, ok := s.matrix[key]

-	if !ok {

-		return 0, ok

-	}

-

-	return set.Cardinality(), ok

-}

-

-func (s *setMatrix) String(key string) (string, bool) {

-	s.Lock()

-	defer s.Unlock()

-	set, ok := s.matrix[key]

-	if !ok {

-		return "", ok

-	}

-	return set.String(), ok

-}

-

-func (s *setMatrix) Keys() []string {

-	s.Lock()

-	defer s.Unlock()

-	keys := make([]string, 0, len(s.matrix))

-	for k := range s.matrix {

-		keys = append(keys, k)

-	}

-	return keys

-}

@@ -121,7 +121,7 @@ type NetworkController interface {

 	// Stop network controller

 	Stop()

-	// ReloadCondfiguration updates the controller configuration

+	// ReloadConfiguration updates the controller configuration

 	ReloadConfiguration(cfgOptions ...config.Option) error

 	// SetClusterProvider sets cluster provider

@@ -1107,6 +1107,8 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S

 		sb.config.hostsPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/hosts")

 		sb.config.resolvConfPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/resolv.conf")

 		sb.id = "ingress_sbox"

+	} else if sb.loadBalancerNID != "" {

+		sb.id = "lb_" + sb.loadBalancerNID

 	}

 	c.Unlock()

@@ -185,7 +185,7 @@ func Key(key ...string) string {

 func ParseKey(key string) ([]string, error) {

 	chain := strings.Split(strings.Trim(key, "/"), "/")

-	// The key must atleast be equal to the rootChain in order to be considered as valid

+	// The key must at least be equal to the rootChain in order to be considered as valid

 	if len(chain) <= len(rootChain) || !reflect.DeepEqual(chain[0:len(rootChain)], rootChain) {

 		return nil, types.BadRequestErrorf("invalid Key : %s", key)

 	}

@@ -589,7 +589,7 @@ func (ds *datastore) DeleteObject(kvObject KVObject) error {

 		defer ds.Unlock()

 	}

-	// cleaup the cache first

+	// cleanup the cache first

 	if ds.cache != nil {

 		// If persistent store is skipped, sequencing needs to

 		// happen in cache.

@@ -645,7 +645,7 @@ func (ds *datastore) DeleteTree(kvObject KVObject) error {

 		defer ds.Unlock()

 	}

-	// cleaup the cache first

+	// cleanup the cache first

 	if ds.cache != nil {

 		// If persistent store is skipped, sequencing needs to

 		// happen in cache.

@@ -8,8 +8,8 @@ import (

 )

 var (

-	// ErrNotImplmented exported

-	ErrNotImplmented = errors.New("Functionality not implemented")

+	// ErrNotImplemented exported

+	ErrNotImplemented = errors.New("Functionality not implemented")

 )

 // MockData exported

@@ -65,7 +65,7 @@ func (s *MockStore) Exists(key string) (bool, error) {

 // List gets a range of values at "directory"

 func (s *MockStore) List(prefix string) ([]*store.KVPair, error) {

-	return nil, ErrNotImplmented

+	return nil, ErrNotImplemented

 }

 // DeleteTree deletes a range of values at "directory"

@@ -76,17 +76,17 @@ func (s *MockStore) DeleteTree(prefix string) error {

 // Watch a single key for modifications

 func (s *MockStore) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) {

-	return nil, ErrNotImplmented

+	return nil, ErrNotImplemented

 }

 // WatchTree triggers a watch on a range of values at "directory"

 func (s *MockStore) WatchTree(prefix string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) {

-	return nil, ErrNotImplmented

+	return nil, ErrNotImplemented

 }

 // NewLock exposed

 func (s *MockStore) NewLock(key string, options *store.LockOptions) (store.Locker, error) {

-	return nil, ErrNotImplmented

+	return nil, ErrNotImplemented

 }

 // AtomicPut put a value at "key" if the key has not been

@@ -9,7 +9,7 @@ import (

 	"sync/atomic"

 	stackdump "github.com/docker/docker/pkg/signal"

-	"github.com/docker/libnetwork/common"

+	"github.com/docker/libnetwork/internal/caller"

 	"github.com/sirupsen/logrus"

 )

@@ -127,7 +127,7 @@ func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) {

 	rsp := WrongCommand("not implemented", fmt.Sprintf("URL path: %s no method implemented check /help\n", r.URL.Path))

 	// audit logs

-	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})

+	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": caller.Name(0), "url": r.URL.String()})

 	log.Info("command not implemented done")

 	HTTPReply(w, rsp, json)

@@ -138,7 +138,7 @@ func help(ctx interface{}, w http.ResponseWriter, r *http.Request) {

 	_, json := ParseHTTPFormOptions(r)

 	// audit logs

-	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})

+	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": caller.Name(0), "url": r.URL.String()})

 	log.Info("help done")

 	n, ok := ctx.(*Server)

@@ -156,7 +156,7 @@ func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) {

 	_, json := ParseHTTPFormOptions(r)

 	// audit logs

-	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})

+	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": caller.Name(0), "url": r.URL.String()})

 	log.Info("ready done")

 	HTTPReply(w, CommandSucceed(&StringCmd{Info: "OK"}), json)

 }

@@ -166,7 +166,7 @@ func stackTrace(ctx interface{}, w http.ResponseWriter, r *http.Request) {

 	_, json := ParseHTTPFormOptions(r)

 	// audit logs

-	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": common.CallerName(0), "url": r.URL.String()})

+	log := logrus.WithFields(logrus.Fields{"component": "diagnostic", "remoteIP": r.RemoteAddr, "method": caller.Name(0), "url": r.URL.String()})

 	log.Info("stack trace")

 	path, err := stackdump.DumpStacks("/tmp/")

@@ -75,10 +75,10 @@ type Driver interface {

 	// DecodeTableEntry passes the driver a key, value pair from table it registered

 	// with libnetwork. Driver should return {object ID, map[string]string} tuple.

 	// If DecodeTableEntry is called for a table associated with NetworkObject or

-	// EndpointObject the return object ID should be the network id or endppoint id

+	// EndpointObject the return object ID should be the network id or endpoint id

 	// associated with that entry. map should have information about the object that

 	// can be presented to the user.

-	// For exampe: overlay driver returns the VTEP IP of the host that has the endpoint

+	// For example: overlay driver returns the VTEP IP of the host that has the endpoint

 	// which is shown in 'network inspect --verbose'

 	DecodeTableEntry(tablename string, key string, value []byte) (string, map[string]string)

@@ -97,7 +97,7 @@ type NetworkInfo interface {

 	TableEventRegister(tableName string, objType ObjectType) error

 }

-// InterfaceInfo provides a go interface for drivers to retrive

+// InterfaceInfo provides a go interface for drivers to retrieve

 // network information to interface resources.

 type InterfaceInfo interface {

 	// SetMacAddress allows the driver to set the mac address to the endpoint interface

@@ -104,7 +104,7 @@ type containerConfiguration struct {

 	ChildEndpoints  []string

 }

-// cnnectivityConfiguration represents the user specified configuration regarding the external connectivity

+// connectivityConfiguration represents the user specified configuration regarding the external connectivity

 type connectivityConfiguration struct {

 	PortBindings []types.PortBinding

 	ExposedPorts []types.TransportPort

@@ -84,7 +84,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 			}

 			v4gw, _, err := net.ParseCIDR(s.GwIP)

 			if err != nil {

-				return fmt.Errorf("gatway %s is not a valid ipv4 address: %v", s.GwIP, err)

+				return fmt.Errorf("gateway %s is not a valid ipv4 address: %v", s.GwIP, err)

 			}

 			err = jinfo.SetGateway(v4gw)

 			if err != nil {

@@ -101,7 +101,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 			}

 			v6gw, _, err := net.ParseCIDR(s.GwIP)

 			if err != nil {

-				return fmt.Errorf("gatway %s is not a valid ipv6 address: %v", s.GwIP, err)

+				return fmt.Errorf("gateway %s is not a valid ipv6 address: %v", s.GwIP, err)

 			}

 			err = jinfo.SetGatewayIPv6(v6gw)

 			if err != nil {

@@ -68,7 +68,7 @@ func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo

 	err = d.storeUpdate(config)

 	if err != nil {

 		d.deleteNetwork(config.ID)

-		logrus.Debugf("encoutered an error rolling back a network create for %s : %v", config.ID, err)

+		logrus.Debugf("encountered an error rolling back a network create for %s : %v", config.ID, err)

 		return err

 	}

@@ -92,7 +92,7 @@ func (d *driver) createNetwork(config *configuration) error {

 				return err

 			}

 			config.CreatedSlaveLink = true

-			// notify the user in logs they have limited comunicatins

+			// notify the user in logs they have limited communications

 			if config.Parent == getDummyName(stringid.TruncateID(config.ID)) {

 				logrus.Debugf("Empty -o parent= and --internal flags limit communications to other containers inside of network: %s",

 					config.Parent)

@@ -30,7 +30,7 @@ func createIPVlan(containerIfName, parent, ipvlanMode string) (string, error) {

 	// Get the link for the master index (Example: the docker host eth iface)

 	parentLink, err := ns.NlHandle().LinkByName(parent)

 	if err != nil {

-		return "", fmt.Errorf("error occoured looking up the %s parent iface %s error: %s", ipvlanType, parent, err)

+		return "", fmt.Errorf("error occurred looking up the %s parent iface %s error: %s", ipvlanType, parent, err)

 	}

 	// Create an ipvlan link

 	ipvlan := &netlink.IPVlan{

@@ -169,7 +169,7 @@ func createDummyLink(dummyName, truncNetID string) error {

 	}

 	parentDummyLink, err := ns.NlHandle().LinkByName(dummyName)

 	if err != nil {

-		return fmt.Errorf("error occoured looking up the %s parent iface %s error: %s", ipvlanType, dummyName, err)

+		return fmt.Errorf("error occurred looking up the %s parent iface %s error: %s", ipvlanType, dummyName, err)

 	}

 	// bring the new netlink iface up

 	if err := ns.NlHandle().LinkSetUp(parentDummyLink); err != nil {

@@ -31,7 +31,7 @@ func (d *driver) deleteNetwork(nid string) {

 	d.Unlock()

 }

-// getNetworks Safely returns a slice of existng networks

+// getNetworks Safely returns a slice of existing networks

 func (d *driver) getNetworks() []*network {

 	d.Lock()

 	defer d.Unlock()

@@ -46,7 +46,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 		}

 		v4gw, _, err := net.ParseCIDR(s.GwIP)

 		if err != nil {

-			return fmt.Errorf("gatway %s is not a valid ipv4 address: %v", s.GwIP, err)

+			return fmt.Errorf("gateway %s is not a valid ipv4 address: %v", s.GwIP, err)

 		}

 		err = jinfo.SetGateway(v4gw)

 		if err != nil {

@@ -63,7 +63,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 		}

 		v6gw, _, err := net.ParseCIDR(s.GwIP)

 		if err != nil {

-			return fmt.Errorf("gatway %s is not a valid ipv6 address: %v", s.GwIP, err)

+			return fmt.Errorf("gateway %s is not a valid ipv6 address: %v", s.GwIP, err)

 		}

 		err = jinfo.SetGatewayIPv6(v6gw)

 		if err != nil {

@@ -72,7 +72,7 @@ func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo

 	err = d.storeUpdate(config)

 	if err != nil {

 		d.deleteNetwork(config.ID)

-		logrus.Debugf("encoutered an error rolling back a network create for %s : %v", config.ID, err)

+		logrus.Debugf("encountered an error rolling back a network create for %s : %v", config.ID, err)

 		return err

 	}

@@ -96,7 +96,7 @@ func (d *driver) createNetwork(config *configuration) error {

 				return err

 			}

 			config.CreatedSlaveLink = true

-			// notify the user in logs they have limited comunicatins

+			// notify the user in logs they have limited communications

 			if config.Parent == getDummyName(stringid.TruncateID(config.ID)) {

 				logrus.Debugf("Empty -o parent= and --internal flags limit communications to other containers inside of network: %s",

 					config.Parent)

@@ -30,7 +30,7 @@ func createMacVlan(containerIfName, parent, macvlanMode string) (string, error)

 	// Get the link for the master index (Example: the docker host eth iface)

 	parentLink, err := ns.NlHandle().LinkByName(parent)

 	if err != nil {

-		return "", fmt.Errorf("error occoured looking up the %s parent iface %s error: %s", macvlanType, parent, err)

+		return "", fmt.Errorf("error occurred looking up the %s parent iface %s error: %s", macvlanType, parent, err)

 	}

 	// Create a macvlan link

 	macvlan := &netlink.Macvlan{

@@ -173,7 +173,7 @@ func createDummyLink(dummyName, truncNetID string) error {

 	}

 	parentDummyLink, err := ns.NlHandle().LinkByName(dummyName)

 	if err != nil {

-		return fmt.Errorf("error occoured looking up the %s parent iface %s error: %s", macvlanType, dummyName, err)

+		return fmt.Errorf("error occurred looking up the %s parent iface %s error: %s", macvlanType, dummyName, err)

 	}

 	// bring the new netlink iface up

 	if err := ns.NlHandle().LinkSetUp(parentDummyLink); err != nil {

@@ -601,7 +601,7 @@ func (n *network) maxMTU() int {

 	mtu -= vxlanEncap

 	if n.secure {

 		// In case of encryption account for the

-		// esp packet espansion and padding

+		// esp packet expansion and padding

 		mtu -= pktExpansion

 		mtu -= (mtu % 4)

 	}

@@ -47,18 +47,10 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 		return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err)

 	}

-	if err := n.joinSandbox(false); err != nil {

+	if err := n.joinSandbox(s, false, true); err != nil {

 		return fmt.Errorf("network sandbox join failed: %v", err)

 	}

-	if err := n.joinSubnetSandbox(s, false); err != nil {

-		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)

-	}

-

-	// joinSubnetSandbox gets called when an endpoint comes up on a new subnet in the

-	// overlay network. Hence the Endpoint count should be updated outside joinSubnetSandbox

-	n.incEndpointCount()

-

 	sbox := n.sandbox()

 	overlayIfName, containerIfName, err := createVethPair()

@@ -39,7 +39,7 @@ var (

 type networkTable map[string]*network

 type subnet struct {

-	once      *sync.Once

+	sboxInit  bool

 	vxlanName string

 	brName    string

 	vni       uint32

@@ -63,7 +63,7 @@ type network struct {

 	endpoints endpointTable

 	driver    *driver

 	joinCnt   int

-	once      *sync.Once

+	sboxInit  bool

 	initEpoch int

 	initErr   error

 	subnets   []*subnet

@@ -150,7 +150,6 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d

 		id:        id,

 		driver:    d,

 		endpoints: endpointTable{},

-		once:      &sync.Once{},

 		subnets:   []*subnet{},

 	}

@@ -193,7 +192,6 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d

 		s := &subnet{

 			subnetIP: ipd.Pool,

 			gwIP:     ipd.Gateway,

-			once:     &sync.Once{},

 		}

 		if len(vnis) != 0 {

@@ -277,7 +275,7 @@ func (d *driver) DeleteNetwork(nid string) error {

 			logrus.Warnf("Failed to delete overlay endpoint %.7s from local store: %v", ep.id, err)

 		}

 	}

-	// flush the peerDB entries

+

 	doPeerFlush = true

 	delete(d.networks, nid)

@@ -304,29 +302,54 @@ func (d *driver) RevokeExternalConnectivity(nid, eid string) error {

 	return nil

 }

-func (n *network) incEndpointCount() {

-	n.Lock()

-	defer n.Unlock()

-	n.joinCnt++

-}

-

-func (n *network) joinSandbox(restore bool) error {

+func (n *network) joinSandbox(s *subnet, restore bool, incJoinCount bool) error {

 	// If there is a race between two go routines here only one will win

 	// the other will wait.

-	n.once.Do(func() {

-		// save the error status of initSandbox in n.initErr so that

-		// all the racing go routines are able to know the status.

+	networkOnce.Do(networkOnceInit)

+

+	n.Lock()

+	// If non-restore initialization occurred and was successful then

+	// tell the peerDB to initialize the sandbox with all the peers

+	// previously received from networkdb.  But only do this after

+	// unlocking the network.  Otherwise we could deadlock with

+	// on the peerDB channel while peerDB is waiting for the network lock.

+	var doInitPeerDB bool

+	defer func() {

+		n.Unlock()

+		if doInitPeerDB {

+			n.driver.initSandboxPeerDB(n.id)

+		}

+	}()

+

+	if !n.sboxInit {

 		n.initErr = n.initSandbox(restore)

-	})

+		doInitPeerDB = n.initErr == nil && !restore

+		// If there was an error, we cannot recover it

+		n.sboxInit = true

+	}

-	return n.initErr

-}

+	if n.initErr != nil {

+		return fmt.Errorf("network sandbox join failed: %v", n.initErr)

+	}

-func (n *network) joinSubnetSandbox(s *subnet, restore bool) error {

-	s.once.Do(func() {

-		s.initErr = n.initSubnetSandbox(s, restore)

-	})

-	return s.initErr

+	subnetErr := s.initErr

+	if !s.sboxInit {

+		subnetErr = n.initSubnetSandbox(s, restore)

+		// We can recover from these errors, but not on restore

+		if restore || subnetErr == nil {

+			s.initErr = subnetErr

+			s.sboxInit = true

+		}

+	}

+	if subnetErr != nil {

+		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), subnetErr)

+	}

+

+	if incJoinCount {

+		n.joinCnt++

+	}

+

+	return nil

 }

 func (n *network) leaveSandbox() {

@@ -337,15 +360,14 @@ func (n *network) leaveSandbox() {

 		return

 	}

-	// We are about to destroy sandbox since the container is leaving the network

-	// Reinitialize the once variable so that we will be able to trigger one time

-	// sandbox initialization(again) when another container joins subsequently.

-	n.once = &sync.Once{}

+	n.destroySandbox()

+

+	n.sboxInit = false

+	n.initErr = nil

 	for _, s := range n.subnets {

-		s.once = &sync.Once{}

+		s.sboxInit = false

+		s.initErr = nil

 	}

-

-	n.destroySandbox()

 }

 // to be called while holding network lock

@@ -478,7 +500,7 @@ func (n *network) generateVxlanName(s *subnet) string {

 		id = n.id[:5]

 	}

-	return "vx-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + id

+	return fmt.Sprintf("vx-%06x-%v", s.vni, id)

 }

 func (n *network) generateBridgeName(s *subnet) string {

@@ -491,7 +513,7 @@ func (n *network) generateBridgeName(s *subnet) string {

 }

 func (n *network) getBridgeNamePrefix(s *subnet) string {

-	return "ov-" + fmt.Sprintf("%06x", n.vxlanID(s))

+	return fmt.Sprintf("ov-%06x", s.vni)

 }

 func checkOverlap(nw *net.IPNet) error {

@@ -513,7 +535,7 @@ func checkOverlap(nw *net.IPNet) error {

 }

 func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) error {

-	sbox := n.sandbox()

+	sbox := n.sbox

 	// restore overlay osl sandbox

 	Ifaces := make(map[string][]osl.IfaceOption)

@@ -542,7 +564,7 @@ func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error

 			deleteInterfaceBySubnet(n.getBridgeNamePrefix(s), s)

 		}

 		// Try to delete the vxlan interface by vni if already present

-		deleteVxlanByVNI("", n.vxlanID(s))

+		deleteVxlanByVNI("", s.vni)

 		if err := checkOverlap(s.subnetIP); err != nil {

 			return err

@@ -556,24 +578,24 @@ func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error

 		// it must a stale namespace from previous

 		// life. Destroy it completely and reclaim resourced.

 		networkMu.Lock()

-		path, ok := vniTbl[n.vxlanID(s)]

+		path, ok := vniTbl[s.vni]

 		networkMu.Unlock()

 		if ok {

-			deleteVxlanByVNI(path, n.vxlanID(s))

+			deleteVxlanByVNI(path, s.vni)

 			if err := syscall.Unmount(path, syscall.MNT_FORCE); err != nil {

 				logrus.Errorf("unmount of %s failed: %v", path, err)

 			}

 			os.Remove(path)

 			networkMu.Lock()

-			delete(vniTbl, n.vxlanID(s))

+			delete(vniTbl, s.vni)

 			networkMu.Unlock()

 		}

 	}

 	// create a bridge and vxlan device for this subnet and move it to the sandbox

-	sbox := n.sandbox()

+	sbox := n.sbox

 	if err := sbox.AddInterface(brName, "br",

 		sbox.InterfaceOptions().Address(s.gwIP),

@@ -581,13 +603,30 @@ func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error

 		return fmt.Errorf("bridge creation in sandbox failed for subnet %q: %v", s.subnetIP.String(), err)

 	}

-	err := createVxlan(vxlanName, n.vxlanID(s), n.maxMTU())

+	err := createVxlan(vxlanName, s.vni, n.maxMTU())

 	if err != nil {

 		return err

 	}

 	if err := sbox.AddInterface(vxlanName, "vxlan",

 		sbox.InterfaceOptions().Master(brName)); err != nil {

+		// If adding vxlan device to the overlay namespace fails, remove the bridge interface we

+		// already added to the namespace. This allows the caller to try the setup again.

+		for _, iface := range sbox.Info().Interfaces() {

+			if iface.SrcName() == brName {

+				if ierr := iface.Remove(); ierr != nil {

+					logrus.Errorf("removing bridge failed from ov ns %v failed, %v", n.sbox.Key(), ierr)

+				}

+			}

+		}

+

+		// Also, delete the vxlan interface. Since a global vni id is associated

+		// with the vxlan interface, an orphaned vxlan interface will result in

+		// failure of vxlan device creation if the vni is assigned to some other

+		// network.

+		if deleteErr := deleteInterface(vxlanName); deleteErr != nil {

+			logrus.Warnf("could not delete vxlan interface, %s, error %v, after config error, %v", vxlanName, deleteErr, err)

+		}

 		return fmt.Errorf("vxlan interface creation failed for subnet %q: %v", s.subnetIP.String(), err)

 	}

@@ -619,6 +658,7 @@ func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error

 	return nil

 }

+// Must be called with the network lock

 func (n *network) initSubnetSandbox(s *subnet, restore bool) error {

 	brName := n.generateBridgeName(s)

 	vxlanName := n.generateVxlanName(s)

@@ -633,10 +673,8 @@ func (n *network) initSubnetSandbox(s *subnet, restore bool) error {

 		}

 	}

-	n.Lock()

 	s.vxlanName = vxlanName

 	s.brName = brName

-	n.Unlock()

 	return nil

 }

@@ -677,11 +715,7 @@ func (n *network) cleanupStaleSandboxes() {

 }

 func (n *network) initSandbox(restore bool) error {

-	n.Lock()

 	n.initEpoch++

-	n.Unlock()

-

-	networkOnce.Do(networkOnceInit)

 	if !restore {

 		if hostMode {

@@ -711,12 +745,7 @@ func (n *network) initSandbox(restore bool) error {

 	}

 	// this is needed to let the peerAdd configure the sandbox

-	n.setSandbox(sbox)

-

-	if !restore {

-		// Initialize the sandbox with all the peers previously received from networkdb

-		n.driver.initSandboxPeerDB(n.id)

-	}

+	n.sbox = sbox

 	// If we are in swarm mode, we don't need anymore the watchMiss routine.

 	// This will save 1 thread and 1 netlink socket per network

@@ -734,7 +763,7 @@ func (n *network) initSandbox(restore bool) error {

 		tv := syscall.NsecToTimeval(soTimeout.Nanoseconds())

 		err = nlSock.SetReceiveTimeout(&tv)

 	})

-	n.setNetlinkSocket(nlSock)

+	n.nlSocket = nlSock

 	if err == nil {

 		go n.watchMiss(nlSock, key)

@@ -836,7 +865,6 @@ func (d *driver) restoreNetworkFromStore(nid string) *network {

 	if n != nil {

 		n.driver = d

 		n.endpoints = endpointTable{}

-		n.once = &sync.Once{}

 		d.networks[nid] = n

 	}

 	return n

@@ -844,11 +872,11 @@ func (d *driver) restoreNetworkFromStore(nid string) *network {

 func (d *driver) network(nid string) *network {

 	d.Lock()

-	defer d.Unlock()

 	n, ok := d.networks[nid]

 	if !ok {

 		n = d.restoreNetworkFromStore(nid)

 	}

+	d.Unlock()

 	return n

 }

@@ -869,26 +897,12 @@ func (d *driver) getNetworkFromStore(nid string) *network {

 func (n *network) sandbox() osl.Sandbox {

 	n.Lock()

 	defer n.Unlock()

-

 	return n.sbox

 }

-func (n *network) setSandbox(sbox osl.Sandbox) {

-	n.Lock()

-	n.sbox = sbox

-	n.Unlock()

-}

-