@@ -121,14 +121,6 @@ func New(config Config) (*Cluster, error) {

 	return c, nil

 }

-func (c *Cluster) checkCompatibility() error {

-	info, _ := c.config.Backend.SystemInfo()

-	if info != nil && (info.ClusterStore != "" || info.ClusterAdvertise != "") {

-		return fmt.Errorf("swarm mode is incompatible with `--cluster-store` and `--cluster-advertise daemon configuration")

-	}

-	return nil

-}

-

 func (c *Cluster) saveState() error {

 	dt, err := json.Marshal(state{ListenAddr: c.listenAddr})

 	if err != nil {

@@ -173,7 +165,7 @@ func (c *Cluster) reconnectOnFailure(ctx context.Context) {

 }

 func (c *Cluster) startNewNode(forceNewCluster bool, listenAddr, joinAddr, secret, cahash string, ismanager bool) (*swarmagent.Node, context.Context, error) {

-	if err := c.checkCompatibility(); err != nil {

+	if err := c.config.Backend.IsSwarmCompatible(); err != nil {

 		return nil, nil, err

 	}

 	c.node = nil

@@ -32,4 +32,5 @@ type Backend interface {

 	ListContainersForNode(nodeID string) []string

 	SetNetworkBootstrapKeys([]*networktypes.EncryptionKey) error

 	SetClusterProvider(provider cluster.Provider)

+	IsSwarmCompatible() error

 }

@@ -37,3 +37,7 @@ func (config *Config) InstallFlags(cmd *flag.FlagSet, usageFn func(string) strin

 	// Then platform-specific install flags

 	config.attachExperimentalFlags(cmd, usageFn)

 }

+

+func (config *Config) isSwarmCompatible() error {

+	return nil

+}

@@ -3,6 +3,7 @@

 package daemon

 import (

+	"fmt"

 	"net"

 	"github.com/docker/docker/opts"

@@ -120,3 +121,13 @@ func (config *Config) GetAllRuntimes() map[string]types.Runtime {

 	config.reloadLock.Unlock()

 	return rts

 }

+

+func (config *Config) isSwarmCompatible() error {

+	if config.IsValueSet("cluster-store") || config.IsValueSet("cluster-advertise") {

+		return fmt.Errorf("--cluster-store and --cluster-advertise daemon configurations are incompatible with swarm mode")

+	}

+	if config.LiveRestore {

+		return fmt.Errorf("--live-restore daemon configuration is incompatible with swarm mode")

+	}

+	return nil

+}

@@ -57,3 +57,7 @@ func (config *Config) GetDefaultRuntimeName() string {

 func (config *Config) GetAllRuntimes() map[string]types.Runtime {

 	return map[string]types.Runtime{}

 }

+

+func (config *Config) isSwarmCompatible() error {

+	return nil

+}

@@ -30,7 +30,7 @@ var (

 	getPortMapInfo    = container.GetSandboxPortMapInfo

 )

-func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libnetwork.Network) ([]libnetwork.SandboxOption, error) {

+func (daemon *Daemon) buildSandboxOptions(container *container.Container) ([]libnetwork.SandboxOption, error) {

 	var (

 		sboxOptions []libnetwork.SandboxOption

 		err         error

@@ -176,16 +176,19 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn

 	// Legacy Link feature is supported only for the default bridge network.

 	// return if this call to build join options is not for default bridge network

-	if n.Name() != defaultNetName {

+	// Legacy Link is only supported by docker run --link

+	if _, ok := container.NetworkSettings.Networks[defaultNetName]; !container.HostConfig.NetworkMode.IsDefault() || !ok {

 		return sboxOptions, nil

 	}

-	ep, _ := container.GetEndpointInNetwork(n)

-	if ep == nil {

+	if container.NetworkSettings.Networks[defaultNetName].EndpointID == "" {

 		return sboxOptions, nil

 	}

-	var childEndpoints, parentEndpoints []string

+	var (

+		childEndpoints, parentEndpoints []string

+		cEndpointID                     string

+	)

 	children := daemon.children(container)

 	for linkAlias, child := range children {

@@ -200,9 +203,9 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn

 			aliasList = aliasList + " " + child.Name[1:]

 		}

 		sboxOptions = append(sboxOptions, libnetwork.OptionExtraHost(aliasList, child.NetworkSettings.Networks[defaultNetName].IPAddress))

-		cEndpoint, _ := child.GetEndpointInNetwork(n)

-		if cEndpoint != nil && cEndpoint.ID() != "" {

-			childEndpoints = append(childEndpoints, cEndpoint.ID())

+		cEndpointID = child.NetworkSettings.Networks[defaultNetName].EndpointID

+		if cEndpointID != "" {

+			childEndpoints = append(childEndpoints, cEndpointID)

 		}

 	}

@@ -219,8 +222,8 @@ func (daemon *Daemon) buildSandboxOptions(container *container.Container, n libn

 			alias,

 			bridgeSettings.IPAddress,

 		))

-		if ep.ID() != "" {

-			parentEndpoints = append(parentEndpoints, ep.ID())

+		if cEndpointID != "" {

+			parentEndpoints = append(parentEndpoints, cEndpointID)

 		}

 	}

@@ -312,7 +315,7 @@ func (daemon *Daemon) updateNetwork(container *container.Container) error {

 		return nil

 	}

-	options, err := daemon.buildSandboxOptions(container, n)

+	options, err := daemon.buildSandboxOptions(container)

 	if err != nil {

 		return fmt.Errorf("Update network failed: %v", err)

 	}

@@ -570,7 +573,7 @@ func (daemon *Daemon) connectToNetwork(container *container.Container, idOrName

 	}

 	if sb == nil {

-		options, err := daemon.buildSandboxOptions(container, n)

+		options, err := daemon.buildSandboxOptions(container)

 		if err != nil {

 			return err

 		}

@@ -709,6 +712,9 @@ func (daemon *Daemon) getNetworkedContainer(containerID, connectedContainerID st

 }

 func (daemon *Daemon) releaseNetwork(container *container.Container) {

+	if daemon.netController == nil {

+		return

+	}

 	if container.HostConfig.NetworkMode.IsContainer() || container.Config.NetworkDisabled {

 		return

 	}

@@ -146,6 +146,7 @@ func (daemon *Daemon) restore() error {

 	var migrateLegacyLinks bool

 	restartContainers := make(map[*container.Container]chan struct{})

+	activeSandboxes := make(map[string]interface{})

 	for _, c := range containers {

 		if err := daemon.registerName(c); err != nil {

 			logrus.Errorf("Failed to register container %s: %s", c.ID, err)

@@ -178,6 +179,16 @@ func (daemon *Daemon) restore() error {

 					logrus.Errorf("Failed to restore with containerd: %q", err)

 					return

 				}

+				if !c.HostConfig.NetworkMode.IsContainer() {

+					options, err := daemon.buildSandboxOptions(c)

+					if err != nil {

+						logrus.Warnf("Failed build sandbox option to restore container %s: %v", c.ID, err)

+					}

+					mapLock.Lock()

+					activeSandboxes[c.NetworkSettings.SandboxID] = options

+					mapLock.Unlock()

+				}

+

 			}

 			// fixme: only if not running

 			// get list of containers we need to restart

@@ -209,6 +220,10 @@ func (daemon *Daemon) restore() error {

 		}(c)

 	}

 	wg.Wait()

+	daemon.netController, err = daemon.initNetworkController(daemon.configStore, activeSandboxes)

+	if err != nil {

+		return fmt.Errorf("Error initializing network controller: %v", err)

+	}

 	// migrate any legacy links from sqlite

 	linkdbFile := filepath.Join(daemon.root, "linkgraph.db")

@@ -356,6 +371,15 @@ func (daemon *Daemon) SetClusterProvider(clusterProvider cluster.Provider) {

 	daemon.netController.SetClusterProvider(clusterProvider)

 }

+// IsSwarmCompatible verifies if the current daemon

+// configuration is compatible with the swarm mode

+func (daemon *Daemon) IsSwarmCompatible() error {

+	if daemon.configStore == nil {

+		return nil

+	}

+	return daemon.configStore.isSwarmCompatible()

+}

+

 // NewDaemon sets up everything for the daemon to be able to service

 // requests from the webserver.

 func NewDaemon(config *Config, registryService registry.Service, containerdRemote libcontainerd.Remote) (daemon *Daemon, err error) {

@@ -530,11 +554,6 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot

 		return nil, err

 	}

-	d.netController, err = d.initNetworkController(config)

-	if err != nil {

-		return nil, fmt.Errorf("Error initializing network controller: %v", err)

-	}

-

 	sysInfo := sysinfo.New(false)

 	// Check if Devices cgroup is mounted, it is hard requirement for container security,

 	// on Linux.

@@ -912,15 +931,17 @@ func (daemon *Daemon) reloadClusterDiscovery(config *Config) error {

 		}

 	}

+	if daemon.clusterProvider != nil {

+		if err := config.isSwarmCompatible(); err != nil {

+			return err

+		}

+	}

+

 	// check discovery modifications

 	if !modifiedDiscoverySettings(daemon.configStore, newAdvertise, newClusterStore, config.ClusterOpts) {

 		return nil

 	}

-	if daemon.clusterProvider != nil {

-		return fmt.Errorf("--cluster-store and --cluster-advertise daemon configurations are incompatible with swarm mode")

-	}

-

 	// enable discovery for the first time if it was not previously enabled

 	if daemon.discoveryWatcher == nil {

 		discoveryWatcher, err := initDiscovery(newClusterStore, newAdvertise, config.ClusterOpts)

@@ -947,7 +968,7 @@ func (daemon *Daemon) reloadClusterDiscovery(config *Config) error {

 	if daemon.netController == nil {

 		return nil

 	}

-	netOptions, err := daemon.networkOptions(daemon.configStore)

+	netOptions, err := daemon.networkOptions(daemon.configStore, nil)

 	if err != nil {

 		logrus.Warnf("Failed to reload configuration with network controller: %v", err)

 		return nil

@@ -964,7 +985,7 @@ func isBridgeNetworkDisabled(config *Config) bool {

 	return config.bridgeConfig.Iface == disableNetworkBridge

 }

-func (daemon *Daemon) networkOptions(dconfig *Config) ([]nwconfig.Option, error) {

+func (daemon *Daemon) networkOptions(dconfig *Config, activeSandboxes map[string]interface{}) ([]nwconfig.Option, error) {

 	options := []nwconfig.Option{}

 	if dconfig == nil {

 		return options, nil

@@ -999,6 +1020,11 @@ func (daemon *Daemon) networkOptions(dconfig *Config) ([]nwconfig.Option, error)

 	options = append(options, nwconfig.OptionLabels(dconfig.Labels))

 	options = append(options, driverOptions(dconfig)...)

+

+	if daemon.configStore != nil && daemon.configStore.LiveRestore && len(activeSandboxes) != 0 {

+		options = append(options, nwconfig.OptionActiveSandboxes(activeSandboxes))

+	}

+

 	return options, nil

 }

@@ -113,7 +113,7 @@ func configureKernelSecuritySupport(config *Config, driverName string) error {

 	return nil

 }

-func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkController, error) {

+func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) {

 	return nil, nil

 }

@@ -627,8 +627,8 @@ func configureKernelSecuritySupport(config *Config, driverName string) error {

 	return nil

 }

-func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkController, error) {

-	netOptions, err := daemon.networkOptions(config)

+func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) {

+	netOptions, err := daemon.networkOptions(config, activeSandboxes)

 	if err != nil {

 		return nil, err

 	}

@@ -638,16 +638,24 @@ func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkC

 		return nil, fmt.Errorf("error obtaining controller instance: %v", err)

 	}

+	if len(activeSandboxes) > 0 {

+		logrus.Infof("There are old running containers, the network config will not take affect")

+		return controller, nil

+	}

+

 	// Initialize default network on "null"

-	if _, err := controller.NewNetwork("null", "none", "", libnetwork.NetworkOptionPersist(false)); err != nil {

-		return nil, fmt.Errorf("Error creating default \"null\" network: %v", err)

+	if n, _ := controller.NetworkByName("none"); n == nil {

+		if _, err := controller.NewNetwork("null", "none", "", libnetwork.NetworkOptionPersist(true)); err != nil {

+			return nil, fmt.Errorf("Error creating default \"null\" network: %v", err)

+		}

 	}

 	// Initialize default network on "host"

-	if _, err := controller.NewNetwork("host", "host", "", libnetwork.NetworkOptionPersist(false)); err != nil {

-		return nil, fmt.Errorf("Error creating default \"host\" network: %v", err)

+	if n, _ := controller.NetworkByName("host"); n == nil {

+		if _, err := controller.NewNetwork("host", "host", "", libnetwork.NetworkOptionPersist(true)); err != nil {

+			return nil, fmt.Errorf("Error creating default \"host\" network: %v", err)

+		}

 	}

-

 	if !config.DisableBridge {

 		// Initialize default driver "bridge"

 		if err := initBridgeDriver(controller, config); err != nil {

@@ -183,7 +183,7 @@ func TestNetworkOptions(t *testing.T) {

 		},

 	}

-	if _, err := daemon.networkOptions(dconfigCorrect); err != nil {

+	if _, err := daemon.networkOptions(dconfigCorrect, nil); err != nil {

 		t.Fatalf("Expect networkOptions success, got error: %v", err)

 	}

@@ -193,7 +193,7 @@ func TestNetworkOptions(t *testing.T) {

 		},

 	}

-	if _, err := daemon.networkOptions(dconfigWrong); err == nil {

+	if _, err := daemon.networkOptions(dconfigWrong, nil); err == nil {

 		t.Fatalf("Expected networkOptions error, got nil")

 	}

 }

@@ -189,8 +189,8 @@ func configureMaxThreads(config *Config) error {

 	return nil

 }

-func (daemon *Daemon) initNetworkController(config *Config) (libnetwork.NetworkController, error) {

-	netOptions, err := daemon.networkOptions(config)

+func (daemon *Daemon) initNetworkController(config *Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) {

+	netOptions, err := daemon.networkOptions(config, nil)

 	if err != nil {

 		return nil, err

 	}

@@ -59,6 +59,9 @@ func (daemon *Daemon) GetNetworkByID(partialID string) (libnetwork.Network, erro

 // GetNetworkByName function returns a network for a given network name.

 func (daemon *Daemon) GetNetworkByName(name string) (libnetwork.Network, error) {

 	c := daemon.netController

+	if c == nil {

+		return nil, libnetwork.ErrNoSuchNetwork(name)

+	}

 	if name == "" {

 		name = c.Config().Daemon.DefaultNetwork

 	}

@@ -68,6 +71,9 @@ func (daemon *Daemon) GetNetworkByName(name string) (libnetwork.Network, error)

 // GetNetworksByID returns a list of networks whose ID partially matches zero or more networks

 func (daemon *Daemon) GetNetworksByID(partialID string) []libnetwork.Network {

 	c := daemon.netController

+	if c == nil {

+		return nil

+	}

 	list := []libnetwork.Network{}

 	l := func(nw libnetwork.Network) bool {

 		if strings.HasPrefix(nw.ID(), partialID) {

@@ -65,7 +65,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837

 clone git github.com/imdario/mergo 0.2.1

 #get libnetwork packages

-clone git github.com/docker/libnetwork e8da32ce5693f0ed6823d59c8415baf76c0809ea

+clone git github.com/docker/libnetwork 452dff166e0abd9455b07c835613197f078a34de

 clone git github.com/docker/go-events 39718a26497694185f8fb58a7d6f31947f3dc42d

 clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80

 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

@@ -1586,3 +1586,67 @@ func (s *DockerNetworkSuite) TestDockerNetworkCreateDeleteSpecialCharacters(c *c

 	dockerCmd(c, "network", "rm", "kiwl$%^")

 	assertNwNotAvailable(c, "kiwl$%^")

 }

+

+func (s *DockerDaemonSuite) TestDaemonRestartRestoreBridgeNetwork(t *check.C) {

+	testRequires(t, DaemonIsLinux)

+	if err := s.d.StartWithBusybox("--live-restore"); err != nil {

+		t.Fatal(err)

+	}

+	defer s.d.Stop()

+	oldCon := "old"

+

+	_, err := s.d.Cmd("run", "-d", "--name", oldCon, "-p", "80:80", "busybox", "top")

+	if err != nil {

+		t.Fatal(err)

+	}

+	oldContainerIP, err := s.d.Cmd("inspect", "-f", "{{ .NetworkSettings.Networks.bridge.IPAddress }}", oldCon)

+	if err != nil {

+		t.Fatal(err)

+	}

+	// Kill the daemon

+	if err := s.d.Kill(); err != nil {

+		t.Fatal(err)

+	}

+

+	// restart the daemon

+	if err := s.d.Start("--live-restore"); err != nil {

+		t.Fatal(err)

+	}

+

+	// start a new container, the new container's ip should not be the same with

+	// old running container.

+	newCon := "new"

+	_, err = s.d.Cmd("run", "-d", "--name", newCon, "busybox", "top")

+	if err != nil {

+		t.Fatal(err)

+	}

+	newContainerIP, err := s.d.Cmd("inspect", "-f", "{{ .NetworkSettings.Networks.bridge.IPAddress }}", newCon)

+	if err != nil {

+		t.Fatal(err)

+	}

+	if strings.Compare(strings.TrimSpace(oldContainerIP), strings.TrimSpace(newContainerIP)) == 0 {

+		t.Fatalf("new container ip should not equal to old running container  ip")

+	}

+

+	// start a new container, the new container should ping old running container

+	_, err = s.d.Cmd("run", "-t", "busybox", "ping", "-c", "1", oldContainerIP)

+	if err != nil {

+		t.Fatal(err)

+	}

+

+	// start a new container try to publist port 80:80 will failed

+	out, err := s.d.Cmd("run", "-p", "80:80", "-d", "busybox", "top")

+	if err == nil || !strings.Contains(out, "Bind for 0.0.0.0:80 failed: port is already allocated") {

+		t.Fatalf("80 port is allocated to old running container, it should failed on allocating to new container")

+	}

+

+	// kill old running container and try to allocate again

+	_, err = s.d.Cmd("kill", oldCon)

+	if err != nil {

+		t.Fatal(err)

+	}

+	_, err = s.d.Cmd("run", "-p", "80:80", "-d", "busybox", "top")

+	if err != nil {

+		t.Fatal(err)

+	}

+}

@@ -15,9 +15,10 @@ import (

 // Config encapsulates configurations of various Libnetwork components

 type Config struct {

-	Daemon  DaemonCfg

-	Cluster ClusterCfg

-	Scopes  map[string]*datastore.ScopeCfg

+	Daemon          DaemonCfg

+	Cluster         ClusterCfg

+	Scopes          map[string]*datastore.ScopeCfg

+	ActiveSandboxes map[string]interface{}

 }

 // DaemonCfg represents libnetwork core configuration

@@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {

 		c.Scopes[datastore.LocalScope].Client.Config = config

 	}

 }

+

+// OptionActiveSandboxes function returns an option setter for passing the sandboxes

+// which were active during previous daemon life

+func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {

+	return func(c *Config) {

+		c.ActiveSandboxes = sandboxes

+	}

+}

@@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {

 		}

 	}

-	// Reserve pools first before doing cleanup. This is because

-	// if the pools are not populated properly, the cleanups of

-	// endpoint/network and sandbox below will not be able to

-	// release ip subnets and addresses properly into the pool

-	// because the pools won't exist.

+	// Reserve pools first before doing cleanup. Otherwise the

+	// cleanups of endpoint/network and sandbox below will

+	// generate many unnecessary warnings

 	c.reservePools()

 	// Cleanup resources

-	c.sandboxCleanup()

+	c.sandboxCleanup(c.cfg.ActiveSandboxes)

 	c.cleanupLocalEndpoints()

 	c.networkCleanup()

@@ -671,9 +669,27 @@ func (c *controller) reservePools() {

 				c.Gateway = n.ipamV6Info[i].Gateway.IP.String()

 			}

 		}

+		// Reserve pools

 		if err := n.ipamAllocate(); err != nil {

 			log.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err)

 		}

+		// Reserve existing endpoints' addresses

+		ipam, _, err := n.getController().getIPAMDriver(n.ipamType)

+		if err != nil {

+			log.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID())

+			continue

+		}

+		epl, err := n.getEndpointsFromStore()

+		if err != nil {

+			log.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID())

+			continue

+		}

+		for _, ep := range epl {

+			if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil {

+				log.Warnf("Failed to reserve current adress for endpoint %q (%s) on network %q (%s)",

+					ep.Name(), ep.ID(), n.Name(), n.ID())

+			}

+		}

 	}

 }

@@ -832,7 +848,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s

 	if sb.config.useDefaultSandBox {

 		c.sboxOnce.Do(func() {

-			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)

+			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)

 		})

 		if err != nil {

@@ -844,7 +860,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s

 	}

 	if sb.osSbox == nil && !sb.config.useExternalKey {

-		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {

+		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {

 			return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)

 		}

 	}

@@ -86,25 +86,52 @@ out:

 	return kmap, nil

 }

-func (c *cache) add(kvObject KVObject) error {

+func (c *cache) add(kvObject KVObject, atomic bool) error {

 	kmap, err := c.kmap(kvObject)

 	if err != nil {

 		return err

 	}

 	c.Lock()

+	// If atomic is true, cache needs to maintain its own index

+	// for atomicity and the add needs to be atomic.

+	if atomic {

+		if prev, ok := kmap[Key(kvObject.Key()...)]; ok {

+			if prev.Index() != kvObject.Index() {

+				c.Unlock()

+				return ErrKeyModified

+			}

+		}

+

+		// Increment index

+		index := kvObject.Index()

+		index++

+		kvObject.SetIndex(index)

+	}

+

 	kmap[Key(kvObject.Key()...)] = kvObject

 	c.Unlock()

 	return nil

 }

-func (c *cache) del(kvObject KVObject) error {

+func (c *cache) del(kvObject KVObject, atomic bool) error {

 	kmap, err := c.kmap(kvObject)

 	if err != nil {

 		return err

 	}

 	c.Lock()

+	// If atomic is true, cache needs to maintain its own index

+	// for atomicity and del needs to be atomic.

+	if atomic {

+		if prev, ok := kmap[Key(kvObject.Key()...)]; ok {

+			if prev.Index() != kvObject.Index() {

+				c.Unlock()

+				return ErrKeyModified

+			}

+		}

+	}

+

 	delete(kmap, Key(kvObject.Key()...))

 	c.Unlock()

 	return nil

@@ -410,7 +410,9 @@ func (ds *datastore) PutObjectAtomic(kvObject KVObject) error {

 add_cache:

 	if ds.cache != nil {

-		return ds.cache.add(kvObject)

+		// If persistent store is skipped, sequencing needs to

+		// happen in cache.

+		return ds.cache.add(kvObject, kvObject.Skip())

 	}

 	return nil

@@ -435,7 +437,9 @@ func (ds *datastore) PutObject(kvObject KVObject) error {

 add_cache:

 	if ds.cache != nil {

-		return ds.cache.add(kvObject)

+		// If persistent store is skipped, sequencing needs to

+		// happen in cache.

+		return ds.cache.add(kvObject, kvObject.Skip())

 	}

 	return nil

@@ -537,7 +541,9 @@ func (ds *datastore) DeleteObject(kvObject KVObject) error {

 	// cleaup the cache first

 	if ds.cache != nil {

-		ds.cache.del(kvObject)

+		// If persistent store is skipped, sequencing needs to

+		// happen in cache.

+		ds.cache.del(kvObject, kvObject.Skip())

 	}

 	if kvObject.Skip() {

@@ -572,7 +578,9 @@ func (ds *datastore) DeleteObjectAtomic(kvObject KVObject) error {

 del_cache:

 	// cleanup the cache only if AtomicDelete went through successfully

 	if ds.cache != nil {

-		return ds.cache.del(kvObject)

+		// If persistent store is skipped, sequencing needs to

+		// happen in cache.

+		return ds.cache.del(kvObject, kvObject.Skip())

 	}

 	return nil

@@ -585,7 +593,9 @@ func (ds *datastore) DeleteTree(kvObject KVObject) error {

 	// cleaup the cache first

 	if ds.cache != nil {

-		ds.cache.del(kvObject)

+		// If persistent store is skipped, sequencing needs to

+		// happen in cache.

+		ds.cache.del(kvObject, kvObject.Skip())

 	}

 	if kvObject.Skip() {

@@ -91,6 +91,7 @@ type connectivityConfiguration struct {

 type bridgeEndpoint struct {

 	id              string

+	nid             string

 	srcName         string

 	addr            *net.IPNet

 	addrv6          *net.IPNet

@@ -99,6 +100,8 @@ type bridgeEndpoint struct {

 	containerConfig *containerConfiguration

 	extConnConfig   *connectivityConfiguration

 	portMapping     []types.PortBinding // Operation port bindings

+	dbIndex         uint64

+	dbExists        bool

 }

 type bridgeNetwork struct {

@@ -882,7 +885,7 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,

 	// Create and add the endpoint

 	n.Lock()

-	endpoint := &bridgeEndpoint{id: eid, config: epConfig}

+	endpoint := &bridgeEndpoint{id: eid, nid: nid, config: epConfig}

 	n.endpoints[eid] = endpoint

 	n.Unlock()

@@ -1009,6 +1012,10 @@ func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo,

 		}

 	}

+	if err = d.storeUpdate(endpoint); err != nil {

+		return fmt.Errorf("failed to save bridge endpoint %s to store: %v", ep.id[0:7], err)

+	}

+

 	return nil

 }

@@ -1069,6 +1076,10 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {

 		d.nlh.LinkDel(link)

 	}

+	if err := d.storeDelete(ep); err != nil {

+		logrus.Warnf("Failed to remove bridge endpoint %s from store: %v", ep.id[0:7], err)

+	}

+

 	return nil

 }

@@ -1225,6 +1236,11 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string

 		return err

 	}

+	if err = d.storeUpdate(endpoint); err != nil {

+		endpoint.portMapping = nil

+		return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)

+	}

+

 	if !network.config.EnableICC {

 		return d.link(network, endpoint, true)

 	}

@@ -12,7 +12,13 @@ import (

 	"github.com/docker/libnetwork/types"

 )

-const bridgePrefix = "bridge"

+const (

+	// network config prefix was not specific enough.

+	// To be backward compatible, need custom endpoint

+	// prefix with different root

+	bridgePrefix         = "bridge"

+	bridgeEndpointPrefix = "bridge-endpoint"

+)

 func (d *driver) initStore(option map[string]interface{}) error {

 	if data, ok := option[netlabel.LocalKVClient]; ok {

@@ -26,7 +32,15 @@ func (d *driver) initStore(option map[string]interface{}) error {

 			return types.InternalErrorf("bridge driver failed to initialize data store: %v", err)

 		}

-		return d.populateNetworks()

+		err = d.populateNetworks()

+		if err != nil {

+			return err

+		}

+

+		err = d.populateEndpoints()

+		if err != nil {

+			return err

+		}

 	}

 	return nil

@@ -48,6 +62,36 @@ func (d *driver) populateNetworks() error {

 		if err = d.createNetwork(ncfg); err != nil {

 			logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state: %v", ncfg.ID, ncfg.BridgeName, err)

 		}

+		logrus.Debugf("Network (%s) restored", ncfg.ID[0:7])

+	}

+

+	return nil

+}

+

+func (d *driver) populateEndpoints() error {

+	kvol, err := d.store.List(datastore.Key(bridgeEndpointPrefix), &bridgeEndpoint{})

+	if err != nil && err != datastore.ErrKeyNotFound {

+		return fmt.Errorf("failed to get bridge endpoints from store: %v", err)

+	}

+

+	if err == datastore.ErrKeyNotFound {

+		return nil

+	}

+

+	for _, kvo := range kvol {

+		ep := kvo.(*bridgeEndpoint)

+		n, ok := d.networks[ep.nid]

+		if !ok {

+			logrus.Debugf("Network (%s) not found for restored bridge endpoint (%s)", ep.nid[0:7], ep.id[0:7])

+			logrus.Debugf("Deleting stale bridge endpoint (%s) from store", ep.nid[0:7])

+			if err := d.storeDelete(ep); err != nil {

+				logrus.Debugf("Failed to delete stale bridge endpoint (%s) from store", ep.nid[0:7])

+			}

+			continue

+		}

+		n.endpoints[ep.id] = ep

+		n.restorePortAllocations(ep)

+		logrus.Debugf("Endpoint (%s) restored to network (%s)", ep.id[0:7], ep.nid[0:7])

 	}

 	return nil

@@ -184,7 +228,7 @@ func (ncfg *networkConfiguration) Exists() bool {

 }

 func (ncfg *networkConfiguration) Skip() bool {

-	return ncfg.DefaultBridge

+	return false

 }

 func (ncfg *networkConfiguration) New() datastore.KVObject {

@@ -200,3 +244,135 @@ func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {

 func (ncfg *networkConfiguration) DataScope() string {

 	return datastore.LocalScope

 }

+

+func (ep *bridgeEndpoint) MarshalJSON() ([]byte, error) {

+	epMap := make(map[string]interface{})

+	epMap["id"] = ep.id

+	epMap["nid"] = ep.nid

+	epMap["SrcName"] = ep.srcName

+	epMap["MacAddress"] = ep.macAddress.String()

+	epMap["Addr"] = ep.addr.String()

+	if ep.addrv6 != nil {

+		epMap["Addrv6"] = ep.addrv6.String()

+	}

+	epMap["Config"] = ep.config

+	epMap["ContainerConfig"] = ep.containerConfig

+	epMap["ExternalConnConfig"] = ep.extConnConfig

+	epMap["PortMapping"] = ep.portMapping

+

+	return json.Marshal(epMap)

+}

+

+func (ep *bridgeEndpoint) UnmarshalJSON(b []byte) error {

+	var (

+		err   error

+		epMap map[string]interface{}

+	)

+

+	if err = json.Unmarshal(b, &epMap); err != nil {

+		return fmt.Errorf("Failed to unmarshal to bridge endpoint: %v", err)

+	}

+

+	if v, ok := epMap["MacAddress"]; ok {

+		if ep.macAddress, err = net.ParseMAC(v.(string)); err != nil {

+			return types.InternalErrorf("failed to decode bridge endpoint MAC address (%s) after json unmarshal: %v", v.(string), err)

+		}

+	}

+	if v, ok := epMap["Addr"]; ok {

+		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {

+			return types.InternalErrorf("failed to decode bridge endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)

+		}

+	}

+	if v, ok := epMap["Addrv6"]; ok {

+		if ep.addrv6, err = types.ParseCIDR(v.(string)); err != nil {

+			return types.InternalErrorf("failed to decode bridge endpoint IPv6 address (%s) after json unmarshal: %v", v.(string), err)

+		}

+	}

+	ep.id = epMap["id"].(string)

+	ep.nid = epMap["nid"].(string)

+	ep.srcName = epMap["SrcName"].(string)

+	d, _ := json.Marshal(epMap["Config"])

+	if err := json.Unmarshal(d, &ep.config); err != nil {

+		logrus.Warnf("Failed to decode endpoint config %v", err)

+	}

+	d, _ = json.Marshal(epMap["ContainerConfig"])

+	if err := json.Unmarshal(d, &ep.containerConfig); err != nil {

+		logrus.Warnf("Failed to decode endpoint container config %v", err)

+	}

+	d, _ = json.Marshal(epMap["ExternalConnConfig"])

+	if err := json.Unmarshal(d, &ep.extConnConfig); err != nil {

+		logrus.Warnf("Failed to decode endpoint external connectivity configuration %v", err)

+	}

+	d, _ = json.Marshal(epMap["PortMapping"])

+	if err := json.Unmarshal(d, &ep.portMapping); err != nil {

+		logrus.Warnf("Failed to decode endpoint port mapping %v", err)

+	}

+

+	return nil

+}

+