@@ -1546,7 +1546,7 @@ func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *check.C) {

 	out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo")

 	c.Assert(err, checker.IsNil, check.Commentf(out))

 	c.Assert(strings.TrimSpace(out), checker.Contains, "foo:bar")

-        c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true")

+	c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true")

 	out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--network=foo", "--name", "top", "busybox", "top")

 	c.Assert(err, checker.IsNil, check.Commentf(out))

@@ -1557,7 +1557,7 @@ func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *check.C) {

 	out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo")

 	c.Assert(err, checker.IsNil, check.Commentf(out))

 	c.Assert(strings.TrimSpace(out), checker.Contains, "foo:bar")

-        c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true")

+	c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true")

 }

 func (s *DockerTrustedSwarmSuite) TestTrustedServiceCreate(c *check.C) {

@@ -31,7 +31,7 @@ github.com/moby/buildkit aaff9d591ef128560018433fe61beb802e149de8

 github.com/tonistiigi/fsutil dea3a0da73aee887fc02142d995be764106ac5e2

 #get libnetwork packages

-github.com/docker/libnetwork 0f08d31bf0e640e0cdc6d5161227f87602d605c5

+github.com/docker/libnetwork 68f1039f172434709a4550fe92e3e058406c74ce 

 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9

 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80

 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

@@ -109,7 +109,7 @@ github.com/containerd/containerd 06b9cb35161009dcb7123345749fef02f7cea8e0

 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4

 # cluster

-github.com/docker/swarmkit 941a01844b89c56aa61086fecb167ab3af1de22b

+github.com/docker/swarmkit 872861d2ae46958af7ead1d5fffb092c73afbaf0

 github.com/gogo/protobuf v0.4

 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a

 github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e

@@ -6,11 +6,9 @@ import (

 	"encoding/json"

 	"fmt"

 	"net"

-	"os"

 	"sort"

 	"sync"

-	"github.com/docker/docker/pkg/stringid"

 	"github.com/docker/go-events"

 	"github.com/docker/libnetwork/cluster"

 	"github.com/docker/libnetwork/datastore"

@@ -282,12 +280,8 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr, d

 	}

 	keys, _ := c.getKeys(subsysGossip)

-	hostname, _ := os.Hostname()

-	nodeName := hostname + "-" + stringid.TruncateID(stringid.GenerateRandomID())

-	logrus.Info("Gossip cluster hostname ", nodeName)

 	netDBConf := networkdb.DefaultConfig()

-	netDBConf.NodeName = nodeName

 	netDBConf.BindAddr = listenAddr

 	netDBConf.AdvertiseAddr = advertiseAddr

 	netDBConf.Keys = keys

@@ -41,6 +41,7 @@ type Handle struct {

 	id         string

 	dbIndex    uint64

 	dbExists   bool

+	curr       uint64

 	store      datastore.DataStore

 	sync.Mutex

 }

@@ -193,26 +194,27 @@ func (h *Handle) getCopy() *Handle {

 		dbIndex:    h.dbIndex,

 		dbExists:   h.dbExists,

 		store:      h.store,

+		curr:       h.curr,

 	}

 }

 // SetAnyInRange atomically sets the first unset bit in the specified range in the sequence and returns the corresponding ordinal

-func (h *Handle) SetAnyInRange(start, end uint64) (uint64, error) {

+func (h *Handle) SetAnyInRange(start, end uint64, serial bool) (uint64, error) {

 	if end < start || end >= h.bits {

 		return invalidPos, fmt.Errorf("invalid bit range [%d, %d]", start, end)

 	}

 	if h.Unselected() == 0 {

 		return invalidPos, ErrNoBitAvailable

 	}

-	return h.set(0, start, end, true, false)

+	return h.set(0, start, end, true, false, serial)

 }

 // SetAny atomically sets the first unset bit in the sequence and returns the corresponding ordinal

-func (h *Handle) SetAny() (uint64, error) {

+func (h *Handle) SetAny(serial bool) (uint64, error) {

 	if h.Unselected() == 0 {

 		return invalidPos, ErrNoBitAvailable

 	}

-	return h.set(0, 0, h.bits-1, true, false)

+	return h.set(0, 0, h.bits-1, true, false, serial)

 }

 // Set atomically sets the corresponding bit in the sequence

@@ -220,7 +222,7 @@ func (h *Handle) Set(ordinal uint64) error {

 	if err := h.validateOrdinal(ordinal); err != nil {

 		return err

 	}

-	_, err := h.set(ordinal, 0, 0, false, false)

+	_, err := h.set(ordinal, 0, 0, false, false, false)

 	return err

 }

@@ -229,7 +231,7 @@ func (h *Handle) Unset(ordinal uint64) error {

 	if err := h.validateOrdinal(ordinal); err != nil {

 		return err

 	}

-	_, err := h.set(ordinal, 0, 0, false, true)

+	_, err := h.set(ordinal, 0, 0, false, true, false)

 	return err

 }

@@ -298,7 +300,7 @@ func (h *Handle) CheckConsistency() error {

 }

 // set/reset the bit

-func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64, error) {

+func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial bool) (uint64, error) {

 	var (

 		bitPos  uint64

 		bytePos uint64

@@ -308,6 +310,7 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64

 	for {

 		var store datastore.DataStore

+		curr := uint64(0)

 		h.Lock()

 		store = h.store

 		h.Unlock()

@@ -318,15 +321,18 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64

 		}

 		h.Lock()

+		if serial {

+			curr = h.curr

+		}

 		// Get position if available

 		if release {

 			bytePos, bitPos = ordinalToPos(ordinal)

 		} else {

 			if any {

-				bytePos, bitPos, err = getFirstAvailable(h.head, start)

+				bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, curr, end)

 				ret = posToOrdinal(bytePos, bitPos)

-				if end < ret {

-					err = ErrNoBitAvailable

+				if err == nil {

+					h.curr = ret + 1

 				}

 			} else {

 				bytePos, bitPos, err = checkIfAvailable(h.head, ordinal)

@@ -515,6 +521,29 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) {

 	return invalidPos, invalidPos, ErrNoBitAvailable

 }

+// getAvailableFromCurrent will look for available ordinal from the current ordinal.

+// If none found then it will loop back to the start to check of the available bit.

+// This can be further optimized to check from start till curr in case of a rollover

+func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) {

+	var bytePos, bitPos uint64

+	if curr != 0 && curr > start {

+		bytePos, bitPos, _ = getFirstAvailable(head, curr)

+		ret := posToOrdinal(bytePos, bitPos)

+		if end < ret {

+			goto begin

+		}

+		return bytePos, bitPos, nil

+	}

+

+begin:

+	bytePos, bitPos, _ = getFirstAvailable(head, start)

+	ret := posToOrdinal(bytePos, bitPos)

+	if end < ret {

+		return invalidPos, invalidPos, ErrNoBitAvailable

+	}

+	return bytePos, bitPos, nil

+}

+

 // checkIfAvailable checks if the bit correspondent to the specified ordinal is unset

 // If the ordinal is beyond the sequence limits, a negative response is returned

 func checkIfAvailable(head *sequence, ordinal uint64) (uint64, uint64, error) {

@@ -87,6 +87,7 @@ func (h *Handle) CopyTo(o datastore.KVObject) error {

 	dstH.dbIndex = h.dbIndex

 	dstH.dbExists = h.dbExists

 	dstH.store = h.store

+	dstH.curr = h.curr

 	dstH.Unlock()

 	return nil

@@ -21,7 +21,6 @@ import (

 const (

 	r            = 0xD0C4E3

-	timeout      = 30

 	pktExpansion = 26 // SPI(4) + SeqN(4) + IV(8) + PadLength(1) + NextHeader(1) + ICV(8)

 )

@@ -68,7 +68,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 	ep.ifName = containerIfName

-	if err := d.writeEndpointToStore(ep); err != nil {

+	if err = d.writeEndpointToStore(ep); err != nil {

 		return fmt.Errorf("failed to update overlay endpoint %s to local data store: %v", ep.id[0:7], err)

 	}

@@ -86,7 +86,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 		return err

 	}

-	if err := sbox.AddInterface(overlayIfName, "veth",

+	if err = sbox.AddInterface(overlayIfName, "veth",

 		sbox.InterfaceOptions().Master(s.brName)); err != nil {

 		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)

 	}

@@ -100,7 +100,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 		return err

 	}

-	if err := nlh.LinkSetHardwareAddr(veth, ep.mac); err != nil {

+	if err = nlh.LinkSetHardwareAddr(veth, ep.mac); err != nil {

 		return fmt.Errorf("could not set mac address (%v) to the container interface: %v", ep.mac, err)

 	}

@@ -108,7 +108,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 		if sub == s {

 			continue

 		}

-		if err := jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {

+		if err = jinfo.AddStaticRoute(sub.subnetIP, types.NEXTHOP, s.gwIP.IP); err != nil {

 			logrus.Errorf("Adding subnet %s static route in network %q failed\n", s.subnetIP, n.id)

 		}

 	}

@@ -122,7 +122,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 	d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), false, false, true)

-	if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {

+	if err = d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {

 		logrus.Warn(err)

 	}

@@ -200,7 +200,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri

 	}

 	if etype == driverapi.Delete {

-		d.peerDelete(nid, eid, addr.IP, addr.Mask, mac, vtep)

+		d.peerDelete(nid, eid, addr.IP, addr.Mask, mac, vtep, false)

 		return

 	}

@@ -232,11 +232,9 @@ func (d *driver) Leave(nid, eid string) error {

 		}

 	}

-	n.leaveSandbox()

+	d.peerDelete(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true)

-	if err := d.checkEncryption(nid, nil, 0, true, false); err != nil {

-		logrus.Warn(err)

-	}

+	n.leaveSandbox()

 	return nil

 }

@@ -119,7 +119,7 @@ func setDefaultVlan() {

 	data := []byte{'0', '\n'}

 	if err = ioutil.WriteFile(path, data, 0644); err != nil {

-		logrus.Errorf("endbling default vlan on bridge %s failed %v", brName, err)

+		logrus.Errorf("enabling default vlan on bridge %s failed %v", brName, err)

 		os.Exit(1)

 	}

 	os.Exit(0)

@@ -251,8 +251,9 @@ func (d *driver) DeleteNetwork(nid string) error {

 		if err := d.deleteEndpointFromStore(ep); err != nil {

 			logrus.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)

 		}

-

 	}

+	// flush the peerDB entries

+	d.peerFlush(nid)

 	d.deleteNetwork(nid)

 	vnis, err := n.releaseVxlanID()

@@ -505,11 +506,7 @@ func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) erro

 	vxlanIfaceOption := make([]osl.IfaceOption, 1)

 	vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName))

 	Ifaces[vxlanName+"+vxlan"] = vxlanIfaceOption

-	err = sbox.Restore(Ifaces, nil, nil, nil)

-	if err != nil {

-		return err

-	}

-	return nil

+	return sbox.Restore(Ifaces, nil, nil, nil)

 }

 func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error {

@@ -760,58 +757,38 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {

 				continue

 			}

-			logrus.Debugf("miss notification: dest IP %v, dest MAC %v", ip, mac)

-

 			if neigh.State&(netlink.NUD_STALE|netlink.NUD_INCOMPLETE) == 0 {

 				continue

 			}

 			if n.driver.isSerfAlive() {

+				logrus.Debugf("miss notification: dest IP %v, dest MAC %v", ip, mac)

 				mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip)

 				if err != nil {

 					logrus.Errorf("could not resolve peer %q: %v", ip, err)

 					continue

 				}

 				n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, l2Miss, l3Miss, false)

-			} else {

-				// If the gc_thresh values are lower kernel might knock off the neighor entries.

-				// When we get a L3 miss check if its a valid peer and reprogram the neighbor

-				// entry again. Rate limit it to once attempt every 500ms, just in case a faulty

-				// container sends a flood of packets to invalid peers

-				if !l3Miss {

-					continue

-				}

-				if time.Since(t) > 500*time.Millisecond {

+			} else if l3Miss && time.Since(t) > time.Second {

+				// All the local peers will trigger a miss notification but this one is expected and the local container will reply

+				// autonomously to the ARP request

+				// In case the gc_thresh3 values is low kernel might reject new entries during peerAdd. This will trigger the following

+				// extra logs that will inform of the possible issue.

+				// Entries created would not be deleted see documentation http://man7.org/linux/man-pages/man7/arp.7.html:

+				// Entries which are marked as permanent are never deleted by the garbage-collector.

+				// The time limit here is to guarantee that the dbSearch is not

+				// done too frequently causing a stall of the peerDB operations.

+				pKey, pEntry, err := n.driver.peerDbSearch(n.id, ip)

+				if err == nil && !pEntry.isLocal {

 					t = time.Now()

-					n.programNeighbor(ip)

+					logrus.Warnf("miss notification for peer:%+v l3Miss:%t l2Miss:%t, if the problem persist check the gc_thresh on the host pKey:%+v pEntry:%+v err:%v",

+						neigh, l3Miss, l2Miss, *pKey, *pEntry, err)

 				}

 			}

 		}

 	}

 }

-func (n *network) programNeighbor(ip net.IP) {

-	peerMac, _, _, err := n.driver.peerDbSearch(n.id, ip)

-	if err != nil {

-		logrus.Errorf("Reprogramming on L3 miss failed for %s, no peer entry", ip)

-		return

-	}

-	s := n.getSubnetforIPAddr(ip)

-	if s == nil {

-		logrus.Errorf("Reprogramming on L3 miss failed for %s, not a valid subnet", ip)

-		return

-	}

-	sbox := n.sandbox()

-	if sbox == nil {

-		logrus.Errorf("Reprogramming on L3 miss failed for %s, overlay sandbox missing", ip)

-		return

-	}

-	if err := sbox.AddNeighbor(ip, peerMac, true, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {

-		logrus.Errorf("Reprogramming on L3 miss failed for %s: %v", ip, err)

-		return

-	}

-}

-

 func (d *driver) addNetwork(n *network) {

 	d.Lock()

 	d.networks[n.id] = n

@@ -1058,7 +1035,7 @@ func (n *network) obtainVxlanID(s *subnet) error {

 		}

 		if s.vni == 0 {

-			vxlanID, err := n.driver.vxlanIdm.GetID()

+			vxlanID, err := n.driver.vxlanIdm.GetID(true)

 			if err != nil {

 				return fmt.Errorf("failed to allocate vxlan id: %v", err)

 			}

@@ -1090,15 +1067,6 @@ func (n *network) contains(ip net.IP) bool {

 	return false

 }

-func (n *network) getSubnetforIPAddr(ip net.IP) *subnet {

-	for _, s := range n.subnets {

-		if s.subnetIP.Contains(ip) {

-			return s

-		}

-	}

-	return nil

-}

-

 // getSubnetforIP returns the subnet to which the given IP belongs

 func (n *network) getSubnetforIP(ip *net.IPNet) *subnet {

 	for _, s := range n.subnets {

@@ -122,7 +122,7 @@ func (d *driver) processEvent(u serf.UserEvent) {

 	case "join":

 		d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), false, false, false)

 	case "leave":

-		d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr))

+		d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), false)

 	}

 }

@@ -135,13 +135,13 @@ func (d *driver) processQuery(q *serf.Query) {

 		fmt.Printf("Failed to scan query payload string: %v\n", err)

 	}

-	peerMac, peerIPMask, vtep, err := d.peerDbSearch(nid, net.ParseIP(ipStr))

+	pKey, pEntry, err := d.peerDbSearch(nid, net.ParseIP(ipStr))

 	if err != nil {

 		return

 	}

-	logrus.Debugf("Sending peer query resp mac %s, mask %s, vtep %s", peerMac, net.IP(peerIPMask), vtep)

-	q.Respond([]byte(fmt.Sprintf("%s %s %s", peerMac.String(), net.IP(peerIPMask).String(), vtep.String())))

+	logrus.Debugf("Sending peer query resp mac %v, mask %s, vtep %s", pKey.peerMac, net.IP(pEntry.peerIPMask).String(), pEntry.vtep)

+	q.Respond([]byte(fmt.Sprintf("%s %s %s", pKey.peerMac.String(), net.IP(pEntry.peerIPMask).String(), pEntry.vtep.String())))

 }

 func (d *driver) resolvePeer(nid string, peerIP net.IP) (net.HardwareAddr, net.IPMask, net.IP, error) {

@@ -262,7 +262,7 @@ func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) {

 		d.Unlock()

 		// If containers are already running on this network update the

-		// advertiseaddress in the peerDB

+		// advertise address in the peerDB

 		d.localJoinOnce.Do(func() {

 			d.peerDBUpdateSelf()

 		})

@@ -24,4 +24,4 @@ message PeerRecord {

 	// which this container is running and can be reached by

 	// building a tunnel to that host IP.

 	string tunnel_endpoint_ip = 3 [(gogoproto.customname) = "TunnelEndpointIP"];

-}

\ No newline at end of file

+}

@@ -165,7 +165,7 @@ func (n *network) obtainVxlanID(s *subnet) error {

 	n.Unlock()

 	if vni == 0 {

-		vni, err = n.driver.vxlanIdm.GetIDInRange(vxlanIDStart, vxlanIDEnd)

+		vni, err = n.driver.vxlanIdm.GetIDInRange(vxlanIDStart, vxlanIDEnd, true)

 		if err != nil {

 			return err

 		}

@@ -8,6 +8,7 @@ import (

 	"syscall"

 	"github.com/docker/libnetwork/common"

+	"github.com/docker/libnetwork/osl"

 	"github.com/sirupsen/logrus"

 )

@@ -22,16 +23,48 @@ type peerEntry struct {

 	eid        string

 	vtep       net.IP

 	peerIPMask net.IPMask

-	inSandbox  bool

 	isLocal    bool

 }

+func (p *peerEntry) MarshalDB() peerEntryDB {

+	ones, bits := p.peerIPMask.Size()

+	return peerEntryDB{

+		eid:            p.eid,

+		vtep:           p.vtep.String(),

+		peerIPMaskOnes: ones,

+		peerIPMaskBits: bits,

+		isLocal:        p.isLocal,

+	}

+}

+

+// This the structure saved into the set (SetMatrix), due to the implementation of it

+// the value inserted in the set has to be Hashable so the []byte had to be converted into

+// strings

+type peerEntryDB struct {

+	eid            string

+	vtep           string

+	peerIPMaskOnes int

+	peerIPMaskBits int

+	isLocal        bool

+}

+

+func (p *peerEntryDB) UnMarshalDB() peerEntry {

+	return peerEntry{

+		eid:        p.eid,

+		vtep:       net.ParseIP(p.vtep),

+		peerIPMask: net.CIDRMask(p.peerIPMaskOnes, p.peerIPMaskBits),

+		isLocal:    p.isLocal,

+	}

+}

+

 type peerMap struct {

-	mp map[string]peerEntry

+	// set of peerEntry, note they have to be objects and not pointers to maintain the proper equality checks

+	mp common.SetMatrix

 	sync.Mutex

 }

 type peerNetworkMap struct {

+	// map with key peerKey

 	mp map[string]*peerMap

 	sync.Mutex

 }

@@ -54,11 +87,7 @@ func (pKey *peerKey) Scan(state fmt.ScanState, verb rune) error {

 	}

 	pKey.peerMac, err = net.ParseMAC(string(macB))

-	if err != nil {

-		return err

-	}

-

-	return nil

+	return err

 }

 func (d *driver) peerDbWalk(f func(string, *peerKey, *peerEntry) bool) error {

@@ -87,10 +116,13 @@ func (d *driver) peerDbNetworkWalk(nid string, f func(*peerKey, *peerEntry) bool

 	}

 	mp := map[string]peerEntry{}

-

 	pMap.Lock()

-	for pKeyStr, pEntry := range pMap.mp {

-		mp[pKeyStr] = pEntry

+	for _, pKeyStr := range pMap.mp.Keys() {

+		entryDBList, ok := pMap.mp.Get(pKeyStr)

+		if ok {

+			peerEntryDB := entryDBList[0].(peerEntryDB)

+			mp[pKeyStr] = peerEntryDB.UnMarshalDB()

+		}

 	}

 	pMap.Unlock()

@@ -107,45 +139,38 @@ func (d *driver) peerDbNetworkWalk(nid string, f func(*peerKey, *peerEntry) bool

 	return nil

 }

-func (d *driver) peerDbSearch(nid string, peerIP net.IP) (net.HardwareAddr, net.IPMask, net.IP, error) {

-	var (

-		peerMac    net.HardwareAddr

-		vtep       net.IP

-		peerIPMask net.IPMask

-		found      bool

-	)

-

+func (d *driver) peerDbSearch(nid string, peerIP net.IP) (*peerKey, *peerEntry, error) {

+	var pKeyMatched *peerKey

+	var pEntryMatched *peerEntry

 	err := d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool {

 		if pKey.peerIP.Equal(peerIP) {

-			peerMac = pKey.peerMac

-			peerIPMask = pEntry.peerIPMask

-			vtep = pEntry.vtep

-			found = true

-			return found

+			pKeyMatched = pKey

+			pEntryMatched = pEntry

+			return true

 		}

-		return found

+		return false

 	})

 	if err != nil {

-		return nil, nil, nil, fmt.Errorf("peerdb search for peer ip %q failed: %v", peerIP, err)

+		return nil, nil, fmt.Errorf("peerdb search for peer ip %q failed: %v", peerIP, err)

 	}

-	if !found {

-		return nil, nil, nil, fmt.Errorf("peer ip %q not found in peerdb", peerIP)

+	if pKeyMatched == nil || pEntryMatched == nil {

+		return nil, nil, fmt.Errorf("peer ip %q not found in peerdb", peerIP)

 	}

-	return peerMac, peerIPMask, vtep, nil

+	return pKeyMatched, pEntryMatched, nil

 }

 func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

-	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) {

+	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) (bool, int) {

 	d.peerDb.Lock()

 	pMap, ok := d.peerDb.mp[nid]

 	if !ok {

 		d.peerDb.mp[nid] = &peerMap{

-			mp: make(map[string]peerEntry),

+			mp: common.NewSetMatrix(),

 		}

 		pMap = d.peerDb.mp[nid]

@@ -165,18 +190,24 @@ func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask

 	}

 	pMap.Lock()

-	pMap.mp[pKey.String()] = pEntry

-	pMap.Unlock()

+	defer pMap.Unlock()

+	b, i := pMap.mp.Insert(pKey.String(), pEntry.MarshalDB())

+	if i != 1 {

+		// Transient case, there is more than one endpoint that is using the same IP,MAC pair

+		s, _ := pMap.mp.String(pKey.String())

+		logrus.Warnf("peerDbAdd transient condition - Key:%s cardinality:%d db state:%s", pKey.String(), i, s)

+	}

+	return b, i

 }

 func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

-	peerMac net.HardwareAddr, vtep net.IP) peerEntry {

+	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) (bool, int) {

 	d.peerDb.Lock()

 	pMap, ok := d.peerDb.mp[nid]

 	if !ok {

 		d.peerDb.Unlock()

-		return peerEntry{}

+		return false, 0

 	}

 	d.peerDb.Unlock()

@@ -185,22 +216,22 @@ func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPM

 		peerMac: peerMac,

 	}

-	pMap.Lock()

-

-	pEntry, ok := pMap.mp[pKey.String()]

-	if ok {

-		// Mismatched endpoint ID(possibly outdated). Do not

-		// delete peerdb

-		if pEntry.eid != eid {

-			pMap.Unlock()

-			return pEntry

-		}

+	pEntry := peerEntry{

+		eid:        eid,

+		vtep:       vtep,

+		peerIPMask: peerIPMask,

+		isLocal:    isLocal,

 	}

-	delete(pMap.mp, pKey.String())

-	pMap.Unlock()

-

-	return pEntry

+	pMap.Lock()

+	defer pMap.Unlock()

+	b, i := pMap.mp.Remove(pKey.String(), pEntry.MarshalDB())

+	if i != 0 {

+		// Transient case, there is more than one endpoint that is using the same IP,MAC pair

+		s, _ := pMap.mp.String(pKey.String())

+		logrus.Warnf("peerDbDelete transient condition - Key:%s cardinality:%d db state:%s", pKey.String(), i, s)

+	}

+	return b, i

 }

 // The overlay uses a lazy initialization approach, this means that when a network is created

@@ -224,6 +255,7 @@ const (

 	peerOperationINIT peerOperationType = iota

 	peerOperationADD

 	peerOperationDELETE

+	peerOperationFLUSH

 )

 type peerOperation struct {

@@ -253,7 +285,9 @@ func (d *driver) peerOpRoutine(ctx context.Context, ch chan *peerOperation) {

 			case peerOperationADD:

 				err = d.peerAddOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.l2Miss, op.l3Miss, true, op.localPeer)

 			case peerOperationDELETE:

-				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP)

+				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.localPeer)

+			case peerOperationFLUSH:

+				err = d.peerFlushOp(op.networkID)

 			}

 			if err != nil {

 				logrus.Warnf("Peer operation failed:%s op:%v", err, op)

@@ -286,7 +320,6 @@ func (d *driver) peerInitOp(nid string) error {

 func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 	peerMac net.HardwareAddr, vtep net.IP, l2Miss, l3Miss, localPeer bool) {

-	callerName := common.CallerName(1)

 	d.peerOpCh <- &peerOperation{

 		opType:     peerOperationADD,

 		networkID:  nid,

@@ -298,24 +331,32 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 		l2Miss:     l2Miss,

 		l3Miss:     l3Miss,

 		localPeer:  localPeer,

-		callerName: callerName,

+		callerName: common.CallerName(1),

 	}

 }

 func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

-	peerMac net.HardwareAddr, vtep net.IP, l2Miss, l3Miss, updateDB, updateOnlyDB bool) error {

+	peerMac net.HardwareAddr, vtep net.IP, l2Miss, l3Miss, updateDB, localPeer bool) error {

 	if err := validateID(nid, eid); err != nil {

 		return err

 	}

+	var dbEntries int

+	var inserted bool

 	if updateDB {

-		d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, false)

-		if updateOnlyDB {

-			return nil

+		inserted, dbEntries = d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, localPeer)

+		if !inserted {

+			logrus.Warnf("Entry already present in db: nid:%s eid:%s peerIP:%v peerMac:%v isLocal:%t vtep:%v",

+				nid, eid, peerIP, peerMac, localPeer, vtep)

 		}

 	}

+	// Local peers do not need any further configuration

+	if localPeer {

+		return nil

+	}

+

 	n := d.network(nid)

 	if n == nil {

 		return nil

@@ -353,21 +394,26 @@ func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask

 	// Add neighbor entry for the peer IP

 	if err := sbox.AddNeighbor(peerIP, peerMac, l3Miss, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {

-		return fmt.Errorf("could not add neighbor entry into the sandbox: %v", err)

+		if _, ok := err.(osl.NeighborSearchError); ok && dbEntries > 1 {

+			// We are in the transient case so only the first configuration is programmed into the kernel

+			// Upon deletion if the active configuration is deleted the next one from the database will be restored

+			// Note we are skipping also the next configuration

+			return nil

+		}

+		return fmt.Errorf("could not add neighbor entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)

 	}

 	// Add fdb entry to the bridge for the peer mac

 	if err := sbox.AddNeighbor(vtep, peerMac, l2Miss, sbox.NeighborOptions().LinkName(s.vxlanName),

 		sbox.NeighborOptions().Family(syscall.AF_BRIDGE)); err != nil {

-		return fmt.Errorf("could not add fdb entry into the sandbox: %v", err)

+		return fmt.Errorf("could not add fdb entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)

 	}

 	return nil

 }

 func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

-	peerMac net.HardwareAddr, vtep net.IP) {

-	callerName := common.CallerName(1)

+	peerMac net.HardwareAddr, vtep net.IP, localPeer bool) {

 	d.peerOpCh <- &peerOperation{

 		opType:     peerOperationDELETE,

 		networkID:  nid,

@@ -376,18 +422,23 @@ func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMas

 		peerIPMask: peerIPMask,

 		peerMac:    peerMac,

 		vtepIP:     vtep,

-		callerName: callerName,

+		callerName: common.CallerName(1),

+		localPeer:  localPeer,

 	}

 }

 func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

-	peerMac net.HardwareAddr, vtep net.IP) error {

+	peerMac net.HardwareAddr, vtep net.IP, localPeer bool) error {

 	if err := validateID(nid, eid); err != nil {

 		return err

 	}

-	pEntry := d.peerDbDelete(nid, eid, peerIP, peerIPMask, peerMac, vtep)

+	deleted, dbEntries := d.peerDbDelete(nid, eid, peerIP, peerIPMask, peerMac, vtep, localPeer)

+	if !deleted {

+		logrus.Warnf("Entry was not in db: nid:%s eid:%s peerIP:%v peerMac:%v isLocal:%t vtep:%v",

+			nid, eid, peerIP, peerMac, localPeer, vtep)

+	}

 	n := d.network(nid)

 	if n == nil {

@@ -399,30 +450,59 @@ func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPM

 		return nil

 	}

-	// Delete fdb entry to the bridge for the peer mac only if the

-	// entry existed in local peerdb. If it is a stale delete

-	// request, still call DeleteNeighbor but only to cleanup any

-	// leftover sandbox neighbor cache and not actually delete the

-	// kernel state.

-	if (eid == pEntry.eid && vtep.Equal(pEntry.vtep)) ||

-		(eid != pEntry.eid && !vtep.Equal(pEntry.vtep)) {

-		if err := sbox.DeleteNeighbor(vtep, peerMac,

-			eid == pEntry.eid && vtep.Equal(pEntry.vtep)); err != nil {

-			return fmt.Errorf("could not delete fdb entry into the sandbox: %v", err)

-		}

+	if err := d.checkEncryption(nid, vtep, 0, localPeer, false); err != nil {

+		logrus.Warn(err)

 	}

-	// Delete neighbor entry for the peer IP

-	if eid == pEntry.eid {

+	// Local peers do not have any local configuration to delete

+	if !localPeer {

+		// Remove fdb entry to the bridge for the peer mac

+		if err := sbox.DeleteNeighbor(vtep, peerMac, true); err != nil {

+			if _, ok := err.(osl.NeighborSearchError); ok && dbEntries > 0 {

+				// We fall in here if there is a transient state and if the neighbor that is being deleted

+				// was never been configured into the kernel (we allow only 1 configuration at the time per <ip,mac> mapping)

+				return nil

+			}

+			return fmt.Errorf("could not delete fdb entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)

+		}

+

+		// Delete neighbor entry for the peer IP

 		if err := sbox.DeleteNeighbor(peerIP, peerMac, true); err != nil {

-			return fmt.Errorf("could not delete neighbor entry into the sandbox: %v", err)

+			return fmt.Errorf("could not delete neighbor entry for nid:%s eid:%s into the sandbox:%v", nid, eid, err)

 		}

 	}

-	if err := d.checkEncryption(nid, vtep, 0, false, false); err != nil {

-		logrus.Warn(err)

+	if dbEntries == 0 {

+		return nil

+	}

+

+	// If there is still an entry into the database and the deletion went through without errors means that there is now no

+	// configuration active in the kernel.

+	// Restore one configuration for the <ip,mac> directly from the database, note that is guaranteed that there is one

+	peerKey, peerEntry, err := d.peerDbSearch(nid, peerIP)

+	if err != nil {

+		logrus.Errorf("peerDeleteOp unable to restore a configuration for nid:%s ip:%v mac:%v err:%s", nid, peerIP, peerMac, err)

+		return err

 	}

+	return d.peerAddOp(nid, peerEntry.eid, peerIP, peerEntry.peerIPMask, peerKey.peerMac, peerEntry.vtep, false, false, false, peerEntry.isLocal)

+}

+func (d *driver) peerFlush(nid string) {

+	d.peerOpCh <- &peerOperation{

+		opType:     peerOperationFLUSH,

+		networkID:  nid,

+		callerName: common.CallerName(1),

+	}

+}

+

+func (d *driver) peerFlushOp(nid string) error {

+	d.peerDb.Lock()

+	defer d.peerDb.Unlock()

+	_, ok := d.peerDb.mp[nid]

+	if !ok {

+		return fmt.Errorf("Unable to find the peerDB for nid:%s", nid)

+	}

+	delete(d.peerDb.mp, nid)

 	return nil

 }

@@ -718,7 +718,7 @@ func (n *network) obtainVxlanID(s *subnet) error {

 		}

 		if s.vni == 0 {

-			vxlanID, err := n.driver.vxlanIdm.GetID()