@@ -27,7 +27,7 @@ github.com/imdario/mergo 0.2.1

 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0

 #get libnetwork packages

-github.com/docker/libnetwork 248fd5ea6a67f8810da322e6e7441e8de96a9045 https://github.com/dmcgowan/libnetwork.git

+github.com/docker/libnetwork 24bb72a8dcfe0b58958414890c8f4138b644b96a

 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9

 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80

 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

@@ -741,11 +741,12 @@ func (n *network) addDriverWatches() {

 			return

 		}

-		agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte) bool {

-			if nid == n.ID() {

+		agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte, deleted bool) bool {

+			// skip the entries that are mark for deletion, this is safe because this function is

+			// called at initialization time so there is no state to delete

+			if nid == n.ID() && !deleted {

 				d.EventNotify(driverapi.Create, nid, table.name, key, value)

 			}

-

 			return false

 		})

 	}

@@ -497,7 +497,10 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) {

 	// Derive the this sequence offsets

 	byteOffset := byteStart - inBlockBytePos

 	bitOffset := inBlockBytePos*8 + bitStart

-

+	var firstOffset uint64

+	if current == head {

+		firstOffset = byteOffset

+	}

 	for current != nil {

 		if current.block != blockMAX {

 			bytePos, bitPos, err := current.getAvailableBit(bitOffset)

@@ -505,7 +508,8 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) {

 		}

 		// Moving to next block: Reset bit offset.

 		bitOffset = 0

-		byteOffset += current.count * blockBytes

+		byteOffset += (current.count * blockBytes) - firstOffset

+		firstOffset = 0

 		current = current.next

 	}

 	return invalidPos, invalidPos, ErrNoBitAvailable

new file mode 100644

@@ -0,0 +1,29 @@

+package common

+

+import (

+	"runtime"

+	"strings"

+)

+

+func callerInfo(i int) string {

+	ptr, _, _, ok := runtime.Caller(i)

+	fName := "unknown"

+	if ok {

+		f := runtime.FuncForPC(ptr)

+		if f != nil {

+			// f.Name() is like: github.com/docker/libnetwork/common.MethodName

+			tmp := strings.Split(f.Name(), ".")

+			if len(tmp) > 0 {

+				fName = tmp[len(tmp)-1]

+			}

+		}

+	}

+

+	return fName

+}

+

+// CallerName returns the name of the function at the specified level

+// level == 0 means current method name

+func CallerName(level int) string {

+	return callerInfo(2 + level)

+}

new file mode 100644

@@ -0,0 +1,133 @@

+package diagnose

+

+import (

+	"fmt"

+	"net"

+	"net/http"

+	"sync"

+

+	"github.com/sirupsen/logrus"

+)

+

+// HTTPHandlerFunc TODO

+type HTTPHandlerFunc func(interface{}, http.ResponseWriter, *http.Request)

+

+type httpHandlerCustom struct {

+	ctx interface{}

+	F   func(interface{}, http.ResponseWriter, *http.Request)

+}

+

+// ServeHTTP TODO

+func (h httpHandlerCustom) ServeHTTP(w http.ResponseWriter, r *http.Request) {

+	h.F(h.ctx, w, r)

+}

+

+var diagPaths2Func = map[string]HTTPHandlerFunc{

+	"/":      notImplemented,

+	"/help":  help,

+	"/ready": ready,

+}

+

+// Server when the debug is enabled exposes a

+// This data structure is protected by the Agent mutex so does not require and additional mutex here

+type Server struct {

+	sk                net.Listener

+	port              int

+	mux               *http.ServeMux

+	registeredHanders []string

+	sync.Mutex

+}

+

+// Init TODO

+func (n *Server) Init() {

+	n.mux = http.NewServeMux()

+

+	// Register local handlers

+	n.RegisterHandler(n, diagPaths2Func)

+}

+

+// RegisterHandler TODO

+func (n *Server) RegisterHandler(ctx interface{}, hdlrs map[string]HTTPHandlerFunc) {

+	n.Lock()

+	defer n.Unlock()

+	for path, fun := range hdlrs {

+		n.mux.Handle(path, httpHandlerCustom{ctx, fun})

+		n.registeredHanders = append(n.registeredHanders, path)

+	}

+}

+

+// EnableDebug opens a TCP socket to debug the passed network DB

+func (n *Server) EnableDebug(ip string, port int) {

+	n.Lock()

+	defer n.Unlock()

+

+	n.port = port

+	logrus.SetLevel(logrus.DebugLevel)

+

+	if n.sk != nil {

+		logrus.Infof("The server is already up and running")

+		return

+	}

+

+	logrus.Infof("Starting the server listening on %d for commands", port)

+

+	// // Create the socket

+	// var err error

+	// n.sk, err = net.Listen("tcp", listeningAddr)

+	// if err != nil {

+	// 	log.Fatal(err)

+	// }

+	//

+	// go func() {

+	// 	http.Serve(n.sk, n.mux)

+	// }()

+	http.ListenAndServe(":8000", n.mux)

+}

+

+// DisableDebug stop the dubug and closes the tcp socket

+func (n *Server) DisableDebug() {

+	n.Lock()

+	defer n.Unlock()

+	n.sk.Close()

+	n.sk = nil

+}

+

+// IsDebugEnable returns true when the debug is enabled

+func (n *Server) IsDebugEnable() bool {

+	n.Lock()

+	defer n.Unlock()

+	return n.sk != nil

+}

+

+func notImplemented(ctx interface{}, w http.ResponseWriter, r *http.Request) {

+	fmt.Fprintf(w, "URL path: %s no method implemented check /help\n", r.URL.Path)

+}

+

+func help(ctx interface{}, w http.ResponseWriter, r *http.Request) {

+	n, ok := ctx.(*Server)

+	if ok {

+		for _, path := range n.registeredHanders {

+			fmt.Fprintf(w, "%s\n", path)

+		}

+	}

+}

+

+func ready(ctx interface{}, w http.ResponseWriter, r *http.Request) {

+	fmt.Fprintf(w, "OK\n")

+}

+

+// DebugHTTPForm TODO

+func DebugHTTPForm(r *http.Request) {

+	r.ParseForm()

+	for k, v := range r.Form {

+		logrus.Debugf("Form[%q] = %q\n", k, v)

+	}

+}

+

+// HTTPReplyError TODO

+func HTTPReplyError(w http.ResponseWriter, message, usage string) {

+	fmt.Fprintf(w, "%s\n", message)

+	if usage != "" {

+		fmt.Fprintf(w, "Usage: %s\n", usage)

+	}

+}

@@ -120,8 +120,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,

 		}

 	}

-	d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,

-		net.ParseIP(d.advertiseAddress), true)

+	d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true)

 	if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {

 		logrus.Warn(err)

@@ -205,7 +204,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri

 		return

 	}

-	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false)

+	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false, false)

 }

 // Leave method is invoked when a Sandbox detaches from an endpoint.

@@ -683,10 +683,12 @@ func (n *network) initSandbox(restore bool) error {

 		return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)

 	}

+	// this is needed to let the peerAdd configure the sandbox

 	n.setSandbox(sbox)

 	if !restore {

-		n.driver.peerDbUpdateSandbox(n.id)

+		// Initialize the sandbox with all the peers previously received from networkdb

+		n.driver.initSandboxPeerDB(n.id)

 	}

 	var nlSock *nl.NetlinkSocket

@@ -765,10 +767,7 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {

 					logrus.Errorf("could not resolve peer %q: %v", ip, err)

 					continue

 				}

-

-				if err := n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss); err != nil {

-					logrus.Errorf("could not add neighbor entry for missed peer %q: %v", ip, err)

-				}

+				n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss, false)

 			} else {

 				// If the gc_thresh values are lower kernel might knock off the neighor entries.

 				// When we get a L3 miss check if its a valid peer and reprogram the neighbor

@@ -120,15 +120,10 @@ func (d *driver) processEvent(u serf.UserEvent) {

 	switch action {

 	case "join":

-		if err := d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac,

-			net.ParseIP(vtepStr), true, false, false); err != nil {

-			logrus.Errorf("Peer add failed in the driver: %v\n", err)

-		}

+		d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr),

+			true, false, false, false)

 	case "leave":

-		if err := d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac,

-			net.ParseIP(vtepStr), true); err != nil {

-			logrus.Errorf("Peer delete failed in the driver: %v\n", err)

-		}

+		d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), true)

 	}

 }

@@ -3,6 +3,7 @@ package overlay

 //go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto

 import (

+	"context"

 	"fmt"

 	"net"

 	"sync"

@@ -50,6 +51,8 @@ type driver struct {

 	joinOnce         sync.Once

 	localJoinOnce    sync.Once

 	keys             []*key

+	peerOpCh         chan *peerOperation

+	peerOpCancel     context.CancelFunc

 	sync.Mutex

 }

@@ -64,10 +67,16 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {

 		peerDb: peerNetworkMap{

 			mp: map[string]*peerMap{},

 		},

-		secMap: &encrMap{nodes: map[string][]*spi{}},

-		config: config,

+		secMap:   &encrMap{nodes: map[string][]*spi{}},

+		config:   config,

+		peerOpCh: make(chan *peerOperation),

 	}

+	// Launch the go routine for processing peer operations

+	ctx, cancel := context.WithCancel(context.Background())

+	d.peerOpCancel = cancel

+	go d.peerOpRoutine(ctx, d.peerOpCh)

+

 	if data, ok := config[netlabel.GlobalKVClient]; ok {

 		var err error

 		dsc, ok := data.(discoverapi.DatastoreConfigData)

@@ -161,7 +170,7 @@ func (d *driver) restoreEndpoints() error {

 		}

 		n.incEndpointCount()

-		d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true)

+		d.peerAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true)

 	}

 	return nil

 }

@@ -170,6 +179,11 @@ func (d *driver) restoreEndpoints() error {

 func Fini(drv driverapi.Driver) {

 	d := drv.(*driver)

+	// Notify the peer go routine to return

+	if d.peerOpCancel != nil {

+		d.peerOpCancel()

+	}

+

 	if d.exitCh != nil {

 		waitCh := make(chan struct{})

@@ -1,11 +1,13 @@

 package overlay

 import (

+	"context"

 	"fmt"

 	"net"

 	"sync"

 	"syscall"

+	"github.com/docker/libnetwork/common"

 	"github.com/sirupsen/logrus"

 )

@@ -59,8 +61,6 @@ func (pKey *peerKey) Scan(state fmt.ScanState, verb rune) error {

 	return nil

 }

-var peerDbWg sync.WaitGroup

-

 func (d *driver) peerDbWalk(f func(string, *peerKey, *peerEntry) bool) error {

 	d.peerDb.Lock()

 	nids := []string{}

@@ -141,8 +141,6 @@ func (d *driver) peerDbSearch(nid string, peerIP net.IP) (net.HardwareAddr, net.

 func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 	peerMac net.HardwareAddr, vtep net.IP, isLocal bool) {

-	peerDbWg.Wait()

-

 	d.peerDb.Lock()

 	pMap, ok := d.peerDb.mp[nid]

 	if !ok {

@@ -173,7 +171,6 @@ func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask

 func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 	peerMac net.HardwareAddr, vtep net.IP) peerEntry {

-	peerDbWg.Wait()

 	d.peerDb.Lock()

 	pMap, ok := d.peerDb.mp[nid]

@@ -206,55 +203,109 @@ func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPM

 	return pEntry

 }

-func (d *driver) peerDbUpdateSandbox(nid string) {

-	d.peerDb.Lock()

-	pMap, ok := d.peerDb.mp[nid]

-	if !ok {

-		d.peerDb.Unlock()

-		return

-	}

-	d.peerDb.Unlock()

+// The overlay uses a lazy initialization approach, this means that when a network is created

+// and the driver registered the overlay does not allocate resources till the moment that a

+// sandbox is actually created.

+// At the moment of this call, that happens when a sandbox is initialized, is possible that

+// networkDB has already delivered some events of peers already available on remote nodes,

+// these peers are saved into the peerDB and this function is used to properly configure

+// the network sandbox with all those peers that got previously notified.

+// Note also that this method sends a single message on the channel and the go routine on the

+// other side, will atomically loop on the whole table of peers and will program their state

+// in one single atomic operation. This is fundamental to guarantee consistency, and avoid that

+// new peerAdd or peerDelete gets reordered during the sandbox init.

+func (d *driver) initSandboxPeerDB(nid string) {

+	d.peerInit(nid)

+}

-	peerDbWg.Add(1)

+type peerOperationType int32

-	var peerOps []func()

-	pMap.Lock()

-	for pKeyStr, pEntry := range pMap.mp {

-		var pKey peerKey

-		if _, err := fmt.Sscan(pKeyStr, &pKey); err != nil {

-			logrus.Errorf("peer key scan failed: %v", err)

-		}

+const (

+	peerOperationINIT peerOperationType = iota

+	peerOperationADD

+	peerOperationDELETE

+)

-		if pEntry.isLocal {

-			continue

-		}

+type peerOperation struct {

+	opType     peerOperationType

+	networkID  string

+	endpointID string

+	peerIP     net.IP

+	peerIPMask net.IPMask

+	peerMac    net.HardwareAddr

+	vtepIP     net.IP

+	updateDB   bool

+	l2Miss     bool

+	l3Miss     bool

+	localPeer  bool

+	callerName string

+}

-		// Go captures variables by reference. The pEntry could be

-		// pointing to the same memory location for every iteration. Make

-		// a copy of pEntry before capturing it in the following closure.

-		entry := pEntry

-		op := func() {

-			if err := d.peerAdd(nid, entry.eid, pKey.peerIP, entry.peerIPMask,

-				pKey.peerMac, entry.vtep,

-				false, false, false); err != nil {

-				logrus.Errorf("peerdbupdate in sandbox failed for ip %s and mac %s: %v",

-					pKey.peerIP, pKey.peerMac, err)

+func (d *driver) peerOpRoutine(ctx context.Context, ch chan *peerOperation) {

+	var err error

+	for {

+		select {

+		case <-ctx.Done():

+			return

+		case op := <-ch:

+			switch op.opType {

+			case peerOperationINIT:

+				err = d.peerInitOp(op.networkID)

+			case peerOperationADD:

+				err = d.peerAddOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.updateDB, op.l2Miss, op.l3Miss, op.localPeer)

+			case peerOperationDELETE:

+				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.localPeer)

+			}

+			if err != nil {

+				logrus.Warnf("Peer operation failed:%s op:%v", err, op)

 			}

 		}

-

-		peerOps = append(peerOps, op)

 	}

-	pMap.Unlock()

+}

-	for _, op := range peerOps {

-		op()

+func (d *driver) peerInit(nid string) {

+	callerName := common.CallerName(1)

+	d.peerOpCh <- &peerOperation{

+		opType:     peerOperationINIT,

+		networkID:  nid,

+		callerName: callerName,

 	}

+}

+

+func (d *driver) peerInitOp(nid string) error {

+	return d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool {

+		// Local entries do not need to be added

+		if pEntry.isLocal {

+			return false

+		}

-	peerDbWg.Done()

+		d.peerAddOp(nid, pEntry.eid, pKey.peerIP, pEntry.peerIPMask, pKey.peerMac, pEntry.vtep, false, false, false, false)

+		// return false to loop on all entries

+		return false

+	})

 }

 func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

-	peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss bool) error {

+	peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss, localPeer bool) {

+	callerName := common.CallerName(1)

+	d.peerOpCh <- &peerOperation{

+		opType:     peerOperationADD,

+		networkID:  nid,

+		endpointID: eid,

+		peerIP:     peerIP,

+		peerIPMask: peerIPMask,

+		peerMac:    peerMac,

+		vtepIP:     vtep,

+		updateDB:   updateDb,

+		l2Miss:     l2Miss,

+		l3Miss:     l3Miss,

+		localPeer:  localPeer,

+		callerName: callerName,

+	}

+}

+

+func (d *driver) peerAddOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

+	peerMac net.HardwareAddr, vtep net.IP, updateDb, l2Miss, l3Miss, updateOnlyDB bool) error {

 	if err := validateID(nid, eid); err != nil {

 		return err

@@ -262,6 +313,9 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 	if updateDb {

 		d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, false)

+		if updateOnlyDB {

+			return nil

+		}

 	}

 	n := d.network(nid)

@@ -271,6 +325,9 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 	sbox := n.sandbox()

 	if sbox == nil {

+		// We are hitting this case for all the events that are arriving before that the sandbox

+		// is being created. The peer got already added into the database and the sanbox init will

+		// call the peerDbUpdateSandbox that will configure all these peers from the database

 		return nil

 	}

@@ -311,6 +368,22 @@ func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 }

 func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

+	peerMac net.HardwareAddr, vtep net.IP, updateDb bool) {

+	callerName := common.CallerName(1)

+	d.peerOpCh <- &peerOperation{

+		opType:     peerOperationDELETE,

+		networkID:  nid,

+		endpointID: eid,

+		peerIP:     peerIP,

+		peerIPMask: peerIPMask,

+		peerMac:    peerMac,

+		vtepIP:     vtep,

+		updateDB:   updateDb,

+		callerName: callerName,

+	}

+}

+

+func (d *driver) peerDeleteOp(nid, eid string, peerIP net.IP, peerIPMask net.IPMask,

 	peerMac net.HardwareAddr, vtep net.IP, updateDb bool) error {

 	if err := validateID(nid, eid); err != nil {

@@ -413,7 +413,7 @@ func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {

 		return err

 	}

 	if v, ok := epMap["gw"]; ok {

-		epj.gw6 = net.ParseIP(v.(string))

+		epj.gw = net.ParseIP(v.(string))

 	}

 	if v, ok := epMap["gw6"]; ok {

 		epj.gw6 = net.ParseIP(v.(string))

@@ -442,6 +442,6 @@ func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {

 	dstEpj.driverTableEntries = make([]*tableEntry, len(epj.driverTableEntries))

 	copy(dstEpj.driverTableEntries, epj.driverTableEntries)

 	dstEpj.gw = types.GetIPCopy(epj.gw)

-	dstEpj.gw = types.GetIPCopy(epj.gw6)

+	dstEpj.gw6 = types.GetIPCopy(epj.gw6)

 	return nil

 }

@@ -114,7 +114,8 @@ type tableEventMessage struct {

 }

 func (m *tableEventMessage) Invalidates(other memberlist.Broadcast) bool {

-	return false

+	otherm := other.(*tableEventMessage)

+	return m.tname == otherm.tname && m.id == otherm.id && m.key == otherm.key

 }

 func (m *tableEventMessage) Message() []byte {

@@ -290,13 +290,6 @@ func (nDB *NetworkDB) reconnectNode() {

 		return

 	}

-	// Update all the local table state to a new time to

-	// force update on the node we are trying to rejoin, just in

-	// case that node has these in deleting state still. This is

-	// facilitate fast convergence after recovering from a gossip

-	// failure.

-	nDB.updateLocalTableTime()

-

 	logrus.Debugf("Initiating bulk sync with node %s after reconnect", node.Name)

 	nDB.bulkSync([]string{node.Name}, true)

 }

@@ -104,6 +104,9 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool {

 	}

 	n = nDB.checkAndGetNode(nEvent)

+	if n == nil {

+		return false

+	}

 	nDB.purgeSameNode(n)

 	n.ltime = nEvent.LTime

@@ -130,25 +133,12 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool {

 }

 func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {

-	var flushEntries bool

 	// Update our local clock if the received messages has newer

 	// time.

 	nDB.networkClock.Witness(nEvent.LTime)

 	nDB.Lock()

-	defer func() {

-		nDB.Unlock()

-		// When a node leaves a network on the last task removal cleanup the

-		// local entries for this network & node combination. When the tasks

-		// on a network are removed we could have missed the gossip updates.

-		// Not doing this cleanup can leave stale entries because bulksyncs

-		// from the node will no longer include this network state.

-		//

-		// deleteNodeNetworkEntries takes nDB lock.

-		if flushEntries {

-			nDB.deleteNodeNetworkEntries(nEvent.NetworkID, nEvent.NodeName)

-		}

-	}()

+	defer nDB.Unlock()

 	if nEvent.NodeName == nDB.config.NodeName {

 		return false

@@ -176,7 +166,12 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {

 		n.leaving = nEvent.Type == NetworkEventTypeLeave

 		if n.leaving {

 			n.reapTime = reapInterval

-			flushEntries = true

+

+			// The remote node is leaving the network, but not the gossip cluster.

+			// Mark all its entries in deleted state, this will guarantee that

+			// if some node bulk sync with us, the deleted state of

+			// these entries will be propagated.

+			nDB.deleteNodeNetworkEntries(nEvent.NetworkID, nEvent.NodeName)

 		}

 		if nEvent.Type == NetworkEventTypeLeave {

@@ -211,17 +206,22 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {

 	nDB.RLock()

 	networks := nDB.networks[nDB.config.NodeName]

 	network, ok := networks[tEvent.NetworkID]

-	nDB.RUnlock()

-	if !ok || network.leaving {

-		return true

+	// Check if the owner of the event is still part of the network

+	nodes := nDB.networkNodes[tEvent.NetworkID]

+	var nodePresent bool

+	for _, node := range nodes {

+		if node == tEvent.NodeName {

+			nodePresent = true

+			break

+		}

 	}

-

-	e, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key)

-	if err != nil && tEvent.Type == TableEventTypeDelete {

-		// If it is a delete event and we don't have the entry here nothing to do.

+	nDB.RUnlock()

+	if !ok || network.leaving || !nodePresent {

+		// I'm out of the network OR the event owner is not anymore part of the network so do not propagate

 		return false

 	}

+	e, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key)

 	if err == nil {

 		// We have the latest state. Ignore the event

 		// since it is stale.

@@ -246,6 +246,11 @@ func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {

 	nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.NetworkID, tEvent.TableName, tEvent.Key), e)

 	nDB.Unlock()

+	if err != nil && tEvent.Type == TableEventTypeDelete {

+		// If it is a delete event and we didn't have the entry here don't repropagate

+		return true

+	}

+

 	var op opType

 	switch tEvent.Type {

 	case TableEventTypeCreate:

@@ -286,8 +291,7 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {

 		return

 	}

-	// Do not rebroadcast a bulk sync

-	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync {

+	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast {

 		var err error

 		buf, err = encodeRawMessage(MessageTypeTableEvent, buf)

 		if err != nil {

@@ -45,9 +45,12 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) {

 	var failed bool

 	logrus.Infof("Node %s/%s, left gossip cluster", mn.Name, mn.Addr)

 	e.broadcastNodeEvent(mn.Addr, opDelete)

-	e.nDB.deleteNodeTableEntries(mn.Name)

-	e.nDB.deleteNetworkEntriesForNode(mn.Name)

+	// The node left or failed, delete all the entries created by it.

+	// If the node was temporary down, deleting the entries will guarantee that the CREATE events will be accepted

+	// If the node instead left because was going down, then it makes sense to just delete all its state

 	e.nDB.Lock()

+	e.nDB.deleteNetworkEntriesForNode(mn.Name)

+	e.nDB.deleteNodeTableEntries(mn.Name)

 	if n, ok := e.nDB.nodes[mn.Name]; ok {

 		delete(e.nDB.nodes, mn.Name)

@@ -61,7 +64,6 @@ func (e *eventDelegate) NotifyLeave(mn *memberlist.Node) {

 	if failed {

 		logrus.Infof("Node %s/%s, added to failed nodes list", mn.Name, mn.Addr)

 	}

-

 }

 func (e *eventDelegate) NotifyUpdate(n *memberlist.Node) {

@@ -108,6 +108,11 @@ type PeerInfo struct {

 	IP   string

 }

+// PeerClusterInfo represents the peer (gossip cluster) nodes

+type PeerClusterInfo struct {

+	PeerInfo

+}

+

 type node struct {

 	memberlist.Node

 	ltime serf.LamportTime

@@ -253,6 +258,20 @@ func (nDB *NetworkDB) Close() {

 	}

 }

+// ClusterPeers returns all the gossip cluster peers.

+func (nDB *NetworkDB) ClusterPeers() []PeerInfo {

+	nDB.RLock()

+	defer nDB.RUnlock()

+	peers := make([]PeerInfo, 0, len(nDB.nodes))

+	for _, node := range nDB.nodes {

+		peers = append(peers, PeerInfo{

+			Name: node.Name,

+			IP:   node.Node.Addr.String(),

+		})

+	}

+	return peers

+}

+

 // Peers returns the gossip peers for a given network.

 func (nDB *NetworkDB) Peers(nid string) []PeerInfo {

 	nDB.RLock()

@@ -399,7 +418,6 @@ func (nDB *NetworkDB) DeleteEntry(tname, nid, key string) error {

 }

 func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) {

-	nDB.Lock()

 	for nid, nodes := range nDB.networkNodes {

 		updatedNodes := make([]string, 0, len(nodes))

 		for _, node := range nodes {

@@ -414,11 +432,25 @@ func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) {

 	}

 	delete(nDB.networks, deletedNode)

-	nDB.Unlock()

 }

+// deleteNodeNetworkEntries is called in 2 conditions with 2 different outcomes:

+// 1) when a notification is coming of a node leaving the network

+//		- Walk all the network entries and mark the leaving node's entries for deletion

+//			These will be garbage collected when the reap timer will expire

+// 2) when the local node is leaving the network

+//		- Walk all the network entries:

+//			A) if the entry is owned by the local node

+//		  then we will mark it for deletion. This will ensure that if a node did not

+//		  yet received the notification that the local node is leaving, will be aware

+//		  of the entries to be deleted.

+//			B) if the entry is owned by a remote node, then we can safely delete it. This

+//			ensures that if we join back this network as we receive the CREATE event for

+//		  entries owned by remote nodes, we will accept them and we notify the application

 func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) {

-	nDB.Lock()

+	// Indicates if the delete is triggered for the local node

+	isNodeLocal := node == nDB.config.NodeName

+

 	nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid),

 		func(path string, v interface{}) bool {

 			oldEntry := v.(*entry)

@@ -427,7 +459,15 @@ func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) {

 			tname := params[1]

 			key := params[2]

-			if oldEntry.node != node {

+			// If the entry is owned by a remote node and this node is not leaving the network

+			if oldEntry.node != node && !isNodeLocal {

+				// Don't do anything because the event is triggered for a node that does not own this entry

+				return false

+			}

+

+			// If this entry is already marked for deletion and this node is not leaving the network

+			if oldEntry.deleting && !isNodeLocal {

+				// Don't do anything this entry will be already garbage collected using the old reapTime

 				return false

 			}

@@ -439,17 +479,29 @@ func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) {

 				reapTime: reapInterval,

 			}

-			nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry)

-			nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry)

+			// we arrived at this point in 2 cases:

+			// 1) this entry is owned by the node that is leaving the network

+			// 2) the local node is leaving the network

+			if oldEntry.node == node {

+				if isNodeLocal {

+					// TODO fcrisciani: this can be removed if there is no way to leave the network

+					// without doing a delete of all the objects

+					entry.ltime++

+				}

+				nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry)

+				nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry)

+			} else {

+				// the local node is leaving the network, all the entries of remote nodes can be safely removed

+				nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key))

+				nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key))

+			}

 			nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value))

 			return false

 		})

-	nDB.Unlock()

 }

 func (nDB *NetworkDB) deleteNodeTableEntries(node string) {

-	nDB.Lock()

 	nDB.indexes[byTable].Walk(func(path string, v interface{}) bool {

 		oldEntry := v.(*entry)

 		if oldEntry.node != node {

@@ -461,27 +513,18 @@ func (nDB *NetworkDB) deleteNodeTableEntries(node string) {

 		nid := params[1]

 		key := params[2]

-		entry := &entry{

-			ltime:    oldEntry.ltime,

-			node:     node,

-			value:    oldEntry.value,

-			deleting: true,

-			reapTime: reapInterval,

-		}

+		nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key))

+		nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key))

-		nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry)

-		nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry)

-

-		nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value))