@@ -24,7 +24,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5

 github.com/imdario/mergo 0.2.1

 #get libnetwork packages

-github.com/docker/libnetwork ab8f7e61743aa7e54c5d0dad0551543adadc33cf

+github.com/docker/libnetwork b13e0604016a4944025aaff521d9c125850b0d04

 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894

 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80

 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

@@ -47,6 +47,7 @@ import (

 	"container/heap"

 	"fmt"

 	"net"

+	"path/filepath"

 	"strings"

 	"sync"

 	"time"

@@ -979,6 +980,8 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s

 	if sb.ingress {

 		c.ingressSandbox = sb

+		sb.config.hostsPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/hosts")

+		sb.config.resolvConfPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/resolv.conf")

 		sb.id = "ingress_sbox"

 	}

 	c.Unlock()

@@ -1346,6 +1346,13 @@ func (d *driver) RevokeExternalConnectivity(nid, eid string) error {

 	endpoint.portMapping = nil

+	// Clean the connection tracker state of the host for the specific endpoint

+	// The host kernel keeps track of the connections (TCP and UDP), so if a new endpoint gets the same IP of

+	// this one (that is going down), is possible that some of the packets would not be routed correctly inside

+	// the new endpoint

+	// Deeper details: https://github.com/docker/docker/issues/8795

+	clearEndpointConnections(d.nlh, endpoint)

+

 	if err = d.storeUpdate(endpoint); err != nil {

 		return fmt.Errorf("failed to update bridge endpoint %s to store: %v", endpoint.id[0:7], err)

 	}

@@ -7,6 +7,7 @@ import (

 	"github.com/Sirupsen/logrus"

 	"github.com/docker/libnetwork/iptables"

+	"github.com/vishvananda/netlink"

 )

 // DockerChain: DOCKER iptable chain name

@@ -348,3 +349,15 @@ func setupInternalNetworkRules(bridgeIface string, addr net.Addr, icc, insert bo

 	}

 	return nil

 }

+

+func clearEndpointConnections(nlh *netlink.Handle, ep *bridgeEndpoint) {

+	var ipv4List []net.IP

+	var ipv6List []net.IP

+	if ep.addr != nil {

+		ipv4List = append(ipv4List, ep.addr.IP)

+	}

+	if ep.addrv6 != nil {

+		ipv6List = append(ipv6List, ep.addrv6.IP)

+	}

+	iptables.DeleteConntrackEntries(nlh, ipv4List, ipv6List)

+}

@@ -665,7 +665,7 @@ func (ep *endpoint) hasInterface(iName string) bool {

 func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {

 	if sbox == nil || sbox.ID() == "" || sbox.Key() == "" {

-		return types.BadRequestErrorf("invalid Sandbox passed to enpoint leave: %v", sbox)

+		return types.BadRequestErrorf("invalid Sandbox passed to endpoint leave: %v", sbox)

 	}

 	sb, ok := sbox.(*sandbox)

@@ -129,7 +129,7 @@ type ActiveEndpointsError struct {

 }

 func (aee *ActiveEndpointsError) Error() string {

-	return fmt.Sprintf("network %s has active endpoints", aee.name)

+	return fmt.Sprintf("network %s id %s has active endpoints", aee.name, aee.id)

 }

 // Forbidden denotes the type of this error

new file mode 100644

@@ -0,0 +1,59 @@

+package iptables

+

+import (

+	"errors"

+	"net"

+	"syscall"

+

+	"github.com/Sirupsen/logrus"

+	"github.com/vishvananda/netlink"

+)

+

+var (

+	// ErrConntrackNotConfigurable means that conntrack module is not loaded or does not have the netlink module loaded

+	ErrConntrackNotConfigurable = errors.New("conntrack is not available")

+)

+

+// IsConntrackProgrammable returns true if the handle supports the NETLINK_NETFILTER and the base modules are loaded

+func IsConntrackProgrammable(nlh *netlink.Handle) bool {

+	return nlh.SupportsNetlinkFamily(syscall.NETLINK_NETFILTER)

+}

+

+// DeleteConntrackEntries deletes all the conntrack connections on the host for the specified IP

+// Returns the number of flows deleted for IPv4, IPv6 else error

+func DeleteConntrackEntries(nlh *netlink.Handle, ipv4List []net.IP, ipv6List []net.IP) (uint, uint, error) {

+	if !IsConntrackProgrammable(nlh) {

+		return 0, 0, ErrConntrackNotConfigurable

+	}

+

+	var totalIPv4FlowPurged uint

+	for _, ipAddress := range ipv4List {

+		flowPurged, err := purgeConntrackState(nlh, syscall.AF_INET, ipAddress)

+		if err != nil {

+			logrus.Warnf("Failed to delete conntrack state for %s: %v", ipAddress, err)

+			continue

+		}

+		totalIPv4FlowPurged += flowPurged

+	}

+

+	var totalIPv6FlowPurged uint

+	for _, ipAddress := range ipv6List {

+		flowPurged, err := purgeConntrackState(nlh, syscall.AF_INET6, ipAddress)

+		if err != nil {

+			logrus.Warnf("Failed to delete conntrack state for %s: %v", ipAddress, err)

+			continue

+		}

+		totalIPv6FlowPurged += flowPurged

+	}

+

+	logrus.Debugf("DeleteConntrackEntries purged ipv4:%d, ipv6:%d", totalIPv4FlowPurged, totalIPv6FlowPurged)

+	return totalIPv4FlowPurged, totalIPv6FlowPurged, nil

+}

+

+func purgeConntrackState(nlh *netlink.Handle, family netlink.InetFamily, ipAddress net.IP) (uint, error) {

+	filter := &netlink.ConntrackFilter{}

+	// NOTE: doing the flush using the ipAddress is safe because today there cannot be multiple networks with the same subnet

+	// so it will not be possible to flush flows that are of other containers

+	filter.AddIP(netlink.ConntrackNatAnyIP, ipAddress)

+	return nlh.ConntrackDeleteFilter(netlink.ConntrackTable, family, filter)

+}

@@ -100,14 +100,14 @@ func detectIptables() {

 	supportsCOpt = supportsCOption(mj, mn, mc)

 }

-func initIptables() {

+func initDependencies() {

 	probe()

 	initFirewalld()

 	detectIptables()

 }

 func initCheck() error {

-	initOnce.Do(initIptables)

+	initOnce.Do(initDependencies)

 	if iptablesPath == "" {

 		return ErrIptablesNotFound

@@ -88,12 +88,25 @@ func (nDB *NetworkDB) handleNodeEvent(nEvent *NodeEvent) bool {

 }

 func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {

+	var flushEntries bool

 	// Update our local clock if the received messages has newer

 	// time.

 	nDB.networkClock.Witness(nEvent.LTime)

 	nDB.Lock()

-	defer nDB.Unlock()

+	defer func() {

+		nDB.Unlock()

+		// When a node leaves a network on the last task removal cleanup the

+		// local entries for this network & node combination. When the tasks

+		// on a network are removed we could have missed the gossip updates.

+		// Not doing this cleanup can leave stale entries because bulksyncs

+		// from the node will no longer include this network state.

+		//

+		// deleteNodeNetworkEntries takes nDB lock.

+		if flushEntries {

+			nDB.deleteNodeNetworkEntries(nEvent.NetworkID, nEvent.NodeName)

+		}

+	}()

 	if nEvent.NodeName == nDB.config.NodeName {

 		return false

@@ -121,6 +134,7 @@ func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {

 		n.leaving = nEvent.Type == NetworkEventTypeLeave

 		if n.leaving {

 			n.reapTime = reapInterval

+			flushEntries = true

 		}

 		nDB.addNetworkNode(nEvent.NetworkID, nEvent.NodeName)

@@ -372,6 +372,37 @@ func (nDB *NetworkDB) deleteNetworkEntriesForNode(deletedNode string) {

 	nDB.Unlock()

 }

+func (nDB *NetworkDB) deleteNodeNetworkEntries(nid, node string) {

+	nDB.Lock()

+	nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid),

+		func(path string, v interface{}) bool {

+			oldEntry := v.(*entry)

+			params := strings.Split(path[1:], "/")

+			nid := params[0]

+			tname := params[1]

+			key := params[2]

+

+			if oldEntry.node != node {

+				return false

+			}

+

+			entry := &entry{

+				ltime:    oldEntry.ltime,

+				node:     node,

+				value:    oldEntry.value,

+				deleting: true,

+				reapTime: reapInterval,

+			}

+

+			nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tname, nid, key), entry)

+			nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", nid, tname, key), entry)

+

+			nDB.broadcaster.Write(makeEvent(opDelete, tname, nid, key, entry.value))

+			return false

+		})

+	nDB.Unlock()

+}

+

 func (nDB *NetworkDB) deleteNodeTableEntries(node string) {

 	nDB.Lock()

 	nDB.indexes[byTable].Walk(func(path string, v interface{}) bool {

@@ -75,13 +75,28 @@ func NlHandle() *netlink.Handle {

 func getSupportedNlFamilies() []int {

 	fams := []int{syscall.NETLINK_ROUTE}

+	// NETLINK_XFRM test

 	if err := loadXfrmModules(); err != nil {

 		if checkXfrmSocket() != nil {

 			logrus.Warnf("Could not load necessary modules for IPSEC rules: %v", err)

-			return fams

+		} else {

+			fams = append(fams, syscall.NETLINK_XFRM)

 		}

+	} else {

+		fams = append(fams, syscall.NETLINK_XFRM)

 	}

-	return append(fams, syscall.NETLINK_XFRM)

+	// NETLINK_NETFILTER test

+	if err := loadNfConntrackModules(); err != nil {

+		if checkNfSocket() != nil {

+			logrus.Warnf("Could not load necessary modules for Conntrack: %v", err)

+		} else {

+			fams = append(fams, syscall.NETLINK_NETFILTER)

+		}

+	} else {

+		fams = append(fams, syscall.NETLINK_NETFILTER)

+	}

+

+	return fams

 }

 func loadXfrmModules() error {

@@ -103,3 +118,23 @@ func checkXfrmSocket() error {

 	syscall.Close(fd)

 	return nil

 }

+

+func loadNfConntrackModules() error {

+	if out, err := exec.Command("modprobe", "-va", "nf_conntrack").CombinedOutput(); err != nil {

+		return fmt.Errorf("Running modprobe nf_conntrack failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)

+	}

+	if out, err := exec.Command("modprobe", "-va", "nf_conntrack_netlink").CombinedOutput(); err != nil {

+		return fmt.Errorf("Running modprobe nf_conntrack_netlink failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)

+	}

+	return nil

+}

+

+// API check on required nf_conntrack* modules (nf_conntrack, nf_conntrack_netlink)

+func checkNfSocket() error {

+	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_NETFILTER)

+	if err != nil {

+		return err

+	}

+	syscall.Close(fd)

+	return nil

+}

@@ -644,13 +644,6 @@ func (sb *sandbox) SetKey(basePath string) error {

 	sb.Lock()

 	sb.osSbox = osSbox

 	sb.Unlock()

-	defer func() {

-		if err != nil {

-			sb.Lock()

-			sb.osSbox = nil

-			sb.Unlock()

-		}

-	}()

 	// If the resolver was setup before stop it and set it up in the

 	// new osl sandbox.