@@ -3,7 +3,7 @@

 # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When

 # updating the binary version, consider updating github.com/docker/libnetwork

 # in vendor.conf accordingly

-LIBNETWORK_COMMIT=20461b8539336a4b5fcf551a86dd24ebae211984

+LIBNETWORK_COMMIT=d7b61745d16675c9f548b19f06fda80d422a74f0

 install_proxy() {

 	case "$1" in

@@ -37,7 +37,7 @@ github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b

 #get libnetwork packages

 # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly

-github.com/docker/libnetwork 20461b8539336a4b5fcf551a86dd24ebae211984

+github.com/docker/libnetwork d7b61745d16675c9f548b19f06fda80d422a74f0

 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9

 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80

 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

@@ -700,6 +700,9 @@ func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver,

 	return nil

 }

+// XXX  This should be made driver agnostic.  See comment below.

+const overlayDSROptionString = "dsr"

+

 // NewNetwork creates a new network of the specified network type. The options

 // are network specific and modeled in a generic way.

 func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) {

@@ -723,15 +726,16 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ...

 	defaultIpam := defaultIpamForNetworkType(networkType)

 	// Construct the network object

 	network := &network{

-		name:        name,

-		networkType: networkType,

-		generic:     map[string]interface{}{netlabel.GenericData: make(map[string]string)},

-		ipamType:    defaultIpam,

-		id:          id,

-		created:     time.Now(),

-		ctrlr:       c,

-		persist:     true,

-		drvOnce:     &sync.Once{},

+		name:             name,

+		networkType:      networkType,

+		generic:          map[string]interface{}{netlabel.GenericData: make(map[string]string)},

+		ipamType:         defaultIpam,

+		id:               id,

+		created:          time.Now(),

+		ctrlr:            c,

+		persist:          true,

+		drvOnce:          &sync.Once{},

+		loadBalancerMode: loadBalancerModeDefault,

 	}

 	network.processOptions(options...)

@@ -829,6 +833,21 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ...

 		}

 	}()

+	// XXX If the driver type is "overlay" check the options for DSR

+	// being set.  If so, set the network's load balancing mode to DSR.

+	// This should really be done in a network option, but due to

+	// time pressure to get this in without adding changes to moby,

+	// swarm and CLI, it is being implemented as a driver-specific

+	// option.  Unfortunately, drivers can't influence the core

+	// "libnetwork.network" data type.  Hence we need this hack code

+	// to implement in this manner.

+	if gval, ok := network.generic[netlabel.GenericData]; ok && network.networkType == "overlay" {

+		optMap := gval.(map[string]string)

+		if _, ok := optMap[overlayDSROptionString]; ok {

+			network.loadBalancerMode = loadBalancerModeDSR

+		}

+	}

+

 addToStore:

 	// First store the endpoint count, then the network. To avoid to

 	// end up with a datastore containing a network and not an epCnt,

@@ -261,7 +261,7 @@ func (ep *hnsEndpoint) UnmarshalJSON(b []byte) error {

 	}

 	if v, ok := epMap["Addr"]; ok {

 		if ep.addr, err = types.ParseCIDR(v.(string)); err != nil {

-			return types.InternalErrorf("failed to decode endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)

+			logrus.Warnf("failed to decode endpoint IPv4 address (%s) after json unmarshal: %v", v.(string), err)

 		}

 	}

 	if v, ok := epMap["gateway"]; ok {

@@ -145,3 +145,23 @@ const (

 	// addresses.

 	SourceHashing = "sh"

 )

+

+const (

+	// ConnFwdMask is a mask for the fwd methods

+	ConnFwdMask = 0x0007

+

+	// ConnFwdMasq denotes forwarding via masquerading/NAT

+	ConnFwdMasq = 0x0000

+

+	// ConnFwdLocalNode denotes forwarding to a local node

+	ConnFwdLocalNode = 0x0001

+

+	// ConnFwdTunnel denotes forwarding via a tunnel

+	ConnFwdTunnel = 0x0002

+

+	// ConnFwdDirectRoute denotes forwarding via direct routing

+	ConnFwdDirectRoute = 0x0003

+

+	// ConnFwdBypass denotes forwarding while bypassing the cache

+	ConnFwdBypass = 0x0004

+)

@@ -62,8 +62,12 @@ type Destination struct {

 	LowerThreshold      uint32

 	ActiveConnections   int

 	InactiveConnections int

+	Stats               DstStats

 }

+// DstStats defines IPVS destination (real server) statistics

+type DstStats SvcStats

+

 // Handle provides a namespace specific ipvs handle to program ipvs

 // rules.

 type Handle struct {

@@ -443,6 +443,12 @@ func assembleDestination(attrs []syscall.NetlinkRouteAttr) (*Destination, error)

 			d.ActiveConnections = int(native.Uint16(attr.Value))

 		case ipvsDestAttrInactiveConnections:

 			d.InactiveConnections = int(native.Uint16(attr.Value))

+		case ipvsSvcAttrStats:

+			stats, err := assembleStats(attr.Value)

+			if err != nil {

+				return nil, err

+			}

+			d.Stats = DstStats(stats)

 		}

 	}

 	return &d, nil

@@ -199,43 +199,50 @@ func (i *IpamInfo) UnmarshalJSON(data []byte) error {

 }

 type network struct {

-	ctrlr          *controller

-	name           string

-	networkType    string

-	id             string

-	created        time.Time

-	scope          string // network data scope

-	labels         map[string]string

-	ipamType       string

-	ipamOptions    map[string]string

-	addrSpace      string

-	ipamV4Config   []*IpamConf

-	ipamV6Config   []*IpamConf

-	ipamV4Info     []*IpamInfo

-	ipamV6Info     []*IpamInfo

-	enableIPv6     bool

-	postIPv6       bool

-	epCnt          *endpointCnt

-	generic        options.Generic

-	dbIndex        uint64

-	dbExists       bool

-	persist        bool

-	stopWatchCh    chan struct{}

-	drvOnce        *sync.Once

-	resolverOnce   sync.Once

-	resolver       []Resolver

-	internal       bool

-	attachable     bool

-	inDelete       bool

-	ingress        bool

-	driverTables   []networkDBTable

-	dynamic        bool

-	configOnly     bool

-	configFrom     string

-	loadBalancerIP net.IP

+	ctrlr            *controller

+	name             string

+	networkType      string

+	id               string

+	created          time.Time

+	scope            string // network data scope

+	labels           map[string]string

+	ipamType         string

+	ipamOptions      map[string]string

+	addrSpace        string

+	ipamV4Config     []*IpamConf

+	ipamV6Config     []*IpamConf

+	ipamV4Info       []*IpamInfo

+	ipamV6Info       []*IpamInfo

+	enableIPv6       bool

+	postIPv6         bool

+	epCnt            *endpointCnt

+	generic          options.Generic

+	dbIndex          uint64

+	dbExists         bool

+	persist          bool

+	stopWatchCh      chan struct{}

+	drvOnce          *sync.Once

+	resolverOnce     sync.Once

+	resolver         []Resolver

+	internal         bool

+	attachable       bool

+	inDelete         bool

+	ingress          bool

+	driverTables     []networkDBTable

+	dynamic          bool

+	configOnly       bool

+	configFrom       string

+	loadBalancerIP   net.IP

+	loadBalancerMode string

 	sync.Mutex

 }

+const (

+	loadBalancerModeNAT     = "NAT"

+	loadBalancerModeDSR     = "DSR"

+	loadBalancerModeDefault = loadBalancerModeNAT

+)

+

 func (n *network) Name() string {

 	n.Lock()

 	defer n.Unlock()

@@ -475,6 +482,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {

 	dstN.configOnly = n.configOnly

 	dstN.configFrom = n.configFrom

 	dstN.loadBalancerIP = n.loadBalancerIP

+	dstN.loadBalancerMode = n.loadBalancerMode

 	// copy labels

 	if dstN.labels == nil {

@@ -592,6 +600,7 @@ func (n *network) MarshalJSON() ([]byte, error) {

 	netMap["configOnly"] = n.configOnly

 	netMap["configFrom"] = n.configFrom

 	netMap["loadBalancerIP"] = n.loadBalancerIP

+	netMap["loadBalancerMode"] = n.loadBalancerMode

 	return json.Marshal(netMap)

 }

@@ -705,6 +714,10 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {

 	if v, ok := netMap["loadBalancerIP"]; ok {

 		n.loadBalancerIP = net.ParseIP(v.(string))

 	}

+	n.loadBalancerMode = loadBalancerModeDefault

+	if v, ok := netMap["loadBalancerMode"]; ok {

+		n.loadBalancerMode = v.(string)

+	}

 	// Reconcile old networks with the recently added `--ipv6` flag

 	if !n.enableIPv6 {

 		n.enableIPv6 = len(n.ipamV6Info) > 0

@@ -384,6 +384,36 @@ func (n *networkNamespace) RemoveAliasIP(ifName string, ip *net.IPNet) error {

 	return n.nlHandle.AddrDel(iface, &netlink.Addr{IPNet: ip})

 }

+func (n *networkNamespace) DisableARPForVIP(srcName string) (Err error) {

+	dstName := ""

+	for _, i := range n.Interfaces() {

+		if i.SrcName() == srcName {

+			dstName = i.DstName()

+			break

+		}

+	}

+	if dstName == "" {

+		return fmt.Errorf("failed to find interface %s in sandbox", srcName)

+	}

+

+	err := n.InvokeFunc(func() {

+		path := filepath.Join("/proc/sys/net/ipv4/conf", dstName, "arp_ignore")

+		if err := ioutil.WriteFile(path, []byte{'1', '\n'}, 0644); err != nil {

+			Err = fmt.Errorf("Failed to set %s to 1: %v", path, err)

+			return

+		}

+		path = filepath.Join("/proc/sys/net/ipv4/conf", dstName, "arp_announce")

+		if err := ioutil.WriteFile(path, []byte{'2', '\n'}, 0644); err != nil {

+			Err = fmt.Errorf("Failed to set %s to 2: %v", path, err)

+			return

+		}

+	})

+	if err != nil {

+		return err

+	}

+	return

+}

+

 func (n *networkNamespace) InvokeFunc(f func()) error {

 	return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error {

 		f()

@@ -51,6 +51,10 @@ type Sandbox interface {

 	// RemoveAliasIP removes the passed IP address from the named interface

 	RemoveAliasIP(ifName string, ip *net.IPNet) error

+	// DisableARPForVIP disables ARP replies and requests for VIP addresses

+	// on a particular interface

+	DisableARPForVIP(ifName string) error

+

 	// Add a static route to the sandbox.

 	AddStaticRoute(*types.StaticRoute) error

@@ -67,7 +67,7 @@ const (

 	maxExtDNS       = 3 //max number of external servers to try

 	extIOTimeout    = 4 * time.Second

 	defaultRespSize = 512

-	maxConcurrent   = 100

+	maxConcurrent   = 1024

 	logInterval     = 2 * time.Second

 )

@@ -111,7 +111,7 @@ func NewResolver(address string, proxyDNS bool, resolverKey string, backend DNSB

 }

 func (r *resolver) SetupFunc(port int) func() {

-	return (func() {

+	return func() {

 		var err error

 		// DNS operates primarily on UDP

@@ -138,7 +138,7 @@ func (r *resolver) SetupFunc(port int) func() {

 			return

 		}

 		r.err = nil

-	})

+	}

 }

 func (r *resolver) Start() error {

@@ -490,35 +490,51 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {

 				continue

 			}

 			r.forwardQueryEnd()

-			if resp != nil {

-				if resp.Rcode == dns.RcodeServerFailure {

-					// for Server Failure response, continue to the next external DNS server

-					logrus.Debugf("[resolver] external DNS %s:%s responded with ServFail for %q", proto, extDNS.IPStr, name)

-					continue

-				}

-				answers := 0

-				for _, rr := range resp.Answer {

-					h := rr.Header()

-					switch h.Rrtype {

-					case dns.TypeA:

-						answers++

-						ip := rr.(*dns.A).A

-						logrus.Debugf("[resolver] received A record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr)

-						r.backend.HandleQueryResp(h.Name, ip)

-					case dns.TypeAAAA:

-						answers++

-						ip := rr.(*dns.AAAA).AAAA

-						logrus.Debugf("[resolver] received AAAA record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr)

-						r.backend.HandleQueryResp(h.Name, ip)

-					}

+

+			if resp == nil {

+				logrus.Debugf("[resolver] external DNS %s:%s returned empty response for %q", proto, extDNS.IPStr, name)

+				break

+			}

+			switch resp.Rcode {

+			case dns.RcodeServerFailure, dns.RcodeRefused:

+				// Server returned FAILURE: continue with the next external DNS server

+				// Server returned REFUSED: this can be a transitional status, so continue with the next external DNS server

+				logrus.Debugf("[resolver] external DNS %s:%s responded with %s for %q", proto, extDNS.IPStr, statusString(resp.Rcode), name)

+				continue

+			case dns.RcodeNameError:

+				// Server returned NXDOMAIN. Stop resolution if it's an authoritative answer (see RFC 8020: https://tools.ietf.org/html/rfc8020#section-2)

+				logrus.Debugf("[resolver] external DNS %s:%s responded with %s for %q", proto, extDNS.IPStr, statusString(resp.Rcode), name)

+				if resp.Authoritative {

+					break

 				}

-				if resp.Answer == nil || answers == 0 {

-					logrus.Debugf("[resolver] external DNS %s:%s did not return any %s records for %q", proto, extDNS.IPStr, queryType, name)

+				continue

+			case dns.RcodeSuccess:

+				// All is well

+			default:

+				// Server gave some error. Log the error, and continue with the next external DNS server

+				logrus.Debugf("[resolver] external DNS %s:%s responded with %s (code %d) for %q", proto, extDNS.IPStr, statusString(resp.Rcode), resp.Rcode, name)

+				continue

+			}

+			answers := 0

+			for _, rr := range resp.Answer {

+				h := rr.Header()

+				switch h.Rrtype {

+				case dns.TypeA:

+					answers++

+					ip := rr.(*dns.A).A

+					logrus.Debugf("[resolver] received A record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr)

+					r.backend.HandleQueryResp(h.Name, ip)

+				case dns.TypeAAAA:

+					answers++

+					ip := rr.(*dns.AAAA).AAAA

+					logrus.Debugf("[resolver] received AAAA record %q for %q from %s:%s", ip, h.Name, proto, extDNS.IPStr)

+					r.backend.HandleQueryResp(h.Name, ip)

 				}

-				resp.Compress = true

-			} else {

-				logrus.Debugf("[resolver] external DNS %s:%s returned empty response for %q", proto, extDNS.IPStr, name)

 			}

+			if resp.Answer == nil || answers == 0 {

+				logrus.Debugf("[resolver] external DNS %s:%s did not return any %s records for %q", proto, extDNS.IPStr, queryType, name)

+			}

+			resp.Compress = true

 			break

 		}

 		if resp == nil {

@@ -531,6 +547,13 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {

 	}

 }

+func statusString(responseCode int) string {

+	if s, ok := dns.RcodeToString[responseCode]; ok {

+		return s

+	}

+	return "UNKNOWN"

+}

+

 func (r *resolver) forwardQueryStart() bool {

 	r.queryLock.Lock()

 	defer r.queryLock.Unlock()

@@ -742,8 +742,17 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) {

 	ep.Lock()

 	joinInfo := ep.joinInfo

+	vip := ep.virtualIP

+	lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR

 	ep.Unlock()

+	if len(vip) > 0 && lbModeIsDSR {

+		ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}

+		if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil {

+			logrus.WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet)

+		}

+	}

+

 	if joinInfo == nil {

 		return

 	}

@@ -831,6 +840,7 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {

 	ep.Lock()

 	joinInfo := ep.joinInfo

 	i := ep.iface

+	lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR

 	ep.Unlock()

 	if ep.needResolver() {

@@ -854,6 +864,18 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {

 		if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil {

 			return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err)

 		}

+

+		if len(ep.virtualIP) > 0 && lbModeIsDSR {

+			if sb.loadBalancerNID == "" {

+				if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil {

+					return fmt.Errorf("failed disable ARP for VIP: %v", err)

+				}

+			}

+			ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)}

+			if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil {

+				return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err)

+			}

+		}

 	}

 	if joinInfo != nil {

@@ -142,7 +142,7 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {

 		}

 		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %.7s (%.7s)", lb.vip, lb.fwMark, lb.service.ingressPorts, sb.ID(), sb.ContainerID())

-		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false); err != nil {

+		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.loadBalancerMode); err != nil {

 			logrus.Errorf("Failed to add firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)

 			return

 		}

@@ -158,6 +158,9 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {

 		Address:       ip,

 		Weight:        1,

 	}

+	if n.loadBalancerMode == loadBalancerModeDSR {

+		d.ConnectionFlags = ipvs.ConnFwdDirectRoute

+	}

 	// Remove the sched name before using the service to add

 	// destination.

@@ -203,6 +206,9 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR

 		Address:       ip,

 		Weight:        1,

 	}

+	if n.loadBalancerMode == loadBalancerModeDSR {

+		d.ConnectionFlags = ipvs.ConnFwdDirectRoute

+	}

 	if fullRemove {

 		if err := i.DelDestination(s, d); err != nil && err != syscall.ENOENT {

@@ -231,7 +237,7 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR

 			}

 		}

-		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true); err != nil {

+		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.loadBalancerMode); err != nil {

 			logrus.Errorf("Failed to delete firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)

 		}

@@ -566,7 +572,7 @@ func readPortsFromFile(fileName string) ([]*PortConfig, error) {

 // Invoke fwmarker reexec routine to mark vip destined packets with

 // the passed firewall mark.

-func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool) error {

+func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, lbMode string) error {

 	var ingressPortsFile string

 	if len(ingressPorts) != 0 {

@@ -586,7 +592,7 @@ func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*Port

 	cmd := &exec.Cmd{

 		Path:   reexec.Self(),

-		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String()),

+		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String(), lbMode),

 		Stdout: os.Stdout,

 		Stderr: os.Stderr,

 	}

@@ -603,7 +609,7 @@ func fwMarker() {

 	runtime.LockOSThread()

 	defer runtime.UnlockOSThread()

-	if len(os.Args) < 7 {

+	if len(os.Args) < 8 {

 		logrus.Error("invalid number of arguments..")

 		os.Exit(1)

 	}

@@ -645,7 +651,8 @@ func fwMarker() {

 		os.Exit(5)

 	}

-	if addDelOpt == "-A" {

+	lbMode := os.Args[7]

+	if addDelOpt == "-A" && lbMode == loadBalancerModeNAT {

 		eIP, subnet, err := net.ParseCIDR(os.Args[6])

 		if err != nil {

 			logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)