GitList

Browse code

bump libnetwork to b0186632

Bump libnetwork to b0186632522c68f4e1222c4f6d7dbe518882024f. This
includes the following changes:
* Dockerize protocol buffer generation and update (78d9390a..e12dd44c)
* Use new plugin interfaces provided by plugin pkg (be94e134)
* Improve linux load-balancing scalability (5111c24e..366b9110)

Signed-off-by: Chris Telfer <ctelfer@docker.com>

Chris Telfer authored on 2018/06/06 23:38:04
Showing 20 changed files

hack/dockerfile/install/proxy.installer index b5748b5..3e2303d 100755
vendor.conf index 866df01..f52aec9 100644
vendor/github.com/docker/libnetwork/controller.go index 5e967b7..b6c5366 100644
vendor/github.com/docker/libnetwork/default_gateway.go index 9a60fd6..cf094b3 100644
vendor/github.com/docker/libnetwork/drivers/overlay/overlay.pb.go index cfa0eee..243c196 100644
vendor/github.com/docker/libnetwork/drivers/remote/driver.go index b521630..9786d9e 100644
vendor/github.com/docker/libnetwork/drivers/windows/overlay/overlay.pb.go index cfa0eee..9d0cf26 100644
vendor/github.com/docker/libnetwork/endpoint.go index e245786..822f88b 100644
vendor/github.com/docker/libnetwork/endpoint_info.go index 68d3e86..80b662d 100644
vendor/github.com/docker/libnetwork/ipams/remote/remote.go index 9f2f1d5..6f2173f 100644
vendor/github.com/docker/libnetwork/network.go index 70d6b1c..6136611 100644
vendor/github.com/docker/libnetwork/networkdb/networkdb.pb.go index 7087a57..043c23b 100644
vendor/github.com/docker/libnetwork/osl/namespace_linux.go index 6389215..a55932b 100644
vendor/github.com/docker/libnetwork/osl/sandbox.go index 6ffc467..0614906 100644
vendor/github.com/docker/libnetwork/sandbox.go index daa0f5e..abea13c 100644
vendor/github.com/docker/libnetwork/service_common.go index 8be2c38..4c30e46 100644
vendor/github.com/docker/libnetwork/service_linux.go index a95ece8..532b8c8 100644
vendor/github.com/docker/libnetwork/service_unsupported.go index 37b9828..ee97506 100644
vendor/github.com/docker/libnetwork/service_windows.go index a9ce244..944f3d3 100644
vendor/github.com/docker/libnetwork/store.go index 95943f6..0a7c575 100644

@@ -3,7 +3,7 @@
                      # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
                      # updating the binary version, consider updating github.com/docker/libnetwork
                      # in vendor.conf accordingly
                     -LIBNETWORK_COMMIT=430c00a6a6b3dfdd774f21e1abd4ad6b0216c629
                     +LIBNETWORK_COMMIT=b0186632522c68f4e1222c4f6d7dbe518882024f
                      install_proxy() {
                      	case "$1" in

@@ -37,7 +37,7 @@ github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b
                      #get libnetwork packages
                      # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly
                     -github.com/docker/libnetwork 430c00a6a6b3dfdd774f21e1abd4ad6b0216c629
                     +github.com/docker/libnetwork b0186632522c68f4e1222c4f6d7dbe518882024f
                      github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
                      github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
                      github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

...	...	@@ -871,7 +871,7 @@ addToStore:
871	871	}
872	872	}()
873	873
874		- if len(network.loadBalancerIP) != 0 {
	874	+ if network.hasLoadBalancerEndpoint() {
875	875	if err = network.createLoadBalancerSandbox(); err != nil {
876	876	return nil, err
877	877	}

@@ -49,9 +49,11 @@ func (sb *sandbox) setupDefaultGW() error {
                      	createOptions := []EndpointOption{CreateOptionAnonymous()}
                     -	eplen := gwEPlen
                     -	if len(sb.containerID) < gwEPlen {
                     -		eplen = len(sb.containerID)
                     +	var gwName string
                     +	if len(sb.containerID) <= gwEPlen {
                     +		gwName = "gateway_" + sb.containerID
                     +	} else {
                     +		gwName = "gateway_" + sb.id[:gwEPlen]
+                     	}
                      	sbLabels := sb.Labels()
@@ -69,7 +71,7 @@ func (sb *sandbox) setupDefaultGW() error {
                      		createOptions = append(createOptions, epOption)
+                     	}
                     -	newEp, err := n.CreateEndpoint("gateway_"+sb.containerID[0:eplen], createOptions...)
                     +	newEp, err := n.CreateEndpoint(gwName, createOptions...)
                      	if err != nil {
                      		return fmt.Errorf("container %s: endpoint create on GW Network failed: %v", sb.containerID, err)
+                     	}

@@ -1,12 +1,11 @@
                     -// Code generated by protoc-gen-gogo.
                     -// source: overlay.proto
                     -// DO NOT EDIT!
                     +// Code generated by protoc-gen-gogo. DO NOT EDIT.
                     +// source: drivers/overlay/overlay.proto
                      /*
                      	Package overlay is a generated protocol buffer package.
                      	It is generated from these files:
                     -		overlay.proto
                     +		drivers/overlay/overlay.proto
                      	It has these top-level messages:
                      		PeerRecord
@@ -19,9 +18,6 @@ import math "math"
                      import _ "github.com/gogo/protobuf/gogoproto"
                      import strings "strings"
                     -import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
                     -import sort "sort"
                     -import strconv "strconv"
                      import reflect "reflect"
                      import io "io"
@@ -33,7 +29,9 @@ var _ = math.Inf
                      // This is a compile-time assertion to ensure that this generated file
                      // is compatible with the proto package it is being compiled against.
                     -const _ = proto.GoGoProtoPackageIsVersion1
                     +// A compilation error at this line likely means your copy of the
                     +// proto package needs to be updated.
                     +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
                      // PeerRecord defines the information corresponding to a peer
                      // container in the overlay network.
@@ -54,6 +52,27 @@ func (m *PeerRecord) Reset()                    { *m = PeerRecord{} }
                      func (*PeerRecord) ProtoMessage()               {}
                      func (*PeerRecord) Descriptor() ([]byte, []int) { return fileDescriptorOverlay, []int{0} }
                     +func (m *PeerRecord) GetEndpointIP() string {
                     +	if m != nil {
                     +		return m.EndpointIP
                     +	}
                     +	return ""
                     +}
+                    +
                     +func (m *PeerRecord) GetEndpointMAC() string {
                     +	if m != nil {
                     +		return m.EndpointMAC
                     +	}
                     +	return ""
                     +}
+                    +
                     +func (m *PeerRecord) GetTunnelEndpointIP() string {
                     +	if m != nil {
                     +		return m.TunnelEndpointIP
                     +	}
                     +	return ""
                     +}
+                    +
                      func init() {
                      	proto.RegisterType((*PeerRecord)(nil), "overlay.PeerRecord")
+                     }
@@ -77,84 +96,49 @@ func valueToGoStringOverlay(v interface{}, typ string) string {
                      	pv := reflect.Indirect(rv).Interface()
                      	return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv)
+                     }
                     -func extensionToGoStringOverlay(e map[int32]github_com_gogo_protobuf_proto.Extension) string {
                     -	if e == nil {
                     -		return "nil"
                     -	}
                     -	s := "map[int32]proto.Extension{"
                     -	keys := make([]int, 0, len(e))
                     -	for k := range e {
                     -		keys = append(keys, int(k))
                     -	}
                     -	sort.Ints(keys)
                     -	ss := []string{}
                     -	for _, k := range keys {
                     -		ss = append(ss, strconv.Itoa(k)+": "+e[int32(k)].GoString())
                     -	}
                     -	s += strings.Join(ss, ",") + "}"
                     -	return s
                     -}
                     -func (m *PeerRecord) Marshal() (data []byte, err error) {
                     +func (m *PeerRecord) Marshal() (dAtA []byte, err error) {
                      	size := m.Size()
                     -	data = make([]byte, size)
                     -	n, err := m.MarshalTo(data)
                     +	dAtA = make([]byte, size)
                     +	n, err := m.MarshalTo(dAtA)
                      	if err != nil {
                      		return nil, err
+                     	}
                     -	return data[:n], nil
                     +	return dAtA[:n], nil
+                     }
                     -func (m *PeerRecord) MarshalTo(data []byte) (int, error) {
                     +func (m *PeerRecord) MarshalTo(dAtA []byte) (int, error) {
                      	var i int
                      	_ = i
                      	var l int
                      	_ = l
                      	if len(m.EndpointIP) > 0 {
                     -		data[i] = 0xa
                     +		dAtA[i] = 0xa
                      		i++
                     -		i = encodeVarintOverlay(data, i, uint64(len(m.EndpointIP)))
                     -		i += copy(data[i:], m.EndpointIP)
                     +		i = encodeVarintOverlay(dAtA, i, uint64(len(m.EndpointIP)))
                     +		i += copy(dAtA[i:], m.EndpointIP)
+                     	}
                      	if len(m.EndpointMAC) > 0 {
                     -		data[i] = 0x12
                     +		dAtA[i] = 0x12
                      		i++
                     -		i = encodeVarintOverlay(data, i, uint64(len(m.EndpointMAC)))
                     -		i += copy(data[i:], m.EndpointMAC)
                     +		i = encodeVarintOverlay(dAtA, i, uint64(len(m.EndpointMAC)))
                     +		i += copy(dAtA[i:], m.EndpointMAC)
+                     	}
                      	if len(m.TunnelEndpointIP) > 0 {
                     -		data[i] = 0x1a
                     +		dAtA[i] = 0x1a
                      		i++
                     -		i = encodeVarintOverlay(data, i, uint64(len(m.TunnelEndpointIP)))
                     -		i += copy(data[i:], m.TunnelEndpointIP)
                     +		i = encodeVarintOverlay(dAtA, i, uint64(len(m.TunnelEndpointIP)))
                     +		i += copy(dAtA[i:], m.TunnelEndpointIP)
+                     	}
                      	return i, nil
+                     }
                     -func encodeFixed64Overlay(data []byte, offset int, v uint64) int {
                     -	data[offset] = uint8(v)
                     -	data[offset+1] = uint8(v >> 8)
                     -	data[offset+2] = uint8(v >> 16)
                     -	data[offset+3] = uint8(v >> 24)
                     -	data[offset+4] = uint8(v >> 32)
                     -	data[offset+5] = uint8(v >> 40)
                     -	data[offset+6] = uint8(v >> 48)
                     -	data[offset+7] = uint8(v >> 56)
                     -	return offset + 8
                     -}
                     -func encodeFixed32Overlay(data []byte, offset int, v uint32) int {
                     -	data[offset] = uint8(v)
                     -	data[offset+1] = uint8(v >> 8)
                     -	data[offset+2] = uint8(v >> 16)
                     -	data[offset+3] = uint8(v >> 24)
                     -	return offset + 4
                     -}
                     -func encodeVarintOverlay(data []byte, offset int, v uint64) int {
                     +func encodeVarintOverlay(dAtA []byte, offset int, v uint64) int {
                      	for v >= 1<<7 {
                     -		data[offset] = uint8(v&0x7f | 0x80)
                     +		dAtA[offset] = uint8(v&0x7f | 0x80)
                      		v >>= 7
                      		offset++
+                     	}
                     -	data[offset] = uint8(v)
                     +	dAtA[offset] = uint8(v)
                      	return offset + 1
+                     }
                      func (m *PeerRecord) Size() (n int) {
@@ -208,8 +192,8 @@ func valueToStringOverlay(v interface{}) string {
                      	pv := reflect.Indirect(rv).Interface()
                      	return fmt.Sprintf("*%v", pv)
+                     }
                     -func (m *PeerRecord) Unmarshal(data []byte) error {
                     -	l := len(data)
                     +func (m *PeerRecord) Unmarshal(dAtA []byte) error {
                     +	l := len(dAtA)
                      	iNdEx := 0
                      	for iNdEx < l {
                      		preIndex := iNdEx
@@ -221,7 +205,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
                      			if iNdEx >= l {
                      				return io.ErrUnexpectedEOF
+                     			}
                     -			b := data[iNdEx]
                     +			b := dAtA[iNdEx]
                      			iNdEx++
                      			wire |= (uint64(b) & 0x7F) << shift
                      			if b < 0x80 {
@@ -249,7 +233,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
                      				if iNdEx >= l {
                      					return io.ErrUnexpectedEOF
+                     				}
                     -				b := data[iNdEx]
                     +				b := dAtA[iNdEx]
                      				iNdEx++
                      				stringLen |= (uint64(b) & 0x7F) << shift
                      				if b < 0x80 {
@@ -264,7 +248,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
                      			if postIndex > l {
                      				return io.ErrUnexpectedEOF
+                     			}
                     -			m.EndpointIP = string(data[iNdEx:postIndex])
                     +			m.EndpointIP = string(dAtA[iNdEx:postIndex])
                      			iNdEx = postIndex
                      		case 2:
                      			if wireType != 2 {
@@ -278,7 +262,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
                      				if iNdEx >= l {
                      					return io.ErrUnexpectedEOF
+                     				}
                     -				b := data[iNdEx]
                     +				b := dAtA[iNdEx]
                      				iNdEx++
                      				stringLen |= (uint64(b) & 0x7F) << shift
                      				if b < 0x80 {
@@ -293,7 +277,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
                      			if postIndex > l {
                      				return io.ErrUnexpectedEOF
+                     			}
                     -			m.EndpointMAC = string(data[iNdEx:postIndex])
                     +			m.EndpointMAC = string(dAtA[iNdEx:postIndex])
                      			iNdEx = postIndex
                      		case 3:
                      			if wireType != 2 {
@@ -307,7 +291,7 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
                      				if iNdEx >= l {
                      					return io.ErrUnexpectedEOF
+                     				}
                     -				b := data[iNdEx]
                     +				b := dAtA[iNdEx]
                      				iNdEx++
                      				stringLen |= (uint64(b) & 0x7F) << shift
                      				if b < 0x80 {
@@ -322,11 +306,11 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
                      			if postIndex > l {
                      				return io.ErrUnexpectedEOF
+                     			}
                     -			m.TunnelEndpointIP = string(data[iNdEx:postIndex])
                     +			m.TunnelEndpointIP = string(dAtA[iNdEx:postIndex])
                      			iNdEx = postIndex
                      		default:
                      			iNdEx = preIndex
                     -			skippy, err := skipOverlay(data[iNdEx:])
                     +			skippy, err := skipOverlay(dAtA[iNdEx:])
                      			if err != nil {
                      				return err
+                     			}
@@ -345,8 +329,8 @@ func (m *PeerRecord) Unmarshal(data []byte) error {
+                     	}
                      	return nil
+                     }
                     -func skipOverlay(data []byte) (n int, err error) {
                     -	l := len(data)
                     +func skipOverlay(dAtA []byte) (n int, err error) {
                     +	l := len(dAtA)
                      	iNdEx := 0
                      	for iNdEx < l {
                      		var wire uint64
@@ -357,7 +341,7 @@ func skipOverlay(data []byte) (n int, err error) {
                      			if iNdEx >= l {
                      				return 0, io.ErrUnexpectedEOF
+                     			}
                     -			b := data[iNdEx]
                     +			b := dAtA[iNdEx]
                      			iNdEx++
                      			wire |= (uint64(b) & 0x7F) << shift
                      			if b < 0x80 {
@@ -375,7 +359,7 @@ func skipOverlay(data []byte) (n int, err error) {
                      					return 0, io.ErrUnexpectedEOF
+                     				}
                      				iNdEx++
                     -				if data[iNdEx-1] < 0x80 {
                     +				if dAtA[iNdEx-1] < 0x80 {
                      					break
+                     				}
+                     			}
@@ -392,7 +376,7 @@ func skipOverlay(data []byte) (n int, err error) {
                      				if iNdEx >= l {
                      					return 0, io.ErrUnexpectedEOF
+                     				}
                     -				b := data[iNdEx]
                     +				b := dAtA[iNdEx]
                      				iNdEx++
                      				length |= (int(b) & 0x7F) << shift
                      				if b < 0x80 {
@@ -415,7 +399,7 @@ func skipOverlay(data []byte) (n int, err error) {
                      					if iNdEx >= l {
                      						return 0, io.ErrUnexpectedEOF
+                     					}
                     -					b := data[iNdEx]
                     +					b := dAtA[iNdEx]
                      					iNdEx++
                      					innerWire |= (uint64(b) & 0x7F) << shift
                      					if b < 0x80 {
@@ -426,7 +410,7 @@ func skipOverlay(data []byte) (n int, err error) {
                      				if innerWireType == 4 {
                      					break
+                     				}
                     -				next, err := skipOverlay(data[start:])
                     +				next, err := skipOverlay(dAtA[start:])
                      				if err != nil {
                      					return 0, err
+                     				}
@@ -450,19 +434,22 @@ var (
                      	ErrIntOverflowOverlay   = fmt.Errorf("proto: integer overflow")
+                     )
                     +func init() { proto.RegisterFile("drivers/overlay/overlay.proto", fileDescriptorOverlay) }
+                    +
                      var fileDescriptorOverlay = []byte{
                     -	// 195 bytes of a gzipped FileDescriptorProto
                     -	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x2f, 0x4b, 0x2d,
                     -	0xca, 0x49, 0xac, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x87, 0x72, 0xa5, 0x44, 0xd2,
                     -	0xf3, 0xd3, 0xf3, 0xc1, 0x62, 0xfa, 0x20, 0x16, 0x44, 0x5a, 0x69, 0x2b, 0x23, 0x17, 0x57, 0x40,
                     -	0x6a, 0x6a, 0x51, 0x50, 0x6a, 0x72, 0x7e, 0x51, 0x8a, 0x90, 0x3e, 0x17, 0x77, 0x6a, 0x5e, 0x4a,
                     -	0x41, 0x7e, 0x66, 0x5e, 0x49, 0x7c, 0x66, 0x81, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0xa7, 0x13, 0xdf,
                     -	0xa3, 0x7b, 0xf2, 0x5c, 0xae, 0x50, 0x61, 0xcf, 0x80, 0x20, 0x2e, 0x98, 0x12, 0xcf, 0x02, 0x21,
                     -	0x23, 0x2e, 0x1e, 0xb8, 0x86, 0xdc, 0xc4, 0x64, 0x09, 0x26, 0xb0, 0x0e, 0x7e, 0xa0, 0x0e, 0x6e,
                     -	0x98, 0x0e, 0x5f, 0x47, 0xe7, 0x20, 0xb8, 0xa9, 0xbe, 0x89, 0xc9, 0x42, 0x4e, 0x5c, 0x42, 0x25,
                     -	0xa5, 0x79, 0x79, 0xa9, 0x39, 0xf1, 0xc8, 0x76, 0x31, 0x83, 0x75, 0x8a, 0x00, 0x75, 0x0a, 0x84,
                     -	0x80, 0x65, 0x91, 0x6c, 0x14, 0x28, 0x41, 0x15, 0x29, 0x70, 0x92, 0xb8, 0xf1, 0x50, 0x8e, 0xe1,
                     -	0xc3, 0x43, 0x39, 0xc6, 0x86, 0x47, 0x72, 0x8c, 0x27, 0x80, 0xf8, 0x02, 0x10, 0x3f, 0x00, 0xe2,
                     -	0x24, 0x36, 0xb0, 0xc7, 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xbf, 0xd7, 0x7d, 0x7d, 0x08,
                     -	0x01, 0x00, 0x00,
                     +	// 212 bytes of a gzipped FileDescriptorProto
                     +	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x29, 0xca, 0x2c,
                     +	0x4b, 0x2d, 0x2a, 0xd6, 0xcf, 0x2f, 0x4b, 0x2d, 0xca, 0x49, 0xac, 0x84, 0xd1, 0x7a, 0x05, 0x45,
                     +	0xf9, 0x25, 0xf9, 0x42, 0xec, 0x50, 0xae, 0x94, 0x48, 0x7a, 0x7e, 0x7a, 0x3e, 0x58, 0x4c, 0x1f,
                     +	0xc4, 0x82, 0x48, 0x2b, 0x6d, 0x65, 0xe4, 0xe2, 0x0a, 0x48, 0x4d, 0x2d, 0x0a, 0x4a, 0x4d, 0xce,
                     +	0x2f, 0x4a, 0x11, 0xd2, 0xe7, 0xe2, 0x4e, 0xcd, 0x4b, 0x29, 0xc8, 0xcf, 0xcc, 0x2b, 0x89, 0xcf,
                     +	0x2c, 0x90, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x74, 0xe2, 0x7b, 0x74, 0x4f, 0x9e, 0xcb, 0x15, 0x2a,
                     +	0xec, 0x19, 0x10, 0xc4, 0x05, 0x53, 0xe2, 0x59, 0x20, 0x64, 0xc4, 0xc5, 0x03, 0xd7, 0x90, 0x9b,
                     +	0x98, 0x2c, 0xc1, 0x04, 0xd6, 0xc1, 0xff, 0xe8, 0x9e, 0x3c, 0x37, 0x4c, 0x87, 0xaf, 0xa3, 0x73,
                     +	0x10, 0xdc, 0x54, 0xdf, 0xc4, 0x64, 0x21, 0x27, 0x2e, 0xa1, 0x92, 0xd2, 0xbc, 0xbc, 0xd4, 0x9c,
                     +	0x78, 0x64, 0xbb, 0x98, 0xc1, 0x3a, 0x45, 0x1e, 0xdd, 0x93, 0x17, 0x08, 0x01, 0xcb, 0x22, 0xd9,
                     +	0x28, 0x50, 0x82, 0x2a, 0x52, 0xe0, 0x24, 0x71, 0xe3, 0xa1, 0x1c, 0xc3, 0x87, 0x87, 0x72, 0x8c,
                     +	0x0d, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1, 0xc1, 0x23, 0x39,
                     +	0xc6, 0x24, 0x36, 0xb0, 0xc7, 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x48, 0x07, 0xf6, 0xf3,
                     +	0x18, 0x01, 0x00, 0x00,
+                     }

@@ -1,16 +1,17 @@
                      package remote
                      import (
                     -	"errors"
                      	"fmt"
                      	"net"
                     +	"github.com/docker/docker/pkg/plugingetter"
                      	"github.com/docker/docker/pkg/plugins"
                      	"github.com/docker/libnetwork/datastore"
                      	"github.com/docker/libnetwork/discoverapi"
                      	"github.com/docker/libnetwork/driverapi"
                      	"github.com/docker/libnetwork/drivers/remote/api"
                      	"github.com/docker/libnetwork/types"
                     +	"github.com/pkg/errors"
                      	"github.com/sirupsen/logrus"
+                     )
@@ -49,7 +50,11 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
                      		handleFunc = pg.Handle
                      		activePlugins := pg.GetAllManagedPluginsByCap(driverapi.NetworkPluginEndpointType)
                      		for _, ap := range activePlugins {
                     -			newPluginHandler(ap.Name(), ap.Client())
                     +			client, err := getPluginClient(ap)
                     +			if err != nil {
                     +				return err
                     +			}
                     +			newPluginHandler(ap.Name(), client)
+                     		}
+                     	}
                      	handleFunc(driverapi.NetworkPluginEndpointType, newPluginHandler)
@@ -57,6 +62,28 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
                      	return nil
+                     }
                     +func getPluginClient(p plugingetter.CompatPlugin) (*plugins.Client, error) {
                     +	if v1, ok := p.(plugingetter.PluginWithV1Client); ok {
                     +		return v1.Client(), nil
                     +	}
+                    +
                     +	pa, ok := p.(plugingetter.PluginAddr)
                     +	if !ok {
                     +		return nil, errors.Errorf("unknown plugin type %T", p)
                     +	}
+                    +
                     +	if pa.Protocol() != plugins.ProtocolSchemeHTTPV1 {
                     +		return nil, errors.Errorf("unsupported plugin protocol %s", pa.Protocol())
                     +	}
+                    +
                     +	addr := pa.Addr()
                     +	client, err := plugins.NewClientWithTimeout(addr.Network()+"://"+addr.String(), nil, pa.Timeout())
                     +	if err != nil {
                     +		return nil, errors.Wrap(err, "error creating plugin client")
                     +	}
                     +	return client, nil
                     +}
+                    +
                      // Get capability from client
                      func (d *driver) getCapabilities() (*driverapi.Capability, error) {
                      	var capResp api.GetCapabilityResponse

@@ -540,6 +540,12 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) {
+                     		}
                      	}()
                     +	// Load balancing endpoints should never have a default gateway nor
                     +	// should they alter the status of a network's default gateway
                     +	if ep.loadBalancer && !sb.ingress {
                     +		return nil
                     +	}
+                    +
                      	if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
                      		return sb.setupDefaultGW()
+                     	}

@@ -49,6 +49,9 @@ type InterfaceInfo interface {
                      	// LinkLocalAddresses returns the list of link-local (IPv4/IPv6) addresses assigned to the endpoint.
                      	LinkLocalAddresses() []*net.IPNet
+                    +
                     +	// SrcName returns the name of the interface w/in the container
                     +	SrcName() string
+                     }
                      type endpointInterface struct {
@@ -272,6 +275,10 @@ func (epi *endpointInterface) LinkLocalAddresses() []*net.IPNet {
                      	return epi.llAddrs
+                     }
                     +func (epi *endpointInterface) SrcName() string {
                     +	return epi.srcName
                     +}
+                    +
                      func (epi *endpointInterface) SetNames(srcName string, dstPrefix string) error {
                      	epi.srcName = srcName
                      	epi.dstPrefix = dstPrefix

@@ -40,7 +40,7 @@ type Network interface {
                      	CreateEndpoint(name string, options ...EndpointOption) (Endpoint, error)
                      	// Delete the network.
                     -	Delete() error
                     +	Delete(options ...NetworkDeleteOption) error
                      	// Endpoints returns the list of Endpoint(s) in this network.
                      	Endpoints() []Endpoint
@@ -875,6 +875,28 @@ func (n *network) processOptions(options ...NetworkOption) {
+                     	}
+                     }
                     +type networkDeleteParams struct {
                     +	rmLBEndpoint bool
                     +}
+                    +
                     +// NetworkDeleteOption is a type for optional parameters to pass to the
                     +// network.Delete() function.
                     +type NetworkDeleteOption func(p *networkDeleteParams)
+                    +
                     +// NetworkDeleteOptionRemoveLB informs a network.Delete() operation that should
                     +// remove the load balancer endpoint for this network.  Note that the Delete()
                     +// method will automatically remove a load balancing endpoint for most networks
                     +// when the network is otherwise empty.  However, this does not occur for some
                     +// networks.  In particular, networks marked as ingress (which are supposed to
                     +// be more permanent than other overlay networks) won't automatically remove
                     +// the LB endpoint on Delete().  This method allows for explicit removal of
                     +// such networks provided there are no other endpoints present in the network.
                     +// If the network still has non-LB endpoints present, Delete() will not
                     +// remove the LB endpoint and will return an error.
                     +func NetworkDeleteOptionRemoveLB(p *networkDeleteParams) {
                     +	p.rmLBEndpoint = true
                     +}
+                    +
                      func (n *network) resolveDriver(name string, load bool) (driverapi.Driver, *driverapi.Capability, error) {
                      	c := n.getController()
@@ -938,11 +960,23 @@ func (n *network) driver(load bool) (driverapi.Driver, error) {
                      	return d, nil
+                     }
                     -func (n *network) Delete() error {
                     -	return n.delete(false)
                     +func (n *network) Delete(options ...NetworkDeleteOption) error {
                     +	var params networkDeleteParams
                     +	for _, opt := range options {
                     +		opt(&params)
                     +	}
                     +	return n.delete(false, params.rmLBEndpoint)
+                     }
                     -func (n *network) delete(force bool) error {
                     +// This function gets called in 3 ways:
                     +//  * Delete() -- (false, false)
                     +//      remove if endpoint count == 0 or endpoint count == 1 and
                     +//      there is a load balancer IP
                     +//  * Delete(libnetwork.NetworkDeleteOptionRemoveLB) -- (false, true)
                     +//      remove load balancer and network if endpoint count == 1
                     +//  * controller.networkCleanup() -- (true, true)
                     +//      remove the network no matter what
                     +func (n *network) delete(force bool, rmLBEndpoint bool) error {
                      	n.Lock()
                      	c := n.ctrlr
                      	name := n.name
@@ -957,10 +991,32 @@ func (n *network) delete(force bool) error {
                      		return &UnknownNetworkError{name: name, id: id}
+                     	}
                     -	if len(n.loadBalancerIP) != 0 {
                     -		endpoints := n.Endpoints()
                     -		if force || (len(endpoints) == 1 && !n.ingress) {
                     -			n.deleteLoadBalancerSandbox()
                     +	// Only remove ingress on force removal or explicit LB endpoint removal
                     +	if n.ingress && !force && !rmLBEndpoint {
                     +		return &ActiveEndpointsError{name: n.name, id: n.id}
                     +	}
+                    +
                     +	// Check that the network is empty
                     +	var emptyCount uint64
                     +	if n.hasLoadBalancerEndpoint() {
                     +		emptyCount = 1
                     +	}
                     +	if !force && n.getEpCnt().EndpointCnt() > emptyCount {
                     +		if n.configOnly {
                     +			return types.ForbiddenErrorf("configuration network %q is in use", n.Name())
                     +		}
                     +		return &ActiveEndpointsError{name: n.name, id: n.id}
                     +	}
+                    +
                     +	if n.hasLoadBalancerEndpoint() {
                     +		// If we got to this point, then the following must hold:
                     +		//  * force is true OR endpoint count == 1
                     +		if err := n.deleteLoadBalancerSandbox(); err != nil {
                     +			if !force {
                     +				return err
                     +			}
                     +			// continue deletion when force is true even on error
                     +			logrus.Warnf("Error deleting load balancer sandbox: %v", err)
+                     		}
                      		//Reload the network from the store to update the epcnt.
                      		n, err = c.getNetworkFromStore(id)
@@ -969,12 +1025,10 @@ func (n *network) delete(force bool) error {
+                     		}
+                     	}
                     -	if !force && n.getEpCnt().EndpointCnt() != 0 {
                     -		if n.configOnly {
                     -			return types.ForbiddenErrorf("configuration network %q is in use", n.Name())
                     -		}
                     -		return &ActiveEndpointsError{name: n.name, id: n.id}
                     -	}
                     +	// Up to this point, errors that we returned were recoverable.
                     +	// From here on, any errors leave us in an inconsistent state.
                     +	// This is unfortunate, but there isn't a safe way to
                     +	// reconstitute a load-balancer endpoint after removing it.
                      	// Mark the network for deletion
                      	n.inDelete = true
@@ -1023,9 +1077,6 @@ func (n *network) delete(force bool) error {
                      	// Cleanup the service discovery for this network
                      	c.cleanupServiceDiscovery(n.ID())
                     -	// Cleanup the load balancer
                     -	c.cleanupServiceBindings(n.ID())
+                    -
                      removeFromStore:
                      	// deleteFromStore performs an atomic delete operation and the
                      	// network.epCnt will help prevent any possible
@@ -1877,6 +1928,10 @@ func (n *network) hasSpecialDriver() bool {
                      	return n.Type() == "host" || n.Type() == "null"
+                     }
                     +func (n *network) hasLoadBalancerEndpoint() bool {
                     +	return len(n.loadBalancerIP) != 0
                     +}
+                    +
                      func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) {
                      	var ipv6Miss bool
@@ -2056,8 +2111,20 @@ func (c *controller) getConfigNetwork(name string) (*network, error) {
                      	return n.(*network), nil
+                     }
                     -func (n *network) createLoadBalancerSandbox() error {
                     -	sandboxName := n.name + "-sbox"
                     +func (n *network) lbSandboxName() string {
                     +	name := "lb-" + n.name
                     +	if n.ingress {
                     +		name = n.name + "-sbox"
                     +	}
                     +	return name
                     +}
+                    +
                     +func (n *network) lbEndpointName() string {
                     +	return n.name + "-endpoint"
                     +}
+                    +
                     +func (n *network) createLoadBalancerSandbox() (retErr error) {
                     +	sandboxName := n.lbSandboxName()
                      	sbOptions := []SandboxOption{}
                      	if n.ingress {
                      		sbOptions = append(sbOptions, OptionIngress())
@@ -2067,26 +2134,30 @@ func (n *network) createLoadBalancerSandbox() error {
                      		return err
+                     	}
                      	defer func() {
                     -		if err != nil {
                     +		if retErr != nil {
                      			if e := n.ctrlr.SandboxDestroy(sandboxName); e != nil {
                     -				logrus.Warnf("could not delete sandbox %s on failure on failure (%v): %v", sandboxName, err, e)
                     +				logrus.Warnf("could not delete sandbox %s on failure on failure (%v): %v", sandboxName, retErr, e)
+                     			}
+                     		}
                      	}()
                     -	endpointName := n.name + "-endpoint"
                     +	endpointName := n.lbEndpointName()
                      	epOptions := []EndpointOption{
                      		CreateOptionIpam(n.loadBalancerIP, nil, nil, nil),
                      		CreateOptionLoadBalancer(),
+                     	}
                     +	if n.hasLoadBalancerEndpoint() && !n.ingress {
                     +		// Mark LB endpoints as anonymous so they don't show up in DNS
                     +		epOptions = append(epOptions, CreateOptionAnonymous())
                     +	}
                      	ep, err := n.createEndpoint(endpointName, epOptions...)
                      	if err != nil {
                      		return err
+                     	}
                      	defer func() {
                     -		if err != nil {
                     +		if retErr != nil {
                      			if e := ep.Delete(true); e != nil {
                     -				logrus.Warnf("could not delete endpoint %s on failure on failure (%v): %v", endpointName, err, e)
                     +				logrus.Warnf("could not delete endpoint %s on failure on failure (%v): %v", endpointName, retErr, e)
+                     			}
+                     		}
                      	}()
@@ -2094,17 +2165,18 @@ func (n *network) createLoadBalancerSandbox() error {
                      	if err := ep.Join(sb, nil); err != nil {
                      		return err
+                     	}
+                    +
                      	return sb.EnableService()
+                     }
                     -func (n *network) deleteLoadBalancerSandbox() {
                     +func (n *network) deleteLoadBalancerSandbox() error {
                      	n.Lock()
                      	c := n.ctrlr
                      	name := n.name
                      	n.Unlock()
                     -	endpointName := name + "-endpoint"
                     -	sandboxName := name + "-sbox"
                     +	sandboxName := n.lbSandboxName()
                     +	endpointName := n.lbEndpointName()
                      	endpoint, err := n.EndpointByName(endpointName)
                      	if err != nil {
@@ -2129,6 +2201,7 @@ func (n *network) deleteLoadBalancerSandbox() {
+                     	}
                      	if err := c.SandboxDestroy(sandboxName); err != nil {
                     -		logrus.Warnf("Failed to delete %s sandbox: %v", sandboxName, err)
                     +		return fmt.Errorf("Failed to delete %s sandbox: %v", sandboxName, err)
+                     	}
                     +	return nil
+                     }

@@ -1,11 +1,11 @@
                      // Code generated by protoc-gen-gogo. DO NOT EDIT.
                     -// source: networkdb.proto
                     +// source: networkdb/networkdb.proto
                      /*
                      	Package networkdb is a generated protocol buffer package.
                      	It is generated from these files:
                     -		networkdb.proto
                     +		networkdb/networkdb.proto
                      	It has these top-level messages:
                      		GossipMessage
@@ -476,7 +476,7 @@ func (m *CompoundMessage) GetMessages() []*CompoundMessage_SimpleMessage {
                      type CompoundMessage_SimpleMessage struct {
                      	// Bytestring payload of a message constructed using
                      	// other message type definitions.
                     -	Payload []byte `protobuf:"bytes,1,opt,name=Payload,proto3" json:"Payload,omitempty"`
                     +	Payload []byte `protobuf:"bytes,1,opt,name=Payload,json=payload,proto3" json:"Payload,omitempty"`
+                     }
                      func (m *CompoundMessage_SimpleMessage) Reset()      { *m = CompoundMessage_SimpleMessage{} }
@@ -997,24 +997,6 @@ func (m *CompoundMessage_SimpleMessage) MarshalTo(dAtA []byte) (int, error) {
                      	return i, nil
+                     }
                     -func encodeFixed64Networkdb(dAtA []byte, offset int, v uint64) int {
                     -	dAtA[offset] = uint8(v)
                     -	dAtA[offset+1] = uint8(v >> 8)
                     -	dAtA[offset+2] = uint8(v >> 16)
                     -	dAtA[offset+3] = uint8(v >> 24)
                     -	dAtA[offset+4] = uint8(v >> 32)
                     -	dAtA[offset+5] = uint8(v >> 40)
                     -	dAtA[offset+6] = uint8(v >> 48)
                     -	dAtA[offset+7] = uint8(v >> 56)
                     -	return offset + 8
                     -}
                     -func encodeFixed32Networkdb(dAtA []byte, offset int, v uint32) int {
                     -	dAtA[offset] = uint8(v)
                     -	dAtA[offset+1] = uint8(v >> 8)
                     -	dAtA[offset+2] = uint8(v >> 16)
                     -	dAtA[offset+3] = uint8(v >> 24)
                     -	return offset + 4
                     -}
                      func encodeVarintNetworkdb(dAtA []byte, offset int, v uint64) int {
                      	for v >= 1<<7 {
                      		dAtA[offset] = uint8(v&0x7f | 0x80)
@@ -2666,68 +2648,68 @@ var (
                      	ErrIntOverflowNetworkdb   = fmt.Errorf("proto: integer overflow")
+                     )
                     -func init() { proto.RegisterFile("networkdb.proto", fileDescriptorNetworkdb) }
                     +func init() { proto.RegisterFile("networkdb/networkdb.proto", fileDescriptorNetworkdb) }
                      var fileDescriptorNetworkdb = []byte{
                     -	// 953 bytes of a gzipped FileDescriptorProto
                     +	// 955 bytes of a gzipped FileDescriptorProto
 x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x96, 0xcd, 0x6e, 0xe3, 0x54,
                     -	0x14, 0xc7, 0x7b, 0xf3, 0xd5, 0xe4, 0x34, 0xa5, 0xe6, 0x4e, 0x67, 0xc6, 0xe3, 0x81, 0xc4, 0x98,
                     -	0x99, 0x2a, 0x53, 0x41, 0x8a, 0x3a, 0x4f, 0xd0, 0x24, 0x16, 0x64, 0x26, 0xe3, 0x44, 0x6e, 0x52,
                     -	0xc4, 0x2a, 0xba, 0xad, 0x2f, 0xa9, 0x55, 0xc7, 0xb6, 0x6c, 0x27, 0x28, 0x2b, 0x10, 0xab, 0x51,
                     -	0x16, 0xbc, 0x41, 0x56, 0xc3, 0x9a, 0x07, 0x40, 0x2c, 0x59, 0xcc, 0x82, 0x05, 0xec, 0x10, 0x8b,
                     -	0x88, 0xe6, 0x09, 0x78, 0x04, 0xe4, 0x6b, 0x3b, 0xb9, 0x49, 0xab, 0x91, 0x10, 0x23, 0xc1, 0x26,
                     -	0xb9, 0x1f, 0xbf, 0x1c, 0x9f, 0xf3, 0xf7, 0xff, 0xdc, 0x1b, 0xd8, 0xb3, 0x69, 0xf0, 0x95, 0xe3,
                     -	0x5d, 0x19, 0xe7, 0x55, 0xd7, 0x73, 0x02, 0x07, 0x17, 0x96, 0x0b, 0xd2, 0xfe, 0xc0, 0x19, 0x38,
                     -	0x6c, 0xf5, 0x28, 0x1c, 0x45, 0x80, 0xd2, 0x86, 0xdd, 0x4f, 0x1d, 0xdf, 0x37, 0xdd, 0x17, 0xd4,
                     -	0xf7, 0xc9, 0x80, 0xe2, 0x43, 0xc8, 0x04, 0x13, 0x97, 0x8a, 0x48, 0x46, 0x95, 0x77, 0x8e, 0xef,
                     -	0x55, 0x57, 0x11, 0x63, 0xa2, 0x3b, 0x71, 0xa9, 0xce, 0x18, 0x8c, 0x21, 0x63, 0x90, 0x80, 0x88,
                     -	0x29, 0x19, 0x55, 0x8a, 0x3a, 0x1b, 0x2b, 0xaf, 0x52, 0x50, 0xd0, 0x1c, 0x83, 0xaa, 0x63, 0x6a,
                     -	0x07, 0xf8, 0xe3, 0xb5, 0x68, 0x0f, 0xb8, 0x68, 0x4b, 0xa6, 0xca, 0x05, 0x6c, 0x42, 0xce, 0xea,
                     -	0x07, 0xe6, 0x90, 0xb2, 0x90, 0x99, 0xda, 0xf1, 0xeb, 0x79, 0x79, 0xeb, 0x8f, 0x79, 0xf9, 0x70,
                     -	0x60, 0x06, 0x97, 0xa3, 0xf3, 0xea, 0x85, 0x33, 0x3c, 0xba, 0x24, 0xfe, 0xa5, 0x79, 0xe1, 0x78,
                     -	0xee, 0x91, 0x4f, 0xbd, 0x2f, 0xd9, 0x47, 0xb5, 0x45, 0x86, 0xae, 0xe3, 0x05, 0x5d, 0x73, 0x48,
                     -	0xf5, 0xac, 0x15, 0x7e, 0xe1, 0x87, 0x50, 0xb0, 0x1d, 0x83, 0xf6, 0x6d, 0x32, 0xa4, 0x62, 0x5a,
                     -	0x46, 0x95, 0x82, 0x9e, 0x0f, 0x17, 0x34, 0x32, 0xa4, 0xca, 0xd7, 0x90, 0x09, 0x9f, 0x8a, 0x1f,
                     -	0xc3, 0x76, 0x53, 0x3b, 0x3b, 0x69, 0x35, 0x1b, 0xc2, 0x96, 0x24, 0x4e, 0x67, 0xf2, 0xfe, 0x32,
                     -	0xad, 0x70, 0xbf, 0x69, 0x8f, 0x89, 0x65, 0x1a, 0xb8, 0x0c, 0x99, 0x67, 0xed, 0xa6, 0x26, 0x20,
                     -	0xe9, 0xee, 0x74, 0x26, 0xbf, 0xbb, 0xc6, 0x3c, 0x73, 0x4c, 0x1b, 0x7f, 0x00, 0xd9, 0x96, 0x7a,
                     -	0x72, 0xa6, 0x0a, 0x29, 0xe9, 0xde, 0x74, 0x26, 0xe3, 0x35, 0xa2, 0x45, 0xc9, 0x98, 0x4a, 0xc5,
                     -	0x97, 0xaf, 0x4a, 0x5b, 0x3f, 0x7e, 0x5f, 0x62, 0x0f, 0x56, 0xae, 0x53, 0x50, 0xd4, 0x22, 0x2d,
                     -	0x22, 0xa1, 0x3e, 0x59, 0x13, 0xea, 0x3d, 0x5e, 0x28, 0x0e, 0xfb, 0x0f, 0xb4, 0xc2, 0x1f, 0x01,
                     -	0xc4, 0xc9, 0xf4, 0x4d, 0x43, 0xcc, 0x84, 0xbb, 0xb5, 0xdd, 0xc5, 0xbc, 0x5c, 0x88, 0x13, 0x6b,
                     -	0x36, 0xf4, 0xc4, 0x65, 0x4d, 0x43, 0x79, 0x89, 0x62, 0x69, 0x2b, 0xbc, 0xb4, 0x0f, 0xa7, 0x33,
                     -	0xf9, 0x3e, 0x5f, 0x08, 0xaf, 0xae, 0xb2, 0x54, 0x37, 0x7a, 0x03, 0x1b, 0x18, 0x13, 0xf8, 0xd1,
                     -	0x4a, 0xe0, 0x07, 0xd3, 0x99, 0x7c, 0x77, 0x13, 0xba, 0x4d, 0xe3, 0x5f, 0xd0, 0x4a, 0x63, 0x3b,
                     -	0xf0, 0x26, 0x1b, 0x95, 0xa0, 0x37, 0x57, 0xf2, 0x36, 0xf5, 0x7d, 0x72, 0x43, 0xdf, 0x5a, 0x71,
                     -	0x31, 0x2f, 0xe7, 0xb5, 0x58, 0x63, 0x4e, 0x6d, 0x11, 0xb6, 0x2d, 0x4a, 0xc6, 0xa6, 0x3d, 0x60,
                     -	0x52, 0xe7, 0xf5, 0x64, 0xaa, 0xfc, 0x84, 0x60, 0x2f, 0x4e, 0xb4, 0x33, 0xf2, 0x2f, 0x3b, 0x23,
                     -	0xcb, 0xe2, 0x72, 0x44, 0xff, 0x36, 0xc7, 0xa7, 0x90, 0x8f, 0x6b, 0xf7, 0xc5, 0x94, 0x9c, 0xae,
                     -	0xec, 0x1c, 0xdf, 0xbf, 0xc5, 0x84, 0xa1, 0x8e, 0xfa, 0x12, 0xfc, 0x07, 0x85, 0x29, 0xdf, 0x65,
                     -	0x00, 0xba, 0xe4, 0xdc, 0x8a, 0x0f, 0x86, 0xea, 0x9a, 0xdf, 0x25, 0xee, 0x51, 0x2b, 0xe8, 0x7f,
                     -	0xef, 0x76, 0xfc, 0x3e, 0x40, 0x10, 0xa6, 0x1b, 0xc5, 0xca, 0xb2, 0x58, 0x05, 0xb6, 0xc2, 0x82,
                     -	0x09, 0x90, 0xbe, 0xa2, 0x13, 0x31, 0xc7, 0xd6, 0xc3, 0x21, 0xde, 0x87, 0xec, 0x98, 0x58, 0x23,
                     -	0x2a, 0x6e, 0xb3, 0x23, 0x33, 0x9a, 0xe0, 0x1a, 0x60, 0x8f, 0xfa, 0xa6, 0x31, 0x22, 0x56, 0xdf,
                     -	0xa3, 0xc4, 0x8d, 0x0a, 0xcd, 0xcb, 0xa8, 0x92, 0xad, 0xed, 0x2f, 0xe6, 0x65, 0x41, 0x8f, 0x77,
                     -	0x75, 0x4a, 0x5c, 0x56, 0x8a, 0xe0, 0x6d, 0xac, 0x28, 0x3f, 0x24, 0x8d, 0x77, 0xc0, 0x37, 0x1e,
                     -	0x6b, 0x96, 0x95, 0xa2, 0x7c, 0xdb, 0x3d, 0x82, 0x5c, 0x5d, 0x57, 0x4f, 0xba, 0x6a, 0xd2, 0x78,
                     -	0xeb, 0x58, 0xdd, 0xa3, 0x24, 0xa0, 0x21, 0xd5, 0xeb, 0x34, 0x42, 0x2a, 0x75, 0x1b, 0xd5, 0x73,
                     -	0x8d, 0x98, 0x6a, 0xa8, 0x2d, 0xb5, 0xab, 0x0a, 0xe9, 0xdb, 0xa8, 0x06, 0xb5, 0x68, 0xb0, 0xd9,
                     -	0x9e, 0xbf, 0x21, 0xd8, 0xab, 0x8d, 0xac, 0xab, 0xd3, 0x89, 0x7d, 0x91, 0x5c, 0x3e, 0x6f, 0xd1,
                     -	0xcf, 0x32, 0xec, 0x8c, 0x6c, 0xdf, 0xb1, 0xcc, 0x0b, 0x33, 0xa0, 0x06, 0x73, 0x4d, 0x5e, 0xe7,
                     -	0x97, 0xde, 0xec, 0x03, 0x89, 0x6b, 0x87, 0x8c, 0x9c, 0x66, 0x7b, 0x89, 0xeb, 0x45, 0xd8, 0x76,
                     -	0xc9, 0xc4, 0x72, 0x88, 0xc1, 0x5e, 0x79, 0x51, 0x4f, 0xa6, 0xca, 0xb7, 0x08, 0xf6, 0xea, 0xce,
                     -	0xd0, 0x75, 0x46, 0xb6, 0x91, 0xd4, 0xd4, 0x80, 0xfc, 0x30, 0x1a, 0xfa, 0x22, 0x62, 0x8d, 0x55,
                     -	0xe1, 0xdc, 0xbe, 0x41, 0x57, 0x4f, 0xcd, 0xa1, 0x6b, 0xd1, 0x78, 0xa6, 0x2f, 0x7f, 0x29, 0x3d,
                     -	0x81, 0xdd, 0xb5, 0xad, 0x30, 0x89, 0x4e, 0x9c, 0x04, 0x8a, 0x92, 0x88, 0xa7, 0x87, 0x3f, 0xa7,
                     -	0x60, 0x87, 0xbb, 0xab, 0xf1, 0x87, 0xbc, 0x21, 0xd8, 0xf5, 0xc4, 0xed, 0x26, 0x6e, 0xa8, 0xc2,
                     -	0xae, 0xa6, 0x76, 0x3f, 0x6f, 0xeb, 0xcf, 0xfb, 0xea, 0x99, 0xaa, 0x75, 0x05, 0x14, 0x1d, 0xda,
                     -	0x1c, 0xba, 0x76, 0x5f, 0x1d, 0xc2, 0x4e, 0xf7, 0xa4, 0xd6, 0x52, 0x63, 0x3a, 0x3e, 0x96, 0x39,
                     -	0x9a, 0xeb, 0xf5, 0x03, 0x28, 0x74, 0x7a, 0xa7, 0x9f, 0xf5, 0x3b, 0xbd, 0x56, 0x4b, 0x48, 0x4b,
                     -	0xf7, 0xa7, 0x33, 0xf9, 0x0e, 0x47, 0x2e, 0x4f, 0xb3, 0x03, 0x28, 0xd4, 0x7a, 0xad, 0xe7, 0xfd,
                     -	0xd3, 0x2f, 0xb4, 0xba, 0x90, 0xb9, 0xc1, 0x25, 0x66, 0xc1, 0x8f, 0x21, 0x5f, 0x6f, 0xbf, 0xe8,
                     -	0xb4, 0x7b, 0x5a, 0x43, 0xc8, 0xde, 0xc0, 0x12, 0x45, 0x71, 0x05, 0x40, 0x6b, 0x37, 0x92, 0x0c,
                     -	0x73, 0x91, 0x31, 0xf9, 0x7a, 0x92, 0x4b, 0x5a, 0xba, 0x13, 0x1b, 0x93, 0x97, 0xad, 0x26, 0xfe,
                     -	0x7e, 0x5d, 0xda, 0xfa, 0xeb, 0xba, 0x84, 0xbe, 0x59, 0x94, 0xd0, 0xeb, 0x45, 0x09, 0xfd, 0xba,
                     -	0x28, 0xa1, 0x3f, 0x17, 0x25, 0x74, 0x9e, 0x63, 0x7f, 0x9d, 0x9e, 0xfe, 0x1d, 0x00, 0x00, 0xff,
                     -	0xff, 0x92, 0x82, 0xdb, 0x1a, 0x6e, 0x09, 0x00, 0x00,
                     +	0x14, 0xc7, 0x7b, 0xf3, 0xd1, 0x26, 0xa7, 0x29, 0x35, 0x77, 0x3a, 0x53, 0xd7, 0x03, 0x89, 0x31,
                     +	0x33, 0x55, 0xa6, 0x82, 0x14, 0x75, 0x9e, 0xa0, 0x49, 0x2c, 0xc8, 0x4c, 0xc6, 0x89, 0xdc, 0xa4,
                     +	0x88, 0x55, 0x74, 0x5b, 0x5f, 0x52, 0xab, 0x8e, 0x6d, 0xd9, 0x4e, 0x50, 0x56, 0x20, 0x56, 0xa3,
                     +	0x2c, 0x78, 0x83, 0xac, 0x86, 0x35, 0x0f, 0x80, 0x58, 0xb2, 0x98, 0x05, 0x0b, 0xd8, 0x21, 0x16,
                     +	0x11, 0xcd, 0x13, 0xf0, 0x08, 0xc8, 0xd7, 0x76, 0x72, 0x93, 0x56, 0x23, 0x21, 0x46, 0x82, 0x4d,
                     +	0x72, 0x3f, 0x7e, 0x39, 0x3e, 0xe7, 0xef, 0xff, 0xb9, 0x37, 0x70, 0x60, 0xd3, 0xe0, 0x2b, 0xc7,
                     +	0xbb, 0x36, 0x2e, 0x8e, 0x17, 0xa3, 0x8a, 0xeb, 0x39, 0x81, 0x83, 0xf3, 0x8b, 0x05, 0x69, 0xaf,
                     +	0xef, 0xf4, 0x1d, 0xb6, 0x7a, 0x1c, 0x8e, 0x22, 0x40, 0x69, 0xc1, 0xce, 0xa7, 0x8e, 0xef, 0x9b,
                     +	0xee, 0x0b, 0xea, 0xfb, 0xa4, 0x4f, 0xf1, 0x11, 0x64, 0x82, 0xb1, 0x4b, 0x45, 0x24, 0xa3, 0xf2,
                     +	0x3b, 0x27, 0x0f, 0x2a, 0xcb, 0x88, 0x31, 0xd1, 0x19, 0xbb, 0x54, 0x67, 0x0c, 0xc6, 0x90, 0x31,
                     +	0x48, 0x40, 0xc4, 0x94, 0x8c, 0xca, 0x05, 0x9d, 0x8d, 0x95, 0x57, 0x29, 0xc8, 0x6b, 0x8e, 0x41,
                     +	0xd5, 0x11, 0xb5, 0x03, 0xfc, 0xf1, 0x4a, 0xb4, 0x03, 0x2e, 0xda, 0x82, 0xa9, 0x70, 0x01, 0x1b,
                     +	0xb0, 0x69, 0xf5, 0x02, 0x73, 0x40, 0x59, 0xc8, 0x4c, 0xf5, 0xe4, 0xf5, 0xac, 0xb4, 0xf1, 0xc7,
                     +	0xac, 0x74, 0xd4, 0x37, 0x83, 0xab, 0xe1, 0x45, 0xe5, 0xd2, 0x19, 0x1c, 0x5f, 0x11, 0xff, 0xca,
                     +	0xbc, 0x74, 0x3c, 0xf7, 0xd8, 0xa7, 0xde, 0x97, 0xec, 0xa3, 0xd2, 0x24, 0x03, 0xd7, 0xf1, 0x82,
                     +	0x8e, 0x39, 0xa0, 0x7a, 0xd6, 0x0a, 0xbf, 0xf0, 0x43, 0xc8, 0xdb, 0x8e, 0x41, 0x7b, 0x36, 0x19,
                     +	0x50, 0x31, 0x2d, 0xa3, 0x72, 0x5e, 0xcf, 0x85, 0x0b, 0x1a, 0x19, 0x50, 0xe5, 0x6b, 0xc8, 0x84,
                     +	0x4f, 0xc5, 0x8f, 0x61, 0xab, 0xa1, 0x9d, 0x9f, 0x36, 0x1b, 0x75, 0x61, 0x43, 0x12, 0x27, 0x53,
                     +	0x79, 0x6f, 0x91, 0x56, 0xb8, 0xdf, 0xb0, 0x47, 0xc4, 0x32, 0x0d, 0x5c, 0x82, 0xcc, 0xb3, 0x56,
                     +	0x43, 0x13, 0x90, 0x74, 0x7f, 0x32, 0x95, 0xdf, 0x5d, 0x61, 0x9e, 0x39, 0xa6, 0x8d, 0x3f, 0x80,
                     +	0x6c, 0x53, 0x3d, 0x3d, 0x57, 0x85, 0x94, 0xf4, 0x60, 0x32, 0x95, 0xf1, 0x0a, 0xd1, 0xa4, 0x64,
                     +	0x44, 0xa5, 0xc2, 0xcb, 0x57, 0xc5, 0x8d, 0x1f, 0xbf, 0x2f, 0xb2, 0x07, 0x2b, 0x37, 0x29, 0x28,
                     +	0x68, 0x91, 0x16, 0x91, 0x50, 0x9f, 0xac, 0x08, 0xf5, 0x1e, 0x2f, 0x14, 0x87, 0xfd, 0x07, 0x5a,
                     +	0xe1, 0x8f, 0x00, 0xe2, 0x64, 0x7a, 0xa6, 0x21, 0x66, 0xc2, 0xdd, 0xea, 0xce, 0x7c, 0x56, 0xca,
                     +	0xc7, 0x89, 0x35, 0xea, 0x7a, 0xe2, 0xb2, 0x86, 0xa1, 0xbc, 0x44, 0xb1, 0xb4, 0x65, 0x5e, 0xda,
                     +	0x87, 0x93, 0xa9, 0xbc, 0xcf, 0x17, 0xc2, 0xab, 0xab, 0x2c, 0xd4, 0x8d, 0xde, 0xc0, 0x1a, 0xc6,
                     +	0x04, 0x7e, 0xb4, 0x14, 0xf8, 0x60, 0x32, 0x95, 0xef, 0xaf, 0x43, 0x77, 0x69, 0xfc, 0x0b, 0x5a,
                     +	0x6a, 0x6c, 0x07, 0xde, 0x78, 0xad, 0x12, 0xf4, 0xe6, 0x4a, 0xde, 0xa6, 0xbe, 0x4f, 0x6e, 0xe9,
                     +	0x5b, 0x2d, 0xcc, 0x67, 0xa5, 0x9c, 0x16, 0x6b, 0xcc, 0xa9, 0x2d, 0xc2, 0x96, 0x45, 0xc9, 0xc8,
                     +	0xb4, 0xfb, 0x4c, 0xea, 0x9c, 0x9e, 0x4c, 0x95, 0x9f, 0x10, 0xec, 0xc6, 0x89, 0xb6, 0x87, 0xfe,
                     +	0x55, 0x7b, 0x68, 0x59, 0x5c, 0x8e, 0xe8, 0xdf, 0xe6, 0xf8, 0x14, 0x72, 0x71, 0xed, 0xbe, 0x98,
                     +	0x92, 0xd3, 0xe5, 0xed, 0x93, 0xfd, 0x3b, 0x4c, 0x18, 0xea, 0xa8, 0x2f, 0xc0, 0x7f, 0x50, 0x98,
                     +	0xf2, 0x5d, 0x06, 0xa0, 0x43, 0x2e, 0xac, 0xf8, 0x60, 0xa8, 0xac, 0xf8, 0x5d, 0xe2, 0x1e, 0xb5,
                     +	0x84, 0xfe, 0xf7, 0x6e, 0xc7, 0xef, 0x03, 0x04, 0x61, 0xba, 0x51, 0xac, 0x2c, 0x8b, 0x95, 0x67,
                     +	0x2b, 0x2c, 0x98, 0x00, 0xe9, 0x6b, 0x3a, 0x16, 0x37, 0xd9, 0x7a, 0x38, 0xc4, 0x7b, 0x90, 0x1d,
                     +	0x11, 0x6b, 0x48, 0xc5, 0x2d, 0x76, 0x64, 0x46, 0x13, 0x5c, 0x05, 0xec, 0x51, 0xdf, 0x34, 0x86,
                     +	0xc4, 0xea, 0x79, 0x94, 0xb8, 0x51, 0xa1, 0x39, 0x19, 0x95, 0xb3, 0xd5, 0xbd, 0xf9, 0xac, 0x24,
                     +	0xe8, 0xf1, 0xae, 0x4e, 0x89, 0xcb, 0x4a, 0x11, 0xbc, 0xb5, 0x15, 0xe5, 0x87, 0xa4, 0xf1, 0x0e,
                     +	0xf9, 0xc6, 0x63, 0xcd, 0xb2, 0x54, 0x94, 0x6f, 0xbb, 0x47, 0xb0, 0x59, 0xd3, 0xd5, 0xd3, 0x8e,
                     +	0x9a, 0x34, 0xde, 0x2a, 0x56, 0xf3, 0x28, 0x09, 0x68, 0x48, 0x75, 0xdb, 0xf5, 0x90, 0x4a, 0xdd,
                     +	0x45, 0x75, 0x5d, 0x23, 0xa6, 0xea, 0x6a, 0x53, 0xed, 0xa8, 0x42, 0xfa, 0x2e, 0xaa, 0x4e, 0x2d,
                     +	0x1a, 0xac, 0xb7, 0xe7, 0x6f, 0x08, 0x76, 0xab, 0x43, 0xeb, 0xfa, 0x6c, 0x6c, 0x5f, 0x26, 0x97,
                     +	0xcf, 0x5b, 0xf4, 0xb3, 0x0c, 0xdb, 0x43, 0xdb, 0x77, 0x2c, 0xf3, 0xd2, 0x0c, 0xa8, 0xc1, 0x5c,
                     +	0x93, 0xd3, 0xf9, 0xa5, 0x37, 0xfb, 0x40, 0xe2, 0xda, 0x21, 0x23, 0xa7, 0xd9, 0x5e, 0xe2, 0x7a,
                     +	0x11, 0xb6, 0x5c, 0x32, 0xb6, 0x1c, 0x62, 0xb0, 0x57, 0x5e, 0xd0, 0x93, 0xa9, 0xf2, 0x2d, 0x82,
                     +	0xdd, 0x9a, 0x33, 0x70, 0x9d, 0xa1, 0x6d, 0x24, 0x35, 0xd5, 0x21, 0x37, 0x88, 0x86, 0xbe, 0x88,
                     +	0x58, 0x63, 0x95, 0x39, 0xb7, 0xaf, 0xd1, 0x95, 0x33, 0x73, 0xe0, 0x5a, 0x34, 0x9e, 0xe9, 0x8b,
                     +	0x5f, 0x4a, 0x4f, 0x60, 0x67, 0x65, 0x2b, 0x4c, 0xa2, 0x1d, 0x27, 0x81, 0x56, 0x92, 0x38, 0xfa,
                     +	0x39, 0x05, 0xdb, 0xdc, 0x5d, 0x8d, 0x3f, 0xe4, 0x0d, 0xc1, 0xae, 0x27, 0x6e, 0x37, 0x71, 0x43,
                     +	0x05, 0x76, 0x34, 0xb5, 0xf3, 0x79, 0x4b, 0x7f, 0xde, 0x53, 0xcf, 0x55, 0xad, 0x23, 0xa0, 0xe8,
                     +	0xd0, 0xe6, 0xd0, 0x95, 0xfb, 0xea, 0x08, 0xb6, 0x3b, 0xa7, 0xd5, 0xa6, 0x1a, 0xd3, 0xf1, 0xb1,
                     +	0xcc, 0xd1, 0x5c, 0xaf, 0x1f, 0x42, 0xbe, 0xdd, 0x3d, 0xfb, 0xac, 0xd7, 0xee, 0x36, 0x9b, 0x42,
                     +	0x5a, 0xda, 0x9f, 0x4c, 0xe5, 0x7b, 0x1c, 0xb9, 0x38, 0xcd, 0x0e, 0x21, 0x5f, 0xed, 0x36, 0x9f,
                     +	0xf7, 0xce, 0xbe, 0xd0, 0x6a, 0x42, 0xe6, 0x16, 0x97, 0x98, 0x05, 0x3f, 0x86, 0x5c, 0xad, 0xf5,
                     +	0xa2, 0xdd, 0xea, 0x6a, 0x75, 0x21, 0x7b, 0x0b, 0x4b, 0x14, 0xc5, 0x65, 0x00, 0xad, 0x55, 0x4f,
                     +	0x32, 0xdc, 0x8c, 0x8c, 0xc9, 0xd7, 0x93, 0x5c, 0xd2, 0xd2, 0xbd, 0xd8, 0x98, 0xbc, 0x6c, 0x55,
                     +	0xf1, 0xf7, 0x9b, 0xe2, 0xc6, 0x5f, 0x37, 0x45, 0xf4, 0xcd, 0xbc, 0x88, 0x5e, 0xcf, 0x8b, 0xe8,
                     +	0xd7, 0x79, 0x11, 0xfd, 0x39, 0x2f, 0xa2, 0x8b, 0x4d, 0xf6, 0xd7, 0xe9, 0xe9, 0xdf, 0x01, 0x00,
                     +	0x00, 0xff, 0xff, 0x02, 0x9d, 0x53, 0x72, 0x78, 0x09, 0x00, 0x00,
+                     }

...	...	@@ -479,7 +479,7 @@ func (c *controller) networkCleanup() {
479	479	for _, n := range networks {
480	480	if n.inDelete {
481	481	logrus.Infof("Removing stale network %s (%s)", n.Name(), n.ID())
482		- if err := n.delete(true); err != nil {
	482	+ if err := n.delete(true, true); err != nil {
483	483	logrus.Debugf("Error while removing stale network: %v", err)
484	484	}
485	485	}

@@ -358,16 +358,20 @@ func (n *networkNamespace) loopbackUp() error {
                      	return n.nlHandle.LinkSetUp(iface)
+                     }
                     -func (n *networkNamespace) AddLoopbackAliasIP(ip *net.IPNet) error {
                     -	iface, err := n.nlHandle.LinkByName("lo")
                     +func (n *networkNamespace) GetLoopbackIfaceName() string {
                     +	return "lo"
                     +}
+                    +
                     +func (n *networkNamespace) AddAliasIP(ifName string, ip *net.IPNet) error {
                     +	iface, err := n.nlHandle.LinkByName(ifName)
                      	if err != nil {
                      		return err
+                     	}
                      	return n.nlHandle.AddrAdd(iface, &netlink.Addr{IPNet: ip})
+                     }
                     -func (n *networkNamespace) RemoveLoopbackAliasIP(ip *net.IPNet) error {
                     -	iface, err := n.nlHandle.LinkByName("lo")
                     +func (n *networkNamespace) RemoveAliasIP(ifName string, ip *net.IPNet) error {
                     +	iface, err := n.nlHandle.LinkByName(ifName)
                      	if err != nil {
                      		return err
+                     	}

@@ -32,11 +32,14 @@ type Sandbox interface {
                      	// Unset the previously set default IPv6 gateway in the sandbox
                      	UnsetGatewayIPv6() error
                     -	// AddLoopbackAliasIP adds the passed IP address to the sandbox loopback interface
                     -	AddLoopbackAliasIP(ip *net.IPNet) error
                     +	// GetLoopbackIfaceName returns the name of the loopback interface
                     +	GetLoopbackIfaceName() string
                     -	// RemoveLoopbackAliasIP removes the passed IP address from the sandbox loopback interface
                     -	RemoveLoopbackAliasIP(ip *net.IPNet) error
                     +	// AddAliasIP adds the passed IP address to the named interface
                     +	AddAliasIP(ifName string, ip *net.IPNet) error
+                    +
                     +	// RemoveAliasIP removes the passed IP address from the named interface
                     +	RemoveAliasIP(ifName string, ip *net.IPNet) error
                      	// Add a static route to the sandbox.
                      	AddStaticRoute(*types.StaticRoute) error

@@ -740,15 +740,8 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) {
                      	ep.Lock()
                      	joinInfo := ep.joinInfo
                     -	vip := ep.virtualIP
                      	ep.Unlock()
                     -	if len(vip) != 0 {
                     -		if err := osSbox.RemoveLoopbackAliasIP(&net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}); err != nil {
                     -			logrus.Warnf("Remove virtual IP %v failed: %v", vip, err)
                     -		}
                     -	}
+                    -
                      	if joinInfo == nil {
                      		return
+                     	}
@@ -861,13 +854,6 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
+                     		}
+                     	}
                     -	if len(ep.virtualIP) != 0 {
                     -		err := sb.osSbox.AddLoopbackAliasIP(&net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)})
                     -		if err != nil {
                     -			return fmt.Errorf("failed to add virtual IP %v: %v", ep.virtualIP, err)
                     -		}
                     -	}
+                    -
                      	if joinInfo != nil {
                      		// Set up non-interface routes.
                      		for _, r := range joinInfo.StaticRoutes {
@@ -893,7 +879,7 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
                      	// information including gateway and other routes so that
                      	// loadbalancers are populated all the network state is in
                      	// place in the sandbox.
                     -	sb.populateLoadbalancers(ep)
                     +	sb.populateLoadBalancers(ep)
                      	// Only update the store if we did not come here as part of
                      	// sandbox delete. If we came here as part of delete then do

@@ -225,6 +225,13 @@ func makeServiceCleanupFunc(c *controller, s *service, nID, eID string, vip net.
                      func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases, taskAliases []string, ip net.IP, method string) error {
                      	var addService bool
                     +	// Failure to lock the network ID on add can result in racing
                     +	// racing against network deletion resulting in inconsistent
                     +	// state in the c.serviceBindings map and it's sub-maps. Also,
                     +	// always lock network ID before services to avoid deadlock.
                     +	c.networkLocker.Lock(nID)
                     +	defer c.networkLocker.Unlock(nID)
+                    +
                      	n, err := c.NetworkByID(nID)
                      	if err != nil {
                      		return err
@@ -289,11 +296,8 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s
                      		logrus.Warnf("addServiceBinding %s possible transient state ok:%t entries:%d set:%t %s", eID, ok, entries, b, setStr)
+                     	}
                     -	// Add loadbalancer service and backend in all sandboxes in
                     -	// the network only if vip is valid.
                     -	if len(vip) != 0 {
                     -		n.(*network).addLBBackend(ip, vip, lb, ingressPorts)
                     -	}
                     +	// Add loadbalancer service and backend to the network
                     +	n.(*network).addLBBackend(ip, lb)
                      	// Add the appropriate name resolutions
                      	c.addEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, addService, "addServiceBinding")
@@ -307,11 +311,6 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
                      	var rmService bool
                     -	n, err := c.NetworkByID(nID)
                     -	if err != nil {
                     -		return err
                     -	}
+                    -
                      	skey := serviceKey{
                      		id:    svcID,
                      		ports: portConfigs(ingressPorts).String(),
@@ -368,8 +367,15 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
                      	// Remove loadbalancer service(if needed) and backend in all
                      	// sandboxes in the network only if the vip is valid.
                     -	if len(vip) != 0 && entries == 0 {
                     -		n.(*network).rmLBBackend(ip, vip, lb, ingressPorts, rmService, fullRemove)
                     +	if entries == 0 {
                     +		// The network may well have been deleted before the last
                     +		// of the service bindings.  That's ok, because removing
                     +		// the network sandbox implicitly removes the backend
                     +		// service bindings.
                     +		n, err := c.NetworkByID(nID)
                     +		if err == nil {
                     +			n.(*network).rmLBBackend(ip, lb, rmService, fullRemove)
                     +		}
+                     	}
                      	// Delete the name resolutions

@@ -30,40 +30,9 @@ func init() {
                      	reexec.Register("redirecter", redirecter)
+                     }
                     -// Get all loadbalancers on this network that is currently discovered
                     -// on this node.
                     -func (n *network) connectedLoadbalancers() []*loadBalancer {
                     -	c := n.getController()
+                    -
                     -	c.Lock()
                     -	serviceBindings := make([]*service, 0, len(c.serviceBindings))
                     -	for _, s := range c.serviceBindings {
                     -		serviceBindings = append(serviceBindings, s)
                     -	}
                     -	c.Unlock()
+                    -
                     -	var lbs []*loadBalancer
                     -	for _, s := range serviceBindings {
                     -		s.Lock()
                     -		// Skip the serviceBindings that got deleted
                     -		if s.deleted {
                     -			s.Unlock()
                     -			continue
                     -		}
                     -		if lb, ok := s.loadBalancers[n.ID()]; ok {
                     -			lbs = append(lbs, lb)
                     -		}
                     -		s.Unlock()
                     -	}
+                    -
                     -	return lbs
                     -}
+                    -
                      // Populate all loadbalancers on the network that the passed endpoint
                      // belongs to, into this sandbox.
                     -func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
                     -	var gwIP net.IP
+                    -
                     +func (sb *sandbox) populateLoadBalancers(ep *endpoint) {
                      	// This is an interface less endpoint. Nothing to do.
                      	if ep.Iface() == nil {
                      		return
@@ -77,96 +46,64 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
                      			logrus.Errorf("Failed to add redirect rules for ep %s (%s): %v", ep.Name(), ep.ID()[0:7], err)
+                     		}
+                     	}
                     +}
                     -	if sb.ingress {
                     -		// For the ingress sandbox if this is not gateway
                     -		// endpoint do nothing.
                     -		if ep != sb.getGatewayEndpoint() {
                     -			return
                     -		}
+                    -
                     -		// This is the gateway endpoint. Now get the ingress
                     -		// network and plumb the loadbalancers.
                     -		gwIP = ep.Iface().Address().IP
                     -		for _, ep := range sb.getConnectedEndpoints() {
                     -			if !ep.endpointInGWNetwork() {
                     -				n = ep.getNetwork()
                     -				eIP = ep.Iface().Address()
                     -			}
                     +func (n *network) findLBEndpointSandbox() (*endpoint, *sandbox, error) {
                     +	// TODO: get endpoint from store?  See EndpointInfo()
                     +	var ep *endpoint
                     +	// Find this node's LB sandbox endpoint:  there should be exactly one
                     +	for _, e := range n.Endpoints() {
                     +		epi := e.Info()
                     +		if epi != nil && epi.LoadBalancer() {
                     +			ep = e.(*endpoint)
                     +			break
+                     		}
+                     	}
+                    -
                     -	for _, lb := range n.connectedLoadbalancers() {
                     -		// Skip if vip is not valid.
                     -		if len(lb.vip) == 0 {
                     -			continue
                     -		}
+                    -
                     -		lb.service.Lock()
                     -		for _, be := range lb.backEnds {
                     -			if !be.disabled {
                     -				sb.addLBBackend(be.ip, lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, gwIP, n.ingress)
                     -			}
                     -		}
                     -		lb.service.Unlock()
                     +	if ep == nil {
                     +		return nil, nil, fmt.Errorf("Unable to find load balancing endpoint for network %s", n.ID())
+                     	}
                     +	// Get the load balancer sandbox itself as well
                     +	sb, ok := ep.getSandbox()
                     +	if !ok {
                     +		return nil, nil, fmt.Errorf("Unable to get sandbox for %s(%s) in for %s", ep.Name(), ep.ID(), n.ID())
                     +	}
                     +	ep = sb.getEndpoint(ep.ID())
                     +	if ep == nil {
                     +		return nil, nil, fmt.Errorf("Load balancing endpoint %s(%s) removed from %s", ep.Name(), ep.ID(), n.ID())
                     +	}
                     +	return ep, sb, nil
+                     }
                     -// Add loadbalancer backend to all sandboxes which has a connection to
                     -// this network. If needed add the service as well.
                     -func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig) {
                     -	n.WalkEndpoints(func(e Endpoint) bool {
                     -		ep := e.(*endpoint)
                     -		if sb, ok := ep.getSandbox(); ok {
                     -			if !sb.isEndpointPopulated(ep) {
                     -				return false
                     -			}
+                    -
                     -			var gwIP net.IP
                     -			if ep := sb.getGatewayEndpoint(); ep != nil {
                     -				gwIP = ep.Iface().Address().IP
                     -			}
+                    -
                     -			sb.addLBBackend(ip, vip, lb.fwMark, ingressPorts, ep.Iface().Address(), gwIP, n.ingress)
                     -		}
+                    -
                     -		return false
                     -	})
                     -}
+                    -
                     -// Remove loadbalancer backend from all sandboxes which has a
                     -// connection to this network. If needed remove the service entry as
                     -// well, as specified by the rmService bool.
                     -func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool, fullRemove bool) {
                     -	n.WalkEndpoints(func(e Endpoint) bool {
                     -		ep := e.(*endpoint)
                     -		if sb, ok := ep.getSandbox(); ok {
                     -			if !sb.isEndpointPopulated(ep) {
                     -				return false
                     -			}
+                    -
                     -			var gwIP net.IP
                     -			if ep := sb.getGatewayEndpoint(); ep != nil {
                     -				gwIP = ep.Iface().Address().IP
                     -			}
+                    -
                     -			sb.rmLBBackend(ip, vip, lb.fwMark, ingressPorts, ep.Iface().Address(), gwIP, rmService, fullRemove, n.ingress)
                     +// Searches the OS sandbox for the name of the endpoint interface
                     +// within the sandbox.   This is required for adding/removing IP
                     +// aliases to the interface.
                     +func findIfaceDstName(sb *sandbox, ep *endpoint) string {
                     +	srcName := ep.Iface().SrcName()
                     +	for _, i := range sb.osSbox.Info().Interfaces() {
                     +		if i.SrcName() == srcName {
                     +			return i.DstName()
+                     		}
+                    -
                     -		return false
                     -	})
                     +	}
                     +	return ""
+                     }
                     -// Add loadbalancer backend into one connected sandbox.
                     -func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, isIngressNetwork bool) {
                     -	if sb.osSbox == nil {
                     +// Add loadbalancer backend to the loadbalncer sandbox for the network.
                     +// If needed add the service as well.
                     +func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {
                     +	if len(lb.vip) == 0 {
                      		return
+                     	}
+                    -
                     -	if isIngressNetwork && !sb.ingress {
                     +	ep, sb, err := n.findLBEndpointSandbox()
                     +	if err != nil {
                     +		logrus.Errorf("addLBBackend %s/%s: %v", n.ID(), n.Name(), err)
                     +		return
                     +	}
                     +	if sb.osSbox == nil {
                      		return
+                     	}
                     +	eIP := ep.Iface().Address()
+                    +
                      	i, err := ipvs.New(sb.Key())
                      	if err != nil {
                      		logrus.Errorf("Failed to create an ipvs handle for sbox %s (%s,%s) for lb addition: %v", sb.ID()[0:7], sb.ContainerID()[0:7], sb.Key(), err)
@@ -176,28 +113,43 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
                      	s := &ipvs.Service{
                      		AddressFamily: nl.FAMILY_V4,
                     -		FWMark:        fwMark,
                     +		FWMark:        lb.fwMark,
                      		SchedName:     ipvs.RoundRobin,
+                     	}
                      	if !i.IsServicePresent(s) {
                     -		var filteredPorts []*PortConfig
                     +		// Add IP alias for the VIP to the endpoint
                     +		ifName := findIfaceDstName(sb, ep)
                     +		if ifName == "" {
                     +			logrus.Errorf("Failed find interface name for endpoint %s(%s) to create LB alias", ep.ID(), ep.Name())
                     +			return
                     +		}
                     +		err := sb.osSbox.AddAliasIP(ifName, &net.IPNet{IP: lb.vip, Mask: net.CIDRMask(32, 32)})
                     +		if err != nil {
                     +			logrus.Errorf("Failed add IP alias %s to network %s LB endpoint interface %s: %v", lb.vip, n.ID(), ifName, err)
                     +			return
                     +		}
+                    +
                      		if sb.ingress {
                     -			filteredPorts = filterPortConfigs(ingressPorts, false)
                     +			var gwIP net.IP
                     +			if ep := sb.getGatewayEndpoint(); ep != nil {
                     +				gwIP = ep.Iface().Address().IP
                     +			}
                     +			filteredPorts := filterPortConfigs(lb.service.ingressPorts, false)
                      			if err := programIngress(gwIP, filteredPorts, false); err != nil {
                      				logrus.Errorf("Failed to add ingress: %v", err)
                      				return
+                     			}
+                     		}
                     -		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %s (%s)", vip, fwMark, ingressPorts, sb.ID()[0:7], sb.ContainerID()[0:7])
                     -		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, false); err != nil {
                     +		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %s (%s)", lb.vip, lb.fwMark, lb.service.ingressPorts, sb.ID()[0:7], sb.ContainerID()[0:7])
                     +		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false); err != nil {
                      			logrus.Errorf("Failed to add firewall mark rule in sbox %s (%s): %v", sb.ID()[0:7], sb.ContainerID()[0:7], err)
                      			return
+                     		}
                      		if err := i.NewService(s); err != nil && err != syscall.EEXIST {
                     -			logrus.Errorf("Failed to create a new service for vip %s fwmark %d in sbox %s (%s): %v", vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
                     +			logrus.Errorf("Failed to create a new service for vip %s fwmark %d in sbox %s (%s): %v", lb.vip, lb.fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
                      			return
+                     		}
+                     	}
@@ -212,20 +164,29 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
                      	// destination.
                      	s.SchedName = ""
                      	if err := i.NewDestination(s, d); err != nil && err != syscall.EEXIST {
                     -		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
                     +		logrus.Errorf("Failed to create real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, lb.vip, lb.fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+                     	}
+                     }
                     -// Remove loadbalancer backend from one connected sandbox.
                     -func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, rmService bool, fullRemove bool, isIngressNetwork bool) {
                     -	if sb.osSbox == nil {
                     +// Remove loadbalancer backend the load balancing endpoint for this
                     +// network. If 'rmService' is true, then remove the service entry as well.
                     +// If 'fullRemove' is true then completely remove the entry, otherwise
                     +// just deweight it for now.
                     +func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullRemove bool) {
                     +	if len(lb.vip) == 0 {
                      		return
+                     	}
+                    -
                     -	if isIngressNetwork && !sb.ingress {
                     +	ep, sb, err := n.findLBEndpointSandbox()
                     +	if err != nil {
                     +		logrus.Debugf("rmLBBackend for %s/%s: %v -- probably transient state", n.ID(), n.Name(), err)
                     +		return
                     +	}
                     +	if sb.osSbox == nil {
                      		return
+                     	}
                     +	eIP := ep.Iface().Address()
+                    +
                      	i, err := ipvs.New(sb.Key())
                      	if err != nil {
                      		logrus.Errorf("Failed to create an ipvs handle for sbox %s (%s,%s) for lb removal: %v", sb.ID()[0:7], sb.ContainerID()[0:7], sb.Key(), err)
@@ -235,7 +196,7 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
                      	s := &ipvs.Service{
                      		AddressFamily: nl.FAMILY_V4,
                     -		FWMark:        fwMark,
                     +		FWMark:        lb.fwMark,
+                     	}
                      	d := &ipvs.Destination{
@@ -246,32 +207,46 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
                      	if fullRemove {
                      		if err := i.DelDestination(s, d); err != nil && err != syscall.ENOENT {
                     -			logrus.Errorf("Failed to delete real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
                     +			logrus.Errorf("Failed to delete real server %s for vip %s fwmark %d in sbox %s (%s): %v", ip, lb.vip, lb.fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+                     		}
                      	} else {
                      		d.Weight = 0
                      		if err := i.UpdateDestination(s, d); err != nil && err != syscall.ENOENT {
                     -			logrus.Errorf("Failed to set LB weight of real server %s to 0 for vip %s fwmark %d in sbox %s (%s): %v", ip, vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
                     +			logrus.Errorf("Failed to set LB weight of real server %s to 0 for vip %s fwmark %d in sbox %s (%s): %v", ip, lb.vip, lb.fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+                     		}
+                     	}
                      	if rmService {
                      		s.SchedName = ipvs.RoundRobin
                      		if err := i.DelService(s); err != nil && err != syscall.ENOENT {
                     -			logrus.Errorf("Failed to delete service for vip %s fwmark %d in sbox %s (%s): %v", vip, fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
                     +			logrus.Errorf("Failed to delete service for vip %s fwmark %d in sbox %s (%s): %v", lb.vip, lb.fwMark, sb.ID()[0:7], sb.ContainerID()[0:7], err)
+                     		}
                     -		var filteredPorts []*PortConfig
                      		if sb.ingress {
                     -			filteredPorts = filterPortConfigs(ingressPorts, true)
                     +			var gwIP net.IP
                     +			if ep := sb.getGatewayEndpoint(); ep != nil {
                     +				gwIP = ep.Iface().Address().IP
                     +			}
                     +			filteredPorts := filterPortConfigs(lb.service.ingressPorts, true)
                      			if err := programIngress(gwIP, filteredPorts, true); err != nil {
                      				logrus.Errorf("Failed to delete ingress: %v", err)
+                     			}
+                     		}
                     -		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, true); err != nil {
                     +		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true); err != nil {
                      			logrus.Errorf("Failed to delete firewall mark rule in sbox %s (%s): %v", sb.ID()[0:7], sb.ContainerID()[0:7], err)
+                     		}
+                    +
                     +		// Remove IP alias from the VIP to the endpoint
                     +		ifName := findIfaceDstName(sb, ep)
                     +		if ifName == "" {
                     +			logrus.Errorf("Failed find interface name for endpoint %s(%s) to create LB alias", ep.ID(), ep.Name())
                     +			return
                     +		}
                     +		err := sb.osSbox.RemoveAliasIP(ifName, &net.IPNet{IP: lb.vip, Mask: net.CIDRMask(32, 32)})
                     +		if err != nil {
                     +			logrus.Errorf("Failed add IP alias %s to network %s LB endpoint interface %s: %v", lb.vip, n.ID(), ifName, err)
                     +		}
+                     	}
+                     }
@@ -617,7 +592,7 @@ func fwMarker() {
                      		ingressPorts, err = readPortsFromFile(os.Args[5])
                      		if err != nil {
                      			logrus.Errorf("Failed reading ingress ports file: %v", err)
                     -			os.Exit(6)
                     +			os.Exit(2)
+                     		}
+                     	}
@@ -625,7 +600,7 @@ func fwMarker() {
                      	fwMark, err := strconv.ParseUint(os.Args[3], 10, 32)
                      	if err != nil {
                      		logrus.Errorf("bad fwmark value(%s) passed: %v", os.Args[3], err)
                     -		os.Exit(2)
                     +		os.Exit(3)
+                     	}
                      	addDelOpt := os.Args[4]
@@ -639,20 +614,20 @@ func fwMarker() {
                      	ns, err := netns.GetFromPath(os.Args[1])
                      	if err != nil {
                      		logrus.Errorf("failed get network namespace %q: %v", os.Args[1], err)
                     -		os.Exit(3)
                     +		os.Exit(4)
+                     	}
                      	defer ns.Close()
                      	if err := netns.Set(ns); err != nil {
                      		logrus.Errorf("setting into container net ns %v failed, %v", os.Args[1], err)
                     -		os.Exit(4)
                     +		os.Exit(5)
+                     	}
                      	if addDelOpt == "-A" {
                      		eIP, subnet, err := net.ParseCIDR(os.Args[6])
                      		if err != nil {
                      			logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)
                     -			os.Exit(9)
                     +			os.Exit(6)
+                     		}
                      		ruleParams := strings.Fields(fmt.Sprintf("-m ipvs --ipvs -d %s -j SNAT --to-source %s", subnet, eIP))
@@ -663,21 +638,18 @@ func fwMarker() {
                      			err := ioutil.WriteFile("/proc/sys/net/ipv4/vs/conntrack", []byte{'1', '\n'}, 0644)
                      			if err != nil {
                      				logrus.Errorf("Failed to write to /proc/sys/net/ipv4/vs/conntrack: %v", err)
                     -				os.Exit(8)
                     +				os.Exit(7)
+                     			}
+                     		}
+                     	}
                     -	rule := strings.Fields(fmt.Sprintf("-t mangle %s OUTPUT -d %s/32 -j MARK --set-mark %d", addDelOpt, vip, fwMark))
                     -	rules = append(rules, rule)
+                    -
                     -	rule = strings.Fields(fmt.Sprintf("-t nat %s OUTPUT -p icmp --icmp echo-request -d %s -j DNAT --to 127.0.0.1", addDelOpt, vip))
                     +	rule := strings.Fields(fmt.Sprintf("-t mangle %s INPUT -d %s/32 -j MARK --set-mark %d", addDelOpt, vip, fwMark))
                      	rules = append(rules, rule)
                      	for _, rule := range rules {
                      		if err := iptables.RawCombinedOutputNative(rule...); err != nil {
                      			logrus.Errorf("setting up rule failed, %v: %v", rule, err)
                     -			os.Exit(5)
                     +			os.Exit(8)
+                     		}
+                     	}
+                     }

...	...	@@ -18,7 +18,7 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, in
18	18	return fmt.Errorf("not supported")
19	19	}
20	20
21		-func (sb sandbox) populateLoadbalancers(ep endpoint) {
	21	+func (sb sandbox) populateLoadBalancers(ep endpoint) {
22	22	}
23	23
24	24	func arrangeIngressFilterRule() {

@@ -19,7 +19,13 @@ func init() {
                      	lbPolicylistMap = make(map[*loadBalancer]*policyLists)
+                     }
                     -func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig) {
                     +func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {
                     +	if len(lb.vip) == 0 {
                     +		return
                     +	}
+                    +
                     +	vip := lb.vip
                     +	ingressPorts := lb.service.ingressPorts
                      	if system.GetOSVersion().Build > 16236 {
                      		lb.Lock()
@@ -117,11 +123,15 @@ func (n *network) addLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []
+                     	}
+                     }
                     -func (n *network) rmLBBackend(ip, vip net.IP, lb *loadBalancer, ingressPorts []*PortConfig, rmService bool, fullRemove bool) {
                     +func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullRemove bool) {
                     +	if len(lb.vip) == 0 {
                     +		return
                     +	}
+                    +
                      	if system.GetOSVersion().Build > 16236 {
                      		if numEnabledBackends(lb) > 0 {
                      			//Reprogram HNS (actually VFP) with the existing backends.
                     -			n.addLBBackend(ip, vip, lb, ingressPorts)
                     +			n.addLBBackend(ip, lb)
                      		} else {
                      			lb.Lock()
                      			defer lb.Unlock()
@@ -156,7 +166,7 @@ func numEnabledBackends(lb *loadBalancer) int {
                      	return nEnabled
+                     }
                     -func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
                     +func (sb *sandbox) populateLoadBalancers(ep *endpoint) {
+                     }
                      func arrangeIngressFilterRule() {