Browse code

Update moby to use scalable-lb libnetwork APIs

This patch is required for the updated version of libnetwork and entails
two minor changes.

First, it uses the new libnetwork.NetworkDeleteOptionRemoveLB option to
the network.Delete() method to automatically remove the load balancing
endpoint for ingress networks. This allows removal of the
deleteLoadBalancerSandbox() function whose functionality is now within
libnetwork.

The second change is to allocate a load balancer endpoint IP address for
all overlay networks rather than just "ingress" and windows overlay
networks. Swarmkit is already performing this allocation, but moby was
not making use of these IP addresses for Linux overlay networks (except
ingress). The current version of libnetwork makes use of these IP
addresses by creating a load balancing sandbox and endpoint similar to
ingress's for all overlay network and putting all load balancing state
for a given node in that sandbox only. This reduces the amount of linux
kernel state required per node.

In the prior scheme, libnetwork would program each container's network
namespace with every piece of load balancing state for every other
container that shared *any* network with the first container. This
meant that the amount of kernel state on a given node scaled with the
square of the number of services in the cluster and with the square of
the number of containers per service. With the new scheme, kernel state
at each node scales linearly with the number of services and the number
of containers per service. This also reduces the number of system calls
required to add or remove tasks and containers. Previously the number
of system calls required grew linearly with the number of other
tasks that shared a network with the container. Now the number of
system calls grows linearly only with the number of networks that the
task/container is attached to. This results in a significant
performance improvement when adding and removing services to a cluster
that already heavily loaded.

The primary disadvantage to this scheme is that it requires the
allocation of an additional IP address per node per subnet for every
node in the cluster that has a task on the given subnet. However, as
mentioned, swarmkit is already allocating these IP addresses for every
node and they are going unused. Future swarmkit modifications should be
examined to only allocate said IP addresses when nodes actually require
them.

Signed-off-by: Chris Telfer <ctelfer@docker.com>

Chris Telfer authored on 2018/04/20 00:39:51
Showing 1 changed files
... ...
@@ -4,7 +4,6 @@ import (
4 4
 	"context"
5 5
 	"fmt"
6 6
 	"net"
7
-	"runtime"
8 7
 	"sort"
9 8
 	"strconv"
10 9
 	"strings"
... ...
@@ -232,9 +231,7 @@ func (daemon *Daemon) releaseIngress(id string) {
232 232
 		return
233 233
 	}
234 234
 
235
-	daemon.deleteLoadBalancerSandbox(n)
236
-
237
-	if err := n.Delete(); err != nil {
235
+	if err := n.Delete(libnetwork.NetworkDeleteOptionRemoveLB); err != nil {
238 236
 		logrus.Errorf("Failed to delete ingress network %s: %v", n.ID(), err)
239 237
 		return
240 238
 	}
... ...
@@ -351,7 +348,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
351 351
 		nwOptions = append(nwOptions, libnetwork.NetworkOptionConfigFrom(create.ConfigFrom.Network))
352 352
 	}
353 353
 
354
-	if agent && driver == "overlay" && (create.Ingress || runtime.GOOS == "windows") {
354
+	if agent && driver == "overlay" {
355 355
 		nodeIP, exists := daemon.GetAttachmentStore().GetIPForNetwork(id)
356 356
 		if !exists {
357 357
 			return nil, fmt.Errorf("Failed to find a load balancer IP to use for network: %v", id)
... ...
@@ -514,37 +511,6 @@ func (daemon *Daemon) DeleteNetwork(networkID string) error {
514 514
 	return daemon.deleteNetwork(n, false)
515 515
 }
516 516
 
517
-func (daemon *Daemon) deleteLoadBalancerSandbox(n libnetwork.Network) {
518
-	controller := daemon.netController
519
-
520
-	//The only endpoint left should be the LB endpoint (nw.Name() + "-endpoint")
521
-	endpoints := n.Endpoints()
522
-	if len(endpoints) == 1 {
523
-		sandboxName := n.Name() + "-sbox"
524
-
525
-		info := endpoints[0].Info()
526
-		if info != nil {
527
-			sb := info.Sandbox()
528
-			if sb != nil {
529
-				if err := sb.DisableService(); err != nil {
530
-					logrus.Warnf("Failed to disable service on sandbox %s: %v", sandboxName, err)
531
-					//Ignore error and attempt to delete the load balancer endpoint
532
-				}
533
-			}
534
-		}
535
-
536
-		if err := endpoints[0].Delete(true); err != nil {
537
-			logrus.Warnf("Failed to delete endpoint %s (%s) in %s: %v", endpoints[0].Name(), endpoints[0].ID(), sandboxName, err)
538
-			//Ignore error and attempt to delete the sandbox.
539
-		}
540
-
541
-		if err := controller.SandboxDestroy(sandboxName); err != nil {
542
-			logrus.Warnf("Failed to delete %s sandbox: %v", sandboxName, err)
543
-			//Ignore error and attempt to delete the network.
544
-		}
545
-	}
546
-}
547
-
548 517
 func (daemon *Daemon) deleteNetwork(nw libnetwork.Network, dynamic bool) error {
549 518
 	if runconfig.IsPreDefinedNetwork(nw.Name()) && !dynamic {
550 519
 		err := fmt.Errorf("%s is a pre-defined network and cannot be removed", nw.Name())