@@ -103,7 +103,7 @@ github.com/docker/containerd 03e5862ec0d8d3b3f750e19fca3ee367e13c090e

 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4

 # cluster

-github.com/docker/swarmkit 62d835f478b2e4fd2768deb88fb3b32e334faaee

+github.com/docker/swarmkit 98620dd1ddfcc03d8f4b0d2910ecf6b52918a731

 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9

 github.com/gogo/protobuf v0.3

 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a

@@ -148,7 +148,7 @@ func (w *worker) Assign(ctx context.Context, assignments []*api.AssignmentChange

 // Update updates the set of tasks and secret for the worker.

 // Tasks in the added set will be added to the worker, and tasks in the removed set

 // will be removed from the worker

-// Serets in the added set will be added to the worker, and secrets in the removed set

+// Secrets in the added set will be added to the worker, and secrets in the removed set

 // will be removed from the worker.

 func (w *worker) Update(ctx context.Context, assignments []*api.AssignmentChange) error {

 	w.mu.Lock()

@@ -140,7 +140,12 @@ type ServiceSpec struct {

 	//	*ServiceSpec_Global

 	Mode isServiceSpec_Mode `protobuf_oneof:"mode"`

 	// UpdateConfig controls the rate and policy of updates.

-	Update   *UpdateConfig              `protobuf:"bytes,6,opt,name=update" json:"update,omitempty"`

+	Update *UpdateConfig `protobuf:"bytes,6,opt,name=update" json:"update,omitempty"`

+	// ServiceSpec.Networks has been deprecated and is replaced by

+	// Networks field in Task (TaskSpec.Networks).

+	// This field (ServiceSpec.Networks) is kept for compatibility.

+	// In case TaskSpec.Networks does not exist, ServiceSpec.Networks

+	// is still honored if it exists.

 	Networks []*NetworkAttachmentConfig `protobuf:"bytes,7,rep,name=networks" json:"networks,omitempty"`

 	// Service endpoint specifies the user provided configuration

 	// to properly discover and load balance a service.

@@ -72,6 +72,11 @@ message ServiceSpec {

 	// UpdateConfig controls the rate and policy of updates.

 	UpdateConfig update = 6;

+	// ServiceSpec.Networks has been deprecated and is replaced by

+	// Networks field in Task (TaskSpec.Networks).

+	// This field (ServiceSpec.Networks) is kept for compatibility.

+	// In case TaskSpec.Networks does not exist, ServiceSpec.Networks

+	// is still honored if it exists.

 	repeated NetworkAttachmentConfig networks = 7 [deprecated=true];

 	// Service endpoint specifies the user provided configuration

@@ -19,7 +19,7 @@ import (

 type localRequestKeyType struct{}

 // LocalRequestKey is a context key to mark a request that originating on the

-// local node. The assocated value is a RemoteNodeInfo structure describing the

+// local node. The associated value is a RemoteNodeInfo structure describing the

 // local node.

 var LocalRequestKey = localRequestKeyType{}

@@ -103,26 +103,9 @@ func makeExternalSignRequest(ctx context.Context, client *http.Client, url strin

 	if err != nil {

 		return nil, recoverableErr{err: errors.Wrap(err, "unable to perform certificate signing request")}

 	}

-

-	doneReading := make(chan struct{})

-	bodyClosed := make(chan struct{})

-	go func() {

-		select {

-		case <-ctx.Done():

-		case <-doneReading:

-		}

-		resp.Body.Close()

-		close(bodyClosed)

-	}()

+	defer resp.Body.Close()

 	body, err := ioutil.ReadAll(resp.Body)

-	close(doneReading)

-	<-bodyClosed

-	select {

-	case <-ctx.Done():

-		return nil, ctx.Err()

-	default:

-	}

 	if err != nil {

 		return nil, recoverableErr{err: errors.Wrap(err, "unable to read CSR response body")}

 	}

@@ -353,7 +353,7 @@ func (s *Server) issueRenewCertificate(ctx context.Context, nodeID string, csr [

 	}, nil

 }

-// GetRootCACertificate returns the certificate of the Root CA. It is used as a convinience for distributing

+// GetRootCACertificate returns the certificate of the Root CA. It is used as a convenience for distributing

 // the root of trust for the swarm. Clients should be using the CA hash to verify if they weren't target to

 // a MiTM. If they fail to do so, node bootstrap works with TOFU semantics.

 func (s *Server) GetRootCACertificate(ctx context.Context, request *api.GetRootCACertificateRequest) (*api.GetRootCACertificateResponse, error) {

@@ -96,9 +96,9 @@ func (c *Conn) Close(success bool) error {

 	}

 	if success {

-		c.remotes.ObserveIfExists(c.peer, -remotes.DefaultObservationWeight)

-	} else {

 		c.remotes.ObserveIfExists(c.peer, remotes.DefaultObservationWeight)

+	} else {

+		c.remotes.ObserveIfExists(c.peer, -remotes.DefaultObservationWeight)

 	}

 	return c.ClientConn.Close()

@@ -29,6 +29,21 @@ func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context {

 	return context.WithValue(ctx, loggerKey{}, logger)

 }

+// WithFields returns a new context with added fields to logger.

+func WithFields(ctx context.Context, fields logrus.Fields) context.Context {

+	logger := ctx.Value(loggerKey{})

+

+	if logger == nil {

+		logger = L

+	}

+	return WithLogger(ctx, logger.(*logrus.Entry).WithFields(fields))

+}

+

+// WithField is convenience wrapper around WithFields.

+func WithField(ctx context.Context, key, value string) context.Context {

+	return WithFields(ctx, logrus.Fields{key: value})

+}

+

 // GetLogger retrieves the current logger from the context. If no logger is

 // available, the default logger is returned.

 func GetLogger(ctx context.Context) *logrus.Entry {

@@ -3,6 +3,7 @@ package allocator

 import (

 	"sync"

+	"github.com/docker/docker/pkg/plugingetter"

 	"github.com/docker/go-events"

 	"github.com/docker/swarmkit/manager/state"

 	"github.com/docker/swarmkit/manager/state/store"

@@ -27,6 +28,9 @@ type Allocator struct {

 	stopChan chan struct{}

 	// doneChan is closed when the allocator is finished running.

 	doneChan chan struct{}

+

+	// pluginGetter provides access to docker's plugin inventory.

+	pluginGetter plugingetter.PluginGetter

 }

 // taskBallot controls how the voting for task allocation is

@@ -67,14 +71,15 @@ type allocActor struct {

 // New returns a new instance of Allocator for use during allocation

 // stage of the manager.

-func New(store *store.MemoryStore) (*Allocator, error) {

+func New(store *store.MemoryStore, pg plugingetter.PluginGetter) (*Allocator, error) {

 	a := &Allocator{

 		store: store,

 		taskBallot: &taskBallot{

 			votes: make(map[string][]string),

 		},

-		stopChan: make(chan struct{}),

-		doneChan: make(chan struct{}),

+		stopChan:     make(chan struct{}),

+		doneChan:     make(chan struct{}),

+		pluginGetter: pg,

 	}

 	return a, nil

@@ -73,7 +73,7 @@ type networkContext struct {

 }

 func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {

-	na, err := networkallocator.New()

+	na, err := networkallocator.New(a.pluginGetter)

 	if err != nil {

 		return err

 	}

@@ -4,7 +4,7 @@ import (

 	"fmt"

 	"net"

-	"github.com/docker/docker/pkg/plugins"

+	"github.com/docker/docker/pkg/plugingetter"

 	"github.com/docker/libnetwork/datastore"

 	"github.com/docker/libnetwork/driverapi"

 	"github.com/docker/libnetwork/drvregistry"

@@ -49,7 +49,7 @@ type NetworkAllocator struct {

 	nodes map[string]struct{}

 }

-// Local in-memory state related to netwok that need to be tracked by NetworkAllocator

+// Local in-memory state related to network that need to be tracked by NetworkAllocator

 type network struct {

 	// A local cache of the store object.

 	nw *api.Network

@@ -69,7 +69,7 @@ type initializer struct {

 }

 // New returns a new NetworkAllocator handle

-func New() (*NetworkAllocator, error) {

+func New(pg plugingetter.PluginGetter) (*NetworkAllocator, error) {

 	na := &NetworkAllocator{

 		networks: make(map[string]*network),

 		services: make(map[string]struct{}),

@@ -79,7 +79,7 @@ func New() (*NetworkAllocator, error) {

 	// There are no driver configurations and notification

 	// functions as of now.

-	reg, err := drvregistry.New(nil, nil, nil, nil, nil)

+	reg, err := drvregistry.New(nil, nil, nil, nil, pg)

 	if err != nil {

 		return nil, err

 	}

@@ -133,7 +133,7 @@ func (na *NetworkAllocator) getNetwork(id string) *network {

 }

 // Deallocate frees all the general and driver specific resources

-// whichs were assigned to the passed network.

+// which were assigned to the passed network.

 func (na *NetworkAllocator) Deallocate(n *api.Network) error {

 	localNet := na.getNetwork(n.ID)

 	if localNet == nil {

@@ -657,7 +657,11 @@ func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, str

 }

 func (na *NetworkAllocator) loadDriver(name string) error {

-	_, err := plugins.Get(name, driverapi.NetworkPluginEndpointType)

+	pg := na.drvRegistry.GetPluginGetter()

+	if pg == nil {

+		return fmt.Errorf("plugin store is unintialized")

+	}

+	_, err := pg.Get(name, driverapi.NetworkPluginEndpointType, plugingetter.Lookup)

 	return err

 }

@@ -17,12 +17,12 @@ const (

 	dynamicPortEnd = 32767

 	// The start of master port range which will hold all the

-	// allocation state of ports allocated so far regerdless of

+	// allocation state of ports allocated so far regardless of

 	// whether it was user defined or not.

 	masterPortStart = 1

 	// The end of master port range which will hold all the

-	// allocation state of ports allocated so far regerdless of

+	// allocation state of ports allocated so far regardless of

 	// whether it was user defined or not.

 	masterPortEnd = 65535

 )

@@ -65,7 +65,7 @@ func (ps allocatedPorts) addState(p *api.PortConfig) {

 // Note multiple dynamically allocated ports might exists. In this case,

 // we will remove only at a time so both allocated ports are tracked.

 //

-// Note becasue of the potential co-existence of user-defined and dynamically

+// Note because of the potential co-existence of user-defined and dynamically

 // allocated ports, delState has to be called for user-defined port first.

 // dynamically allocated ports should be removed later.

 func (ps allocatedPorts) delState(p *api.PortConfig) *api.PortConfig {

@@ -277,7 +277,7 @@ func (pa *portAllocator) isPortsAllocated(s *api.Service) bool {

 	}

 	// If service has allocated endpoint while has no user-defined endpoint,

-	// we assume allocated endpoints are redudant, and they need deallocated.

+	// we assume allocated endpoints are redundant, and they need deallocated.

 	// If service has no allocated endpoint while has user-defined endpoint,

 	// we assume it is not allocated.

 	if (s.Endpoint != nil && s.Spec.Endpoint == nil) ||

@@ -502,7 +502,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe

 		}

 		if !reflect.DeepEqual(requestSpecNetworks, specNetworks) {

-			return errNetworkUpdateNotSupported

+			return grpc.Errorf(codes.Unimplemented, errNetworkUpdateNotSupported.Error())

 		}

 		// Check to see if all the secrets being added exist as objects

@@ -516,11 +516,11 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe

 		// with service mode change (comparing current config with previous config).

 		// proper way to change service mode is to delete and re-add.

 		if reflect.TypeOf(service.Spec.Mode) != reflect.TypeOf(request.Spec.Mode) {

-			return errModeChangeNotAllowed

+			return grpc.Errorf(codes.Unimplemented, errModeChangeNotAllowed.Error())

 		}

 		if service.Spec.Annotations.Name != request.Spec.Annotations.Name {

-			return errRenameNotSupported

+			return grpc.Errorf(codes.Unimplemented, errRenameNotSupported.Error())

 		}

 		service.Meta.Version = *request.ServiceVersion

@@ -106,6 +106,7 @@ type nodeUpdate struct {

 // Dispatcher is responsible for dispatching tasks and tracking agent health.

 type Dispatcher struct {

 	mu                   sync.Mutex

+	wg                   sync.WaitGroup

 	nodes                *nodeStore

 	store                *store.MemoryStore

 	mgrQueue             *watch.Queue

@@ -216,6 +217,9 @@ func (d *Dispatcher) Run(ctx context.Context) error {

 	defer cancel()

 	d.ctx, d.cancel = context.WithCancel(ctx)

+	ctx = d.ctx

+	d.wg.Add(1)

+	defer d.wg.Done()

 	d.mu.Unlock()

 	publishManagers := func(peers []*api.Peer) {

@@ -240,10 +244,10 @@ func (d *Dispatcher) Run(ctx context.Context) error {

 		case ev := <-peerWatcher:

 			publishManagers(ev.([]*api.Peer))

 		case <-d.processUpdatesTrigger:

-			d.processUpdates()

+			d.processUpdates(ctx)

 			batchTimer.Reset(maxBatchInterval)

 		case <-batchTimer.C:

-			d.processUpdates()

+			d.processUpdates(ctx)

 			batchTimer.Reset(maxBatchInterval)

 		case v := <-configWatcher:

 			cluster := v.(state.EventUpdateCluster)

@@ -260,7 +264,7 @@ func (d *Dispatcher) Run(ctx context.Context) error {

 			d.networkBootstrapKeys = cluster.Cluster.NetworkBootstrapKeys

 			d.mu.Unlock()

 			d.keyMgrQueue.Publish(cluster.Cluster.NetworkBootstrapKeys)

-		case <-d.ctx.Done():

+		case <-ctx.Done():

 			return nil

 		}

 	}

@@ -287,17 +291,20 @@ func (d *Dispatcher) Stop() error {

 	d.mgrQueue.Close()

 	d.keyMgrQueue.Close()

+	d.wg.Wait()

+

 	return nil

 }

-func (d *Dispatcher) isRunningLocked() error {

+func (d *Dispatcher) isRunningLocked() (context.Context, error) {

 	d.mu.Lock()

 	if !d.isRunning() {

 		d.mu.Unlock()

-		return grpc.Errorf(codes.Aborted, "dispatcher is stopped")

+		return nil, grpc.Errorf(codes.Aborted, "dispatcher is stopped")

 	}

+	ctx := d.ctx

 	d.mu.Unlock()

-	return nil

+	return ctx, nil

 }

 func (d *Dispatcher) markNodesUnknown(ctx context.Context) error {

@@ -377,7 +384,7 @@ func (d *Dispatcher) isRunning() bool {

 // markNodeReady updates the description of a node, updates its address, and sets status to READY

 // this is used during registration when a new node description is provided

 // and during node updates when the node description changes

-func (d *Dispatcher) markNodeReady(nodeID string, description *api.NodeDescription, addr string) error {

+func (d *Dispatcher) markNodeReady(ctx context.Context, nodeID string, description *api.NodeDescription, addr string) error {

 	d.nodeUpdatesLock.Lock()

 	d.nodeUpdates[nodeID] = nodeUpdate{

 		status: &api.NodeStatus{

@@ -396,8 +403,8 @@ func (d *Dispatcher) markNodeReady(nodeID string, description *api.NodeDescripti

 	if numUpdates >= maxBatchItems {

 		select {

 		case d.processUpdatesTrigger <- struct{}{}:

-		case <-d.ctx.Done():

-			return d.ctx.Err()

+		case <-ctx.Done():

+			return ctx.Err()

 		}

 	}

@@ -405,8 +412,8 @@ func (d *Dispatcher) markNodeReady(nodeID string, description *api.NodeDescripti

 	// Wait until the node update batch happens before unblocking register.

 	d.processUpdatesLock.Lock()

 	select {

-	case <-d.ctx.Done():

-		return d.ctx.Err()

+	case <-ctx.Done():

+		return ctx.Err()

 	default:

 	}

 	d.processUpdatesCond.Wait()

@@ -431,7 +438,8 @@ func nodeIPFromContext(ctx context.Context) (string, error) {

 // register is used for registration of node with particular dispatcher.

 func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, error) {

 	// prevent register until we're ready to accept it

-	if err := d.isRunningLocked(); err != nil {

+	dctx, err := d.isRunningLocked()

+	if err != nil {

 		return "", err

 	}

@@ -453,7 +461,7 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a

 		log.G(ctx).Debugf(err.Error())

 	}

-	if err := d.markNodeReady(nodeID, description, addr); err != nil {

+	if err := d.markNodeReady(dctx, nodeID, description, addr); err != nil {

 		return "", err

 	}

@@ -496,7 +504,8 @@ func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStat

 	}

 	log := log.G(ctx).WithFields(fields)

-	if err := d.isRunningLocked(); err != nil {

+	dctx, err := d.isRunningLocked()

+	if err != nil {

 		return nil, err

 	}

@@ -542,13 +551,13 @@ func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStat

 	if numUpdates >= maxBatchItems {

 		select {

 		case d.processUpdatesTrigger <- struct{}{}:

-		case <-d.ctx.Done():

+		case <-dctx.Done():

 		}

 	}

 	return nil, nil

 }

-func (d *Dispatcher) processUpdates() {

+func (d *Dispatcher) processUpdates(ctx context.Context) {

 	var (

 		taskUpdates map[string]*api.TaskStatus

 		nodeUpdates map[string]nodeUpdate

@@ -571,7 +580,7 @@ func (d *Dispatcher) processUpdates() {

 		return

 	}

-	log := log.G(d.ctx).WithFields(logrus.Fields{

+	log := log.G(ctx).WithFields(logrus.Fields{

 		"method": "(*Dispatcher).processUpdates",

 	})

@@ -661,7 +670,8 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe

 	}

 	nodeID := nodeInfo.NodeID

-	if err := d.isRunningLocked(); err != nil {

+	dctx, err := d.isRunningLocked()

+	if err != nil {

 		return err

 	}

@@ -763,8 +773,8 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe

 				break batchingLoop

 			case <-stream.Context().Done():

 				return stream.Context().Err()

-			case <-d.ctx.Done():

-				return d.ctx.Err()

+			case <-dctx.Done():

+				return dctx.Err()

 			}

 		}

@@ -783,7 +793,8 @@ func (d *Dispatcher) Assignments(r *api.AssignmentsRequest, stream api.Dispatche

 	}

 	nodeID := nodeInfo.NodeID

-	if err := d.isRunningLocked(); err != nil {

+	dctx, err := d.isRunningLocked()

+	if err != nil {

 		return err

 	}

@@ -1075,8 +1086,8 @@ func (d *Dispatcher) Assignments(r *api.AssignmentsRequest, stream api.Dispatche

 				break batchingLoop

 			case <-stream.Context().Done():

 				return stream.Context().Err()

-			case <-d.ctx.Done():

-				return d.ctx.Err()

+			case <-dctx.Done():

+				return dctx.Err()

 			}

 		}

@@ -1197,16 +1208,14 @@ func (d *Dispatcher) moveTasksToOrphaned(nodeID string) error {

 // markNodeNotReady sets the node state to some state other than READY

 func (d *Dispatcher) markNodeNotReady(id string, state api.NodeStatus_State, message string) error {

-	if err := d.isRunningLocked(); err != nil {

+	dctx, err := d.isRunningLocked()

+	if err != nil {

 		return err

 	}

 	// Node is down. Add it to down nodes so that we can keep

 	// track of tasks assigned to the node.

-	var (

-		node *api.Node

-		err  error

-	)

+	var node *api.Node

 	d.store.View(func(readTx store.ReadTx) {

 		node = store.GetNode(readTx, id)

 		if node == nil {

@@ -1219,7 +1228,7 @@ func (d *Dispatcher) markNodeNotReady(id string, state api.NodeStatus_State, mes

 	expireFunc := func() {

 		if err := d.moveTasksToOrphaned(id); err != nil {

-			log.G(context.TODO()).WithError(err).Error(`failed to move all tasks to "ORPHANED" state`)

+			log.G(dctx).WithError(err).Error(`failed to move all tasks to "ORPHANED" state`)

 		}

 		d.downNodes.Delete(id)

@@ -1243,7 +1252,7 @@ func (d *Dispatcher) markNodeNotReady(id string, state api.NodeStatus_State, mes

 	if numUpdates >= maxBatchItems {

 		select {

 		case d.processUpdatesTrigger <- struct{}{}:

-		case <-d.ctx.Done():

+		case <-dctx.Done():

 		}

 	}

@@ -1291,7 +1300,8 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio

 	}

 	nodeID := nodeInfo.NodeID

-	if err := d.isRunningLocked(); err != nil {

+	dctx, err := d.isRunningLocked()

+	if err != nil {

 		return err

 	}

@@ -1310,7 +1320,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio

 			log.G(ctx).Debugf(err.Error())

 		}

 		// update the node description

-		if err := d.markNodeReady(nodeID, r.Description, addr); err != nil {

+		if err := d.markNodeReady(dctx, nodeID, r.Description, addr); err != nil {

 			return err

 		}

 	}

@@ -1401,7 +1411,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio

 			return stream.Context().Err()

 		case <-node.Disconnect:

 			disconnect = true

-		case <-d.ctx.Done():

+		case <-dctx.Done():

 			disconnect = true

 		case ev := <-keyMgrUpdates:

 			netKeys = ev.([]*api.EncryptionKey)

@@ -14,6 +14,7 @@ import (

 	"time"

 	"github.com/Sirupsen/logrus"

+	"github.com/docker/docker/pkg/plugingetter"

 	"github.com/docker/go-events"

 	"github.com/docker/swarmkit/api"

 	"github.com/docker/swarmkit/ca"

@@ -105,6 +106,9 @@ type Config struct {

 	// Availability allows a user to control the current scheduling status of a node

 	Availability api.NodeSpec_Availability

+

+	// PluginGetter provides access to docker's plugin inventory.

+	PluginGetter plugingetter.PluginGetter

 }

 // Manager is the cluster manager for Swarm.

@@ -478,7 +482,7 @@ func (m *Manager) Stop(ctx context.Context, clearData bool) {

 	// starting up.

 	<-m.started

-	// the mutex stops us from trying to stop while we're alrady stopping, or

+	// the mutex stops us from trying to stop while we're already stopping, or

 	// from returning before we've finished stopping.

 	m.mu.Lock()

 	defer m.mu.Unlock()

@@ -833,7 +837,7 @@ func (m *Manager) becomeLeader(ctx context.Context) {

 	// shutdown underlying manager processes when leadership is

 	// lost.

-	m.allocator, err = allocator.New(s)

+	m.allocator, err = allocator.New(s, m.config.PluginGetter)

 	if err != nil {

 		log.G(ctx).WithError(err).Error("failed to create allocator")

 		// TODO(stevvooe): It doesn't seem correct here to fail

@@ -406,7 +406,11 @@ func (u *Updater) updateTask(ctx context.Context, slot orchestrator.Slot, update

 	}

 	if delayStartCh != nil {

-		<-delayStartCh

+		select {

+		case <-delayStartCh:

+		case <-u.stopChan:

+			return nil

+		}

 	}

 	// Wait for the new task to come up.

@@ -456,7 +460,11 @@ func (u *Updater) useExistingTask(ctx context.Context, slot orchestrator.Slot, e

 		}

 		if delayStartCh != nil {

-			<-delayStartCh

+			select {

+			case <-delayStartCh:

+			case <-u.stopChan:

+				return nil

+			}

 		}

 	}

@@ -39,7 +39,7 @@ func newNodeInfo(n *api.Node, tasks map[string]*api.Task, availableResources api

 	return nodeInfo

 }

-// addTask removes a task from nodeInfo if it's tracked there, and returns true

+// removeTask removes a task from nodeInfo if it's tracked there, and returns true

 // if nodeInfo was modified.

 func (nodeInfo *NodeInfo) removeTask(t *api.Task) bool {

 	oldTask, ok := nodeInfo.Tasks[t.ID]

@@ -532,7 +532,8 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]

 	}

 	nodes := s.nodeSet.findBestNodes(len(taskGroup), s.pipeline.Process, nodeLess)

-	if len(nodes) == 0 {

+	nodeCount := len(nodes)

+	if nodeCount == 0 {

 		s.noSuitableNode(ctx, taskGroup, schedulingDecisions)

 		return

 	}

@@ -540,7 +541,7 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]

 	failedConstraints := make(map[int]bool) // key is index in nodes slice

 	nodeIter := 0

 	for taskID, t := range taskGroup {

-		n := &nodes[nodeIter%len(nodes)]

+		n := &nodes[nodeIter%nodeCount]

 		log.G(ctx).WithField("task.id", t.ID).Debugf("assigning to node %s", n.ID)

 		newT := *t

@@ -555,16 +556,16 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]

 		nodeInfo, err := s.nodeSet.nodeInfo(n.ID)

 		if err == nil && nodeInfo.addTask(&newT) {

 			s.nodeSet.updateNode(nodeInfo)

-			nodes[nodeIter%len(nodes)] = nodeInfo

+			nodes[nodeIter%nodeCount] = nodeInfo

 		}

 		schedulingDecisions[taskID] = schedulingDecision{old: t, new: &newT}

 		delete(taskGroup, taskID)

-		if nodeIter+1 < len(nodes) {

+		if nodeIter+1 < nodeCount {

 			// First pass fills the nodes until they have the same

 			// number of tasks from this service.

-			nextNode := nodes[(nodeIter+1)%len(nodes)]

+			nextNode := nodes[(nodeIter+1)%nodeCount]

 			if nodeLess(&nextNode, &nodeInfo) {

 				nodeIter++

 			}

@@ -575,10 +576,10 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]

 		}

 		origNodeIter := nodeIter

-		for failedConstraints[nodeIter%len(nodes)] || !s.pipeline.Process(&nodes[nodeIter%len(nodes)]) {

-			failedConstraints[nodeIter%len(nodes)] = true

+		for failedConstraints[nodeIter%nodeCount] || !s.pipeline.Process(&nodes[nodeIter%nodeCount]) {

+			failedConstraints[nodeIter%nodeCount] = true

 			nodeIter++

-			if nodeIter-origNodeIter == len(nodes) {

+			if nodeIter-origNodeIter == nodeCount {

 				// None of the nodes meet the constraints anymore.

 				s.noSuitableNode(ctx, taskGroup, schedulingDecisions)

 				return

@@ -2,16 +2,12 @@ package membership

 import (

 	"errors"

-	"fmt"

 	"sync"

-	"google.golang.org/grpc"

-

 	"github.com/coreos/etcd/raft/raftpb"

 	"github.com/docker/swarmkit/api"

 	"github.com/docker/swarmkit/watch"

 	"github.com/gogo/protobuf/proto"

-	"golang.org/x/net/context"

 )

 var (

@@ -25,26 +21,19 @@ var (

 	ErrConfigChangeInvalid = errors.New("membership: ConfChange type should be either AddNode, RemoveNode or UpdateNode")

 	// ErrCannotUnmarshalConfig is thrown when a node cannot unmarshal a configuration change

 	ErrCannotUnmarshalConfig = errors.New("membership: cannot unmarshal configuration change")

+	// ErrMemberRemoved is thrown when a node was removed from the cluster

+	ErrMemberRemoved = errors.New("raft: member was removed from the cluster")

 )

-// deferredConn used to store removed members connection for some time.

-// We need this in case if removed node is redirector or endpoint of ControlAPI call.

-type deferredConn struct {

-	tick int

-	conn *grpc.ClientConn

-}

-

 // Cluster represents a set of active

 // raft Members

 type Cluster struct {

-	mu           sync.RWMutex

-	members      map[uint64]*Member

-	deferedConns map[*deferredConn]struct{}

+	mu      sync.RWMutex

+	members map[uint64]*Member

 	// removed contains the list of removed Members,

 	// those ids cannot be reused

-	removed        map[uint64]bool

-	heartbeatTicks int

+	removed map[uint64]bool

 	PeersBroadcast *watch.Queue

 }

@@ -52,74 +41,19 @@ type Cluster struct {

 // Member represents a raft Cluster Member

 type Member struct {

 	*api.RaftMember

-

-	Conn         *grpc.ClientConn

-	tick         int

-	active       bool

-	lastSeenHost string

-}

-

-// HealthCheck sends a health check RPC to the member and returns the response.

-func (member *Member) HealthCheck(ctx context.Context) error {

-	healthClient := api.NewHealthClient(member.Conn)

-	resp, err := healthClient.Check(ctx, &api.HealthCheckRequest{Service: "Raft"})

-	if err != nil {

-		return err

-	}

-	if resp.Status != api.HealthCheckResponse_SERVING {

-		return fmt.Errorf("health check returned status %s", resp.Status.String())

-	}

-	return nil

 }

 // NewCluster creates a new Cluster neighbors list for a raft Member.

-// Member marked as inactive if there was no call ReportActive for heartbeatInterval.

-func NewCluster(heartbeatTicks int) *Cluster {

+func NewCluster() *Cluster {

 	// TODO(abronan): generate Cluster ID for federation

 	return &Cluster{

 		members:        make(map[uint64]*Member),

 		removed:        make(map[uint64]bool),

-		deferedConns:   make(map[*deferredConn]struct{}),

-		heartbeatTicks: heartbeatTicks,

 		PeersBroadcast: watch.NewQueue(),

 	}

 }

-func (c *Cluster) handleInactive() {

-	for _, m := range c.members {

-		if !m.active {

-			continue

-		}

-		m.tick++

-		if m.tick > c.heartbeatTicks {

-			m.active = false

-			if m.Conn != nil {

-				m.Conn.Close()

-			}

-		}

-	}

-}

-

-func (c *Cluster) handleDeferredConns() {

-	for dc := range c.deferedConns {

-		dc.tick++

-		if dc.tick > c.heartbeatTicks {

-			dc.conn.Close()

-			delete(c.deferedConns, dc)

-		}

-	}

-}

-

-// Tick increases ticks for all members. After heartbeatTicks node marked as

-// inactive.

-func (c *Cluster) Tick() {

-	c.mu.Lock()

-	defer c.mu.Unlock()

-	c.handleInactive()

-	c.handleDeferredConns()

-}

-

 // Members returns the list of raft Members in the Cluster.

 func (c *Cluster) Members() map[uint64]*Member {

 	members := make(map[uint64]*Member)

@@ -168,8 +102,6 @@ func (c *Cluster) AddMember(member *Member) error {

 	if c.removed[member.RaftID] {

 		return ErrIDRemoved

 	}

-	member.active = true

-	member.tick = 0

 	c.members[member.RaftID] = member

@@ -187,55 +119,47 @@ func (c *Cluster) RemoveMember(id uint64) error {

 	return c.clearMember(id)

 }

-// ClearMember removes a node from the Cluster Memberlist, but does NOT add it

-// to the removed list.

-func (c *Cluster) ClearMember(id uint64) error {

+// UpdateMember updates member address.

+func (c *Cluster) UpdateMember(id uint64, m *api.RaftMember) error {

 	c.mu.Lock()

 	defer c.mu.Unlock()

-	return c.clearMember(id)

-}

-

-func (c *Cluster) clearMember(id uint64) error {

-	m, ok := c.members[id]

-	if ok {

-		if m.Conn != nil {

-			// defer connection close to after heartbeatTicks

-			dConn := &deferredConn{conn: m.Conn}

-			c.deferedConns[dConn] = struct{}{}

-		}

-		delete(c.members, id)

+	if c.removed[id] {

+		return ErrIDRemoved

 	}

-	c.broadcastUpdate()

-	return nil

-}

-

-// ReplaceMemberConnection replaces the member's GRPC connection.

-func (c *Cluster) ReplaceMemberConnection(id uint64, oldConn *Member, newConn *Member, newAddr string, force bool) error {

-	c.mu.Lock()

-	defer c.mu.Unlock()

 	oldMember, ok := c.members[id]

 	if !ok {

 		return ErrIDNotFound

 	}

-	if !force && oldConn.Conn != oldMember.Conn {

-		// The connection was already replaced. Don't do it again.

-		newConn.Conn.Close()

-		return nil

+	if oldMember.NodeID != m.NodeID {

+		// Should never happen; this is a sanity check

+		return errors.New("node ID mismatch match on node update")

 	}

-	if oldMember.Conn != nil {

-		oldMember.Conn.Close()

+	if oldMember.Addr == m.Addr {

+		// nothing to do

+		return nil

 	}

+	oldMember.RaftMember = m

+	return nil

+}

+

+// ClearMember removes a node from the Cluster Memberlist, but does NOT add it

+// to the removed list.

+func (c *Cluster) ClearMember(id uint64) error {

+	c.mu.Lock()

+	defer c.mu.Unlock()

-	newMember := *oldMember

-	newMember.RaftMember = oldMember.RaftMember.Copy()

-	newMember.RaftMember.Addr = newAddr

-	newMember.Conn = newConn.Conn

-	c.members[id] = &newMember