@@ -101,7 +101,7 @@ github.com/docker/containerd 4ab9917febca54791c5f071a9d1f404867857fcc

 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4

 # cluster

-github.com/docker/swarmkit 17756457ad6dc4d8a639a1f0b7a85d1b65a617bb

+github.com/docker/swarmkit e68072200ebbba6ce9745b3a3e49fdba3eb71ff8

 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9

 github.com/gogo/protobuf v0.3

 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a

@@ -264,8 +264,8 @@ func (a *Agent) run(ctx context.Context) {

 			sessionq = a.sessionq

 		case err := <-session.errs:

 			// TODO(stevvooe): This may actually block if a session is closed

-			// but no error was sent. Session.close must only be called here

-			// for this to work.

+			// but no error was sent. This must be the only place

+			// session.close is called in response to errors, for this to work.

 			if err != nil {

 				log.G(ctx).WithError(err).Error("agent: session failed")

 				backoff = initialSessionFailureBackoff + 2*backoff

@@ -315,7 +315,11 @@ func (a *Agent) run(ctx context.Context) {

 				nodeDescription = newNodeDescription

 				// close the session

 				log.G(ctx).Info("agent: found node update")

-				session.sendError(nil)

+				if err := session.close(); err != nil {

+					log.G(ctx).WithError(err).Error("agent: closing session failed")

+				}

+				sessionq = nil

+				registered = nil

 			}

 		case <-a.stopped:

 			// TODO(stevvooe): Wait on shutdown and cleanup. May need to pump

@@ -165,7 +165,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, kw KeyWrit

 	// responding properly (for example, it may have just been demoted).

 	var signedCert []byte

 	for i := 0; i != 5; i++ {

-		signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, r, transport, nodeInfo)

+		signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, r, transport, nodeInfo, 0)

 		if err == nil {

 			break

 		}

@@ -545,7 +545,7 @@ func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) {

 // GetRemoteSignedCertificate submits a CSR to a remote CA server address,

 // and that is part of a CA identified by a specific certificate pool.

-func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, rootCAPool *x509.CertPool, r remotes.Remotes, creds credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) ([]byte, error) {

+func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, rootCAPool *x509.CertPool, r remotes.Remotes, creds credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse, nodeCertificateStatusRequestTimeout time.Duration) ([]byte, error) {

 	if rootCAPool == nil {

 		return nil, errors.New("valid root CA pool required")

 	}

@@ -560,7 +560,6 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r

 	if err != nil {

 		return nil, err

 	}

-	defer conn.Close()

 	// Create a CAClient to retrieve a new Certificate

 	caClient := api.NewNodeCAClient(conn)

@@ -570,6 +569,7 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r

 	issueResponse, err := caClient.IssueNodeCertificate(ctx, issueRequest)

 	if err != nil {

 		r.Observe(peer, -remotes.DefaultObservationWeight)

+		conn.Close()

 		return nil, err

 	}

@@ -587,18 +587,31 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r

 	// Exponential backoff with Max of 30 seconds to wait for a new retry

 	for {

+		timeout := 5 * time.Second

+		if nodeCertificateStatusRequestTimeout > 0 {

+			timeout = nodeCertificateStatusRequestTimeout

+		}

 		// Send the Request and retrieve the certificate

-		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)

+		stateCtx, cancel := context.WithTimeout(ctx, timeout)

 		defer cancel()

-		statusResponse, err := caClient.NodeCertificateStatus(ctx, statusRequest)

-		if err != nil {

+		statusResponse, err := caClient.NodeCertificateStatus(stateCtx, statusRequest)

+		switch {

+		case err != nil && grpc.Code(err) != codes.DeadlineExceeded:

+			// Because IssueNodeCertificate succeeded, if this call failed likely it is due to an issue with this

+			// particular connection, so we need to get another.

 			r.Observe(peer, -remotes.DefaultObservationWeight)

-			return nil, err

-		}

+			conn.Close()

+			conn, peer, err = getGRPCConnection(creds, r)

+			if err != nil {

+				return nil, err

+			}

+			caClient = api.NewNodeCAClient(conn)

-		// If the certificate was issued, return

-		if statusResponse.Status.State == api.IssuanceStateIssued {

+		// If there was no deadline exceeded error, and the certificate was issued, return

+		case err == nil && statusResponse.Status.State == api.IssuanceStateIssued:

 			if statusResponse.Certificate == nil {

+				r.Observe(peer, -remotes.DefaultObservationWeight)

+				conn.Close()

 				return nil, errors.New("no certificate in CertificateStatus response")

 			}

@@ -609,14 +622,20 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r

 			// current request.

 			if bytes.Equal(statusResponse.Certificate.CSR, csr) {

 				r.Observe(peer, remotes.DefaultObservationWeight)

+				conn.Close()

 				return statusResponse.Certificate.Certificate, nil

 			}

 		}

 		// If we're still pending, the issuance failed, or the state is unknown

-		// let's continue trying.

+		// let's continue trying after an exponential backoff

 		expBackoff.Failure(nil, nil)

-		time.Sleep(expBackoff.Proceed(nil))

+		select {

+		case <-ctx.Done():

+			conn.Close()

+			return nil, err

+		case <-time.After(expBackoff.Proceed(nil)):

+		}

 	}

 }

@@ -7,6 +7,7 @@ import (

 	"io/ioutil"

 	"net/http"

 	"sync"

+	"time"

 	"github.com/Sirupsen/logrus"

 	"github.com/cloudflare/cfssl/api"

@@ -23,6 +24,8 @@ var ErrNoExternalCAURLs = errors.New("no external CA URLs")

 // ExternalCA is able to make certificate signing requests to one of a list

 // remote CFSSL API endpoints.

 type ExternalCA struct {

+	ExternalRequestTimeout time.Duration

+

 	mu     sync.Mutex

 	rootCA *RootCA

 	urls   []string

@@ -33,8 +36,9 @@ type ExternalCA struct {

 // authenticate to any of the given URLS of CFSSL API endpoints.

 func NewExternalCA(rootCA *RootCA, tlsConfig *tls.Config, urls ...string) *ExternalCA {

 	return &ExternalCA{

-		rootCA: rootCA,

-		urls:   urls,

+		ExternalRequestTimeout: 5 * time.Second,

+		rootCA:                 rootCA,

+		urls:                   urls,

 		client: &http.Client{

 			Transport: &http.Transport{

 				TLSClientConfig: tlsConfig,

@@ -87,7 +91,9 @@ func (eca *ExternalCA) Sign(ctx context.Context, req signer.SignRequest) (cert [

 	// Try each configured proxy URL. Return after the first success. If

 	// all fail then the last error will be returned.

 	for _, url := range urls {

-		cert, err = makeExternalSignRequest(ctx, client, url, csrJSON)

+		requestCtx, cancel := context.WithTimeout(ctx, eca.ExternalRequestTimeout)

+		cert, err = makeExternalSignRequest(requestCtx, client, url, csrJSON)

+		cancel()

 		if err == nil {

 			return eca.rootCA.AppendFirstRootPEM(cert)

 		}

@@ -243,8 +243,9 @@ func (g *Orchestrator) reconcileServices(ctx context.Context, serviceIDs []strin

 	updates := make(map[*api.Service][]orchestrator.Slot)

 	_, err := g.store.Batch(func(batch *store.Batch) error {

-		var updateTasks []orchestrator.Slot

 		for _, serviceID := range serviceIDs {

+			var updateTasks []orchestrator.Slot

+

 			if _, exists := nodeTasks[serviceID]; !exists {

 				continue

 			}

@@ -298,7 +299,6 @@ func (g *Orchestrator) reconcileServices(ctx context.Context, serviceIDs []strin

 	for service, updateTasks := range updates {

 		g.updater.Update(ctx, g.cluster, service, updateTasks)

 	}

-

 }

 // updateNode updates g.nodes based on the current node value

@@ -406,7 +406,11 @@ func (u *Updater) updateTask(ctx context.Context, slot orchestrator.Slot, update

 	}

 	if delayStartCh != nil {

-		<-delayStartCh

+		select {

+		case <-delayStartCh:

+		case <-u.stopChan:

+			return nil

+		}

 	}

 	// Wait for the new task to come up.

@@ -456,7 +460,11 @@ func (u *Updater) useExistingTask(ctx context.Context, slot orchestrator.Slot, e

 		}

 		if delayStartCh != nil {

-			<-delayStartCh

+			select {

+			case <-delayStartCh:

+			case <-u.stopChan:

+				return nil

+			}

 		}

 	}