@@ -1,16 +1,52 @@

 package cluster

+//

+// ## Swarmkit integration

+//

+// Cluster - static configurable object for accessing everything swarm related.

+// Contains methods for connecting and controlling the cluster. Exists always,

+// even if swarm mode is not enabled.

+//

+// NodeRunner - Manager for starting the swarmkit node. Is present only and

+// always if swarm mode is enabled. Implements backoff restart loop in case of

+// errors.

+//

+// NodeState - Information about the current node status including access to

+// gRPC clients if a manager is active.

+//

+// ### Locking

+//

+// `cluster.controlMutex` - taken for the whole lifecycle of the processes that

+// can reconfigure cluster(init/join/leave etc). Protects that one

+// reconfiguration action has fully completed before another can start.

+//

+// `cluster.mu` - taken when the actual changes in cluster configurations

+// happen. Different from `controlMutex` because in some cases we need to

+// access current cluster state even if the long-running reconfiguration is

+// going on. For example network stack may ask for the current cluster state in

+// the middle of the shutdown. Any time current cluster state is asked you

+// should take the read lock of `cluster.mu`. If you are writing an API

+// responder that returns synchronously, hold `cluster.mu.RLock()` for the

+// duration of the whole handler function. That ensures that node will not be

+// shut down until the handler has finished.

+//

+// NodeRunner implements its internal locks that should not be used outside of

+// the struct. Instead, you should just call `nodeRunner.State()` method to get

+// the current state of the cluster(still need `cluster.mu.RLock()` to access

+// `cluster.nr` reference itself). Most of the changes in NodeRunner happen

+// because of an external event(network problem, unexpected swarmkit error) and

+// Docker shouldn't take any locks that delay these changes from happening.

+//

+

 import (

 	"crypto/x509"

 	"encoding/base64"

 	"encoding/json"

 	"fmt"

 	"io"

-	"io/ioutil"

 	"net"

 	"os"

 	"path/filepath"

-	"runtime"

 	"strings"

 	"sync"

 	"time"

@@ -25,7 +61,6 @@ import (

 	types "github.com/docker/docker/api/types/swarm"

 	"github.com/docker/docker/daemon/cluster/convert"

 	executorpkg "github.com/docker/docker/daemon/cluster/executor"

-	"github.com/docker/docker/daemon/cluster/executor/container"

 	"github.com/docker/docker/daemon/logger"

 	"github.com/docker/docker/opts"

 	"github.com/docker/docker/pkg/ioutils"

@@ -39,7 +74,6 @@ import (

 	"github.com/docker/swarmkit/protobuf/ptypes"

 	"github.com/pkg/errors"

 	"golang.org/x/net/context"

-	"google.golang.org/grpc"

 )

 const swarmDirName = "swarm"

@@ -95,19 +129,14 @@ type Config struct {

 // Cluster provides capabilities to participate in a cluster as a worker or a

 // manager.

 type Cluster struct {

-	sync.RWMutex

-	*node

-	root            string

-	runtimeRoot     string

-	config          Config

-	configEvent     chan struct{} // todo: make this array and goroutine safe

-	actualLocalAddr string        // after resolution, not persisted

-	stop            bool

-	err             error

-	cancelDelay     func()

-	attachers       map[string]*attacher

-	locked          bool

-	lastNodeConfig  *nodeStartConfig

+	mu           sync.RWMutex

+	controlMutex sync.RWMutex // protect init/join/leave user operations

+	nr           *nodeRunner

+	root         string

+	runtimeRoot  string

+	config       Config

+	configEvent  chan struct{} // todo: make this array and goroutine safe

+	attachers    map[string]*attacher

 }

 // attacher manages the in-memory attachment state of a container

@@ -122,38 +151,6 @@ type attacher struct {

 	detachWaitCh     chan struct{}

 }

-type node struct {

-	*swarmnode.Node

-	done           chan struct{}

-	ready          bool

-	conn           *grpc.ClientConn

-	client         swarmapi.ControlClient

-	logs           swarmapi.LogsClient

-	reconnectDelay time.Duration

-	config         nodeStartConfig

-}

-

-// nodeStartConfig holds configuration needed to start a new node. Exported

-// fields of this structure are saved to disk in json. Unexported fields

-// contain data that shouldn't be persisted between daemon reloads.

-type nodeStartConfig struct {

-	// LocalAddr is this machine's local IP or hostname, if specified.

-	LocalAddr string

-	// RemoteAddr is the address that was given to "swarm join". It is used

-	// to find LocalAddr if necessary.

-	RemoteAddr string

-	// ListenAddr is the address we bind to, including a port.

-	ListenAddr string

-	// AdvertiseAddr is the address other nodes should connect to,

-	// including a port.

-	AdvertiseAddr   string

-	joinAddr        string

-	forceNewCluster bool

-	joinToken       string

-	lockKey         []byte

-	autolock        bool

-}

-

 // New creates a new Cluster instance using provided config.

 func New(config Config) (*Cluster, error) {

 	root := filepath.Join(config.Root, swarmDirName)

@@ -174,7 +171,7 @@ func New(config Config) (*Cluster, error) {

 		attachers:   make(map[string]*attacher),

 	}

-	nodeConfig, err := c.loadState()

+	nodeConfig, err := loadPersistentState(root)

 	if err != nil {

 		if os.IsNotExist(err) {

 			return c, nil

@@ -182,95 +179,30 @@ func New(config Config) (*Cluster, error) {

 		return nil, err

 	}

-	n, err := c.startNewNode(*nodeConfig)

+	nr, err := c.newNodeRunner(*nodeConfig)

 	if err != nil {

 		return nil, err

 	}

+	c.nr = nr

 	select {

 	case <-time.After(swarmConnectTimeout):

 		logrus.Error("swarm component could not be started before timeout was reached")

-	case <-n.Ready():

-	case <-n.done:

-		if errors.Cause(c.err) == ErrSwarmLocked {

-			return c, nil

-		}

-		if err, ok := errors.Cause(c.err).(x509.CertificateInvalidError); ok && err.Reason == x509.Expired {

-			c.err = ErrSwarmCertificatesExpired

-			return c, nil

-		}

-		return nil, fmt.Errorf("swarm component could not be started: %v", c.err)

-	}

-	go c.reconnectOnFailure(n)

-	return c, nil

-}

-

-func (c *Cluster) loadState() (*nodeStartConfig, error) {

-	dt, err := ioutil.ReadFile(filepath.Join(c.root, stateFile))

-	if err != nil {

-		return nil, err

-	}

-	// missing certificate means no actual state to restore from

-	if _, err := os.Stat(filepath.Join(c.root, "certificates/swarm-node.crt")); err != nil {

-		if os.IsNotExist(err) {

-			c.clearState()

-		}

-		return nil, err

-	}

-	var st nodeStartConfig

-	if err := json.Unmarshal(dt, &st); err != nil {

-		return nil, err

-	}

-	return &st, nil

-}

-

-func (c *Cluster) saveState(config nodeStartConfig) error {

-	dt, err := json.Marshal(config)

-	if err != nil {

-		return err

-	}

-	return ioutils.AtomicWriteFile(filepath.Join(c.root, stateFile), dt, 0600)

-}

-

-func (c *Cluster) reconnectOnFailure(n *node) {

-	for {

-		<-n.done

-		c.Lock()

-		if c.stop || c.node != nil {

-			c.Unlock()

-			return

-		}

-		n.reconnectDelay *= 2

-		if n.reconnectDelay > maxReconnectDelay {

-			n.reconnectDelay = maxReconnectDelay

-		}

-		logrus.Warnf("Restarting swarm in %.2f seconds", n.reconnectDelay.Seconds())

-		delayCtx, cancel := context.WithTimeout(context.Background(), n.reconnectDelay)

-		c.cancelDelay = cancel

-		c.Unlock()

-		<-delayCtx.Done()

-		if delayCtx.Err() != context.DeadlineExceeded {

-			return

-		}

-		c.Lock()

-		if c.node != nil {

-			c.Unlock()

-			return

-		}

-		var err error

-		config := n.config

-		config.RemoteAddr = c.getRemoteAddress()

-		config.joinAddr = config.RemoteAddr

-		n, err = c.startNewNode(config)

+	case err := <-nr.Ready():

 		if err != nil {

-			c.err = err

-			close(n.done)

+			if errors.Cause(err) == ErrSwarmLocked {

+				return c, nil

+			}

+			if err, ok := errors.Cause(c.nr.err).(x509.CertificateInvalidError); ok && err.Reason == x509.Expired {

+				return c, nil

+			}

+			return nil, errors.Wrap(err, "swarm component could not be started")

 		}

-		c.Unlock()

 	}

+	return c, nil

 }

-func (c *Cluster) startNewNode(conf nodeStartConfig) (*node, error) {

+func (c *Cluster) newNodeRunner(conf nodeStartConfig) (*nodeRunner, error) {

 	if err := c.config.Backend.IsSwarmCompatible(); err != nil {

 		return nil, err

 	}

@@ -304,128 +236,47 @@ func (c *Cluster) startNewNode(conf nodeStartConfig) (*node, error) {

 		}

 	}

-	var control string

-	if runtime.GOOS == "windows" {

-		control = `\\.\pipe\` + controlSocket

-	} else {

-		control = filepath.Join(c.runtimeRoot, controlSocket)

-	}

-

-	c.node = nil

-	c.cancelDelay = nil

-	c.stop = false

-	n, err := swarmnode.New(&swarmnode.Config{

-		Hostname:           c.config.Name,

-		ForceNewCluster:    conf.forceNewCluster,

-		ListenControlAPI:   control,

-		ListenRemoteAPI:    conf.ListenAddr,

-		AdvertiseRemoteAPI: conf.AdvertiseAddr,

-		JoinAddr:           conf.joinAddr,

-		StateDir:           c.root,

-		JoinToken:          conf.joinToken,

-		Executor:           container.NewExecutor(c.config.Backend),

-		HeartbeatTick:      1,

-		ElectionTick:       3,

-		UnlockKey:          conf.lockKey,

-		AutoLockManagers:   conf.autolock,

-	})

+	nr := &nodeRunner{cluster: c}

+	nr.actualLocalAddr = actualLocalAddr

-	if err != nil {

+	if err := nr.Start(conf); err != nil {

 		return nil, err

 	}

-	ctx := context.Background()

-	if err := n.Start(ctx); err != nil {

-		return nil, err

-	}

-	node := &node{

-		Node:           n,

-		done:           make(chan struct{}),

-		reconnectDelay: initialReconnectDelay,

-		config:         conf,

-	}

-	c.node = node

-	c.actualLocalAddr = actualLocalAddr // not saved

-	c.saveState(conf)

 	c.config.Backend.SetClusterProvider(c)

-	go func() {

-		err := detectLockedError(n.Err(ctx))

-		if err != nil {

-			logrus.Errorf("cluster exited with error: %v", err)

-		}

-		c.Lock()

-		c.node = nil

-		c.err = err

-		if errors.Cause(err) == ErrSwarmLocked {

-			c.locked = true

-			confClone := conf

-			c.lastNodeConfig = &confClone

-		}

-		c.Unlock()

-		close(node.done)

-	}()

-

-	go func() {

-		select {

-		case <-n.Ready():

-			c.Lock()

-			node.ready = true

-			c.err = nil

-			c.Unlock()

-		case <-ctx.Done():

-		}

-		c.configEvent <- struct{}{}

-	}()

-

-	go func() {

-		for conn := range n.ListenControlSocket(ctx) {

-			c.Lock()

-			if node.conn != conn {

-				if conn == nil {

-					node.client = nil

-					node.logs = nil

-				} else {

-					node.client = swarmapi.NewControlClient(conn)

-					node.logs = swarmapi.NewLogsClient(conn)

-				}

-			}

-			node.conn = conn

-			c.Unlock()

-			c.configEvent <- struct{}{}

-		}

-	}()

-	return node, nil

+	return nr, nil

 }

 // Init initializes new cluster from user provided request.

 func (c *Cluster) Init(req types.InitRequest) (string, error) {

-	c.Lock()

-	if c.swarmExists() {

-		if !req.ForceNewCluster {

-			c.Unlock()

+	c.controlMutex.Lock()

+	defer c.controlMutex.Unlock()

+	c.mu.Lock()

+	if c.nr != nil {

+		if req.ForceNewCluster {

+			if err := c.nr.Stop(); err != nil {

+				c.mu.Unlock()

+				return "", err

+			}

+		} else {

+			c.mu.Unlock()

 			return "", ErrSwarmExists

 		}

-		if err := c.stopNode(); err != nil {

-			c.Unlock()

-			return "", err

-		}

 	}

+	c.mu.Unlock()

 	if err := validateAndSanitizeInitRequest(&req); err != nil {

-		c.Unlock()

 		return "", err

 	}

 	listenHost, listenPort, err := resolveListenAddr(req.ListenAddr)

 	if err != nil {

-		c.Unlock()

 		return "", err

 	}

 	advertiseHost, advertisePort, err := c.resolveAdvertiseAddr(req.AdvertiseAddr, listenPort)

 	if err != nil {

-		c.Unlock()

 		return "", err

 	}

@@ -451,7 +302,6 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {

 		if !found {

 			ip, err := c.resolveSystemAddr()

 			if err != nil {

-				c.Unlock()

 				logrus.Warnf("Could not find a local address: %v", err)

 				return "", errMustSpecifyListenAddr

 			}

@@ -459,8 +309,11 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {

 		}

 	}

-	// todo: check current state existing

-	n, err := c.startNewNode(nodeStartConfig{

+	if !req.ForceNewCluster {

+		clearPersistentState(c.root)

+	}

+

+	nr, err := c.newNodeRunner(nodeStartConfig{

 		forceNewCluster: req.ForceNewCluster,

 		autolock:        req.AutoLockManagers,

 		LocalAddr:       localAddr,

@@ -468,45 +321,52 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {

 		AdvertiseAddr:   net.JoinHostPort(advertiseHost, advertisePort),

 	})

 	if err != nil {

-		c.Unlock()

 		return "", err

 	}

-	c.Unlock()

+	c.mu.Lock()

+	c.nr = nr

+	c.mu.Unlock()

-	select {

-	case <-n.Ready():

-		if err := initClusterSpec(n, req.Spec); err != nil {

-			return "", err

-		}

-		go c.reconnectOnFailure(n)

-		return n.NodeID(), nil

-	case <-n.done:

-		c.RLock()

-		defer c.RUnlock()

+	if err := <-nr.Ready(); err != nil {

 		if !req.ForceNewCluster { // if failure on first attempt don't keep state

-			if err := c.clearState(); err != nil {

+			if err := clearPersistentState(c.root); err != nil {

 				return "", err

 			}

 		}

-		return "", c.err

+		if err != nil {

+			c.mu.Lock()

+			c.nr = nil

+			c.mu.Unlock()

+		}

+		return "", err

 	}

+	state := nr.State()

+	if state.swarmNode == nil { // should never happen but protect from panic

+		return "", errors.New("invalid cluster state for spec initialization")

+	}

+	if err := initClusterSpec(state.swarmNode, req.Spec); err != nil {

+		return "", err

+	}

+	return state.NodeID(), nil

 }

 // Join makes current Cluster part of an existing swarm cluster.

 func (c *Cluster) Join(req types.JoinRequest) error {

-	c.Lock()

-	if c.swarmExists() {

-		c.Unlock()

+	c.controlMutex.Lock()

+	defer c.controlMutex.Unlock()

+	c.mu.Lock()

+	if c.nr != nil {

+		c.mu.Unlock()

 		return ErrSwarmExists

 	}

+	c.mu.Unlock()

+

 	if err := validateAndSanitizeJoinRequest(&req); err != nil {

-		c.Unlock()

 		return err

 	}

 	listenHost, listenPort, err := resolveListenAddr(req.ListenAddr)

 	if err != nil {

-		c.Unlock()

 		return err

 	}

@@ -520,8 +380,9 @@ func (c *Cluster) Join(req types.JoinRequest) error {

 		}

 	}

-	// todo: check current state existing

-	n, err := c.startNewNode(nodeStartConfig{

+	clearPersistentState(c.root)

+

+	nr, err := c.newNodeRunner(nodeStartConfig{

 		RemoteAddr:    req.RemoteAddrs[0],

 		ListenAddr:    net.JoinHostPort(listenHost, listenPort),

 		AdvertiseAddr: advertiseAddr,

@@ -529,46 +390,40 @@ func (c *Cluster) Join(req types.JoinRequest) error {

 		joinToken:     req.JoinToken,

 	})

 	if err != nil {

-		c.Unlock()

 		return err

 	}

-	c.Unlock()

+

+	c.mu.Lock()

+	c.nr = nr

+	c.mu.Unlock()

 	select {

 	case <-time.After(swarmConnectTimeout):

-		// attempt to connect will continue in background, but reconnect only if it didn't fail

-		go func() {

-			select {

-			case <-n.Ready():

-				c.reconnectOnFailure(n)

-			case <-n.done:

-				logrus.Errorf("failed to join the cluster: %+v", c.err)

-			}

-		}()

 		return ErrSwarmJoinTimeoutReached

-	case <-n.Ready():

-		go c.reconnectOnFailure(n)

-		return nil

-	case <-n.done:

-		c.RLock()

-		defer c.RUnlock()

-		return c.err

+	case err := <-nr.Ready():

+		if err != nil {

+			c.mu.Lock()

+			c.nr = nil

+			c.mu.Unlock()

+		}

+		return err

 	}

 }

 // GetUnlockKey returns the unlock key for the swarm.

 func (c *Cluster) GetUnlockKey() (string, error) {

-	c.RLock()

-	defer c.RUnlock()

+	c.mu.RLock()

+	defer c.mu.RUnlock()

-	if !c.isActiveManager() {

-		return "", c.errNoManager()

+	state := c.currentNodeState()

+	if !state.IsActiveManager() {

+		return "", c.errNoManager(state)

 	}

 	ctx, cancel := c.getRequestContext()

 	defer cancel()

-	client := swarmapi.NewCAClient(c.conn)

+	client := swarmapi.NewCAClient(state.grpcConn)

 	r, err := client.GetUnlockKey(ctx, &swarmapi.GetUnlockKeyRequest{})

 	if err != nil {

@@ -585,141 +440,104 @@ func (c *Cluster) GetUnlockKey() (string, error) {

 // UnlockSwarm provides a key to decrypt data that is encrypted at rest.

 func (c *Cluster) UnlockSwarm(req types.UnlockRequest) error {

-	c.RLock()

-	if !c.isActiveManager() {

-		if err := c.errNoManager(); err != ErrSwarmLocked {

-			c.RUnlock()

-			return err

-		}

-	}

-

-	if c.node != nil || c.locked != true {

-		c.RUnlock()

+	c.controlMutex.Lock()

+	defer c.controlMutex.Unlock()

+

+	c.mu.RLock()

+	state := c.currentNodeState()

+	nr := c.nr

+	c.mu.RUnlock()

+	if nr == nil || errors.Cause(state.err) != ErrSwarmLocked {

 		return errors.New("swarm is not locked")

 	}

-	c.RUnlock()

-

 	key, err := encryption.ParseHumanReadableKey(req.UnlockKey)

 	if err != nil {

 		return err

 	}

-	c.Lock()

-	config := *c.lastNodeConfig

+	config := nr.config

 	config.lockKey = key

-	n, err := c.startNewNode(config)

+	if err := nr.Stop(); err != nil {

+		return err

+	}

+	nr, err = c.newNodeRunner(config)

 	if err != nil {

-		c.Unlock()

 		return err

 	}

-	c.Unlock()

-	select {

-	case <-n.Ready():

-	case <-n.done:

-		if errors.Cause(c.err) == ErrSwarmLocked {

+

+	c.mu.Lock()

+	c.nr = nr

+	c.mu.Unlock()

+

+	if err := <-nr.Ready(); err != nil {

+		if errors.Cause(err) == ErrSwarmLocked {

 			return errors.New("swarm could not be unlocked: invalid key provided")

 		}

-		return fmt.Errorf("swarm component could not be started: %v", c.err)

+		return fmt.Errorf("swarm component could not be started: %v", err)

 	}

-	go c.reconnectOnFailure(n)

 	return nil

 }

-// stopNode is a helper that stops the active c.node and waits until it has

-// shut down. Call while keeping the cluster lock.

-func (c *Cluster) stopNode() error {

-	if c.node == nil {

-		return nil

-	}

-	c.stop = true

-	if c.cancelDelay != nil {

-		c.cancelDelay()

-		c.cancelDelay = nil

-	}

-	node := c.node

-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)

-	defer cancel()

-	// TODO: can't hold lock on stop because it calls back to network

-	c.Unlock()

-	defer c.Lock()

-	if err := node.Stop(ctx); err != nil && !strings.Contains(err.Error(), "context canceled") {

-		return err

-	}

-	<-node.done

-	return nil

-}

-

-func removingManagerCausesLossOfQuorum(reachable, unreachable int) bool {

-	return reachable-2 <= unreachable

-}

-

-func isLastManager(reachable, unreachable int) bool {

-	return reachable == 1 && unreachable == 0

-}

-

 // Leave shuts down Cluster and removes current state.

 func (c *Cluster) Leave(force bool) error {

-	c.Lock()

-	node := c.node

-	if node == nil {

-		if c.locked {

-			c.locked = false

-			c.lastNodeConfig = nil

-			c.Unlock()

-		} else if c.err == ErrSwarmCertificatesExpired {

-			c.err = nil

-			c.Unlock()

-		} else {

-			c.Unlock()

-			return ErrNoSwarm

-		}

-	} else {

-		if node.Manager() != nil && !force {

-			msg := "You are attempting to leave the swarm on a node that is participating as a manager. "

-			if c.isActiveManager() {

-				active, reachable, unreachable, err := c.managerStats()

-				if err == nil {

-					if active && removingManagerCausesLossOfQuorum(reachable, unreachable) {

-						if isLastManager(reachable, unreachable) {

-							msg += "Removing the last manager erases all current state of the swarm. Use `--force` to ignore this message. "

-							c.Unlock()

-							return fmt.Errorf(msg)

-						}

-						msg += fmt.Sprintf("Removing this node leaves %v managers out of %v. Without a Raft quorum your swarm will be inaccessible. ", reachable-1, reachable+unreachable)

+	c.controlMutex.Lock()

+	defer c.controlMutex.Unlock()

+

+	c.mu.Lock()

+	nr := c.nr

+	if nr == nil {

+		c.mu.Unlock()

+		return ErrNoSwarm

+	}

+	state := c.currentNodeState()

+	if state.IsManager() && !force {

+		msg := "You are attempting to leave the swarm on a node that is participating as a manager. "

+		if state.IsActiveManager() {

+			active, reachable, unreachable, err := managerStats(state.controlClient, state.NodeID())

+			if err == nil {

+				if active && removingManagerCausesLossOfQuorum(reachable, unreachable) {

+					if isLastManager(reachable, unreachable) {

+						msg += "Removing the last manager erases all current state of the swarm. Use `--force` to ignore this message. "

+						c.mu.Unlock()

+						return fmt.Errorf(msg)

 					}

+					msg += fmt.Sprintf("Removing this node leaves %v managers out of %v. Without a Raft quorum your swarm will be inaccessible. ", reachable-1, reachable+unreachable)

 				}

-			} else {

-				msg += "Doing so may lose the consensus of your cluster. "

 			}

-

-			msg += "The only way to restore a swarm that has lost consensus is to reinitialize it with `--force-new-cluster`. Use `--force` to suppress this message."

-			c.Unlock()

-			return fmt.Errorf(msg)

+		} else {

+			msg += "Doing so may lose the consensus of your cluster. "

 		}

-		if err := c.stopNode(); err != nil {

-			logrus.Errorf("failed to shut down cluster node: %v", err)

-			signal.DumpStacks("")

-			c.Unlock()

+

+		msg += "The only way to restore a swarm that has lost consensus is to reinitialize it with `--force-new-cluster`. Use `--force` to suppress this message."

+		c.mu.Unlock()

+		return fmt.Errorf(msg)

+	}

+	// release readers in here

+	if err := nr.Stop(); err != nil {

+		logrus.Errorf("failed to shut down cluster node: %v", err)

+		signal.DumpStacks("")

+		c.mu.Unlock()

+		return err

+	}

+	c.nr = nil

+	c.mu.Unlock()

+	if nodeID := state.NodeID(); nodeID != "" {

+		nodeContainers, err := c.listContainerForNode(nodeID)

+		if err != nil {

 			return err

 		}

-		c.Unlock()

-		if nodeID := node.NodeID(); nodeID != "" {

-			nodeContainers, err := c.listContainerForNode(nodeID)

-			if err != nil {

-				return err

-			}

-			for _, id := range nodeContainers {

-				if err := c.config.Backend.ContainerRm(id, &apitypes.ContainerRmConfig{ForceRemove: true}); err != nil {

-					logrus.Errorf("error removing %v: %v", id, err)

-				}

+		for _, id := range nodeContainers {

+			if err := c.config.Backend.ContainerRm(id, &apitypes.ContainerRmConfig{ForceRemove: true}); err != nil {

+				logrus.Errorf("error removing %v: %v", id, err)

 			}

 		}

 	}

 	c.configEvent <- struct{}{}

 	// todo: cleanup optional?

-	if err := c.clearState(); err != nil {

+	if err := clearPersistentState(c.root); err != nil {

 		return err

 	}

+	c.config.Backend.SetClusterProvider(nil)

 	return nil

 }

@@ -739,35 +557,24 @@ func (c *Cluster) listContainerForNode(nodeID string) ([]string, error) {

 	return ids, nil

 }

-func (c *Cluster) clearState() error {

-	// todo: backup this data instead of removing?

-	if err := os.RemoveAll(c.root); err != nil {

-		return err

-	}

-	if err := os.MkdirAll(c.root, 0700); err != nil {

-		return err

-	}

-	c.config.Backend.SetClusterProvider(nil)

-	return nil

-}

-

 func (c *Cluster) getRequestContext() (context.Context, func()) { // TODO: not needed when requests don't block on qourum lost

 	return context.WithTimeout(context.Background(), swarmRequestTimeout)

 }

 // Inspect retrieves the configuration properties of a managed swarm cluster.

 func (c *Cluster) Inspect() (types.Swarm, error) {

-	c.RLock()

-	defer c.RUnlock()

+	c.mu.RLock()

+	defer c.mu.RUnlock()

-	if !c.isActiveManager() {

-		return types.Swarm{}, c.errNoManager()

+	state := c.currentNodeState()

+	if !state.IsActiveManager() {

+		return types.Swarm{}, c.errNoManager(state)

 	}

 	ctx, cancel := c.getRequestContext()

 	defer cancel()

-	swarm, err := getSwarm(ctx, c.client)

+	swarm, err := getSwarm(ctx, state.controlClient)

 	if err != nil {

 		return types.Swarm{}, err

 	}

@@ -777,17 +584,18 @@ func (c *Cluster) Inspect() (types.Swarm, error) {

 // Update updates configuration of a managed swarm cluster.

 func (c *Cluster) Update(version uint64, spec types.Spec, flags types.UpdateFlags) error {

-	c.RLock()

-	defer c.RUnlock()

+	c.mu.RLock()

+	defer c.mu.RUnlock()

-	if !c.isActiveManager() {

-		return c.errNoManager()

+	state := c.currentNodeState()

+	if !state.IsActiveManager() {

+		return c.errNoManager(state)

 	}

 	ctx, cancel := c.getRequestContext()

 	defer cancel()

-	swarm, err := getSwarm(ctx, c.client)

+	swarm, err := getSwarm(ctx, state.controlClient)

 	if err != nil {

 		return err

 	}

@@ -800,7 +608,7 @@ func (c *Cluster) Update(version uint64, spec types.Spec, flags types.UpdateFlag

 		return err

 	}

-	_, err = c.client.UpdateCluster(

+	_, err = state.controlClient.UpdateCluster(

 		ctx,

 		&swarmapi.UpdateClusterRequest{

 			ClusterID: swarm.ID,

@@ -820,61 +628,62 @@ func (c *Cluster) Update(version uint64, spec types.Spec, flags types.UpdateFlag

 // IsManager returns true if Cluster is participating as a manager.

 func (c *Cluster) IsManager() bool {

-	c.RLock()

-	defer c.RUnlock()

-	return c.isActiveManager()

+	c.mu.RLock()

+	defer c.mu.RUnlock()

+	return c.currentNodeState().IsActiveManager()

 }

 // IsAgent returns true if Cluster is participating as a worker/agent.

 func (c *Cluster) IsAgent() bool {

-	c.RLock()

-	defer c.RUnlock()

-	return c.node != nil && c.ready

+	c.mu.RLock()

+	defer c.mu.RUnlock()

+	return c.currentNodeState().status == types.LocalNodeStateActive

 }

 // GetLocalAddress returns the local address.

 func (c *Cluster) GetLocalAddress() string {

-	c.RLock()

-	defer c.RUnlock()

-	return c.actualLocalAddr

+	c.mu.RLock()

+	defer c.mu.RUnlock()

+	return c.currentNodeState().actualLocalAddr

 }

 // GetListenAddress returns the listen address.

 func (c *Cluster) GetListenAddress() string {

-	c.RLock()

-	defer c.RUnlock()