1. docker
  2. Commit 3b555a575dfe9badbcd31fd8bc1f03fde746fe88
Browse code

vendor: update swarmkit to cb6d8131

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2016/08/10 03:49:39
Showing 38 changed files
hack/vendor.sh
History View file @ 3b555a5
... ... 
@@ -140,7 +140,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0
140 140 
 clone git github.com/docker/containerd 0ac3cd1be170d180b2baed755e8f0da547ceb267
141 141
142 142 
 # cluster
143 
-clone git github.com/docker/swarmkit e1c0d64515d839b76e2ef33d396c74933753ffaf
143 
+clone git github.com/docker/swarmkit cb6d81316727941665594f153434e5ce2e425c9b
144 144 
 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
145 145 
 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028
146 146 
 clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b
vendor/src/github.com/docker/swarmkit/agent/agent.go
History View file @ 3b555a5
... ... 
@@ -4,6 +4,7 @@ import (
4 4 
 	"fmt"
5 5 
 	"math/rand"
6 6 
 	"reflect"
7 
+	"sync"
7 8 
 	"time"
8 9
9 10 
 	"github.com/docker/swarmkit/api"
... ... 
@@ -31,11 +32,13 @@ type Agent struct {
31 31 
 	sessionq chan sessionOperation
32 32 
 	worker   Worker
33 33
34 
-	started chan struct{}
35 
-	ready   chan struct{}
36 
-	stopped chan struct{} // requests shutdown
37 
-	closed  chan struct{} // only closed in run
38 
-	err     error         // read only after closed is closed
34 
+	started   chan struct{}
35 
+	startOnce sync.Once // start only once
36 
+	ready     chan struct{}
37 
+	stopped   chan struct{} // requests shutdown
38 
+	stopOnce  sync.Once     // only allow stop to be called once
39 
+	closed    chan struct{} // only closed in run
40 
+	err       error         // read only after closed is closed
39 41 
 }
40 42
41 43 
 // New returns a new agent, ready for task dispatch.
... ... 
@@ -59,57 +62,50 @@ func New(config *Config) (*Agent, error) {
59 59
60 60 
 // Start begins execution of the agent in the provided context, if not already
61 61 
 // started.
62 
+//
63 
+// Start returns an error if the agent has already started.
62 64 
 func (a *Agent) Start(ctx context.Context) error {
63 
-	select {
64 
-	case <-a.started:
65 
-		select {
66 
-		case <-a.closed:
67 
-			return a.err
68 
-		case <-a.stopped:
69 
-			return errAgentStopped
70 
-		case <-ctx.Done():
71 
-			return ctx.Err()
72 
-		default:
73 
-			return errAgentStarted
74 
-		}
75 
-	case <-ctx.Done():
76 
-		return ctx.Err()
77 
-	default:
78 
-	}
65 
+	err := errAgentStarted
79 66
80 
-	close(a.started)
81 
-	go a.run(ctx)
67 
+	a.startOnce.Do(func() {
68 
+		close(a.started)
69 
+		go a.run(ctx)
70 
+		err = nil // clear error above, only once.
71 
+	})
82 72
83 
-	return nil
73 
+	return err
84 74 
 }
85 75
86 76 
 // Stop shuts down the agent, blocking until full shutdown. If the agent is not
87 
-// started, Stop will block until Started.
77 
+// started, Stop will block until the agent has fully shutdown.
88 78 
 func (a *Agent) Stop(ctx context.Context) error {
89 79 
 	select {
90 80 
 	case <-a.started:
91 
-		select {
92 
-		case <-a.closed:
93 
-			return a.err
94 
-		case <-a.stopped:
95 
-			select {
96 
-			case <-a.closed:
97 
-				return a.err
98 
-			case <-ctx.Done():
99 
-				return ctx.Err()
100 
-			}
101 
-		case <-ctx.Done():
102 
-			return ctx.Err()
103 
-		default:
104 
-			close(a.stopped)
105 
-			// recurse and wait for closure
106 
-			return a.Stop(ctx)
107 
-		}
108 
-	case <-ctx.Done():
109 
-		return ctx.Err()
110 81 
 	default:
111 82 
 		return errAgentNotStarted
112 83 
 	}
84 
+
85 
+	a.stop()
86 
+
87 
+	// wait till closed or context cancelled
88 
+	select {
89 
+	case <-a.closed:
90 
+		return nil
91 
+	case <-ctx.Done():
92 
+		return ctx.Err()
93 
+	}
94 
+}
95 
+
96 
+// stop signals the agent shutdown process, returning true if this call was the
97 
+// first to actually shutdown the agent.
98 
+func (a *Agent) stop() bool {
99 
+	var stopped bool
100 
+	a.stopOnce.Do(func() {
101 
+		close(a.stopped)
102 
+		stopped = true
103 
+	})
104 
+
105 
+	return stopped
113 106 
 }
114 107
115 108 
 // Err returns the error that caused the agent to shutdown or nil. Err blocks
... ... 
@@ -133,7 +129,7 @@ func (a *Agent) run(ctx context.Context) {
133 133 
 	defer cancel()
134 134 
 	defer close(a.closed) // full shutdown.
135 135
136 
-	ctx = log.WithLogger(ctx, log.G(ctx).WithField("module", "agent"))
136 
+	ctx = log.WithModule(ctx, "agent")
137 137
138 138 
 	log.G(ctx).Debugf("(*Agent).run")
139 139 
 	defer log.G(ctx).Debugf("(*Agent).run exited")
... ... 
@@ -197,11 +193,6 @@ func (a *Agent) run(ctx context.Context) {
197 197 
 			sessionq = nil
198 198 
 			// if we're here before <-registered, do nothing for that event
199 199 
 			registered = nil
200 
-
201 
-			// Bounce the connection.
202 
-			if a.config.Picker != nil {
203 
-				a.config.Picker.Reset()
204 
-			}
205 200 
 		case <-session.closed:
206 201 
 			log.G(ctx).Debugf("agent: rebuild session")
207 202
... ... 
@@ -218,6 +209,7 @@ func (a *Agent) run(ctx context.Context) {
218 218 
 			if a.err == nil {
219 219 
 				a.err = ctx.Err()
220 220 
 			}
221 
+			session.close()
221 222
222 223 
 			return
223 224 
 		}
vendor/src/github.com/docker/swarmkit/agent/config.go
History View file @ 3b555a5
... ... 
@@ -7,7 +7,7 @@ import (
7 7 
 	"github.com/docker/swarmkit/agent/exec"
8 8 
 	"github.com/docker/swarmkit/api"
9 9 
 	"github.com/docker/swarmkit/picker"
10 
-	"google.golang.org/grpc"
10 
+	"google.golang.org/grpc/credentials"
11 11 
 )
12 12
13 13 
 // Config provides values for an Agent.
... ... 
@@ -19,15 +19,6 @@ type Config struct {
19 19 
 	// updated with managers weights as observed by the agent.
20 20 
 	Managers picker.Remotes
21 21
22 
-	// Conn specifies the client connection Agent will use.
23 
-	Conn *grpc.ClientConn
24 
-
25 
-	// Picker is the picker used by Conn.
26 
-	// TODO(aaronl): This is only part of the config to allow resetting the
27 
-	// GRPC connection. This should be refactored to address the coupling
28 
-	// between Conn and Picker.
29 
-	Picker *picker.Picker
30 
-
31 22 
 	// Executor specifies the executor to use for the agent.
32 23 
 	Executor exec.Executor
33 24
... ... 
@@ -36,11 +27,14 @@ type Config struct {
36 36
37 37 
 	// NotifyRoleChange channel receives new roles from session messages.
38 38 
 	NotifyRoleChange chan<- api.NodeRole
39 
+
40 
+	// Credentials is credentials for grpc connection to manager.
41 
+	Credentials credentials.TransportAuthenticator
39 42 
 }
40 43
41 44 
 func (c *Config) validate() error {
42 
-	if c.Conn == nil {
43 
-		return fmt.Errorf("agent: Connection is required")
45 
+	if c.Credentials == nil {
46 
+		return fmt.Errorf("agent: Credentials is required")
44 47 
 	}
45 48
46 49 
 	if c.Executor == nil {
vendor/src/github.com/docker/swarmkit/agent/errors.go
History View file @ 3b555a5
... ... 
@@ -10,10 +10,11 @@ var (
10 10 
 	ErrClosed = errors.New("agent: closed")
11 11
12 12 
 	errNodeNotRegistered = fmt.Errorf("node not registered")
13 
+	errNodeStarted       = errors.New("node: already started")
14 
+	errNodeNotStarted    = errors.New("node: not started")
13 15
14 
-	errAgentNotStarted = errors.New("agent: not started")
15 16 
 	errAgentStarted    = errors.New("agent: already started")
16 
-	errAgentStopped    = errors.New("agent: stopped")
17 
+	errAgentNotStarted = errors.New("agent: not started")
17 18
18 19 
 	errTaskNoContoller          = errors.New("agent: no task controller")
19 20 
 	errTaskNotAssigned          = errors.New("agent: task not assigned")
vendor/src/github.com/docker/swarmkit/agent/exec/controller.go
History View file @ 3b555a5
... ... 
@@ -2,11 +2,11 @@ package exec
2 2
3 3 
 import (
4 4 
 	"fmt"
5 
-	"reflect"
6 5 
 	"time"
7 6
8 7 
 	"github.com/Sirupsen/logrus"
9 8 
 	"github.com/docker/swarmkit/api"
9 
+	"github.com/docker/swarmkit/api/equality"
10 10 
 	"github.com/docker/swarmkit/log"
11 11 
 	"github.com/docker/swarmkit/protobuf/ptypes"
12 12 
 	"github.com/pkg/errors"
... ... 
@@ -186,7 +186,7 @@ func Do(ctx context.Context, task *api.Task, ctlr Controller) (*api.TaskStatus,
186 186 
 	defer func() {
187 187 
 		logStateChange(ctx, task.DesiredState, task.Status.State, status.State)
188 188
189 
-		if !reflect.DeepEqual(status, task.Status) {
189 
+		if !equality.TaskStatusesEqualStable(status, &task.Status) {
190 190 
 			status.Timestamp = ptypes.MustTimestampProto(time.Now())
191 191 
 		}
192 192 
 	}()
vendor/src/github.com/docker/swarmkit/agent/helpers.go
History View file @ 3b555a5
... ... 
@@ -3,7 +3,7 @@ package agent
3 3 
 import "golang.org/x/net/context"
4 4
5 5 
 // runctx blocks until the function exits, closed is closed, or the context is
6 
-// cancelled. Call as part os go statement.
6 
+// cancelled. Call as part of go statement.
7 7 
 func runctx(ctx context.Context, closed chan struct{}, errs chan error, fn func(ctx context.Context) error) {
8 8 
 	select {
9 9 
 	case errs <- fn(ctx):
vendor/src/github.com/docker/swarmkit/agent/node.go
History View file @ 3b555a5
... ... 
@@ -89,7 +89,9 @@ type Node struct {
89 89 
 	nodeID               string
90 90 
 	nodeMembership       api.NodeSpec_Membership
91 91 
 	started              chan struct{}
92 
+	startOnce            sync.Once
92 93 
 	stopped              chan struct{}
94 
+	stopOnce             sync.Once
93 95 
 	ready                chan struct{} // closed when agent has completed registration and manager(if enabled) is ready to receive control requests
94 96 
 	certificateRequested chan struct{} // closed when certificate issue request has been sent by node
95 97 
 	closed               chan struct{}
... ... 
@@ -137,26 +139,15 @@ func NewNode(c *NodeConfig) (*Node, error) {
137 137
138 138 
 // Start starts a node instance.
139 139 
 func (n *Node) Start(ctx context.Context) error {
140 
-	select {
141 
-	case <-n.started:
142 
-		select {
143 
-		case <-n.closed:
144 
-			return n.err
145 
-		case <-n.stopped:
146 
-			return errAgentStopped
147 
-		case <-ctx.Done():
148 
-			return ctx.Err()
149 
-		default:
150 
-			return errAgentStarted
151 
-		}
152 
-	case <-ctx.Done():
153 
-		return ctx.Err()
154 
-	default:
155 
-	}
140 
+	err := errNodeStarted
156 141
157 
-	close(n.started)
158 
-	go n.run(ctx)
159 
-	return nil
142 
+	n.startOnce.Do(func() {
143 
+		close(n.started)
144 
+		go n.run(ctx)
145 
+		err = nil // clear error above, only once.
146 
+	})
147 
+
148 
+	return err
160 149 
 }
161 150
162 151 
 func (n *Node) run(ctx context.Context) (err error) {
... ... 
@@ -166,7 +157,7 @@ func (n *Node) run(ctx context.Context) (err error) {
166 166 
 	}()
167 167 
 	ctx, cancel := context.WithCancel(ctx)
168 168 
 	defer cancel()
169 
-	ctx = log.WithLogger(ctx, log.G(ctx).WithField("module", "node"))
169 
+	ctx = log.WithModule(ctx, "node")
170 170
171 171 
 	go func() {
172 172 
 		select {
... ... 
@@ -325,27 +316,19 @@ func (n *Node) run(ctx context.Context) (err error) {
325 325 
 func (n *Node) Stop(ctx context.Context) error {
326 326 
 	select {
327 327 
 	case <-n.started:
328 
-		select {
329 
-		case <-n.closed:
330 
-			return n.err
331 
-		case <-n.stopped:
332 
-			select {
333 
-			case <-n.closed:
334 
-				return n.err
335 
-			case <-ctx.Done():
336 
-				return ctx.Err()
337 
-			}
338 
-		case <-ctx.Done():
339 
-			return ctx.Err()
340 
-		default:
341 
-			close(n.stopped)
342 
-			// recurse and wait for closure
343 
-			return n.Stop(ctx)
344 
-		}
328 
+	default:
329 
+		return errNodeNotStarted
330 
+	}
331 
+
332 
+	n.stopOnce.Do(func() {
333 
+		close(n.stopped)
334 
+	})
335 
+
336 
+	select {
337 
+	case <-n.closed:
338 
+		return nil
345 339 
 	case <-ctx.Done():
346 340 
 		return ctx.Err()
347 
-	default:
348 
-		return errAgentNotStarted
349 341 
 	}
350 342 
 }
351 343
... ... 
@@ -361,31 +344,21 @@ func (n *Node) Err(ctx context.Context) error {
361 361 
 }
362 362
363 363 
 func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.TransportAuthenticator, ready chan<- struct{}) error {
364 
-	var manager api.Peer
365 364 
 	select {
366 365 
 	case <-ctx.Done():
367 
-	case manager = <-n.remotes.WaitSelect(ctx):
366 
+	case <-n.remotes.WaitSelect(ctx):
368 367 
 	}
369 368 
 	if ctx.Err() != nil {
370 369 
 		return ctx.Err()
371 370 
 	}
372 
-	picker := picker.NewPicker(n.remotes, manager.Addr)
373 
-	conn, err := grpc.Dial(manager.Addr,
374 
-		grpc.WithPicker(picker),
375 
-		grpc.WithTransportCredentials(creds),
376 
-		grpc.WithBackoffMaxDelay(maxSessionFailureBackoff))
377 
-	if err != nil {
378 
-		return err
379 
-	}
380 371
381 372 
 	agent, err := New(&Config{
382 373 
 		Hostname:         n.config.Hostname,
383 374 
 		Managers:         n.remotes,
384 375 
 		Executor:         n.config.Executor,
385 376 
 		DB:               db,
386 
-		Conn:             conn,
387 
-		Picker:           picker,
388 377 
 		NotifyRoleChange: n.roleChangeReq,
378 
+		Credentials:      creds,
389 379 
 	})
390 380 
 	if err != nil {
391 381 
 		return err
vendor/src/github.com/docker/swarmkit/agent/session.go
History View file @ 3b555a5
... ... 
@@ -6,6 +6,7 @@ import (
6 6
7 7 
 	"github.com/docker/swarmkit/api"
8 8 
 	"github.com/docker/swarmkit/log"
9 
+	"github.com/docker/swarmkit/picker"
9 10 
 	"github.com/docker/swarmkit/protobuf/ptypes"
10 11 
 	"golang.org/x/net/context"
11 12 
 	"google.golang.org/grpc"
... ... 
@@ -27,6 +28,9 @@ var (
27 27 
 // flow into the agent, such as task assignment, are called back into the
28 28 
 // agent through errs, messages and tasks.
29 29 
 type session struct {
30 
+	conn *grpc.ClientConn
31 
+	addr string
32 
+
30 33 
 	agent     *Agent
31 34 
 	sessionID string
32 35 
 	session   api.Dispatcher_SessionClient
... ... 
@@ -41,12 +45,27 @@ type session struct {
41 41 
 func newSession(ctx context.Context, agent *Agent, delay time.Duration) *session {
42 42 
 	s := &session{
43 43 
 		agent:      agent,
44 
-		errs:       make(chan error),
44 
+		errs:       make(chan error, 1),
45 45 
 		messages:   make(chan *api.SessionMessage),
46 46 
 		tasks:      make(chan *api.TasksMessage),
47 47 
 		registered: make(chan struct{}),
48 48 
 		closed:     make(chan struct{}),
49 49 
 	}
50 
+	peer, err := agent.config.Managers.Select()
51 
+	if err != nil {
52 
+		s.errs <- err
53 
+		return s
54 
+	}
55 
+	cc, err := grpc.Dial(peer.Addr,
56 
+		grpc.WithTransportCredentials(agent.config.Credentials),
57 
+		grpc.WithTimeout(dispatcherRPCTimeout),
58 
+	)
59 
+	if err != nil {
60 
+		s.errs <- err
61 
+		return s
62 
+	}
63 
+	s.addr = peer.Addr
64 
+	s.conn = cc
50 65
51 66 
 	go s.run(ctx, delay)
52 67 
 	return s
... ... 
@@ -77,8 +96,6 @@ func (s *session) run(ctx context.Context, delay time.Duration) {
77 77 
 func (s *session) start(ctx context.Context) error {
78 78 
 	log.G(ctx).Debugf("(*session).start")
79 79
80 
-	client := api.NewDispatcherClient(s.agent.config.Conn)
81 
-
82 80 
 	description, err := s.agent.config.Executor.Describe(ctx)
83 81 
 	if err != nil {
84 82 
 		log.G(ctx).WithError(err).WithField("executor", s.agent.config.Executor).
... ... 
@@ -103,6 +120,8 @@ func (s *session) start(ctx context.Context) error {
103 103 
 	// Need to run Session in a goroutine since there's no way to set a
104 104 
 	// timeout for an individual Recv call in a stream.
105 105 
 	go func() {
106 
+		client := api.NewDispatcherClient(s.conn)
107 
+
106 108 
 		stream, err = client.Session(sessionCtx, &api.SessionRequest{
107 109 
 			Description: description,
108 110 
 		})
... ... 
@@ -133,7 +152,7 @@ func (s *session) start(ctx context.Context) error {
133 133
134 134 
 func (s *session) heartbeat(ctx context.Context) error {
135 135 
 	log.G(ctx).Debugf("(*session).heartbeat")
136 
-	client := api.NewDispatcherClient(s.agent.config.Conn)
136 
+	client := api.NewDispatcherClient(s.conn)
137 137 
 	heartbeat := time.NewTimer(1) // send out a heartbeat right away
138 138 
 	defer heartbeat.Stop()
139 139
... ... 
@@ -195,7 +214,7 @@ func (s *session) handleSessionMessage(ctx context.Context, msg *api.SessionMess
195 195
196 196 
 func (s *session) watch(ctx context.Context) error {
197 197 
 	log.G(ctx).Debugf("(*session).watch")
198 
-	client := api.NewDispatcherClient(s.agent.config.Conn)
198 
+	client := api.NewDispatcherClient(s.conn)
199 199 
 	watch, err := client.Tasks(ctx, &api.TasksRequest{
200 200 
 		SessionID: s.sessionID})
201 201 
 	if err != nil {
... ... 
@@ -221,7 +240,7 @@ func (s *session) watch(ctx context.Context) error {
221 221 
 // sendTaskStatus uses the current session to send the status of a single task.
222 222 
 func (s *session) sendTaskStatus(ctx context.Context, taskID string, status *api.TaskStatus) error {
223 223
224 
-	client := api.NewDispatcherClient(s.agent.config.Conn)
224 
+	client := api.NewDispatcherClient(s.conn)
225 225 
 	if _, err := client.UpdateTaskStatus(ctx, &api.UpdateTaskStatusRequest{
226 226 
 		SessionID: s.sessionID,
227 227 
 		Updates: []*api.UpdateTaskStatusRequest_TaskStatusUpdate{
... ... 
@@ -262,7 +281,7 @@ func (s *session) sendTaskStatuses(ctx context.Context, updates ...*api.UpdateTa
262 262 
 		return updates, ctx.Err()
263 263 
 	}
264 264
265 
-	client := api.NewDispatcherClient(s.agent.config.Conn)
265 
+	client := api.NewDispatcherClient(s.conn)
266 266 
 	n := batchSize
267 267
268 268 
 	if len(updates) < n {
... ... 
@@ -285,6 +304,10 @@ func (s *session) close() error {
285 285 
 	case <-s.closed:
286 286 
 		return errSessionClosed
287 287 
 	default:
288 
+		if s.conn != nil {
289 
+			s.agent.config.Managers.ObserveIfExists(api.Peer{Addr: s.addr}, -picker.DefaultObservationWeight)
290 
+			s.conn.Close()
291 
+		}
288 292 
 		close(s.closed)
289 293 
 		return nil
290 294 
 	}
vendor/src/github.com/docker/swarmkit/agent/task.go
History View file @ 3b555a5
... ... 
@@ -68,7 +68,7 @@ func (tm *taskManager) run(ctx context.Context) {
68 68 
 	ctx, cancelAll := context.WithCancel(ctx)
69 69 
 	defer cancelAll() // cancel all child operations on exit.
70 70
71 
-	ctx = log.WithLogger(ctx, log.G(ctx).WithField("module", "taskmanager"))
71 
+	ctx = log.WithModule(ctx, "taskmanager")
72 72
73 73 
 	var (
74 74 
 		opctx    context.Context
vendor/src/github.com/docker/swarmkit/agent/worker.go
History View file @ 3b555a5
... ... 
@@ -57,7 +57,7 @@ func (w *worker) Init(ctx context.Context) error {
57 57 
 	w.mu.Lock()
58 58 
 	defer w.mu.Unlock()
59 59
60 
-	ctx = log.WithLogger(ctx, log.G(ctx).WithField("module", "worker"))
60 
+	ctx = log.WithModule(ctx, "worker")
61 61
62 62 
 	// TODO(stevvooe): Start task cleanup process.
63 63
vendor/src/github.com/docker/swarmkit/api/ca.pb.go
History View file @ 3b555a5
... ... 
@@ -668,12 +668,12 @@ func encodeVarintCa(data []byte, offset int, v uint64) int {
668 668
669 669 
 type raftProxyCAServer struct {
670 670 
 	local        CAServer
671 
-	connSelector *raftpicker.ConnSelector
671 
+	connSelector raftpicker.Interface
672 672 
 	cluster      raftpicker.RaftCluster
673 673 
 	ctxMods      []func(context.Context) (context.Context, error)
674 674 
 }
675 675
676 
-func NewRaftProxyCAServer(local CAServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) CAServer {
676 
+func NewRaftProxyCAServer(local CAServer, connSelector raftpicker.Interface, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) CAServer {
677 677 
 	redirectChecker := func(ctx context.Context) (context.Context, error) {
678 678 
 		s, ok := transport.StreamFromContext(ctx)
679 679 
 		if !ok {
... ... 
@@ -724,17 +724,30 @@ func (p *raftProxyCAServer) GetRootCACertificate(ctx context.Context, r *GetRoot
724 724 
 	if err != nil {
725 725 
 		return nil, err
726 726 
 	}
727 
+
728 
+	defer func() {
729 
+		if err != nil {
730 
+			errStr := err.Error()
731 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
732 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
733 
+				strings.Contains(errStr, "connection error") ||
734 
+				grpc.Code(err) == codes.Internal {
735 
+				p.connSelector.Reset()
736 
+			}
737 
+		}
738 
+	}()
739 
+
727 740 
 	return NewCAClient(conn).GetRootCACertificate(ctx, r)
728 741 
 }
729 742
730 743 
 type raftProxyNodeCAServer struct {
731 744 
 	local        NodeCAServer
732 
-	connSelector *raftpicker.ConnSelector
745 
+	connSelector raftpicker.Interface
733 746 
 	cluster      raftpicker.RaftCluster
734 747 
 	ctxMods      []func(context.Context) (context.Context, error)
735 748 
 }
736 749
737 
-func NewRaftProxyNodeCAServer(local NodeCAServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) NodeCAServer {
750 
+func NewRaftProxyNodeCAServer(local NodeCAServer, connSelector raftpicker.Interface, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) NodeCAServer {
738 751 
 	redirectChecker := func(ctx context.Context) (context.Context, error) {
739 752 
 		s, ok := transport.StreamFromContext(ctx)
740 753 
 		if !ok {
... ... 
@@ -785,6 +798,19 @@ func (p *raftProxyNodeCAServer) IssueNodeCertificate(ctx context.Context, r *Iss
785 785 
 	if err != nil {
786 786 
 		return nil, err
787 787 
 	}
788 
+
789 
+	defer func() {
790 
+		if err != nil {
791 
+			errStr := err.Error()
792 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
793 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
794 
+				strings.Contains(errStr, "connection error") ||
795 
+				grpc.Code(err) == codes.Internal {
796 
+				p.connSelector.Reset()
797 
+			}
798 
+		}
799 
+	}()
800 
+
788 801 
 	return NewNodeCAClient(conn).IssueNodeCertificate(ctx, r)
789 802 
 }
790 803
... ... 
@@ -801,6 +827,19 @@ func (p *raftProxyNodeCAServer) NodeCertificateStatus(ctx context.Context, r *No
801 801 
 	if err != nil {
802 802 
 		return nil, err
803 803 
 	}
804 
+
805 
+	defer func() {
806 
+		if err != nil {
807 
+			errStr := err.Error()
808 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
809 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
810 
+				strings.Contains(errStr, "connection error") ||
811 
+				grpc.Code(err) == codes.Internal {
812 
+				p.connSelector.Reset()
813 
+			}
814 
+		}
815 
+	}()
816 
+
804 817 
 	return NewNodeCAClient(conn).NodeCertificateStatus(ctx, r)
805 818 
 }
806 819
vendor/src/github.com/docker/swarmkit/api/control.pb.go
History View file @ 3b555a5
... ... 
@@ -4239,12 +4239,12 @@ func encodeVarintControl(data []byte, offset int, v uint64) int {
4239 4239
4240 4240 
 type raftProxyControlServer struct {
4241 4241 
 	local        ControlServer
4242 
-	connSelector *raftpicker.ConnSelector
4242 
+	connSelector raftpicker.Interface
4243 4243 
 	cluster      raftpicker.RaftCluster
4244 4244 
 	ctxMods      []func(context.Context) (context.Context, error)
4245 4245 
 }
4246 4246
4247 
-func NewRaftProxyControlServer(local ControlServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) ControlServer {
4247 
+func NewRaftProxyControlServer(local ControlServer, connSelector raftpicker.Interface, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) ControlServer {
4248 4248 
 	redirectChecker := func(ctx context.Context) (context.Context, error) {
4249 4249 
 		s, ok := transport.StreamFromContext(ctx)
4250 4250 
 		if !ok {
... ... 
@@ -4295,6 +4295,19 @@ func (p *raftProxyControlServer) GetNode(ctx context.Context, r *GetNodeRequest)
4295 4295 
 	if err != nil {
4296 4296 
 		return nil, err
4297 4297 
 	}
4298 
+
4299 
+	defer func() {
4300 
+		if err != nil {
4301 
+			errStr := err.Error()
4302 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4303 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4304 
+				strings.Contains(errStr, "connection error") ||
4305 
+				grpc.Code(err) == codes.Internal {
4306 
+				p.connSelector.Reset()
4307 
+			}
4308 
+		}
4309 
+	}()
4310 
+
4298 4311 
 	return NewControlClient(conn).GetNode(ctx, r)
4299 4312 
 }
4300 4313
... ... 
@@ -4311,6 +4324,19 @@ func (p *raftProxyControlServer) ListNodes(ctx context.Context, r *ListNodesRequ
4311 4311 
 	if err != nil {
4312 4312 
 		return nil, err
4313 4313 
 	}
4314 
+
4315 
+	defer func() {
4316 
+		if err != nil {
4317 
+			errStr := err.Error()
4318 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4319 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4320 
+				strings.Contains(errStr, "connection error") ||
4321 
+				grpc.Code(err) == codes.Internal {
4322 
+				p.connSelector.Reset()
4323 
+			}
4324 
+		}
4325 
+	}()
4326 
+
4314 4327 
 	return NewControlClient(conn).ListNodes(ctx, r)
4315 4328 
 }
4316 4329
... ... 
@@ -4327,6 +4353,19 @@ func (p *raftProxyControlServer) UpdateNode(ctx context.Context, r *UpdateNodeRe
4327 4327 
 	if err != nil {
4328 4328 
 		return nil, err
4329 4329 
 	}
4330 
+
4331 
+	defer func() {
4332 
+		if err != nil {
4333 
+			errStr := err.Error()
4334 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4335 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4336 
+				strings.Contains(errStr, "connection error") ||
4337 
+				grpc.Code(err) == codes.Internal {
4338 
+				p.connSelector.Reset()
4339 
+			}
4340 
+		}
4341 
+	}()
4342 
+
4330 4343 
 	return NewControlClient(conn).UpdateNode(ctx, r)
4331 4344 
 }
4332 4345
... ... 
@@ -4343,6 +4382,19 @@ func (p *raftProxyControlServer) RemoveNode(ctx context.Context, r *RemoveNodeRe
4343 4343 
 	if err != nil {
4344 4344 
 		return nil, err
4345 4345 
 	}
4346 
+
4347 
+	defer func() {
4348 
+		if err != nil {
4349 
+			errStr := err.Error()
4350 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4351 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4352 
+				strings.Contains(errStr, "connection error") ||
4353 
+				grpc.Code(err) == codes.Internal {
4354 
+				p.connSelector.Reset()
4355 
+			}
4356 
+		}
4357 
+	}()
4358 
+
4346 4359 
 	return NewControlClient(conn).RemoveNode(ctx, r)
4347 4360 
 }
4348 4361
... ... 
@@ -4359,6 +4411,19 @@ func (p *raftProxyControlServer) GetTask(ctx context.Context, r *GetTaskRequest)
4359 4359 
 	if err != nil {
4360 4360 
 		return nil, err
4361 4361 
 	}
4362 
+
4363 
+	defer func() {
4364 
+		if err != nil {
4365 
+			errStr := err.Error()
4366 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4367 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4368 
+				strings.Contains(errStr, "connection error") ||
4369 
+				grpc.Code(err) == codes.Internal {
4370 
+				p.connSelector.Reset()
4371 
+			}
4372 
+		}
4373 
+	}()
4374 
+
4362 4375 
 	return NewControlClient(conn).GetTask(ctx, r)
4363 4376 
 }
4364 4377
... ... 
@@ -4375,6 +4440,19 @@ func (p *raftProxyControlServer) ListTasks(ctx context.Context, r *ListTasksRequ
4375 4375 
 	if err != nil {
4376 4376 
 		return nil, err
4377 4377 
 	}
4378 
+
4379 
+	defer func() {
4380 
+		if err != nil {
4381 
+			errStr := err.Error()
4382 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4383 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4384 
+				strings.Contains(errStr, "connection error") ||
4385 
+				grpc.Code(err) == codes.Internal {
4386 
+				p.connSelector.Reset()
4387 
+			}
4388 
+		}
4389 
+	}()
4390 
+
4378 4391 
 	return NewControlClient(conn).ListTasks(ctx, r)
4379 4392 
 }
4380 4393
... ... 
@@ -4391,6 +4469,19 @@ func (p *raftProxyControlServer) RemoveTask(ctx context.Context, r *RemoveTaskRe
4391 4391 
 	if err != nil {
4392 4392 
 		return nil, err
4393 4393 
 	}
4394 
+
4395 
+	defer func() {
4396 
+		if err != nil {
4397 
+			errStr := err.Error()
4398 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4399 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4400 
+				strings.Contains(errStr, "connection error") ||
4401 
+				grpc.Code(err) == codes.Internal {
4402 
+				p.connSelector.Reset()
4403 
+			}
4404 
+		}
4405 
+	}()
4406 
+
4394 4407 
 	return NewControlClient(conn).RemoveTask(ctx, r)
4395 4408 
 }
4396 4409
... ... 
@@ -4407,6 +4498,19 @@ func (p *raftProxyControlServer) GetService(ctx context.Context, r *GetServiceRe
4407 4407 
 	if err != nil {
4408 4408 
 		return nil, err
4409 4409 
 	}
4410 
+
4411 
+	defer func() {
4412 
+		if err != nil {
4413 
+			errStr := err.Error()
4414 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4415 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4416 
+				strings.Contains(errStr, "connection error") ||
4417 
+				grpc.Code(err) == codes.Internal {
4418 
+				p.connSelector.Reset()
4419 
+			}
4420 
+		}
4421 
+	}()
4422 
+
4410 4423 
 	return NewControlClient(conn).GetService(ctx, r)
4411 4424 
 }
4412 4425
... ... 
@@ -4423,6 +4527,19 @@ func (p *raftProxyControlServer) ListServices(ctx context.Context, r *ListServic
4423 4423 
 	if err != nil {
4424 4424 
 		return nil, err
4425 4425 
 	}
4426 
+
4427 
+	defer func() {
4428 
+		if err != nil {
4429 
+			errStr := err.Error()
4430 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4431 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4432 
+				strings.Contains(errStr, "connection error") ||
4433 
+				grpc.Code(err) == codes.Internal {
4434 
+				p.connSelector.Reset()
4435 
+			}
4436 
+		}
4437 
+	}()
4438 
+
4426 4439 
 	return NewControlClient(conn).ListServices(ctx, r)
4427 4440 
 }
4428 4441
... ... 
@@ -4439,6 +4556,19 @@ func (p *raftProxyControlServer) CreateService(ctx context.Context, r *CreateSer
4439 4439 
 	if err != nil {
4440 4440 
 		return nil, err
4441 4441 
 	}
4442 
+
4443 
+	defer func() {
4444 
+		if err != nil {
4445 
+			errStr := err.Error()
4446 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4447 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4448 
+				strings.Contains(errStr, "connection error") ||
4449 
+				grpc.Code(err) == codes.Internal {
4450 
+				p.connSelector.Reset()
4451 
+			}
4452 
+		}
4453 
+	}()
4454 
+
4442 4455 
 	return NewControlClient(conn).CreateService(ctx, r)
4443 4456 
 }
4444 4457
... ... 
@@ -4455,6 +4585,19 @@ func (p *raftProxyControlServer) UpdateService(ctx context.Context, r *UpdateSer
4455 4455 
 	if err != nil {
4456 4456 
 		return nil, err
4457 4457 
 	}
4458 
+
4459 
+	defer func() {
4460 
+		if err != nil {
4461 
+			errStr := err.Error()
4462 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4463 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4464 
+				strings.Contains(errStr, "connection error") ||
4465 
+				grpc.Code(err) == codes.Internal {
4466 
+				p.connSelector.Reset()
4467 
+			}
4468 
+		}
4469 
+	}()
4470 
+
4458 4471 
 	return NewControlClient(conn).UpdateService(ctx, r)
4459 4472 
 }
4460 4473
... ... 
@@ -4471,6 +4614,19 @@ func (p *raftProxyControlServer) RemoveService(ctx context.Context, r *RemoveSer
4471 4471 
 	if err != nil {
4472 4472 
 		return nil, err
4473 4473 
 	}
4474 
+
4475 
+	defer func() {
4476 
+		if err != nil {
4477 
+			errStr := err.Error()
4478 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4479 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4480 
+				strings.Contains(errStr, "connection error") ||
4481 
+				grpc.Code(err) == codes.Internal {
4482 
+				p.connSelector.Reset()
4483 
+			}
4484 
+		}
4485 
+	}()
4486 
+
4474 4487 
 	return NewControlClient(conn).RemoveService(ctx, r)
4475 4488 
 }
4476 4489
... ... 
@@ -4487,6 +4643,19 @@ func (p *raftProxyControlServer) GetNetwork(ctx context.Context, r *GetNetworkRe
4487 4487 
 	if err != nil {
4488 4488 
 		return nil, err
4489 4489 
 	}
4490 
+
4491 
+	defer func() {
4492 
+		if err != nil {
4493 
+			errStr := err.Error()
4494 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4495 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4496 
+				strings.Contains(errStr, "connection error") ||
4497 
+				grpc.Code(err) == codes.Internal {
4498 
+				p.connSelector.Reset()
4499 
+			}
4500 
+		}
4501 
+	}()
4502 
+
4490 4503 
 	return NewControlClient(conn).GetNetwork(ctx, r)
4491 4504 
 }
4492 4505
... ... 
@@ -4503,6 +4672,19 @@ func (p *raftProxyControlServer) ListNetworks(ctx context.Context, r *ListNetwor
4503 4503 
 	if err != nil {
4504 4504 
 		return nil, err
4505 4505 
 	}
4506 
+
4507 
+	defer func() {
4508 
+		if err != nil {
4509 
+			errStr := err.Error()
4510 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4511 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4512 
+				strings.Contains(errStr, "connection error") ||
4513 
+				grpc.Code(err) == codes.Internal {
4514 
+				p.connSelector.Reset()
4515 
+			}
4516 
+		}
4517 
+	}()
4518 
+
4506 4519 
 	return NewControlClient(conn).ListNetworks(ctx, r)
4507 4520 
 }
4508 4521
... ... 
@@ -4519,6 +4701,19 @@ func (p *raftProxyControlServer) CreateNetwork(ctx context.Context, r *CreateNet
4519 4519 
 	if err != nil {
4520 4520 
 		return nil, err
4521 4521 
 	}
4522 
+
4523 
+	defer func() {
4524 
+		if err != nil {
4525 
+			errStr := err.Error()
4526 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4527 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4528 
+				strings.Contains(errStr, "connection error") ||
4529 
+				grpc.Code(err) == codes.Internal {
4530 
+				p.connSelector.Reset()
4531 
+			}
4532 
+		}
4533 
+	}()
4534 
+
4522 4535 
 	return NewControlClient(conn).CreateNetwork(ctx, r)
4523 4536 
 }
4524 4537
... ... 
@@ -4535,6 +4730,19 @@ func (p *raftProxyControlServer) RemoveNetwork(ctx context.Context, r *RemoveNet
4535 4535 
 	if err != nil {
4536 4536 
 		return nil, err
4537 4537 
 	}
4538 
+
4539 
+	defer func() {
4540 
+		if err != nil {
4541 
+			errStr := err.Error()
4542 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4543 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4544 
+				strings.Contains(errStr, "connection error") ||
4545 
+				grpc.Code(err) == codes.Internal {
4546 
+				p.connSelector.Reset()
4547 
+			}
4548 
+		}
4549 
+	}()
4550 
+
4538 4551 
 	return NewControlClient(conn).RemoveNetwork(ctx, r)
4539 4552 
 }
4540 4553
... ... 
@@ -4551,6 +4759,19 @@ func (p *raftProxyControlServer) GetCluster(ctx context.Context, r *GetClusterRe
4551 4551 
 	if err != nil {
4552 4552 
 		return nil, err
4553 4553 
 	}
4554 
+
4555 
+	defer func() {
4556 
+		if err != nil {
4557 
+			errStr := err.Error()
4558 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4559 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4560 
+				strings.Contains(errStr, "connection error") ||
4561 
+				grpc.Code(err) == codes.Internal {
4562 
+				p.connSelector.Reset()
4563 
+			}
4564 
+		}
4565 
+	}()
4566 
+
4554 4567 
 	return NewControlClient(conn).GetCluster(ctx, r)
4555 4568 
 }
4556 4569
... ... 
@@ -4567,6 +4788,19 @@ func (p *raftProxyControlServer) ListClusters(ctx context.Context, r *ListCluste
4567 4567 
 	if err != nil {
4568 4568 
 		return nil, err
4569 4569 
 	}
4570 
+
4571 
+	defer func() {
4572 
+		if err != nil {
4573 
+			errStr := err.Error()
4574 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4575 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4576 
+				strings.Contains(errStr, "connection error") ||
4577 
+				grpc.Code(err) == codes.Internal {
4578 
+				p.connSelector.Reset()
4579 
+			}
4580 
+		}
4581 
+	}()
4582 
+
4570 4583 
 	return NewControlClient(conn).ListClusters(ctx, r)
4571 4584 
 }
4572 4585
... ... 
@@ -4583,6 +4817,19 @@ func (p *raftProxyControlServer) UpdateCluster(ctx context.Context, r *UpdateClu
4583 4583 
 	if err != nil {
4584 4584 
 		return nil, err
4585 4585 
 	}
4586 
+
4587 
+	defer func() {
4588 
+		if err != nil {
4589 
+			errStr := err.Error()
4590 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
4591 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
4592 
+				strings.Contains(errStr, "connection error") ||
4593 
+				grpc.Code(err) == codes.Internal {
4594 
+				p.connSelector.Reset()
4595 
+			}
4596 
+		}
4597 
+	}()
4598 
+
4586 4599 
 	return NewControlClient(conn).UpdateCluster(ctx, r)
4587 4600 
 }
4588 4601
vendor/src/github.com/docker/swarmkit/api/dispatcher.pb.go
History View file @ 3b555a5
... ... 
@@ -1072,12 +1072,12 @@ func encodeVarintDispatcher(data []byte, offset int, v uint64) int {
1072 1072
1073 1073 
 type raftProxyDispatcherServer struct {
1074 1074 
 	local        DispatcherServer
1075 
-	connSelector *raftpicker.ConnSelector
1075 
+	connSelector raftpicker.Interface
1076 1076 
 	cluster      raftpicker.RaftCluster
1077 1077 
 	ctxMods      []func(context.Context) (context.Context, error)
1078 1078 
 }
1079 1079
1080 
-func NewRaftProxyDispatcherServer(local DispatcherServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) DispatcherServer {
1080 
+func NewRaftProxyDispatcherServer(local DispatcherServer, connSelector raftpicker.Interface, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) DispatcherServer {
1081 1081 
 	redirectChecker := func(ctx context.Context) (context.Context, error) {
1082 1082 
 		s, ok := transport.StreamFromContext(ctx)
1083 1083 
 		if !ok {
... ... 
@@ -1128,6 +1128,19 @@ func (p *raftProxyDispatcherServer) Session(r *SessionRequest, stream Dispatcher
1128 1128 
 	if err != nil {
1129 1129 
 		return err
1130 1130 
 	}
1131 
+
1132 
+	defer func() {
1133 
+		if err != nil {
1134 
+			errStr := err.Error()
1135 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1136 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1137 
+				strings.Contains(errStr, "connection error") ||
1138 
+				grpc.Code(err) == codes.Internal {
1139 
+				p.connSelector.Reset()
1140 
+			}
1141 
+		}
1142 
+	}()
1143 
+
1131 1144 
 	clientStream, err := NewDispatcherClient(conn).Session(ctx, r)
1132 1145
1133 1146 
 	if err != nil {
... ... 
@@ -1162,6 +1175,19 @@ func (p *raftProxyDispatcherServer) Heartbeat(ctx context.Context, r *HeartbeatR
1162 1162 
 	if err != nil {
1163 1163 
 		return nil, err
1164 1164 
 	}
1165 
+
1166 
+	defer func() {
1167 
+		if err != nil {
1168 
+			errStr := err.Error()
1169 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1170 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1171 
+				strings.Contains(errStr, "connection error") ||
1172 
+				grpc.Code(err) == codes.Internal {
1173 
+				p.connSelector.Reset()
1174 
+			}
1175 
+		}
1176 
+	}()
1177 
+
1165 1178 
 	return NewDispatcherClient(conn).Heartbeat(ctx, r)
1166 1179 
 }
1167 1180
... ... 
@@ -1178,6 +1204,19 @@ func (p *raftProxyDispatcherServer) UpdateTaskStatus(ctx context.Context, r *Upd
1178 1178 
 	if err != nil {
1179 1179 
 		return nil, err
1180 1180 
 	}
1181 
+
1182 
+	defer func() {
1183 
+		if err != nil {
1184 
+			errStr := err.Error()
1185 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1186 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1187 
+				strings.Contains(errStr, "connection error") ||
1188 
+				grpc.Code(err) == codes.Internal {
1189 
+				p.connSelector.Reset()
1190 
+			}
1191 
+		}
1192 
+	}()
1193 
+
1181 1194 
 	return NewDispatcherClient(conn).UpdateTaskStatus(ctx, r)
1182 1195 
 }
1183 1196
... ... 
@@ -1194,6 +1233,19 @@ func (p *raftProxyDispatcherServer) Tasks(r *TasksRequest, stream Dispatcher_Tas
1194 1194 
 	if err != nil {
1195 1195 
 		return err
1196 1196 
 	}
1197 
+
1198 
+	defer func() {
1199 
+		if err != nil {
1200 
+			errStr := err.Error()
1201 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1202 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1203 
+				strings.Contains(errStr, "connection error") ||
1204 
+				grpc.Code(err) == codes.Internal {
1205 
+				p.connSelector.Reset()
1206 
+			}
1207 
+		}
1208 
+	}()
1209 
+
1197 1210 
 	clientStream, err := NewDispatcherClient(conn).Tasks(ctx, r)
1198 1211
1199 1212 
 	if err != nil {
vendor/src/github.com/docker/swarmkit/api/equality/equality.go
History View file @ 3b555a5
... ... 
@@ -19,3 +19,11 @@ func TasksEqualStable(a, b *api.Task) bool {
19 19
20 20 
 	return reflect.DeepEqual(&copyA, &copyB)
21 21 
 }
22 
+
23 
+// TaskStatusesEqualStable compares the task status excluding timestamp fields.
24 
+func TaskStatusesEqualStable(a, b *api.TaskStatus) bool {
25 
+	copyA, copyB := *a, *b
26 
+
27 
+	copyA.Timestamp, copyB.Timestamp = nil, nil
28 
+	return reflect.DeepEqual(&copyA, &copyB)
29 
+}
vendor/src/github.com/docker/swarmkit/api/health.pb.go
History View file @ 3b555a5
... ... 
@@ -319,12 +319,12 @@ func encodeVarintHealth(data []byte, offset int, v uint64) int {
319 319
320 320 
 type raftProxyHealthServer struct {
321 321 
 	local        HealthServer
322 
-	connSelector *raftpicker.ConnSelector
322 
+	connSelector raftpicker.Interface
323 323 
 	cluster      raftpicker.RaftCluster
324 324 
 	ctxMods      []func(context.Context) (context.Context, error)
325 325 
 }
326 326
327 
-func NewRaftProxyHealthServer(local HealthServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) HealthServer {
327 
+func NewRaftProxyHealthServer(local HealthServer, connSelector raftpicker.Interface, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) HealthServer {
328 328 
 	redirectChecker := func(ctx context.Context) (context.Context, error) {
329 329 
 		s, ok := transport.StreamFromContext(ctx)
330 330 
 		if !ok {
... ... 
@@ -375,6 +375,19 @@ func (p *raftProxyHealthServer) Check(ctx context.Context, r *HealthCheckRequest
375 375 
 	if err != nil {
376 376 
 		return nil, err
377 377 
 	}
378 
+
379 
+	defer func() {
380 
+		if err != nil {
381 
+			errStr := err.Error()
382 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
383 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
384 
+				strings.Contains(errStr, "connection error") ||
385 
+				grpc.Code(err) == codes.Internal {
386 
+				p.connSelector.Reset()
387 
+			}
388 
+		}
389 
+	}()
390 
+
378 391 
 	return NewHealthClient(conn).Check(ctx, r)
379 392 
 }
380 393
vendor/src/github.com/docker/swarmkit/api/raft.pb.go
History View file @ 3b555a5
... ... 
@@ -1438,12 +1438,12 @@ func encodeVarintRaft(data []byte, offset int, v uint64) int {
1438 1438
1439 1439 
 type raftProxyRaftServer struct {
1440 1440 
 	local        RaftServer
1441 
-	connSelector *raftpicker.ConnSelector
1441 
+	connSelector raftpicker.Interface
1442 1442 
 	cluster      raftpicker.RaftCluster
1443 1443 
 	ctxMods      []func(context.Context) (context.Context, error)
1444 1444 
 }
1445 1445
1446 
-func NewRaftProxyRaftServer(local RaftServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) RaftServer {
1446 
+func NewRaftProxyRaftServer(local RaftServer, connSelector raftpicker.Interface, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) RaftServer {
1447 1447 
 	redirectChecker := func(ctx context.Context) (context.Context, error) {
1448 1448 
 		s, ok := transport.StreamFromContext(ctx)
1449 1449 
 		if !ok {
... ... 
@@ -1494,6 +1494,19 @@ func (p *raftProxyRaftServer) ProcessRaftMessage(ctx context.Context, r *Process
1494 1494 
 	if err != nil {
1495 1495 
 		return nil, err
1496 1496 
 	}
1497 
+
1498 
+	defer func() {
1499 
+		if err != nil {
1500 
+			errStr := err.Error()
1501 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1502 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1503 
+				strings.Contains(errStr, "connection error") ||
1504 
+				grpc.Code(err) == codes.Internal {
1505 
+				p.connSelector.Reset()
1506 
+			}
1507 
+		}
1508 
+	}()
1509 
+
1497 1510 
 	return NewRaftClient(conn).ProcessRaftMessage(ctx, r)
1498 1511 
 }
1499 1512
... ... 
@@ -1510,17 +1523,30 @@ func (p *raftProxyRaftServer) ResolveAddress(ctx context.Context, r *ResolveAddr
1510 1510 
 	if err != nil {
1511 1511 
 		return nil, err
1512 1512 
 	}
1513 
+
1514 
+	defer func() {
1515 
+		if err != nil {
1516 
+			errStr := err.Error()
1517 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1518 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1519 
+				strings.Contains(errStr, "connection error") ||
1520 
+				grpc.Code(err) == codes.Internal {
1521 
+				p.connSelector.Reset()
1522 
+			}
1523 
+		}
1524 
+	}()
1525 
+
1513 1526 
 	return NewRaftClient(conn).ResolveAddress(ctx, r)
1514 1527 
 }
1515 1528
1516 1529 
 type raftProxyRaftMembershipServer struct {
1517 1530 
 	local        RaftMembershipServer
1518 
-	connSelector *raftpicker.ConnSelector
1531 
+	connSelector raftpicker.Interface
1519 1532 
 	cluster      raftpicker.RaftCluster
1520 1533 
 	ctxMods      []func(context.Context) (context.Context, error)
1521 1534 
 }
1522 1535
1523 
-func NewRaftProxyRaftMembershipServer(local RaftMembershipServer, connSelector *raftpicker.ConnSelector, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) RaftMembershipServer {
1536 
+func NewRaftProxyRaftMembershipServer(local RaftMembershipServer, connSelector raftpicker.Interface, cluster raftpicker.RaftCluster, ctxMod func(context.Context) (context.Context, error)) RaftMembershipServer {
1524 1537 
 	redirectChecker := func(ctx context.Context) (context.Context, error) {
1525 1538 
 		s, ok := transport.StreamFromContext(ctx)
1526 1539 
 		if !ok {
... ... 
@@ -1571,6 +1597,19 @@ func (p *raftProxyRaftMembershipServer) Join(ctx context.Context, r *JoinRequest
1571 1571 
 	if err != nil {
1572 1572 
 		return nil, err
1573 1573 
 	}
1574 
+
1575 
+	defer func() {
1576 
+		if err != nil {
1577 
+			errStr := err.Error()
1578 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1579 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1580 
+				strings.Contains(errStr, "connection error") ||
1581 
+				grpc.Code(err) == codes.Internal {
1582 
+				p.connSelector.Reset()
1583 
+			}
1584 
+		}
1585 
+	}()
1586 
+
1574 1587 
 	return NewRaftMembershipClient(conn).Join(ctx, r)
1575 1588 
 }
1576 1589
... ... 
@@ -1587,6 +1626,19 @@ func (p *raftProxyRaftMembershipServer) Leave(ctx context.Context, r *LeaveReque
1587 1587 
 	if err != nil {
1588 1588 
 		return nil, err
1589 1589 
 	}
1590 
+
1591 
+	defer func() {
1592 
+		if err != nil {
1593 
+			errStr := err.Error()
1594 
+			if strings.Contains(errStr, grpc.ErrClientConnClosing.Error()) ||
1595 
+				strings.Contains(errStr, grpc.ErrClientConnTimeout.Error()) ||
1596 
+				strings.Contains(errStr, "connection error") ||
1597 
+				grpc.Code(err) == codes.Internal {
1598 
+				p.connSelector.Reset()
1599 
+			}
1600 
+		}
1601 
+	}()
1602 
+
1590 1603 
 	return NewRaftMembershipClient(conn).Leave(ctx, r)
1591 1604 
 }
1592 1605
vendor/src/github.com/docker/swarmkit/api/specs.pb.go
History View file @ 3b555a5
... ... 
@@ -432,8 +432,14 @@ type ContainerSpec struct {
432 432 
 	// Dir defines the working directory to set for the container process.
433 433 
 	Dir string `protobuf:"bytes,6,opt,name=dir,proto3" json:"dir,omitempty"`
434 434 
 	// User specifies the user that should be employed to run the container.
435 
-	User   string  `protobuf:"bytes,7,opt,name=user,proto3" json:"user,omitempty"`
436 
-	Mounts []Mount `protobuf:"bytes,8,rep,name=mounts" json:"mounts"`
435 
+	//
436 
+	// Note that the primary group may be specified by appending the group name
437 
+	// or id to the user name, separated by a `:`. This syntax is
438 
+	// `<user>:<group>`.
439 
+	User string `protobuf:"bytes,7,opt,name=user,proto3" json:"user,omitempty"`
440 
+	// Groups specifies supplementary groups available to the user.
441 
+	Groups []string `protobuf:"bytes,11,rep,name=groups" json:"groups,omitempty"`
442 
+	Mounts []Mount  `protobuf:"bytes,8,rep,name=mounts" json:"mounts"`
437 443 
 	// StopGracePeriod the grace period for stopping the container before
438 444 
 	// forcefully killing the container.
439 445 
 	StopGracePeriod *docker_swarmkit_v11.Duration `protobuf:"bytes,9,opt,name=stop_grace_period,json=stopGracePeriod" json:"stop_grace_period,omitempty"`
... ... 
@@ -688,6 +694,13 @@ func (m *ContainerSpec) Copy() *ContainerSpec {
688 688 
 		}
689 689 
 	}
690 690
691 
+	if m.Groups != nil {
692 
+		o.Groups = make([]string, 0, len(m.Groups))
693 
+		for _, v := range m.Groups {
694 
+			o.Groups = append(o.Groups, v)
695 
+		}
696 
+	}
697 
+
691 698 
 	if m.Mounts != nil {
692 699 
 		o.Mounts = make([]Mount, 0, len(m.Mounts))
693 700 
 		for _, v := range m.Mounts {
... ... 
@@ -881,7 +894,7 @@ func (this *ContainerSpec) GoString() string {
881 881 
 	if this == nil {
882 882 
 		return "nil"
883 883 
 	}
884 
-	s := make([]string, 0, 14)
884 
+	s := make([]string, 0, 15)
885 885 
 	s = append(s, "&api.ContainerSpec{")
886 886 
 	s = append(s, "Image: "+fmt.Sprintf("%#v", this.Image)+",\n")
887 887 
 	keysForLabels := make([]string, 0, len(this.Labels))
... ... 
@@ -902,6 +915,7 @@ func (this *ContainerSpec) GoString() string {
902 902 
 	s = append(s, "Env: "+fmt.Sprintf("%#v", this.Env)+",\n")
903 903 
 	s = append(s, "Dir: "+fmt.Sprintf("%#v", this.Dir)+",\n")
904 904 
 	s = append(s, "User: "+fmt.Sprintf("%#v", this.User)+",\n")
905 
+	s = append(s, "Groups: "+fmt.Sprintf("%#v", this.Groups)+",\n")
905 906 
 	if this.Mounts != nil {
906 907 
 		s = append(s, "Mounts: "+fmt.Sprintf("%#v", this.Mounts)+",\n")
907 908 
 	}
... ... 
@@ -1424,6 +1438,21 @@ func (m *ContainerSpec) MarshalTo(data []byte) (int, error) {
1424 1424 
 		}
1425 1425 
 		i += n16
1426 1426 
 	}
1427 
+	if len(m.Groups) > 0 {
1428 
+		for _, s := range m.Groups {
1429 
+			data[i] = 0x5a
1430 
+			i++
1431 
+			l = len(s)
1432 
+			for l >= 1<<7 {
1433 
+				data[i] = uint8(uint64(l)&0x7f | 0x80)
1434 
+				l >>= 7
1435 
+				i++
1436 
+			}
1437 
+			data[i] = uint8(l)
1438 
+			i++
1439 
+			i += copy(data[i:], s)
1440 
+		}
1441 
+	}
1427 1442 
 	return i, nil
1428 1443 
 }
1429 1444
... ... 
@@ -1838,6 +1867,12 @@ func (m *ContainerSpec) Size() (n int) {
1838 1838 
 		l = m.PullOptions.Size()
1839 1839 
 		n += 1 + l + sovSpecs(uint64(l))
1840 1840 
 	}
1841 
+	if len(m.Groups) > 0 {
1842 
+		for _, s := range m.Groups {
1843 
+			l = len(s)
1844 
+			n += 1 + l + sovSpecs(uint64(l))
1845 
+		}
1846 
+	}
1841 1847 
 	return n
1842 1848 
 }
1843 1849
... ... 
@@ -2048,6 +2083,7 @@ func (this *ContainerSpec) String() string {
2048 2048 
 		`Mounts:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.Mounts), "Mount", "Mount", 1), `&`, ``, 1) + `,`,
2049 2049 
 		`StopGracePeriod:` + strings.Replace(fmt.Sprintf("%v", this.StopGracePeriod), "Duration", "docker_swarmkit_v11.Duration", 1) + `,`,
2050 2050 
 		`PullOptions:` + strings.Replace(fmt.Sprintf("%v", this.PullOptions), "ContainerSpec_PullOptions", "ContainerSpec_PullOptions", 1) + `,`,
2051 
+		`Groups:` + fmt.Sprintf("%v", this.Groups) + `,`,
2051 2052 
 		`}`,
2052 2053 
 	}, "")
2053 2054 
 	return s
... ... 
@@ -3371,6 +3407,35 @@ func (m *ContainerSpec) Unmarshal(data []byte) error {
3371 3371 
 				return err
3372 3372 
 			}
3373 3373 
 			iNdEx = postIndex
3374 
+		case 11:
3375 
+			if wireType != 2 {
3376 
+				return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType)
3377 
+			}
3378 
+			var stringLen uint64
3379 
+			for shift := uint(0); ; shift += 7 {
3380 
+				if shift >= 64 {
3381 
+					return ErrIntOverflowSpecs
3382 
+				}
3383 
+				if iNdEx >= l {
3384 
+					return io.ErrUnexpectedEOF
3385 
+				}
3386 
+				b := data[iNdEx]
3387 
+				iNdEx++
3388 
+				stringLen |= (uint64(b) & 0x7F) << shift
3389 
+				if b < 0x80 {
3390 
+					break
3391 
+				}
3392 
+			}
3393 
+			intStringLen := int(stringLen)
3394 
+			if intStringLen < 0 {
3395 
+				return ErrInvalidLengthSpecs
3396 
+			}
3397 
+			postIndex := iNdEx + intStringLen
3398 
+			if postIndex > l {
3399 
+				return io.ErrUnexpectedEOF
3400 
+			}
3401 
+			m.Groups = append(m.Groups, string(data[iNdEx:postIndex]))
3402 
+			iNdEx = postIndex
3374 3403 
 		default:
3375 3404 
 			iNdEx = preIndex
3376 3405 
 			skippy, err := skipSpecs(data[iNdEx:])
... ... 
@@ -4123,89 +4188,89 @@ var (
4123 4123 
 )
4124 4124
4125 4125 
 var fileDescriptorSpecs = []byte{
4126 
-	// 1332 bytes of a gzipped FileDescriptorProto
4126 
+	// 1344 bytes of a gzipped FileDescriptorProto
4127 4127 
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45,
4128 4128 
 	0x14, 0x8f, 0x93, 0x8d, 0xe3, 0xbc, 0x75, 0xda, 0x74, 0x54, 0x5a, 0xd7, 0x2d, 0x49, 0x6a, 0x0a,
4129 
-	0x14, 0x24, 0x1c, 0x30, 0xa8, 0x7f, 0xf8, 0x23, 0x70, 0x6c, 0x93, 0x86, 0x92, 0x74, 0x35, 0x69,
4130 
-	0x2b, 0x71, 0xb2, 0x26, 0xbb, 0x53, 0x67, 0x95, 0xf5, 0xee, 0x32, 0x3b, 0xeb, 0x2a, 0x37, 0x8e,
4131 
-	0x15, 0x07, 0x6e, 0x70, 0xe3, 0x84, 0xc4, 0x47, 0xe0, 0x33, 0xf4, 0xc8, 0x05, 0x89, 0x53, 0x45,
4132 
-	0xfb, 0x09, 0x90, 0xf8, 0x02, 0xbc, 0x99, 0x1d, 0xdb, 0x6b, 0xba, 0x69, 0x39, 0xf4, 0x60, 0x69,
4133 
-	0xe6, 0xcd, 0xef, 0xf7, 0x66, 0xe6, 0xbd, 0xdf, 0xbc, 0xb7, 0x06, 0x3b, 0x89, 0xb9, 0x9b, 0x34,
4134 
-	0x63, 0x11, 0xc9, 0x88, 0x10, 0x2f, 0x72, 0x8f, 0xb8, 0x68, 0x26, 0x0f, 0x99, 0x18, 0x1e, 0xf9,
4135 
-	0xb2, 0x39, 0xfa, 0xa0, 0x6e, 0xcb, 0xe3, 0x98, 0x1b, 0x40, 0xfd, 0xec, 0x20, 0x1a, 0x44, 0x7a,
4136 
-	0xb8, 0xa9, 0x46, 0xc6, 0x7a, 0xde, 0x4b, 0x05, 0x93, 0x7e, 0x14, 0x6e, 0x8e, 0x07, 0xd9, 0x42,
4137 
-	0xe3, 0x07, 0x0b, 0x2a, 0x7b, 0x91, 0xc7, 0xf7, 0x71, 0x0f, 0xb2, 0x0d, 0x36, 0x0b, 0xc3, 0x48,
4138 
-	0x6a, 0x40, 0x52, 0x2b, 0x6d, 0x94, 0xae, 0xda, 0xad, 0xf5, 0xe6, 0xf3, 0x5b, 0x36, 0xdb, 0x53,
4139 
-	0xd8, 0x96, 0xf5, 0xf8, 0xc9, 0xfa, 0x1c, 0xcd, 0x33, 0xc9, 0xfb, 0x60, 0x89, 0x28, 0xe0, 0xb5,
4140 
-	0x79, 0xf4, 0x70, 0xaa, 0x75, 0xa9, 0xc8, 0x83, 0xda, 0x94, 0x22, 0x86, 0x6a, 0x24, 0x6e, 0x0d,
4141 
-	0x43, 0x3e, 0x3c, 0xe0, 0x22, 0x39, 0xf4, 0xe3, 0xda, 0x82, 0xe6, 0xbd, 0x7d, 0x12, 0x4f, 0x1d,
4142 
-	0xb6, 0xb9, 0x3b, 0x81, 0xd3, 0x1c, 0x95, 0xec, 0x42, 0x95, 0x8d, 0x98, 0x1f, 0xb0, 0x03, 0x3f,
4143 
-	0xf0, 0xe5, 0x71, 0xcd, 0xd2, 0xae, 0xde, 0x79, 0xa1, 0xab, 0x76, 0x8e, 0x40, 0x67, 0xe8, 0x0d,
4144 
-	0x0f, 0x60, 0xba, 0x11, 0x79, 0x0b, 0x96, 0x9c, 0xde, 0x5e, 0x77, 0x67, 0x6f, 0x7b, 0x75, 0xae,
4145 
-	0x7e, 0xe1, 0xfb, 0x9f, 0x37, 0x5e, 0x53, 0x3e, 0xa6, 0x00, 0x87, 0x87, 0x9e, 0x1f, 0x0e, 0xc8,
4146 
-	0x55, 0xa8, 0xb4, 0x3b, 0x9d, 0x9e, 0x73, 0xb7, 0xd7, 0x5d, 0x2d, 0xd5, 0xeb, 0x08, 0x3c, 0x37,
4147 
-	0x0b, 0x6c, 0xbb, 0x2e, 0x8f, 0x25, 0xf7, 0xea, 0xd6, 0xa3, 0x5f, 0xd6, 0xe6, 0x1a, 0x8f, 0x4a,
4148 
-	0x50, 0xcd, 0x1f, 0x02, 0x37, 0x2a, 0xb7, 0x3b, 0x77, 0x77, 0xee, 0xf7, 0x70, 0x9f, 0x09, 0x3d,
4149 
-	0x8f, 0x68, 0xbb, 0xd2, 0x1f, 0x71, 0x72, 0x05, 0x16, 0x9d, 0xf6, 0xbd, 0xfd, 0x1e, 0xee, 0x32,
4150 
-	0x39, 0x4e, 0x1e, 0xe6, 0xb0, 0x34, 0xd1, 0xa8, 0x2e, 0x6d, 0xef, 0xec, 0xad, 0xce, 0x17, 0xa3,
4151 
-	0xba, 0x82, 0xf9, 0xa1, 0x39, 0xca, 0x6f, 0x16, 0xd8, 0xfb, 0x5c, 0x8c, 0x7c, 0xf7, 0x15, 0x6b,
4152 
-	0xe2, 0x1a, 0x58, 0x92, 0x25, 0x47, 0x5a, 0x13, 0x76, 0xb1, 0x26, 0xee, 0xe2, 0xba, 0xda, 0xd4,
4153 
-	0xd0, 0x35, 0x5e, 0x29, 0x43, 0xf0, 0x38, 0xf0, 0x5d, 0x86, 0xf1, 0xd2, 0xca, 0xb0, 0x5b, 0x6f,
4154 
-	0x16, 0xb1, 0xe9, 0x04, 0x65, 0xce, 0x7f, 0x6b, 0x8e, 0xe6, 0xa8, 0xe4, 0x13, 0x28, 0x0f, 0x82,
4155 
-	0xe8, 0x80, 0x05, 0x5a, 0x13, 0x76, 0xeb, 0x72, 0x91, 0x93, 0x6d, 0x8d, 0x98, 0x3a, 0x30, 0x14,
4156 
-	0x72, 0x03, 0xca, 0x69, 0xec, 0xa1, 0x9f, 0x5a, 0x59, 0x93, 0x37, 0x8a, 0xc8, 0xf7, 0x34, 0xa2,
4157 
-	0x13, 0x85, 0x0f, 0xfc, 0x01, 0x35, 0x78, 0xb2, 0x0f, 0x95, 0x90, 0xcb, 0x87, 0x91, 0x38, 0x4a,
4158 
-	0x6a, 0x4b, 0x1b, 0x0b, 0xc8, 0xbd, 0x5e, 0xc4, 0xcd, 0xc5, 0xbc, 0xb9, 0x97, 0xe1, 0xdb, 0x52,
4159 
-	0x32, 0xf7, 0x70, 0xc8, 0x43, 0x69, 0x5c, 0x4e, 0x1c, 0x91, 0x4f, 0xa1, 0x82, 0x52, 0x8b, 0x23,
4160 
-	0x3f, 0x94, 0xb5, 0xca, 0xc9, 0x07, 0xea, 0x19, 0x8c, 0xf2, 0x4a, 0x27, 0x8c, 0xfa, 0x6d, 0x38,
4161 
-	0x7f, 0xc2, 0x16, 0xe4, 0x1c, 0x94, 0x25, 0x13, 0x03, 0x2e, 0x75, 0xa6, 0x97, 0xa9, 0x99, 0x91,
4162 
-	0x1a, 0x2c, 0xb1, 0xc0, 0x67, 0x09, 0x4f, 0x30, 0x81, 0x0b, 0xb8, 0x30, 0x9e, 0x6e, 0x95, 0xc1,
4163 
-	0x1a, 0xa2, 0x9e, 0x1a, 0x9b, 0x70, 0xe6, 0xb9, 0x0c, 0x90, 0x3a, 0x54, 0x4c, 0x06, 0x32, 0xe9,
4164 
-	0x58, 0x74, 0x32, 0x6f, 0x9c, 0x86, 0x95, 0x99, 0x68, 0x37, 0xfe, 0x98, 0x87, 0xca, 0x58, 0x02,
4165 
-	0xa4, 0x0d, 0xcb, 0x6e, 0x14, 0x4a, 0x14, 0x26, 0x17, 0x46, 0x75, 0x85, 0x09, 0xeb, 0x8c, 0x41,
4166 
-	0x8a, 0x85, 0x09, 0x9b, 0xb2, 0xc8, 0x97, 0xb0, 0x2c, 0x78, 0x12, 0xa5, 0xc2, 0xd5, 0xa7, 0x56,
4167 
-	0x2e, 0xae, 0x16, 0x0b, 0x27, 0x03, 0x51, 0xfe, 0x6d, 0xea, 0x0b, 0xae, 0xa2, 0x91, 0xd0, 0x29,
4168 
-	0x15, 0x85, 0xb3, 0x84, 0x13, 0x0c, 0x84, 0x7c, 0x91, 0x72, 0x68, 0x06, 0x71, 0x22, 0xbc, 0xdd,
4169 
-	0x31, 0x1d, 0x33, 0x90, 0xbc, 0x1c, 0x07, 0xcc, 0xd5, 0x5e, 0x6b, 0x8b, 0x9a, 0xfe, 0x7a, 0x11,
4170 
-	0xdd, 0x19, 0x83, 0xe8, 0x14, 0x4f, 0x6e, 0x02, 0x04, 0xd1, 0xa0, 0xef, 0x09, 0x7c, 0xeb, 0xc2,
4171 
-	0x28, 0xaf, 0x5e, 0xc4, 0xee, 0x6a, 0x04, 0x5d, 0x46, 0x74, 0x36, 0xdc, 0x5a, 0xc6, 0x43, 0xa7,
4172 
-	0xa1, 0xf4, 0x87, 0xbc, 0xf1, 0x93, 0x05, 0x2b, 0x33, 0x61, 0x22, 0x67, 0x61, 0xd1, 0x1f, 0xb2,
4173 
-	0x01, 0x37, 0x49, 0xce, 0x26, 0xa4, 0x07, 0x65, 0xac, 0x08, 0x3c, 0xc8, 0x52, 0x6c, 0xb7, 0xde,
4174 
-	0x7b, 0x69, 0xbc, 0x9b, 0x5f, 0x6b, 0x7c, 0x2f, 0x94, 0xe2, 0x98, 0x1a, 0xb2, 0x92, 0x8a, 0x1b,
4175 
-	0x0d, 0x87, 0x2c, 0x54, 0xaf, 0x55, 0x4b, 0xc5, 0x4c, 0x09, 0x01, 0x0b, 0xd5, 0x94, 0x60, 0x14,
4176 
-	0x95, 0x59, 0x8f, 0xc9, 0x2a, 0x2c, 0xf0, 0x70, 0x84, 0x91, 0x51, 0x26, 0x35, 0x54, 0x16, 0xcf,
4177 
-	0xcf, 0x6e, 0x8b, 0x16, 0x1c, 0x2a, 0x1e, 0x96, 0x31, 0x81, 0xcf, 0x47, 0x99, 0xf4, 0x98, 0x5c,
4178 
-	0x87, 0xf2, 0x30, 0xc2, 0x0b, 0x26, 0xa8, 0x7f, 0x75, 0xd8, 0x0b, 0x45, 0x87, 0xdd, 0x55, 0x08,
4179 
-	0x53, 0x4d, 0x0c, 0x9c, 0xdc, 0x82, 0x33, 0x89, 0x8c, 0xe2, 0xfe, 0x40, 0x60, 0x94, 0xfb, 0x31,
4180 
-	0x17, 0x7e, 0xe4, 0xd5, 0x96, 0x4f, 0x2e, 0x4a, 0x5d, 0xd3, 0x30, 0xe9, 0x69, 0x45, 0xdb, 0x56,
4181 
-	0x2c, 0x47, 0x93, 0x88, 0x03, 0xd5, 0x38, 0x0d, 0x82, 0x7e, 0x14, 0x67, 0xb5, 0x11, 0xb4, 0x93,
4182 
-	0xff, 0x11, 0x35, 0x07, 0x59, 0x77, 0x32, 0x12, 0xb5, 0xe3, 0xe9, 0xa4, 0x7e, 0x13, 0xec, 0x5c,
4183 
-	0x44, 0x55, 0x24, 0x8e, 0xf8, 0xb1, 0x49, 0x92, 0x1a, 0xaa, 0xc4, 0x8d, 0x58, 0x90, 0x66, 0x9d,
4184 
-	0x15, 0x13, 0xa7, 0x27, 0x1f, 0xcf, 0xdf, 0x28, 0xd5, 0x5b, 0x60, 0xe7, 0xdc, 0x92, 0x37, 0x60,
4185 
-	0x45, 0xf0, 0x81, 0x9f, 0xa0, 0x9b, 0x3e, 0x4b, 0xe5, 0x61, 0xed, 0x0b, 0x4d, 0xa8, 0x8e, 0x8d,
4186 
-	0x6d, 0xb4, 0x35, 0xfe, 0xc1, 0xb6, 0x93, 0x2f, 0x11, 0xa4, 0x93, 0xbd, 0x65, 0xbd, 0xe3, 0xa9,
4187 
-	0xd6, 0xe6, 0xcb, 0x4a, 0x8a, 0x7e, 0x39, 0x41, 0xaa, 0x76, 0xdc, 0x55, 0xed, 0x5c, 0x93, 0xc9,
4188 
-	0x47, 0xb0, 0x18, 0x47, 0x42, 0x8e, 0x55, 0xb4, 0x56, 0xa8, 0x76, 0x04, 0x98, 0xa2, 0x96, 0x81,
4189 
-	0x1b, 0x87, 0x70, 0x6a, 0xd6, 0x1b, 0x76, 0xad, 0x85, 0xfb, 0x3b, 0x0e, 0x36, 0xc0, 0x8b, 0xd8,
4190 
-	0xb3, 0xce, 0xcf, 0x2e, 0xde, 0xf7, 0x85, 0x4c, 0x59, 0xb0, 0xe3, 0x90, 0x77, 0xb1, 0xb7, 0xed,
4191 
-	0xed, 0x53, 0x8a, 0x1d, 0x70, 0x1d, 0x71, 0x17, 0x67, 0x71, 0x6a, 0x09, 0xd3, 0xee, 0xd1, 0xe8,
4192 
-	0x60, 0xd2, 0xe1, 0x7e, 0x9c, 0x07, 0xdb, 0x94, 0xbf, 0x57, 0xdb, 0xe1, 0x3e, 0x87, 0x95, 0xec,
4193 
-	0xa5, 0xf6, 0x5d, 0x7d, 0x35, 0x53, 0x73, 0x5e, 0xf4, 0x60, 0xab, 0x19, 0xc1, 0x14, 0xdf, 0xcb,
4194 
-	0x50, 0xf5, 0xe3, 0xd1, 0xb5, 0x3e, 0x0f, 0xd9, 0x41, 0x60, 0x9a, 0x5d, 0x85, 0xda, 0xca, 0xd6,
4195 
-	0xcb, 0x4c, 0xaa, 0xa0, 0x62, 0xf0, 0xb9, 0x08, 0x4d, 0x1b, 0xab, 0xd0, 0xc9, 0x9c, 0x7c, 0x06,
4196 
-	0x96, 0x1f, 0xb3, 0xa1, 0xa9, 0x32, 0x85, 0x37, 0xd8, 0x71, 0xda, 0xbb, 0x46, 0x22, 0x5b, 0x95,
4197 
-	0x67, 0x4f, 0xd6, 0x2d, 0x65, 0xa0, 0x9a, 0xd6, 0xf8, 0x15, 0x3b, 0x7f, 0x27, 0x48, 0x13, 0x69,
4198 
-	0x8a, 0xc4, 0x2b, 0x8b, 0xcb, 0x37, 0x70, 0x86, 0xe9, 0xef, 0x1d, 0x16, 0xaa, 0x17, 0xa7, 0x0b,
4199 
-	0xa4, 0x89, 0xcd, 0x95, 0x42, 0x77, 0x13, 0x70, 0x56, 0x4c, 0xb7, 0xca, 0xca, 0x67, 0xad, 0x44,
4200 
-	0x57, 0xd9, 0x7f, 0x56, 0xb0, 0xb9, 0xae, 0x44, 0xc2, 0x3d, 0xc4, 0x5a, 0x9b, 0x3d, 0x52, 0xf3,
4201 
-	0x7d, 0x50, 0xf8, 0xe5, 0x78, 0x27, 0x0f, 0xcc, 0x22, 0x6e, 0x4e, 0x3b, 0xeb, 0x03, 0x7b, 0xbd,
4202 
-	0x25, 0xd8, 0x83, 0x71, 0xb1, 0x2f, 0xd4, 0x2f, 0xc5, 0xf5, 0x19, 0x17, 0x9a, 0x41, 0xbe, 0x02,
4203 
-	0xf0, 0xfc, 0x24, 0x66, 0x12, 0xdd, 0x09, 0x93, 0x87, 0xc2, 0x2b, 0x76, 0x27, 0xa8, 0x19, 0x2f,
4204 
-	0x39, 0x36, 0xb9, 0x8d, 0x0d, 0x90, 0x8d, 0x95, 0x54, 0x3e, 0xb9, 0x3e, 0x75, 0xda, 0xc6, 0xc5,
4205 
-	0xaa, 0x72, 0x81, 0x39, 0xad, 0x8c, 0x2d, 0xb4, 0xe2, 0x32, 0xa3, 0xac, 0xdb, 0xb0, 0xa2, 0x3e,
4206 
-	0xa6, 0xfa, 0x1e, 0x7f, 0xc0, 0xd2, 0x40, 0x26, 0xba, 0x94, 0x9e, 0xf0, 0xd1, 0xa0, 0x5a, 0x70,
4207 
-	0xd7, 0xe0, 0xcc, 0xb9, 0xaa, 0x32, 0x6f, 0xbb, 0xf4, 0xf8, 0xe9, 0xda, 0xdc, 0x9f, 0xf8, 0xfb,
4208 
-	0xfb, 0xe9, 0x5a, 0xe9, 0xbb, 0x67, 0x6b, 0xa5, 0xc7, 0xf8, 0xfb, 0x1d, 0x7f, 0x7f, 0xe1, 0xef,
4209 
-	0xa0, 0xac, 0xff, 0x58, 0x7c, 0xf8, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xcc, 0x10, 0x79, 0x5b,
4210 
-	0xb7, 0x0c, 0x00, 0x00,
4129 
+	0x14, 0x24, 0x1c, 0x30, 0xa8, 0x7f, 0xf8, 0x23, 0x70, 0x6c, 0x93, 0x86, 0x92, 0xd4, 0x9a, 0xb4,
4130 
+	0x95, 0x38, 0x59, 0x93, 0xf5, 0xd4, 0x59, 0x65, 0xbd, 0xb3, 0xcc, 0xce, 0xba, 0xca, 0x8d, 0x63,
4131 
+	0xc5, 0x81, 0x1b, 0x47, 0x4e, 0x48, 0x1c, 0x39, 0xf2, 0x19, 0x7a, 0xe4, 0x82, 0xc4, 0xa9, 0xa2,
4132 
+	0xfd, 0x04, 0x48, 0x7c, 0x01, 0xde, 0xcc, 0x8e, 0xed, 0x35, 0xdd, 0xb4, 0x1c, 0x7a, 0xb0, 0x34,
4133 
+	0x7f, 0x7e, 0xbf, 0xf7, 0x66, 0xdf, 0xfb, 0xcd, 0x7b, 0x63, 0x70, 0xe3, 0x88, 0x7b, 0x71, 0x3d,
4134 
+	0x92, 0x42, 0x09, 0x42, 0xfa, 0xc2, 0x3b, 0xe2, 0xb2, 0x1e, 0x3f, 0x64, 0x72, 0x78, 0xe4, 0xab,
4135 
+	0xfa, 0xe8, 0x83, 0xaa, 0xab, 0x8e, 0x23, 0x6e, 0x01, 0xd5, 0xb3, 0x03, 0x31, 0x10, 0x66, 0xb8,
4136 
+	0xa9, 0x47, 0x76, 0xf5, 0x7c, 0x3f, 0x91, 0x4c, 0xf9, 0x22, 0xdc, 0x1c, 0x0f, 0xd2, 0x8d, 0xda,
4137 
+	0x0f, 0x0e, 0x94, 0xf6, 0x44, 0x9f, 0xef, 0xa3, 0x0f, 0xb2, 0x0d, 0x2e, 0x0b, 0x43, 0xa1, 0x0c,
4138 
+	0x20, 0xae, 0x14, 0x36, 0x0a, 0x57, 0xdd, 0xc6, 0x7a, 0xfd, 0x79, 0x97, 0xf5, 0xe6, 0x14, 0xb6,
4139 
+	0xe5, 0x3c, 0x7e, 0xb2, 0x3e, 0x47, 0xb3, 0x4c, 0xf2, 0x3e, 0x38, 0x52, 0x04, 0xbc, 0x32, 0x8f,
4140 
+	0x16, 0x4e, 0x35, 0x2e, 0xe5, 0x59, 0xd0, 0x4e, 0x29, 0x62, 0xa8, 0x41, 0xa2, 0x6b, 0x18, 0xf2,
4141 
+	0xe1, 0x01, 0x97, 0xf1, 0xa1, 0x1f, 0x55, 0x16, 0x0c, 0xef, 0xed, 0x93, 0x78, 0xfa, 0xb0, 0xf5,
4142 
+	0xdd, 0x09, 0x9c, 0x66, 0xa8, 0x64, 0x17, 0xca, 0x6c, 0xc4, 0xfc, 0x80, 0x1d, 0xf8, 0x81, 0xaf,
4143 
+	0x8e, 0x2b, 0x8e, 0x31, 0xf5, 0xce, 0x0b, 0x4d, 0x35, 0x33, 0x04, 0x3a, 0x43, 0xaf, 0xf5, 0x01,
4144 
+	0xa6, 0x8e, 0xc8, 0x5b, 0xb0, 0xd4, 0xed, 0xec, 0xb5, 0x77, 0xf6, 0xb6, 0x57, 0xe7, 0xaa, 0x17,
4145 
+	0xbe, 0xff, 0x69, 0xe3, 0x35, 0x6d, 0x63, 0x0a, 0xe8, 0xf2, 0xb0, 0xef, 0x87, 0x03, 0x72, 0x15,
4146 
+	0x4a, 0xcd, 0x56, 0xab, 0xd3, 0xbd, 0xdb, 0x69, 0xaf, 0x16, 0xaa, 0x55, 0x04, 0x9e, 0x9b, 0x05,
4147 
+	0x36, 0x3d, 0x8f, 0x47, 0x8a, 0xf7, 0xab, 0xce, 0xa3, 0x9f, 0xd7, 0xe6, 0x6a, 0x8f, 0x0a, 0x50,
4148 
+	0xce, 0x1e, 0x02, 0x1d, 0x15, 0x9b, 0xad, 0xbb, 0x3b, 0xf7, 0x3b, 0xe8, 0x67, 0x42, 0xcf, 0x22,
4149 
+	0x9a, 0x9e, 0xf2, 0x47, 0x9c, 0x5c, 0x81, 0xc5, 0x6e, 0xf3, 0xde, 0x7e, 0x07, 0xbd, 0x4c, 0x8e,
4150 
+	0x93, 0x85, 0x75, 0x59, 0x12, 0x1b, 0x54, 0x9b, 0x36, 0x77, 0xf6, 0x56, 0xe7, 0xf3, 0x51, 0x6d,
4151 
+	0xc9, 0xfc, 0xd0, 0x1e, 0xe5, 0x37, 0x07, 0xdc, 0x7d, 0x2e, 0x47, 0xbe, 0xf7, 0x8a, 0x35, 0x71,
4152 
+	0x0d, 0x1c, 0xc5, 0xe2, 0x23, 0xa3, 0x09, 0x37, 0x5f, 0x13, 0x77, 0x71, 0x5f, 0x3b, 0xb5, 0x74,
4153 
+	0x83, 0xd7, 0xca, 0x90, 0x3c, 0x0a, 0x7c, 0x8f, 0x61, 0xbc, 0x8c, 0x32, 0xdc, 0xc6, 0x9b, 0x79,
4154 
+	0x6c, 0x3a, 0x41, 0xd9, 0xf3, 0xdf, 0x9a, 0xa3, 0x19, 0x2a, 0xf9, 0x04, 0x8a, 0x83, 0x40, 0x1c,
4155 
+	0xb0, 0xc0, 0x68, 0xc2, 0x6d, 0x5c, 0xce, 0x33, 0xb2, 0x6d, 0x10, 0x53, 0x03, 0x96, 0x42, 0x6e,
4156 
+	0x40, 0x31, 0x89, 0xfa, 0x68, 0xa7, 0x52, 0x34, 0xe4, 0x8d, 0x3c, 0xf2, 0x3d, 0x83, 0x68, 0x89,
4157 
+	0xf0, 0x81, 0x3f, 0xa0, 0x16, 0x4f, 0xf6, 0xa1, 0x14, 0x72, 0xf5, 0x50, 0xc8, 0xa3, 0xb8, 0xb2,
4158 
+	0xb4, 0xb1, 0x80, 0xdc, 0xeb, 0x79, 0xdc, 0x4c, 0xcc, 0xeb, 0x7b, 0x29, 0xbe, 0xa9, 0x14, 0xf3,
4159 
+	0x0e, 0x87, 0x3c, 0x54, 0xd6, 0xe4, 0xc4, 0x10, 0xf9, 0x14, 0x4a, 0x28, 0xb5, 0x48, 0xf8, 0xa1,
4160 
+	0xaa, 0x94, 0x4e, 0x3e, 0x50, 0xc7, 0x62, 0xb4, 0x55, 0x3a, 0x61, 0x54, 0x6f, 0xc3, 0xf9, 0x13,
4161 
+	0x5c, 0x90, 0x73, 0x50, 0x54, 0x4c, 0x0e, 0xb8, 0x32, 0x99, 0x5e, 0xa6, 0x76, 0x46, 0x2a, 0xb0,
4162 
+	0xc4, 0x02, 0x9f, 0xc5, 0x3c, 0xc6, 0x04, 0x2e, 0xe0, 0xc6, 0x78, 0xba, 0x55, 0x04, 0x67, 0x88,
4163 
+	0x7a, 0xaa, 0x6d, 0xc2, 0x99, 0xe7, 0x32, 0x40, 0xaa, 0x50, 0xb2, 0x19, 0x48, 0xa5, 0xe3, 0xd0,
4164 
+	0xc9, 0xbc, 0x76, 0x1a, 0x56, 0x66, 0xa2, 0x5d, 0xfb, 0x63, 0x1e, 0x4a, 0x63, 0x09, 0x90, 0x26,
4165 
+	0x2c, 0x7b, 0x22, 0x54, 0x28, 0x4c, 0x2e, 0xad, 0xea, 0x72, 0x13, 0xd6, 0x1a, 0x83, 0x34, 0x0b,
4166 
+	0x13, 0x36, 0x65, 0x91, 0x2f, 0x61, 0x59, 0xf2, 0x58, 0x24, 0xd2, 0x33, 0xa7, 0xd6, 0x26, 0xae,
4167 
+	0xe6, 0x0b, 0x27, 0x05, 0x51, 0xfe, 0x6d, 0xe2, 0x4b, 0xae, 0xa3, 0x11, 0xd3, 0x29, 0x15, 0x85,
4168 
+	0xb3, 0x84, 0x13, 0x0c, 0x84, 0x7a, 0x91, 0x72, 0x68, 0x0a, 0xe9, 0x0a, 0xfc, 0xba, 0x63, 0x3a,
4169 
+	0x66, 0x20, 0x79, 0x39, 0x0a, 0x98, 0x67, 0xac, 0x56, 0x16, 0x0d, 0xfd, 0xf5, 0x3c, 0x7a, 0x77,
4170 
+	0x0c, 0xa2, 0x53, 0x3c, 0xb9, 0x09, 0x10, 0x88, 0x41, 0xaf, 0x2f, 0xf1, 0xae, 0x4b, 0xab, 0xbc,
4171 
+	0x6a, 0x1e, 0xbb, 0x6d, 0x10, 0x74, 0x19, 0xd1, 0xe9, 0x70, 0x6b, 0x19, 0x0f, 0x9d, 0x84, 0xca,
4172 
+	0x1f, 0xf2, 0xda, 0xaf, 0x0e, 0xac, 0xcc, 0x84, 0x89, 0x9c, 0x85, 0x45, 0x7f, 0xc8, 0x06, 0xdc,
4173 
+	0x26, 0x39, 0x9d, 0x90, 0x0e, 0x14, 0xb1, 0x22, 0xf0, 0x20, 0x4d, 0xb1, 0xdb, 0x78, 0xef, 0xa5,
4174 
+	0xf1, 0xae, 0x7f, 0x6d, 0xf0, 0x9d, 0x50, 0xc9, 0x63, 0x6a, 0xc9, 0x5a, 0x2a, 0x9e, 0x18, 0x0e,
4175 
+	0x59, 0xa8, 0x6f, 0xab, 0x91, 0x8a, 0x9d, 0x12, 0x02, 0x0e, 0xaa, 0x29, 0xc6, 0x28, 0xea, 0x65,
4176 
+	0x33, 0x26, 0xab, 0xb0, 0xc0, 0xc3, 0x11, 0x46, 0x46, 0x2f, 0xe9, 0xa1, 0x5e, 0xe9, 0xfb, 0xe9,
4177 
+	0xd7, 0xe2, 0x0a, 0x0e, 0x35, 0x0f, 0xcb, 0x98, 0xc4, 0xeb, 0xa3, 0x97, 0xcc, 0x98, 0x5c, 0x87,
4178 
+	0xe2, 0x50, 0xe0, 0x07, 0xc6, 0xa8, 0x7f, 0x7d, 0xd8, 0x0b, 0x79, 0x87, 0xdd, 0xd5, 0x08, 0x5b,
4179 
+	0x4d, 0x2c, 0x9c, 0xdc, 0x82, 0x33, 0xb1, 0x12, 0x51, 0x6f, 0x20, 0x31, 0xca, 0xbd, 0x88, 0x4b,
4180 
+	0x5f, 0xf4, 0x2b, 0xcb, 0x27, 0x17, 0xa5, 0xb6, 0x6d, 0x98, 0xf4, 0xb4, 0xa6, 0x6d, 0x6b, 0x56,
4181 
+	0xd7, 0x90, 0x48, 0x17, 0xca, 0x51, 0x12, 0x04, 0x3d, 0x11, 0xa5, 0xb5, 0x11, 0x8c, 0x91, 0xff,
4182 
+	0x11, 0xb5, 0x2e, 0xb2, 0xee, 0xa4, 0x24, 0xea, 0x46, 0xd3, 0x89, 0xbe, 0x7d, 0x03, 0x29, 0x92,
4183 
+	0x28, 0xae, 0xb8, 0x26, 0x1e, 0x76, 0x56, 0xbd, 0x09, 0x6e, 0x26, 0xd2, 0x3a, 0x42, 0x47, 0xfc,
4184 
+	0xd8, 0x26, 0x4f, 0x0f, 0x75, 0x42, 0x47, 0x2c, 0x48, 0xd2, 0x8e, 0x8b, 0x09, 0x35, 0x93, 0x8f,
4185 
+	0xe7, 0x6f, 0x14, 0xaa, 0x0d, 0x70, 0x33, 0xee, 0xc8, 0x1b, 0xb0, 0x22, 0xf9, 0xc0, 0x8f, 0xd1,
4186 
+	0x4c, 0x8f, 0x25, 0xea, 0xb0, 0xf2, 0x85, 0x21, 0x94, 0xc7, 0x8b, 0x4d, 0x5c, 0xab, 0xfd, 0x83,
4187 
+	0xed, 0x28, 0x5b, 0x3a, 0x48, 0x2b, 0xbd, 0xe3, 0xc6, 0xe3, 0xa9, 0xc6, 0xe6, 0xcb, 0x4a, 0x8d,
4188 
+	0xb9, 0x51, 0x41, 0xa2, 0x3d, 0xee, 0xea, 0x36, 0x6f, 0xc8, 0xe4, 0x23, 0x58, 0x8c, 0x84, 0x54,
4189 
+	0x63, 0x75, 0xad, 0xe5, 0xde, 0x02, 0x04, 0xd8, 0x62, 0x97, 0x82, 0x6b, 0x87, 0x70, 0x6a, 0xd6,
4190 
+	0x1a, 0x76, 0xb3, 0x85, 0xfb, 0x3b, 0x5d, 0x6c, 0x8c, 0x17, 0xb1, 0x97, 0x9d, 0x9f, 0xdd, 0xbc,
4191 
+	0xef, 0x4b, 0x95, 0xb0, 0x60, 0xa7, 0x4b, 0xde, 0xc5, 0x9e, 0xb7, 0xb7, 0x4f, 0x29, 0x76, 0xc6,
4192 
+	0x75, 0xc4, 0x5d, 0x9c, 0xc5, 0xe9, 0x2d, 0x94, 0x43, 0x9f, 0x8a, 0x83, 0x49, 0xe7, 0xfb, 0x71,
4193 
+	0x1e, 0x5c, 0x5b, 0x16, 0x5f, 0x6d, 0xe7, 0xfb, 0x1c, 0x56, 0xd2, 0x1b, 0xdc, 0xf3, 0xcc, 0xa7,
4194 
+	0xd9, 0x5a, 0xf4, 0xa2, 0x8b, 0x5c, 0x4e, 0x09, 0xb6, 0x28, 0x5f, 0x86, 0xb2, 0x1f, 0x8d, 0xae,
4195 
+	0xf5, 0x78, 0xc8, 0x0e, 0x02, 0xdb, 0x04, 0x4b, 0xd4, 0xd5, 0x6b, 0x9d, 0x74, 0x49, 0x17, 0x5a,
4196 
+	0x0c, 0x3e, 0x97, 0xa1, 0x6d, 0x6f, 0x25, 0x3a, 0x99, 0x93, 0xcf, 0xc0, 0xf1, 0x23, 0x36, 0xb4,
4197 
+	0xd5, 0x27, 0xf7, 0x0b, 0x76, 0xba, 0xcd, 0x5d, 0x2b, 0x91, 0xad, 0xd2, 0xb3, 0x27, 0xeb, 0x8e,
4198 
+	0x5e, 0xa0, 0x86, 0x56, 0xfb, 0x05, 0x5f, 0x04, 0xad, 0x20, 0x89, 0x95, 0x2d, 0x1e, 0xaf, 0x2c,
4199 
+	0x2e, 0xdf, 0xc0, 0x19, 0x66, 0xde, 0x41, 0x2c, 0xd4, 0x37, 0xd1, 0x14, 0x4e, 0x1b, 0x9b, 0x2b,
4200 
+	0xb9, 0xe6, 0x26, 0xe0, 0xb4, 0xc8, 0x6e, 0x15, 0xb5, 0xcd, 0x4a, 0x81, 0xae, 0xb2, 0xff, 0xec,
4201 
+	0x60, 0xd3, 0x5d, 0x11, 0xd2, 0x3b, 0xc4, 0x1a, 0x9c, 0x5e, 0x5e, 0xfb, 0x6e, 0xc8, 0x7d, 0x51,
4202 
+	0xde, 0xc9, 0x02, 0xd3, 0x88, 0xdb, 0xd3, 0xce, 0xda, 0xc0, 0x37, 0x80, 0x23, 0xd9, 0x83, 0x71,
4203 
+	0x13, 0xc8, 0xd5, 0x2f, 0xc5, 0xfd, 0x19, 0x13, 0x86, 0x41, 0xbe, 0x02, 0xe8, 0xfb, 0x71, 0xc4,
4204 
+	0x14, 0x9a, 0x93, 0x36, 0x0f, 0xb9, 0x9f, 0xd8, 0x9e, 0xa0, 0x66, 0xac, 0x64, 0xd8, 0xe4, 0x36,
4205 
+	0x36, 0x46, 0x36, 0x56, 0x52, 0xf1, 0xe4, 0xba, 0xd5, 0x6a, 0x5a, 0x13, 0xab, 0xda, 0x04, 0xe6,
4206 
+	0xb4, 0x34, 0x5e, 0xa1, 0x25, 0x8f, 0x59, 0x65, 0xdd, 0x86, 0x15, 0xfd, 0xc8, 0xea, 0xf5, 0xf9,
4207 
+	0x03, 0x96, 0x04, 0x2a, 0x36, 0x25, 0xf6, 0x84, 0xc7, 0x84, 0x6e, 0xcd, 0x6d, 0x8b, 0xb3, 0xe7,
4208 
+	0x2a, 0xab, 0xec, 0xda, 0xa5, 0xc7, 0x4f, 0xd7, 0xe6, 0xfe, 0xc4, 0xdf, 0xdf, 0x4f, 0xd7, 0x0a,
4209 
+	0xdf, 0x3d, 0x5b, 0x2b, 0x3c, 0xc6, 0xdf, 0xef, 0xf8, 0xfb, 0x0b, 0x7f, 0x07, 0x45, 0xf3, 0x87,
4210 
+	0xe3, 0xc3, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xa4, 0x46, 0x7c, 0xf6, 0xcf, 0x0c, 0x00, 0x00,
4211 4211 
 }
vendor/src/github.com/docker/swarmkit/api/specs.proto
History View file @ 3b555a5
... ... 
@@ -160,8 +160,15 @@ message ContainerSpec {
160 160 
 	string dir = 6;
161 161
162 162 
 	// User specifies the user that should be employed to run the container.
163 
+	//
164 
+	// Note that the primary group may be specified by appending the group name
165 
+	// or id to the user name, separated by a `:`. This syntax is
166 
+	// `<user>:<group>`.
163 167 
 	string user = 7;
164 168
169 
+	// Groups specifies supplementary groups available to the user.
170 
+	repeated string groups = 11;
171 
+
165 172 
 	repeated Mount mounts = 8 [(gogoproto.nullable) = false];
166 173
167 174 
 	// StopGracePeriod the grace period for stopping the container before
vendor/src/github.com/docker/swarmkit/ca/certificates.go
History View file @ 3b555a5
... ... 
@@ -57,19 +57,17 @@ const (
57 57 
 	RootCAExpiration = "630720000s"
58 58 
 	// DefaultNodeCertExpiration represents the default expiration for node certificates (3 months)
59 59 
 	DefaultNodeCertExpiration = 2160 * time.Hour
60 
+	// CertBackdate represents the amount of time each certificate is backdated to try to avoid
61 
+	// clock drift issues.
62 
+	CertBackdate = 1 * time.Hour
60 63 
 	// CertLowerRotationRange represents the minimum fraction of time that we will wait when randomly
61 64 
 	// choosing our next certificate rotation
62 65 
 	CertLowerRotationRange = 0.5
63 66 
 	// CertUpperRotationRange represents the maximum fraction of time that we will wait when randomly
64 67 
 	// choosing our next certificate rotation
65 68 
 	CertUpperRotationRange = 0.8
66 
-	// MinNodeCertExpiration represents the minimum expiration for node certificates (25 + 5 minutes)
67 
-	// X - 5 > CertUpperRotationRange * X <=> X < 5/(1 - CertUpperRotationRange)
68 
-	// Since we're issuing certificates 5 minutes in the past to get around clock drifts, and
69 
-	// we're selecting a random rotation distribution range from CertLowerRotationRange to
70 
-	// CertUpperRotationRange, we need to ensure that we don't accept an expiration time that will
71 
-	// make a node able to randomly choose the next rotation after the expiration of the certificate.
72 
-	MinNodeCertExpiration = 30 * time.Minute
69 
+	// MinNodeCertExpiration represents the minimum expiration for node certificates
70 
+	MinNodeCertExpiration = 1 * time.Hour
73 71 
 )
74 72
75 73 
 // ErrNoLocalRootCA is an error type used to indicate that the local root CA
vendor/src/github.com/docker/swarmkit/ca/config.go
History View file @ 3b555a5
... ... 
@@ -109,12 +109,6 @@ func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration
109 109 
 	return err
110 110 
 }
111 111
112 
-// DefaultPolicy is the default policy used by the signers to ensure that the only fields
113 
-// from the remote CSRs we trust are: PublicKey, PublicKeyAlgorithm and SignatureAlgorithm.
114 
-func DefaultPolicy() *cfconfig.Signing {
115 
-	return SigningPolicy(DefaultNodeCertExpiration)
116 
-}
117 
-
118 112 
 // SigningPolicy creates a policy used by the signer to ensure that the only fields
119 113 
 // from the remote CSRs we trust are: PublicKey, PublicKeyAlgorithm and SignatureAlgorithm.
120 114 
 // It receives the duration a certificate will be valid for
... ... 
@@ -124,10 +118,14 @@ func SigningPolicy(certExpiry time.Duration) *cfconfig.Signing {
124 124 
 		certExpiry = DefaultNodeCertExpiration
125 125 
 	}
126 126
127 
+	// Add the backdate
128 
+	certExpiry = certExpiry + CertBackdate
129 
+
127 130 
 	return &cfconfig.Signing{
128 131 
 		Default: &cfconfig.SigningProfile{
129 
-			Usage:  []string{"signing", "key encipherment", "server auth", "client auth"},
130 
-			Expiry: certExpiry,
132 
+			Usage:    []string{"signing", "key encipherment", "server auth", "client auth"},
133 
+			Expiry:   certExpiry,
134 
+			Backdate: CertBackdate,
131 135 
 			// Only trust the key components from the CSR. Everything else should
132 136 
 			// come directly from API call params.
133 137 
 			CSRWhitelist: &cfconfig.CSRWhitelist{
... ... 
@@ -396,7 +394,7 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string,
396 396 
 // calculateRandomExpiry returns a random duration between 50% and 80% of the original
397 397 
 // duration
398 398 
 func calculateRandomExpiry(expiresIn time.Duration) time.Duration {
399 
-	if expiresIn.Minutes() < 1 {
399 
+	if expiresIn.Minutes() <= 1 {
400 400 
 		return time.Second
401 401 
 	}
402 402
vendor/src/github.com/docker/swarmkit/ca/server.go
History View file @ 3b555a5
... ... 
@@ -306,8 +306,7 @@ func (s *Server) Run(ctx context.Context) error {
306 306 
 	s.mu.Unlock()
307 307
308 308 
 	defer s.wg.Done()
309 
-	logger := log.G(ctx).WithField("module", "ca")
310 
-	ctx = log.WithLogger(ctx, logger)
309 
+	ctx = log.WithModule(ctx, "ca")
311 310
312 311 
 	// Retrieve the channels to keep track of changes in the cluster
313 312 
 	// Retrieve all the currently registered nodes
vendor/src/github.com/docker/swarmkit/log/context.go
History View file @ 3b555a5
... ... 
@@ -1,6 +1,8 @@
1 1 
 package log
2 2
3 3 
 import (
4 
+	"path"
5 
+
4 6 
 	"github.com/Sirupsen/logrus"
5 7 
 	"golang.org/x/net/context"
6 8 
 )
... ... 
@@ -16,7 +18,10 @@ var (
16 16 
 	L = logrus.NewEntry(logrus.StandardLogger())
17 17 
 )
18 18
19 
-type loggerKey struct{}
19 
+type (
20 
+	loggerKey struct{}
21 
+	moduleKey struct{}
22 
+)
20 23
21 24 
 // WithLogger returns a new context with the provided logger. Use in
22 25 
 // combination with logger.WithField(s) for great effect.
... ... 
@@ -35,3 +40,42 @@ func GetLogger(ctx context.Context) *logrus.Entry {
35 35
36 36 
 	return logger.(*logrus.Entry)
37 37 
 }
38 
+
39 
+// WithModule adds the module to the context, appending it with a slash if a
40 
+// module already exists. A module is just an roughly correlated defined by the
41 
+// call tree for a given context.
42 
+//
43 
+// As an example, we might have a "node" module already part of a context. If
44 
+// this function is called with "tls", the new value of module will be
45 
+// "node/tls".
46 
+//
47 
+// Modules represent the call path. If the new module and last module are the
48 
+// same, a new module entry will not be created. If the new module and old
49 
+// older module are the same but separated by other modules, the cycle will be
50 
+// represented by the module path.
51 
+func WithModule(ctx context.Context, module string) context.Context {
52 
+	parent := GetModulePath(ctx)
53 
+
54 
+	if parent != "" {
55 
+		// don't re-append module when module is the same.
56 
+		if path.Base(parent) == module {
57 
+			return ctx
58 
+		}
59 
+
60 
+		module = path.Join(parent, module)
61 
+	}
62 
+
63 
+	ctx = WithLogger(ctx, GetLogger(ctx).WithField("module", module))
64 
+	return context.WithValue(ctx, moduleKey{}, module)
65 
+}
66 
+
67 
+// GetModulePath returns the module path for the provided context. If no module
68 
+// is set, an empty string is returned.
69 
+func GetModulePath(ctx context.Context) string {
70 
+	module := ctx.Value(moduleKey{})
71 
+	if module == nil {
72 
+		return ""
73 
+	}
74 
+
75 
+	return module.(string)
76 
+}
vendor/src/github.com/docker/swarmkit/manager/allocator/doc.go
History View file @ 3b555a5
... ... 
@@ -9,10 +9,10 @@
9 9 
 // allocation, they all have to agree on that. The way this achieved
10 10 
 // in `allocator` is by creating a `taskBallot` to which all task
11 11 
 // allocators register themselves as mandatory voters. For each task
12 
-// that needs allocation, each allocator indepdently votes to indicate
12 
+// that needs allocation, each allocator independently votes to indicate
13 13 
 // the completion of their allocation. Once all registered voters have
14 14 
 // voted then the task is moved to ALLOCATED state.
15 15 
 //
16 16 
 // Other than the coordination needed for task ALLOCATED state, all
17 
-// the allocators function fairly indepdently.
17 
+// the allocators function fairly independently.
18 18 
 package allocator
vendor/src/github.com/docker/swarmkit/manager/allocator/network.go
History View file @ 3b555a5
... ... 
@@ -193,7 +193,7 @@ func (a *Allocator) doNetworkInit(ctx context.Context) error {
193 193 
 	}
194 194
195 195 
 	for _, s := range services {
196 
-		if !serviceAllocationNeeded(s, nc) {
196 
+		if nc.nwkAllocator.IsServiceAllocated(s) {
197 197 
 			continue
198 198 
 		}
199 199
... ... 
@@ -304,7 +304,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
304 304 
 	case state.EventCreateService:
305 305 
 		s := v.Service.Copy()
306 306
307 
-		if !serviceAllocationNeeded(s, nc) {
307 
+		if nc.nwkAllocator.IsServiceAllocated(s) {
308 308 
 			break
309 309 
 		}
310 310
... ... 
@@ -315,7 +315,7 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
315 315 
 	case state.EventUpdateService:
316 316 
 		s := v.Service.Copy()
317 317
318 
-		if !serviceAllocationNeeded(s, nc) {
318 
+		if nc.nwkAllocator.IsServiceAllocated(s) {
319 319 
 			break
320 320 
 		}
321 321
... ... 
@@ -326,13 +326,13 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
326 326 
 	case state.EventDeleteService:
327 327 
 		s := v.Service.Copy()
328 328
329 
-		if serviceAllocationNeeded(s, nc) {
330 
-			break
331 
-		}
332 
-
333 329 
 		if err := nc.nwkAllocator.ServiceDeallocate(s); err != nil {
334 330 
 			log.G(ctx).Errorf("Failed deallocation during delete of service %s: %v", s.ID, err)
335 331 
 		}
332 
+
333 
+		// Remove it from unallocatedServices just in case
334 
+		// it's still there.
335 
+		delete(nc.unallocatedServices, s.ID)
336 336 
 	case state.EventCreateNode, state.EventUpdateNode, state.EventDeleteNode:
337 337 
 		a.doNodeAlloc(ctx, nc, ev)
338 338 
 	case state.EventCreateTask, state.EventUpdateTask, state.EventDeleteTask:
... ... 
@@ -382,23 +382,6 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even
382 382 
 	}
383 383 
 }
384 384
385 
-// serviceAllocationNeeded returns if a service needs allocation or not.
386 
-func serviceAllocationNeeded(s *api.Service, nc *networkContext) bool {
387 
-	// Service needs allocation if:
388 
-	// Spec has network attachments and endpoint resolution mode is VIP OR
389 
-	// Spec has non-zero number of exposed ports and ingress routing is SwarmPort
390 
-	if (len(s.Spec.Networks) != 0 &&
391 
-		(s.Spec.Endpoint == nil ||
392 
-			(s.Spec.Endpoint != nil &&
393 
-				s.Spec.Endpoint.Mode == api.ResolutionModeVirtualIP))) ||
394 
-		(s.Spec.Endpoint != nil &&
395 
-			len(s.Spec.Endpoint.Ports) != 0) {
396 
-		return !nc.nwkAllocator.IsServiceAllocated(s)
397 
-	}
398 
-
399 
-	return false
400 
-}
401 
-
402 385 
 // taskRunning checks whether a task is either actively running, or in the
403 386 
 // process of starting up.
404 387 
 func taskRunning(t *api.Task) bool {
... ... 
@@ -420,7 +403,7 @@ func taskReadyForNetworkVote(t *api.Task, s *api.Service, nc *networkContext) bo
420 420 
 	// network configured or service endpoints have been
421 421 
 	// allocated.
422 422 
 	return (len(t.Networks) == 0 || nc.nwkAllocator.IsTaskAllocated(t)) &&
423 
-		(s == nil || !serviceAllocationNeeded(s, nc))
423 
+		(s == nil || nc.nwkAllocator.IsServiceAllocated(s))
424 424 
 }
425 425
426 426 
 func taskUpdateNetworks(t *api.Task, networks []*api.NetworkAttachment) {
... ... 
@@ -599,6 +582,22 @@ func (a *Allocator) allocateService(ctx context.Context, nc *networkContext, s *
599 599 
 		return err
600 600 
 	}
601 601
602 
+	// If the service doesn't expose ports any more and if we have
603 
+	// any lingering virtual IP references for ingress network
604 
+	// clean them up here.
605 
+	if s.Spec.Endpoint == nil || len(s.Spec.Endpoint.Ports) == 0 {
606 
+		if s.Endpoint != nil {
607 
+			for i, vip := range s.Endpoint.VirtualIPs {
608 
+				if vip.NetworkID == nc.ingressNetwork.ID {
609 
+					n := len(s.Endpoint.VirtualIPs)
610 
+					s.Endpoint.VirtualIPs[i], s.Endpoint.VirtualIPs[n-1] = s.Endpoint.VirtualIPs[n-1], nil
611 
+					s.Endpoint.VirtualIPs = s.Endpoint.VirtualIPs[:n-1]
612 
+					break
613 
+				}
614 
+			}
615 
+		}
616 
+	}
617 
+
602 618 
 	if err := a.store.Update(func(tx store.Tx) error {
603 619 
 		for {
604 620 
 			err := store.UpdateService(tx, s)
... ... 
@@ -670,7 +669,7 @@ func (a *Allocator) allocateTask(ctx context.Context, nc *networkContext, tx sto
670 670 
 				return nil, fmt.Errorf("could not find service %s", t.ServiceID)
671 671 
 			}
672 672
673 
-			if serviceAllocationNeeded(s, nc) {
673 
+			if !nc.nwkAllocator.IsServiceAllocated(s) {
674 674 
 				return nil, fmt.Errorf("service %s to which this task %s belongs has pending allocations", s.ID, t.ID)
675 675 
 			}
676 676
... ... 
@@ -733,7 +732,7 @@ func (a *Allocator) procUnallocatedNetworks(ctx context.Context, nc *networkCont
733 733
734 734 
 func (a *Allocator) procUnallocatedServices(ctx context.Context, nc *networkContext) {
735 735 
 	for _, s := range nc.unallocatedServices {
736 
-		if serviceAllocationNeeded(s, nc) {
736 
+		if !nc.nwkAllocator.IsServiceAllocated(s) {
737 737 
 			if err := a.allocateService(ctx, nc, s); err != nil {
738 738 
 				log.G(ctx).Debugf("Failed allocation of unallocated service %s: %v", s.ID, err)
739 739 
 				continue
vendor/src/github.com/docker/swarmkit/manager/allocator/networkallocator/networkallocator.go
History View file @ 3b555a5
... ... 
@@ -165,15 +165,29 @@ func (na *NetworkAllocator) ServiceAllocate(s *api.Service) (err error) {
165 165 
 		}
166 166 
 	}()
167 167
168 
-	// If ResolutionMode is DNSRR do not try allocating VIPs.
169 
-	if s.Spec.Endpoint != nil && s.Spec.Endpoint.Mode == api.ResolutionModeDNSRoundRobin {
170 
-		return
168 
+	if s.Endpoint == nil {
169 
+		s.Endpoint = &api.Endpoint{}
171 170 
 	}
171 
+	s.Endpoint.Spec = s.Spec.Endpoint.Copy()
172 172
173 
-	if s.Endpoint == nil {
174 
-		s.Endpoint = &api.Endpoint{
175 
-			Spec: s.Spec.Endpoint.Copy(),
173 
+	// If ResolutionMode is DNSRR do not try allocating VIPs, but
174 
+	// free any VIP from previous state.
175 
+	if s.Spec.Endpoint != nil && s.Spec.Endpoint.Mode == api.ResolutionModeDNSRoundRobin {
176 
+		if s.Endpoint != nil {
177 
+			for _, vip := range s.Endpoint.VirtualIPs {
178 
+				if err := na.deallocateVIP(vip); err != nil {
179 
+					// don't bail here, deallocate as many as possible.
180 
+					log.L.WithError(err).
181 
+						WithField("vip.network", vip.NetworkID).
182 
+						WithField("vip.addr", vip.Addr).Error("error deallocating vip")
183 
+				}
184 
+			}
185 
+
186 
+			s.Endpoint.VirtualIPs = nil
176 187 
 		}
188 
+
189 
+		delete(na.services, s.ID)
190 
+		return
177 191 
 	}
178 192
179 193 
 	// First allocate VIPs for all the pre-populated endpoint attachments
... ... 
@@ -198,7 +212,6 @@ outer:
198 198
199 199 
 		s.Endpoint.VirtualIPs = append(s.Endpoint.VirtualIPs, vip)
200 200 
 	}
201 
-	s.Endpoint.Spec = s.Spec.Endpoint.Copy()
202 201
203 202 
 	na.services[s.ID] = struct{}{}
204 203 
 	return
... ... 
@@ -232,7 +245,7 @@ func (na *NetworkAllocator) IsAllocated(n *api.Network) bool {
232 232 
 	return ok
233 233 
 }
234 234
235 
-// IsTaskAllocated returns if the passed task has it's network resources allocated or not.
235 
+// IsTaskAllocated returns if the passed task has its network resources allocated or not.
236 236 
 func (na *NetworkAllocator) IsTaskAllocated(t *api.Task) bool {
237 237 
 	// If the task is not found in the allocated set, then it is
238 238 
 	// not allocated.
... ... 
@@ -245,7 +258,7 @@ func (na *NetworkAllocator) IsTaskAllocated(t *api.Task) bool {
245 245 
 		return false
246 246 
 	}
247 247
248 
-	// To determine whether the task has it's resources allocated,
248 
+	// To determine whether the task has its resources allocated,
249 249 
 	// we just need to look at one network(in case of
250 250 
 	// multi-network attachment).  This is because we make sure we
251 251 
 	// allocate for every network or we allocate for none.
... ... 
@@ -269,13 +282,30 @@ func (na *NetworkAllocator) IsTaskAllocated(t *api.Task) bool {
269 269 
 	return true
270 270 
 }
271 271
272 
-// IsServiceAllocated returns if the passed service has it's network resources allocated or not.
272 
+// IsServiceAllocated returns if the passed service has its network resources allocated or not.
273 273 
 func (na *NetworkAllocator) IsServiceAllocated(s *api.Service) bool {
274 
-	if _, ok := na.services[s.ID]; !ok {
275 
-		return false
274 
+	// If endpoint mode is VIP and allocator does not have the
275 
+	// service in VIP allocated set then it is not allocated.
276 
+	if len(s.Spec.Networks) != 0 &&
277 
+		(s.Spec.Endpoint == nil ||
278 
+			s.Spec.Endpoint.Mode == api.ResolutionModeVirtualIP) {
279 
+		if _, ok := na.services[s.ID]; !ok {
280 
+			return false
281 
+		}
282 
+	}
283 
+
284 
+	// If the endpoint mode is DNSRR and allocator has the service
285 
+	// in VIP allocated set then we return not allocated to make
286 
+	// sure the allocator triggers networkallocator to free up the
287 
+	// resources if any.
288 
+	if s.Spec.Endpoint != nil && s.Spec.Endpoint.Mode == api.ResolutionModeDNSRoundRobin {
289 
+		if _, ok := na.services[s.ID]; ok {
290 
+			return false
291 
+		}
276 292 
 	}
277 293
278 
-	if s.Spec.Endpoint != nil {
294 
+	if (s.Spec.Endpoint != nil && len(s.Spec.Endpoint.Ports) != 0) ||
295 
+		(s.Endpoint != nil && len(s.Endpoint.Ports) != 0) {
279 296 
 		return na.portAllocator.isPortsAllocated(s)
280 297 
 	}
281 298
vendor/src/github.com/docker/swarmkit/manager/controlapi/cluster.go
History View file @ 3b555a5
... ... 
@@ -186,7 +186,7 @@ func (s *Server) ListClusters(ctx context.Context, request *api.ListClustersRequ
186 186 
 }
187 187
188 188 
 // redactClusters is a method that enforces a whitelist of fields that are ok to be
189 
-// returned in the Cluster object. It should filter out all senstive information.
189 
+// returned in the Cluster object. It should filter out all sensitive information.
190 190 
 func redactClusters(clusters []*api.Cluster) []*api.Cluster {
191 191 
 	var redactedClusters []*api.Cluster
192 192 
 	// Only add public fields to the new clusters
vendor/src/github.com/docker/swarmkit/manager/controlapi/hackpicker/cluster.go
History View file @ 3b555a5
193 193 
new file mode 100644
... ... 
@@ -0,0 +1,12 @@
0 
+package hackpicker
1 
+
2 
+// AddrSelector is interface which should track cluster for its leader address.
3 
+type AddrSelector interface {
4 
+	LeaderAddr() (string, error)
5 
+}
6 
+
7 
+// RaftCluster is interface which combines useful methods for clustering.
8 
+type RaftCluster interface {
9 
+	AddrSelector
10 
+	IsLeader() bool
11 
+}
vendor/src/github.com/docker/swarmkit/manager/controlapi/hackpicker/raftpicker.go
History View file @ 3b555a5
0 12 
new file mode 100644
... ... 
@@ -0,0 +1,141 @@
0 
+// Package hackpicker is temporary solution to provide more seamless experience
1 
+// for controlapi. It has drawback of slow reaction to leader change, but it
2 
+// tracks leader automatically without erroring out to client.
3 
+package hackpicker
4 
+
5 
+import (
6 
+	"sync"
7 
+
8 
+	"golang.org/x/net/context"
9 
+	"google.golang.org/grpc"
10 
+	"google.golang.org/grpc/transport"
11 
+)
12 
+
13 
+// picker always picks address of cluster leader.
14 
+type picker struct {
15 
+	mu   sync.Mutex
16 
+	addr string
17 
+	raft AddrSelector
18 
+	conn *grpc.Conn
19 
+	cc   *grpc.ClientConn
20 
+}
21 
+
22 
+// Init does initial processing for the Picker, e.g., initiate some connections.
23 
+func (p *picker) Init(cc *grpc.ClientConn) error {
24 
+	p.cc = cc
25 
+	return nil
26 
+}
27 
+
28 
+func (p *picker) initConn() error {
29 
+	if p.conn == nil {
30 
+		conn, err := grpc.NewConn(p.cc)
31 
+		if err != nil {
32 
+			return err
33 
+		}
34 
+		p.conn = conn
35 
+	}
36 
+	return nil
37 
+}
38 
+
39 
+// Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC
40 
+// or some error happens.
41 
+func (p *picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
42 
+	p.mu.Lock()
43 
+	if err := p.initConn(); err != nil {
44 
+		p.mu.Unlock()
45 
+		return nil, err
46 
+	}
47 
+	p.mu.Unlock()
48 
+
49 
+	addr, err := p.raft.LeaderAddr()
50 
+	if err != nil {
51 
+		return nil, err
52 
+	}
53 
+	p.mu.Lock()
54 
+	if p.addr != addr {
55 
+		p.addr = addr
56 
+		p.conn.NotifyReset()
57 
+	}
58 
+	p.mu.Unlock()
59 
+	return p.conn.Wait(ctx)
60 
+}
61 
+
62 
+// PickAddr picks a peer address for connecting. This will be called repeated for
63 
+// connecting/reconnecting.
64 
+func (p *picker) PickAddr() (string, error) {
65 
+	addr, err := p.raft.LeaderAddr()
66 
+	if err != nil {
67 
+		return "", err
68 
+	}
69 
+	p.mu.Lock()
70 
+	p.addr = addr
71 
+	p.mu.Unlock()
72 
+	return addr, nil
73 
+}
74 
+
75 
+// State returns the connectivity state of the underlying connections.
76 
+func (p *picker) State() (grpc.ConnectivityState, error) {
77 
+	return p.conn.State(), nil
78 
+}
79 
+
80 
+// WaitForStateChange blocks until the state changes to something other than
81 
+// the sourceState. It returns the new state or error.
82 
+func (p *picker) WaitForStateChange(ctx context.Context, sourceState grpc.ConnectivityState) (grpc.ConnectivityState, error) {
83 
+	return p.conn.WaitForStateChange(ctx, sourceState)
84 
+}
85 
+
86 
+// Reset the current connection and force a reconnect to another address.
87 
+func (p *picker) Reset() error {
88 
+	p.conn.NotifyReset()
89 
+	return nil
90 
+}
91 
+
92 
+// Close closes all the Conn's owned by this Picker.
93 
+func (p *picker) Close() error {
94 
+	return p.conn.Close()
95 
+}
96 
+
97 
+// ConnSelector is struct for obtaining connection with raftpicker.
98 
+type ConnSelector struct {
99 
+	mu      sync.Mutex
100 
+	cc      *grpc.ClientConn
101 
+	cluster RaftCluster
102 
+	opts    []grpc.DialOption
103 
+}
104 
+
105 
+// NewConnSelector returns new ConnSelector with cluster and grpc.DialOpts which
106 
+// will be used for Dial on first call of Conn.
107 
+func NewConnSelector(cluster RaftCluster, opts ...grpc.DialOption) *ConnSelector {
108 
+	return &ConnSelector{
109 
+		cluster: cluster,
110 
+		opts:    opts,
111 
+	}
112 
+}
113 
+
114 
+// Conn returns *grpc.ClientConn with picker which picks raft cluster leader.
115 
+// Internal connection estabilished lazily on this call.
116 
+// It can return error if cluster wasn't ready at the moment of initial call.
117 
+func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
118 
+	c.mu.Lock()
119 
+	defer c.mu.Unlock()
120 
+	if c.cc != nil {
121 
+		return c.cc, nil
122 
+	}
123 
+	addr, err := c.cluster.LeaderAddr()
124 
+	if err != nil {
125 
+		return nil, err
126 
+	}
127 
+	picker := &picker{raft: c.cluster, addr: addr}
128 
+	opts := append(c.opts, grpc.WithPicker(picker))
129 
+	cc, err := grpc.Dial(addr, opts...)
130 
+	if err != nil {
131 
+		return nil, err
132 
+	}
133 
+	c.cc = cc
134 
+	return c.cc, nil
135 
+}
136 
+
137 
+// Reset does nothing for hackpicker.
138 
+func (c *ConnSelector) Reset() error {
139 
+	return nil
140 
+}
vendor/src/github.com/docker/swarmkit/manager/controlapi/service.go
History View file @ 3b555a5
... ... 
@@ -182,7 +182,7 @@ func validateServiceSpec(spec *api.ServiceSpec) error {
182 182
183 183 
 // checkPortConflicts does a best effort to find if the passed in spec has port
184 184 
 // conflicts with existing services.
185 
-func (s *Server) checkPortConflicts(spec *api.ServiceSpec) error {
185 
+func (s *Server) checkPortConflicts(spec *api.ServiceSpec, serviceID string) error {
186 186 
 	if spec.Endpoint == nil {
187 187 
 		return nil
188 188 
 	}
... ... 
@@ -215,17 +215,21 @@ func (s *Server) checkPortConflicts(spec *api.ServiceSpec) error {
215 215 
 	}
216 216
217 217 
 	for _, service := range services {
218 
+		// If service ID is the same (and not "") then this is an update
219 
+		if serviceID != "" && serviceID == service.ID {
220 
+			continue
221 
+		}
218 222 
 		if service.Spec.Endpoint != nil {
219 223 
 			for _, pc := range service.Spec.Endpoint.Ports {
220 224 
 				if reqPorts[pcToString(pc)] {
221 
-					return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service %s", pc.PublishedPort, service.ID)
225 
+					return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s)", pc.PublishedPort, service.Spec.Annotations.Name, service.ID)
222 226 
 				}
223 227 
 			}
224 228 
 		}
225 229 
 		if service.Endpoint != nil {
226 230 
 			for _, pc := range service.Endpoint.Ports {
227 231 
 				if reqPorts[pcToString(pc)] {
228 
-					return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service %s", pc.PublishedPort, service.ID)
232 
+					return grpc.Errorf(codes.InvalidArgument, "port '%d' is already in use by service '%s' (%s)", pc.PublishedPort, service.Spec.Annotations.Name, service.ID)
229 233 
 				}
230 234 
 			}
231 235 
 		}
... ... 
@@ -243,7 +247,7 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe
243 243 
 		return nil, err
244 244 
 	}
245 245
246 
-	if err := s.checkPortConflicts(request.Spec); err != nil {
246 
+	if err := s.checkPortConflicts(request.Spec, ""); err != nil {
247 247 
 		return nil, err
248 248 
 	}
249 249
... ... 
@@ -309,7 +313,7 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
309 309 
 	}
310 310
311 311 
 	if request.Spec.Endpoint != nil && !reflect.DeepEqual(request.Spec.Endpoint, service.Spec.Endpoint) {
312 
-		if err := s.checkPortConflicts(request.Spec); err != nil {
312 
+		if err := s.checkPortConflicts(request.Spec, request.ServiceID); err != nil {
313 313 
 			return nil, err
314 314 
 		}
315 315 
 	}
vendor/src/github.com/docker/swarmkit/manager/dispatcher/dispatcher.go
History View file @ 3b555a5
... ... 
@@ -59,7 +59,7 @@ var (
59 59 
 )
60 60
61 61 
 // Config is configuration for Dispatcher. For default you should use
62 
-// DefautConfig.
62 
+// DefaultConfig.
63 63 
 type Config struct {
64 64 
 	HeartbeatPeriod  time.Duration
65 65 
 	HeartbeatEpsilon time.Duration
... ... 
@@ -79,13 +79,20 @@ func DefaultConfig() *Config {
79 79 
 	}
80 80 
 }
81 81
82 
-// Cluster is interface which represent raft cluster. mananger/state/raft.Node
83 
-// is implenents it. This interface needed only for easier unit-testing.
82 
+// Cluster is interface which represent raft cluster. manager/state/raft.Node
83 
+// is implements it. This interface needed only for easier unit-testing.
84 84 
 type Cluster interface {
85 85 
 	GetMemberlist() map[uint64]*api.RaftMember
86 86 
 	MemoryStore() *store.MemoryStore
87 87 
 }
88 88
89 
+// nodeUpdate provides a new status and/or description to apply to a node
90 
+// object.
91 
+type nodeUpdate struct {
92 
+	status      *api.NodeStatus
93 
+	description *api.NodeDescription
94 
+}
95 
+
89 96 
 // Dispatcher is responsible for dispatching tasks and tracking agent health.
90 97 
 type Dispatcher struct {
91 98 
 	mu                   sync.Mutex
... ... 
@@ -103,7 +110,14 @@ type Dispatcher struct {
103 103 
 	taskUpdates     map[string]*api.TaskStatus // indexed by task ID
104 104 
 	taskUpdatesLock sync.Mutex
105 105
106 
-	processTaskUpdatesTrigger chan struct{}
106 
+	nodeUpdates     map[string]nodeUpdate // indexed by node ID
107 
+	nodeUpdatesLock sync.Mutex
108 
+
109 
+	processUpdatesTrigger chan struct{}
110 
+
111 
+	// for waiting for the next task/node batch update
112 
+	processUpdatesLock sync.Mutex
113 
+	processUpdatesCond *sync.Cond
107 114 
 }
108 115
109 116 
 // weightedPeerByNodeID is a sort wrapper for []*api.WeightedPeer
... ... 
@@ -118,16 +132,21 @@ func (b weightedPeerByNodeID) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
118 118 
 // New returns Dispatcher with cluster interface(usually raft.Node).
119 119 
 // NOTE: each handler which does something with raft must add to Dispatcher.wg
120 120 
 func New(cluster Cluster, c *Config) *Dispatcher {
121 
-	return &Dispatcher{
122 
-		nodes:                     newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod),
123 
-		store:                     cluster.MemoryStore(),
124 
-		cluster:                   cluster,
125 
-		mgrQueue:                  watch.NewQueue(16),
126 
-		keyMgrQueue:               watch.NewQueue(16),
127 
-		taskUpdates:               make(map[string]*api.TaskStatus),
128 
-		processTaskUpdatesTrigger: make(chan struct{}, 1),
129 
-		config: c,
121 
+	d := &Dispatcher{
122 
+		nodes:                 newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod),
123 
+		store:                 cluster.MemoryStore(),
124 
+		cluster:               cluster,
125 
+		mgrQueue:              watch.NewQueue(16),
126 
+		keyMgrQueue:           watch.NewQueue(16),
127 
+		taskUpdates:           make(map[string]*api.TaskStatus),
128 
+		nodeUpdates:           make(map[string]nodeUpdate),
129 
+		processUpdatesTrigger: make(chan struct{}, 1),
130 
+		config:                c,
130 131 
 	}
132 
+
133 
+	d.processUpdatesCond = sync.NewCond(&d.processUpdatesLock)
134 
+
135 
+	return d
131 136 
 }
132 137
133 138 
 func getWeightedPeers(cluster Cluster) []*api.WeightedPeer {
... ... 
@@ -157,10 +176,9 @@ func (d *Dispatcher) Run(ctx context.Context) error {
157 157 
 		d.mu.Unlock()
158 158 
 		return fmt.Errorf("dispatcher is already running")
159 159 
 	}
160 
-	logger := log.G(ctx).WithField("module", "dispatcher")
161 
-	ctx = log.WithLogger(ctx, logger)
160 
+	ctx = log.WithModule(ctx, "dispatcher")
162 161 
 	if err := d.markNodesUnknown(ctx); err != nil {
163 
-		logger.Errorf(`failed to move all nodes to "unknown" state: %v`, err)
162 
+		log.G(ctx).Errorf(`failed to move all nodes to "unknown" state: %v`, err)
164 163 
 	}
165 164 
 	configWatcher, cancel, err := store.ViewAndWatch(
166 165 
 		d.store,
... ... 
@@ -214,11 +232,11 @@ func (d *Dispatcher) Run(ctx context.Context) error {
214 214 
 		select {
215 215 
 		case <-publishTicker.C:
216 216 
 			publishManagers()
217 
-		case <-d.processTaskUpdatesTrigger:
218 
-			d.processTaskUpdates()
217 
+		case <-d.processUpdatesTrigger:
218 
+			d.processUpdates()
219 219 
 			batchTimer.Reset(maxBatchInterval)
220 220 
 		case <-batchTimer.C:
221 
-			d.processTaskUpdates()
221 
+			d.processUpdates()
222 222 
 			batchTimer.Reset(maxBatchInterval)
223 223 
 		case v := <-configWatcher:
224 224 
 			cluster := v.(state.EventUpdateCluster)
... ... 
@@ -251,6 +269,14 @@ func (d *Dispatcher) Stop() error {
251 251 
 	d.cancel()
252 252 
 	d.mu.Unlock()
253 253 
 	d.nodes.Clean()
254 
+
255 
+	d.processUpdatesLock.Lock()
256 
+	// In case there are any waiters. There is no chance of any starting
257 
+	// after this point, because they check if the context is canceled
258 
+	// before waiting.
259 
+	d.processUpdatesCond.Broadcast()
260 
+	d.processUpdatesLock.Unlock()
261 
+
254 262 
 	return nil
255 263 
 }
256 264
... ... 
@@ -340,26 +366,39 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a
340 340 
 		return "", err
341 341 
 	}
342 342
343 
-	// create or update node in store
344 343 
 	// TODO(stevvooe): Validate node specification.
345 344 
 	var node *api.Node
346 
-	err := d.store.Update(func(tx store.Tx) error {
345 
+	d.store.View(func(tx store.ReadTx) {
347 346 
 		node = store.GetNode(tx, nodeID)
348 
-		if node == nil {
349 
-			return ErrNodeNotFound
350 
-		}
347 
+	})
348 
+	if node == nil {
349 
+		return "", ErrNodeNotFound
350 
+	}
351 351
352 
-		node.Description = description
353 
-		node.Status = api.NodeStatus{
354 
-			State: api.NodeStatus_READY,
352 
+	d.nodeUpdatesLock.Lock()
353 
+	d.nodeUpdates[nodeID] = nodeUpdate{status: &api.NodeStatus{State: api.NodeStatus_READY}, description: description}
354 
+	numUpdates := len(d.nodeUpdates)
355 
+	d.nodeUpdatesLock.Unlock()
356 
+
357 
+	if numUpdates >= maxBatchItems {
358 
+		select {
359 
+		case d.processUpdatesTrigger <- struct{}{}:
360 
+		case <-d.ctx.Done():
361 
+			return "", d.ctx.Err()
355 362 
 		}
356 
-		return store.UpdateNode(tx, node)
357 363
358 
-	})
359 
-	if err != nil {
360 
-		return "", err
361 364 
 	}
362 365
366 
+	// Wait until the node update batch happens before unblocking register.
367 
+	d.processUpdatesLock.Lock()
368 
+	select {
369 
+	case <-d.ctx.Done():
370 
+		return "", d.ctx.Err()
371 
+	default:
372 
+	}
373 
+	d.processUpdatesCond.Wait()
374 
+	d.processUpdatesLock.Unlock()
375 
+
363 376 
 	expireFunc := func() {
364 377 
 		nodeStatus := api.NodeStatus{State: api.NodeStatus_DOWN, Message: "heartbeat failure"}
365 378 
 		log.G(ctx).Debugf("heartbeat expiration")
... ... 
@@ -444,23 +483,39 @@ func (d *Dispatcher) UpdateTaskStatus(ctx context.Context, r *api.UpdateTaskStat
444 444 
 	d.taskUpdatesLock.Unlock()
445 445
446 446 
 	if numUpdates >= maxBatchItems {
447 
-		d.processTaskUpdatesTrigger <- struct{}{}
447 
+		select {
448 
+		case d.processUpdatesTrigger <- struct{}{}:
449 
+		case <-d.ctx.Done():
450 
+		}
448 451 
 	}
449 452 
 	return nil, nil
450 453 
 }
451 454
452 
-func (d *Dispatcher) processTaskUpdates() {
455 
+func (d *Dispatcher) processUpdates() {
456 
+	var (
457 
+		taskUpdates map[string]*api.TaskStatus
458 
+		nodeUpdates map[string]nodeUpdate
459 
+	)
453 460 
 	d.taskUpdatesLock.Lock()
454 
-	if len(d.taskUpdates) == 0 {
455 
-		d.taskUpdatesLock.Unlock()
456 
-		return
461 
+	if len(d.taskUpdates) != 0 {
462 
+		taskUpdates = d.taskUpdates
463 
+		d.taskUpdates = make(map[string]*api.TaskStatus)
457 464 
 	}
458 
-	taskUpdates := d.taskUpdates
459 
-	d.taskUpdates = make(map[string]*api.TaskStatus)
460 465 
 	d.taskUpdatesLock.Unlock()
461 466
467 
+	d.nodeUpdatesLock.Lock()
468 
+	if len(d.nodeUpdates) != 0 {
469 
+		nodeUpdates = d.nodeUpdates
470 
+		d.nodeUpdates = make(map[string]nodeUpdate)
471 
+	}
472 
+	d.nodeUpdatesLock.Unlock()
473 
+
474 
+	if len(taskUpdates) == 0 && len(nodeUpdates) == 0 {
475 
+		return
476 
+	}
477 
+
462 478 
 	log := log.G(d.ctx).WithFields(logrus.Fields{
463 
-		"method": "(*Dispatcher).processTaskUpdates",
479 
+		"method": "(*Dispatcher).processUpdates",
464 480 
 	})
465 481
466 482 
 	_, err := d.store.Batch(func(batch *store.Batch) error {
... ... 
@@ -494,14 +549,45 @@ func (d *Dispatcher) processTaskUpdates() {
494 494 
 				return nil
495 495 
 			})
496 496 
 			if err != nil {
497 
-				log.WithError(err).Error("dispatcher transaction failed")
497 
+				log.WithError(err).Error("dispatcher task update transaction failed")
498 
+			}
499 
+		}
500 
+
501 
+		for nodeID, nodeUpdate := range nodeUpdates {
502 
+			err := batch.Update(func(tx store.Tx) error {
503 
+				logger := log.WithField("node.id", nodeID)
504 
+				node := store.GetNode(tx, nodeID)
505 
+				if node == nil {
506 
+					logger.Errorf("node unavailable")
507 
+					return nil
508 
+				}
509 
+
510 
+				if nodeUpdate.status != nil {
511 
+					node.Status = *nodeUpdate.status
512 
+				}
513 
+				if nodeUpdate.description != nil {
514 
+					node.Description = nodeUpdate.description
515 
+				}
516 
+
517 
+				if err := store.UpdateNode(tx, node); err != nil {
518 
+					logger.WithError(err).Error("failed to update node status")
519 
+					return nil
520 
+				}
521 
+				logger.Debug("node status updated")
522 
+				return nil
523 
+			})
524 
+			if err != nil {
525 
+				log.WithError(err).Error("dispatcher node update transaction failed")
498 526 
 			}
499 527 
 		}
528 
+
500 529 
 		return nil
501 530 
 	})
502 531 
 	if err != nil {
503 532 
 		log.WithError(err).Error("dispatcher batch failed")
504 533 
 	}
534 
+
535 
+	d.processUpdatesCond.Broadcast()
505 536 
 }
506 537
507 538 
 // Tasks is a stream of tasks state for node. Each message contains full list
... ... 
@@ -595,7 +681,10 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
595 595 
 					modificationCnt++
596 596 
 				case state.EventUpdateTask:
597 597 
 					if oldTask, exists := tasksMap[v.Task.ID]; exists {
598 
-						if equality.TasksEqualStable(oldTask, v.Task) {
598 
+						// States ASSIGNED and below are set by the orchestrator/scheduler,
599 
+						// not the agent, so tasks in these states need to be sent to the
600 
+						// agent even if nothing else has changed.
601 
+						if equality.TasksEqualStable(oldTask, v.Task) && v.Task.Status.State > api.TaskStateAssigned {
599 602 
 							// this update should not trigger action at agent
600 603 
 							tasksMap[v.Task.ID] = v.Task
601 604 
 							continue
... ... 
@@ -632,17 +721,17 @@ func (d *Dispatcher) nodeRemove(id string, status api.NodeStatus) error {
632 632 
 	if err := d.isRunningLocked(); err != nil {
633 633 
 		return err
634 634 
 	}
635 
-	// TODO(aaronl): Is it worth batching node removals?
636 
-	err := d.store.Update(func(tx store.Tx) error {
637 
-		node := store.GetNode(tx, id)
638 
-		if node == nil {
639 
-			return errors.New("node not found")
635 
+
636 
+	d.nodeUpdatesLock.Lock()
637 
+	d.nodeUpdates[id] = nodeUpdate{status: status.Copy(), description: d.nodeUpdates[id].description}
638 
+	numUpdates := len(d.nodeUpdates)
639 
+	d.nodeUpdatesLock.Unlock()
640 
+
641 
+	if numUpdates >= maxBatchItems {
642 
+		select {
643 
+		case d.processUpdatesTrigger <- struct{}{}:
644 
+		case <-d.ctx.Done():
640 645 
 		}
641 
-		node.Status = status
642 
-		return store.UpdateNode(tx, node)
643 
-	})
644 
-	if err != nil {
645 
-		return fmt.Errorf("failed to update node %s status to down: %v", id, err)
646 646 
 	}
647 647
648 648 
 	if rn := d.nodes.Delete(id); rn == nil {
vendor/src/github.com/docker/swarmkit/manager/keymanager/keymanager.go
History View file @ 3b555a5
... ... 
@@ -122,7 +122,6 @@ func (k *KeyManager) updateKey(cluster *api.Cluster) error {
122 122 
 }
123 123
124 124 
 func (k *KeyManager) rotateKey(ctx context.Context) error {
125 
-	log := log.G(ctx).WithField("module", "keymanager")
126 125 
 	var (
127 126 
 		clusters []*api.Cluster
128 127 
 		err      error
... ... 
@@ -132,7 +131,7 @@ func (k *KeyManager) rotateKey(ctx context.Context) error {
132 132 
 	})
133 133
134 134 
 	if err != nil {
135 
-		log.Errorf("reading cluster config failed, %v", err)
135 
+		log.G(ctx).Errorf("reading cluster config failed, %v", err)
136 136 
 		return err
137 137 
 	}
138 138
... ... 
@@ -173,7 +172,7 @@ func (k *KeyManager) rotateKey(ctx context.Context) error {
173 173 
 // Run starts the keymanager, it doesn't return
174 174 
 func (k *KeyManager) Run(ctx context.Context) error {
175 175 
 	k.mu.Lock()
176 
-	log := log.G(ctx).WithField("module", "keymanager")
176 
+	ctx = log.WithModule(ctx, "keymanager")
177 177 
 	var (
178 178 
 		clusters []*api.Cluster
179 179 
 		err      error
... ... 
@@ -183,7 +182,7 @@ func (k *KeyManager) Run(ctx context.Context) error {
183 183 
 	})
184 184
185 185 
 	if err != nil {
186 
-		log.Errorf("reading cluster config failed, %v", err)
186 
+		log.G(ctx).Errorf("reading cluster config failed, %v", err)
187 187 
 		k.mu.Unlock()
188 188 
 		return err
189 189 
 	}
... ... 
@@ -196,7 +195,7 @@ func (k *KeyManager) Run(ctx context.Context) error {
196 196 
 			}
197 197 
 		}
198 198 
 		if err := k.updateKey(cluster); err != nil {
199 
-			log.Errorf("store update failed %v", err)
199 
+			log.G(ctx).Errorf("store update failed %v", err)
200 200 
 		}
201 201 
 	} else {
202 202 
 		k.keyRing.lClock = cluster.EncryptionKeyLamportClock
vendor/src/github.com/docker/swarmkit/manager/manager.go
History View file @ 3b555a5
... ... 
@@ -17,6 +17,7 @@ import (
17 17 
 	"github.com/docker/swarmkit/log"
18 18 
 	"github.com/docker/swarmkit/manager/allocator"
19 19 
 	"github.com/docker/swarmkit/manager/controlapi"
20 
+	"github.com/docker/swarmkit/manager/controlapi/hackpicker"
20 21 
 	"github.com/docker/swarmkit/manager/dispatcher"
21 22 
 	"github.com/docker/swarmkit/manager/health"
22 23 
 	"github.com/docker/swarmkit/manager/keymanager"
... ... 
@@ -350,11 +351,13 @@ func (m *Manager) Run(parent context.Context) error {
350 350 
 					// creating the allocator but then use it anyway.
351 351 
 				}
352 352
353 
-				go func(keyManager *keymanager.KeyManager) {
354 
-					if err := keyManager.Run(ctx); err != nil {
355 
-						log.G(ctx).WithError(err).Error("keymanager failed with an error")
356 
-					}
357 
-				}(m.keyManager)
353 
+				if m.keyManager != nil {
354 
+					go func(keyManager *keymanager.KeyManager) {
355 
+						if err := keyManager.Run(ctx); err != nil {
356 
+							log.G(ctx).WithError(err).Error("keymanager failed with an error")
357 
+						}
358 
+					}(m.keyManager)
359 
+				}
358 360
359 361 
 				go func(d *dispatcher.Dispatcher) {
360 362 
 					if err := d.Run(ctx); err != nil {
... ... 
@@ -385,14 +388,17 @@ func (m *Manager) Run(parent context.Context) error {
385 385 
 						log.G(ctx).WithError(err).Error("scheduler exited with an error")
386 386 
 					}
387 387 
 				}(m.scheduler)
388 
+
388 389 
 				go func(taskReaper *orchestrator.TaskReaper) {
389 390 
 					taskReaper.Run()
390 391 
 				}(m.taskReaper)
392 
+
391 393 
 				go func(orchestrator *orchestrator.ReplicatedOrchestrator) {
392 394 
 					if err := orchestrator.Run(ctx); err != nil {
393 395 
 						log.G(ctx).WithError(err).Error("replicated orchestrator exited with an error")
394 396 
 					}
395 397 
 				}(m.replicatedOrchestrator)
398 
+
396 399 
 				go func(globalOrchestrator *orchestrator.GlobalOrchestrator) {
397 400 
 					if err := globalOrchestrator.Run(ctx); err != nil {
398 401 
 						log.G(ctx).WithError(err).Error("global orchestrator exited with an error")
... ... 
@@ -420,21 +426,34 @@ func (m *Manager) Run(parent context.Context) error {
420 420 
 				m.scheduler.Stop()
421 421 
 				m.scheduler = nil
422 422
423 
-				m.keyManager.Stop()
424 
-				m.keyManager = nil
423 
+				if m.keyManager != nil {
424 
+					m.keyManager.Stop()
425 
+					m.keyManager = nil
426 
+				}
425 427 
 			}
426 428 
 			m.mu.Unlock()
427 429 
 		}
428 430 
 	}()
429 431
430 432 
 	proxyOpts := []grpc.DialOption{
431 
-		grpc.WithBackoffMaxDelay(time.Second),
433 
+		grpc.WithTimeout(5 * time.Second),
432 434 
 		grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds),
433 435 
 	}
434 436
435 437 
 	cs := raftpicker.NewConnSelector(m.RaftNode, proxyOpts...)
436 438 
 	m.connSelector = cs
437 439
440 
+	// We need special connSelector for controlapi because it provides automatic
441 
+	// leader tracking.
442 
+	// Other APIs are using connSelector which errors out on leader change, but
443 
+	// allows to react quickly to reelections.
444 
+	controlAPIProxyOpts := []grpc.DialOption{
445 
+		grpc.WithBackoffMaxDelay(time.Second),
446 
+		grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds),
447 
+	}
448 
+
449 
+	controlAPIConnSelector := hackpicker.NewConnSelector(m.RaftNode, controlAPIProxyOpts...)
450 
+
438 451 
 	authorize := func(ctx context.Context, roles []string) error {
439 452 
 		// Authorize the remote roles, ensure they can only be forwarded by managers
440 453 
 		_, err := ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, m.config.SecurityConfig.ClientTLSCreds.Organization())
... ... 
@@ -464,7 +483,7 @@ func (m *Manager) Run(parent context.Context) error {
464 464 
 	// this manager rather than forwarded requests (it has no TLS
465 465 
 	// information to put in the metadata map).
466 466 
 	forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil }
467 
-	localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, cs, m.RaftNode, forwardAsOwnRequest)
467 
+	localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, controlAPIConnSelector, m.RaftNode, forwardAsOwnRequest)
468 468
469 469 
 	// Everything registered on m.server should be an authenticated
470 470 
 	// wrapper, or a proxy wrapping an authenticated wrapper!
vendor/src/github.com/docker/swarmkit/manager/orchestrator/restart.go
History View file @ 3b555a5
... ... 
@@ -2,6 +2,7 @@ package orchestrator
2 2
3 3 
 import (
4 4 
 	"container/list"
5 
+	"errors"
5 6 
 	"sync"
6 7 
 	"time"
7 8
... ... 
@@ -76,6 +77,9 @@ func (r *RestartSupervisor) waitRestart(ctx context.Context, oldDelay *delayedSt
76 76 
 		if t == nil {
77 77 
 			return nil
78 78 
 		}
79 
+		if t.DesiredState > api.TaskStateRunning {
80 
+			return nil
81 
+		}
79 82 
 		service := store.GetService(tx, t.ServiceID)
80 83 
 		if service == nil {
81 84 
 			return nil
... ... 
@@ -108,6 +112,13 @@ func (r *RestartSupervisor) Restart(ctx context.Context, tx store.Tx, cluster *a
108 108 
 	}
109 109 
 	r.mu.Unlock()
110 110
111 
+	// Sanity check: was the task shut down already by a separate call to
112 
+	// Restart? If so, we must avoid restarting it, because this will create
113 
+	// an extra task. This should never happen unless there is a bug.
114 
+	if t.DesiredState > api.TaskStateRunning {
115 
+		return errors.New("Restart called on task that was already shut down")
116 
+	}
117 
+
111 118 
 	t.DesiredState = api.TaskStateShutdown
112 119 
 	err := store.UpdateTask(tx, &t)
113 120 
 	if err != nil {
vendor/src/github.com/docker/swarmkit/manager/orchestrator/tasks.go
History View file @ 3b555a5
... ... 
@@ -163,7 +163,7 @@ func (r *ReplicatedOrchestrator) tickTasks(ctx context.Context) {
163 163 
 		})
164 164
165 165 
 		if err != nil {
166 
-			log.G(ctx).WithError(err).Errorf("orchestator task removal batch failed")
166 
+			log.G(ctx).WithError(err).Errorf("orchestrator task removal batch failed")
167 167 
 		}
168 168
169 169 
 		r.restartTasks = make(map[string]struct{})
vendor/src/github.com/docker/swarmkit/manager/raftpicker/raftpicker.go
History View file @ 3b555a5
... ... 
@@ -4,136 +4,43 @@ import (
4 4 
 	"sync"
5 5 
 	"time"
6 6
7 
-	"golang.org/x/net/context"
7 
+	"github.com/Sirupsen/logrus"
8 
+
8 9 
 	"google.golang.org/grpc"
9 
-	"google.golang.org/grpc/transport"
10 10 
 )
11 11
12 
-// picker always picks address of cluster leader.
13 
-type picker struct {
14 
-	mu   sync.Mutex
15 
-	addr string
16 
-	raft AddrSelector
17 
-	conn *grpc.Conn
18 
-
19 
-	stop chan struct{}
20 
-	done chan struct{}
21 
-}
22 
-
23 
-func newPicker(raft AddrSelector, addr string) *picker {
24 
-	return &picker{
25 
-		raft: raft,
26 
-		addr: addr,
27 
-
28 
-		stop: make(chan struct{}),
29 
-		done: make(chan struct{}),
30 
-	}
31 
-}
32 
-
33 
-// Init does initial processing for the Picker, e.g., initiate some connections.
34 
-func (p *picker) Init(cc *grpc.ClientConn) error {
35 
-	conn, err := grpc.NewConn(cc)
36 
-	if err != nil {
37 
-		return err
38 
-	}
39 
-	p.conn = conn
40 
-	return nil
41 
-}
42 
-
43 
-// Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC
44 
-// or some error happens.
45 
-func (p *picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
46 
-	if err := p.updateConn(); err != nil {
47 
-		return nil, err
48 
-	}
49 
-	return p.conn.Wait(ctx)
50 
-}
51 
-
52 
-// PickAddr picks a peer address for connecting. This will be called repeated for
53 
-// connecting/reconnecting.
54 
-func (p *picker) PickAddr() (string, error) {
55 
-	addr, err := p.raft.LeaderAddr()
56 
-	if err != nil {
57 
-		return "", err
58 
-	}
59 
-	p.mu.Lock()
60 
-	p.addr = addr
61 
-	p.mu.Unlock()
62 
-	return addr, nil
63 
-}
64 
-
65 
-// State returns the connectivity state of the underlying connections.
66 
-func (p *picker) State() (grpc.ConnectivityState, error) {
67 
-	return p.conn.State(), nil
68 
-}
69 
-
70 
-// WaitForStateChange blocks until the state changes to something other than
71 
-// the sourceState. It returns the new state or error.
72 
-func (p *picker) WaitForStateChange(ctx context.Context, sourceState grpc.ConnectivityState) (grpc.ConnectivityState, error) {
73 
-	return p.conn.WaitForStateChange(ctx, sourceState)
74 
-}
75 
-
76 
-// Reset the current connection and force a reconnect to another address.
77 
-func (p *picker) Reset() error {
78 
-	p.conn.NotifyReset()
79 
-	return nil
80 
-}
81 
-
82 
-// Close closes all the Conn's owned by this Picker.
83 
-func (p *picker) Close() error {
84 
-	close(p.stop)
85 
-	<-p.done
86 
-	return p.conn.Close()
87 
-}
88 
-
89 
-func (p *picker) updateConn() error {
90 
-	addr, err := p.raft.LeaderAddr()
91 
-	if err != nil {
92 
-		return err
93 
-	}
94 
-	p.mu.Lock()
95 
-	if p.addr != addr {
96 
-		p.addr = addr
97 
-		p.Reset()
98 
-	}
99 
-	p.mu.Unlock()
100 
-	return nil
101 
-}
102 
-
103 
-func (p *picker) updateLoop() {
104 
-	defer close(p.done)
105 
-	ticker := time.NewTicker(1 * time.Second)
106 
-	defer ticker.Stop()
107 
-	for {
108 
-		select {
109 
-		case <-ticker.C:
110 
-			p.updateConn()
111 
-		case <-p.stop:
112 
-			return
113 
-		}
114 
-	}
12 
+// Interface is interface to replace implementation with controlapi/hackpicker.
13 
+// TODO: it should be done cooler.
14 
+type Interface interface {
15 
+	Conn() (*grpc.ClientConn, error)
16 
+	Reset() error
115 17 
 }
116 18
117 
-// ConnSelector is struct for obtaining connection with raftpicker.
19 
+// ConnSelector is struct for obtaining connection connected to cluster leader.
118 20 
 type ConnSelector struct {
119 21 
 	mu      sync.Mutex
120 
-	cc      *grpc.ClientConn
121 22 
 	cluster RaftCluster
122 23 
 	opts    []grpc.DialOption
123 
-	picker  *picker
24 
+
25 
+	cc   *grpc.ClientConn
26 
+	addr string
27 
+
28 
+	stop chan struct{}
124 29 
 }
125 30
126 31 
 // NewConnSelector returns new ConnSelector with cluster and grpc.DialOpts which
127 
-// will be used for Dial on first call of Conn.
32 
+// will be used for connection create.
128 33 
 func NewConnSelector(cluster RaftCluster, opts ...grpc.DialOption) *ConnSelector {
129 
-	return &ConnSelector{
34 
+	cs := &ConnSelector{
130 35 
 		cluster: cluster,
131 36 
 		opts:    opts,
37 
+		stop:    make(chan struct{}),
132 38 
 	}
39 
+	go cs.updateLoop()
40 
+	return cs
133 41 
 }
134 42
135 
-// Conn returns *grpc.ClientConn with picker which picks raft cluster leader.
136 
-// Internal connection estabilished lazily on this call.
43 
+// Conn returns *grpc.ClientConn which connected to cluster leader.
137 44 
 // It can return error if cluster wasn't ready at the moment of initial call.
138 45 
 func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
139 46 
 	c.mu.Lock()
... ... 
@@ -145,23 +52,76 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
145 145 
 	if err != nil {
146 146 
 		return nil, err
147 147 
 	}
148 
-	c.picker = newPicker(c.cluster, addr)
149 
-	go c.picker.updateLoop()
150 
-	opts := append(c.opts, grpc.WithPicker(c.picker))
151 
-	cc, err := grpc.Dial(addr, opts...)
148 
+	cc, err := grpc.Dial(addr, c.opts...)
152 149 
 	if err != nil {
153 150 
 		return nil, err
154 151 
 	}
155 152 
 	c.cc = cc
156 
-	return c.cc, nil
153 
+	c.addr = addr
154 
+	return cc, nil
157 155 
 }
158 156
159 
-// Stop cancels tracking loop for raftpicker and closes it.
157 
+// Reset recreates underlying connection.
158 
+func (c *ConnSelector) Reset() error {
159 
+	c.mu.Lock()
160 
+	defer c.mu.Unlock()
161 
+	if c.cc != nil {
162 
+		c.cc.Close()
163 
+		c.cc = nil
164 
+	}
165 
+	addr, err := c.cluster.LeaderAddr()
166 
+	if err != nil {
167 
+		logrus.WithError(err).Errorf("error obtaining leader address")
168 
+		return err
169 
+	}
170 
+	cc, err := grpc.Dial(addr, c.opts...)
171 
+	if err != nil {
172 
+		logrus.WithError(err).Errorf("error reestabilishing connection to leader")
173 
+		return err
174 
+	}
175 
+	c.cc = cc
176 
+	c.addr = addr
177 
+	return nil
178 
+}
179 
+
180 
+// Stop cancels updating connection loop.
160 181 
 func (c *ConnSelector) Stop() {
182 
+	close(c.stop)
183 
+}
184 
+
185 
+func (c *ConnSelector) updateConn() error {
186 
+	addr, err := c.cluster.LeaderAddr()
187 
+	if err != nil {
188 
+		return err
189 
+	}
161 190 
 	c.mu.Lock()
162 191 
 	defer c.mu.Unlock()
163 
-	if c.picker == nil {
164 
-		return
192 
+	if c.addr != addr {
193 
+		if c.cc != nil {
194 
+			c.cc.Close()
195 
+			c.cc = nil
196 
+		}
197 
+		conn, err := grpc.Dial(addr, c.opts...)
198 
+		if err != nil {
199 
+			return err
200 
+		}
201 
+		c.cc = conn
202 
+		c.addr = addr
203 
+	}
204 
+	return nil
205 
+}
206 
+
207 
+func (c *ConnSelector) updateLoop() {
208 
+	ticker := time.NewTicker(1 * time.Second)
209 
+	defer ticker.Stop()
210 
+	for {
211 
+		select {
212 
+		case <-ticker.C:
213 
+			if err := c.updateConn(); err != nil {
214 
+				logrus.WithError(err).Errorf("error reestabilishing connection to leader")
215 
+			}
216 
+		case <-c.stop:
217 
+			return
218 
+		}
165 219 
 	}
166 
-	c.picker.Close()
167 220 
 }
vendor/src/github.com/docker/swarmkit/manager/scheduler/scheduler.go
History View file @ 3b555a5
... ... 
@@ -61,8 +61,8 @@ func (s *Scheduler) setupTasksList(tx store.ReadTx) error {
61 61 
 	tasksByNode := make(map[string]map[string]*api.Task)
62 62 
 	for _, t := range tasks {
63 63 
 		// Ignore all tasks that have not reached ALLOCATED
64 
-		// state.
65 
-		if t.Status.State < api.TaskStateAllocated {
64 
+		// state and tasks that no longer consume resources.
65 
+		if t.Status.State < api.TaskStateAllocated || t.Status.State > api.TaskStateRunning {
66 66 
 			continue
67 67 
 		}
68 68
... ... 
@@ -109,8 +109,31 @@ func (s *Scheduler) Run(ctx context.Context) error {
109 109 
 	// Queue all unassigned tasks before processing changes.
110 110 
 	s.tick(ctx)
111 111
112 
+	const (
113 
+		// commitDebounceGap is the amount of time to wait between
114 
+		// commit events to debounce them.
115 
+		commitDebounceGap = 50 * time.Millisecond
116 
+		// maxLatency is a time limit on the debouncing.
117 
+		maxLatency = time.Second
118 
+	)
119 
+	var (
120 
+		debouncingStarted     time.Time
121 
+		commitDebounceTimer   *time.Timer
122 
+		commitDebounceTimeout <-chan time.Time
123 
+	)
124 
+
112 125 
 	pendingChanges := 0
113 126
127 
+	schedule := func() {
128 
+		if len(s.preassignedTasks) > 0 {
129 
+			s.processPreassignedTasks(ctx)
130 
+		}
131 
+		if pendingChanges > 0 {
132 
+			s.tick(ctx)
133 
+			pendingChanges = 0
134 
+		}
135 
+	}
136 
+
114 137 
 	// Watch for changes.
115 138 
 	for {
116 139 
 		select {
... ... 
@@ -131,15 +154,25 @@ func (s *Scheduler) Run(ctx context.Context) error {
131 131 
 			case state.EventDeleteNode:
132 132 
 				s.nodeHeap.remove(v.Node.ID)
133 133 
 			case state.EventCommit:
134 
-				if len(s.preassignedTasks) > 0 {
135 
-					s.processPreassignedTasks(ctx)
136 
-				}
137 
-				if pendingChanges > 0 {
138 
-					s.tick(ctx)
139 
-					pendingChanges = 0
134 
+				if commitDebounceTimer != nil {
135 
+					if time.Since(debouncingStarted) > maxLatency {
136 
+						commitDebounceTimer.Stop()
137 
+						commitDebounceTimer = nil
138 
+						commitDebounceTimeout = nil
139 
+						schedule()
140 
+					} else {
141 
+						commitDebounceTimer.Reset(commitDebounceGap)
142 
+					}
143 
+				} else {
144 
+					commitDebounceTimer = time.NewTimer(commitDebounceGap)
145 
+					commitDebounceTimeout = commitDebounceTimer.C
146 
+					debouncingStarted = time.Now()
140 147 
 				}
141 148 
 			}
142 
-
149 
+		case <-commitDebounceTimeout:
150 
+			schedule()
151 
+			commitDebounceTimer = nil
152 
+			commitDebounceTimeout = nil
143 153 
 		case <-s.stopChan:
144 154 
 			return nil
145 155 
 		}
vendor/src/github.com/docker/swarmkit/manager/state/raft/raft.go
History View file @ 3b555a5
... ... 
@@ -87,27 +87,23 @@ type Node struct {
87 87 
 	StateDir string
88 88 
 	Error    error
89 89
90 
-	raftStore   *raft.MemoryStorage
91 
-	memoryStore *store.MemoryStore
92 
-	Config      *raft.Config
93 
-	opts        NewNodeOptions
94 
-	reqIDGen    *idutil.Generator
95 
-	wait        *wait
96 
-	wal         *wal.WAL
97 
-	snapshotter *snap.Snapshotter
98 
-	wasLeader   bool
99 
-	restored    bool
100 
-	isMember    uint32
101 
-	joinAddr    string
90 
+	raftStore           *raft.MemoryStorage
91 
+	memoryStore         *store.MemoryStore
92 
+	Config              *raft.Config
93 
+	opts                NewNodeOptions
94 
+	reqIDGen            *idutil.Generator
95 
+	wait                *wait
96 
+	wal                 *wal.WAL
97 
+	snapshotter         *snap.Snapshotter
98 
+	restored            bool
99 
+	signalledLeadership uint32
100 
+	isMember            uint32
101 
+	joinAddr            string
102 102
103 103 
 	// waitProp waits for all the proposals to be terminated before
104 104 
 	// shutting down the node.
105 105 
 	waitProp sync.WaitGroup
106 106
107 
-	// forceNewCluster is a special flag used to recover from disaster
108 
-	// scenario by pointing to an existing or backed up data directory.
109 
-	forceNewCluster bool
110 
-
111 107 
 	confState     raftpb.ConfState
112 108 
 	appliedIndex  uint64
113 109 
 	snapshotIndex uint64
... ... 
@@ -118,7 +114,7 @@ type Node struct {
118 118 
 	doneCh      chan struct{}
119 119 
 	// removeRaftCh notifies about node deletion from raft cluster
120 120 
 	removeRaftCh        chan struct{}
121 
-	removeRaftOnce      sync.Once
121 
+	removeRaftFunc      func()
122 122 
 	leadershipBroadcast *events.Broadcaster
123 123
124 124 
 	// used to coordinate shutdown
... ... 
@@ -192,7 +188,6 @@ func NewNode(ctx context.Context, opts NewNodeOptions) *Node {
192 192 
 			MaxInflightMsgs: cfg.MaxInflightMsgs,
193 193 
 			Logger:          cfg.Logger,
194 194 
 		},
195 
-		forceNewCluster:     opts.ForceNewCluster,
196 195 
 		stopCh:              make(chan struct{}),
197 196 
 		doneCh:              make(chan struct{}),
198 197 
 		removeRaftCh:        make(chan struct{}),
... ... 
@@ -215,6 +210,15 @@ func NewNode(ctx context.Context, opts NewNodeOptions) *Node {
215 215 
 	n.reqIDGen = idutil.NewGenerator(uint16(n.Config.ID), time.Now())
216 216 
 	n.wait = newWait()
217 217
218 
+	n.removeRaftFunc = func(n *Node) func() {
219 
+		var removeRaftOnce sync.Once
220 
+		return func() {
221 
+			removeRaftOnce.Do(func() {
222 
+				close(n.removeRaftCh)
223 
+			})
224 
+		}
225 
+	}(n)
226 
+
218 227 
 	return n
219 228 
 }
220 229
... ... 
@@ -329,6 +333,8 @@ func (n *Node) Run(ctx context.Context) error {
329 329 
 		close(n.doneCh)
330 330 
 	}()
331 331
332 
+	wasLeader := false
333 
+
332 334 
 	for {
333 335 
 		select {
334 336 
 		case <-n.ticker.C():
... ... 
@@ -348,9 +354,11 @@ func (n *Node) Run(ctx context.Context) error {
348 348 
 				n.Config.Logger.Error(err)
349 349 
 			}
350 350
351 
-			// Send raft messages to peers
352 
-			if err := n.send(rd.Messages); err != nil {
353 
-				n.Config.Logger.Error(err)
351 
+			if len(rd.Messages) != 0 {
352 
+				// Send raft messages to peers
353 
+				if err := n.send(rd.Messages); err != nil {
354 
+					n.Config.Logger.Error(err)
355 
+				}
354 356 
 			}
355 357
356 358 
 			// Apply snapshot to memory store. The snapshot
... ... 
@@ -358,7 +366,7 @@ func (n *Node) Run(ctx context.Context) error {
358 358 
 			// saveToStorage.
359 359 
 			if !raft.IsEmptySnap(rd.Snapshot) {
360 360 
 				// Load the snapshot data into the store
361 
-				if err := n.restoreFromSnapshot(rd.Snapshot.Data, n.forceNewCluster); err != nil {
361 
+				if err := n.restoreFromSnapshot(rd.Snapshot.Data, false); err != nil {
362 362 
 					n.Config.Logger.Error(err)
363 363 
 				}
364 364 
 				n.appliedIndex = rd.Snapshot.Metadata.Index
... ... 
@@ -387,12 +395,23 @@ func (n *Node) Run(ctx context.Context) error {
387 387 
 			// if that happens we will apply them as any
388 388 
 			// follower would.
389 389 
 			if rd.SoftState != nil {
390 
-				if n.wasLeader && rd.SoftState.RaftState != raft.StateLeader {
391 
-					n.wasLeader = false
390 
+				if wasLeader && rd.SoftState.RaftState != raft.StateLeader {
391 
+					wasLeader = false
392 392 
 					n.wait.cancelAll()
393 
-					n.leadershipBroadcast.Write(IsFollower)
394 
-				} else if !n.wasLeader && rd.SoftState.RaftState == raft.StateLeader {
395 
-					n.wasLeader = true
393 
+					if atomic.LoadUint32(&n.signalledLeadership) == 1 {
394 
+						atomic.StoreUint32(&n.signalledLeadership, 0)
395 
+						n.leadershipBroadcast.Write(IsFollower)
396 
+					}
397 
+				} else if !wasLeader && rd.SoftState.RaftState == raft.StateLeader {
398 
+					wasLeader = true
399 
+				}
400 
+			}
401 
+
402 
+			if wasLeader && atomic.LoadUint32(&n.signalledLeadership) != 1 {
403 
+				// If all the entries in the log have become
404 
+				// committed, broadcast our leadership status.
405 
+				if n.caughtUp() {
406 
+					atomic.StoreUint32(&n.signalledLeadership, 1)
396 407 
 					n.leadershipBroadcast.Write(IsLeader)
397 408 
 				}
398 409 
 			}
... ... 
@@ -451,17 +470,6 @@ func (n *Node) Shutdown() {
451 451 
 	}
452 452 
 }
453 453
454 
-// isShutdown indicates if node was shut down.
455 
-// This method should be called under n.stopMu to avoid races with n.stop().
456 
-func (n *Node) isShutdown() bool {
457 
-	select {
458 
-	case <-n.Ctx.Done():
459 
-		return true
460 
-	default:
461 
-		return false
462 
-	}
463 
-}
464 
-
465 454 
 func (n *Node) stop() {
466 455 
 	n.stopMu.Lock()
467 456 
 	defer n.stopMu.Unlock()
... ... 
@@ -476,16 +484,18 @@ func (n *Node) stop() {
476 476 
 			_ = member.Conn.Close()
477 477 
 		}
478 478 
 	}
479 
+
479 480 
 	n.Stop()
480 481 
 	n.ticker.Stop()
481 482 
 	if err := n.wal.Close(); err != nil {
482 483 
 		n.Config.Logger.Errorf("raft: error closing WAL: %v", err)
483 484 
 	}
485 
+	atomic.StoreUint32(&n.isMember, 0)
484 486 
 	// TODO(stevvooe): Handle ctx.Done()
485 487 
 }
486 488
487 
-// IsLeader checks if we are the leader or not
488 
-func (n *Node) IsLeader() bool {
489 
+// isLeader checks if we are the leader or not, without the protection of lock
490 
+func (n *Node) isLeader() bool {
489 491 
 	if !n.IsMember() {
490 492 
 		return false
491 493 
 	}
... ... 
@@ -496,14 +506,43 @@ func (n *Node) IsLeader() bool {
496 496 
 	return false
497 497 
 }
498 498
499 
-// Leader returns the id of the leader
500 
-func (n *Node) Leader() uint64 {
499 
+// IsLeader checks if we are the leader or not, with the protection of lock
500 
+func (n *Node) IsLeader() bool {
501 
+	n.stopMu.RLock()
502 
+	defer n.stopMu.RUnlock()
503 
+
504 
+	return n.isLeader()
505 
+}
506 
+
507 
+// leader returns the id of the leader, without the protection of lock
508 
+func (n *Node) leader() uint64 {
501 509 
 	if !n.IsMember() {
502 510 
 		return 0
503 511 
 	}
504 512 
 	return n.Node.Status().Lead
505 513 
 }
506 514
515 
+// Leader returns the id of the leader, with the protection of lock
516 
+func (n *Node) Leader() uint64 {
517 
+	n.stopMu.RLock()
518 
+	defer n.stopMu.RUnlock()
519 
+
520 
+	return n.leader()
521 
+}
522 
+
523 
+// ReadyForProposals returns true if the node has broadcasted a message
524 
+// saying that it has become the leader. This means it is ready to accept
525 
+// proposals.
526 
+func (n *Node) ReadyForProposals() bool {
527 
+	return atomic.LoadUint32(&n.signalledLeadership) == 1
528 
+}
529 
+
530 
+func (n *Node) caughtUp() bool {
531 
+	// obnoxious function that always returns a nil error
532 
+	lastIndex, _ := n.raftStore.LastIndex()
533 
+	return n.appliedIndex >= lastIndex
534 
+}
535 
+
507 536 
 // Join asks to a member of the raft to propose
508 537 
 // a configuration change and add us as a member thus
509 538 
 // beginning the log replication process. This method
... ... 
@@ -534,12 +573,7 @@ func (n *Node) Join(ctx context.Context, req *api.JoinRequest) (*api.JoinRespons
534 534 
 		return nil, ErrNoRaftMember
535 535 
 	}
536 536
537 
-	if n.IsStopped() {
538 
-		log.WithError(ErrStopped).Errorf(ErrStopped.Error())
539 
-		return nil, ErrStopped
540 
-	}
541 
-
542 
-	if !n.IsLeader() {
537 
+	if !n.isLeader() {
543 538 
 		return nil, ErrLostLeadership
544 539 
 	}
545 540
... ... 
@@ -670,11 +704,7 @@ func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResp
670 670 
 		return nil, ErrNoRaftMember
671 671 
 	}
672 672
673 
-	if n.IsStopped() {
674 
-		return nil, ErrStopped
675 
-	}
676 
-
677 
-	if !n.IsLeader() {
673 
+	if !n.isLeader() {
678 674 
 		return nil, ErrLostLeadership
679 675 
 	}
680 676
... ... 
@@ -717,12 +747,24 @@ func (n *Node) RemoveMember(ctx context.Context, id uint64) error {
717 717 
 // raft state machine with the provided message on the
718 718 
 // receiving node
719 719 
 func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessageRequest) (*api.ProcessRaftMessageResponse, error) {
720 
+	if msg == nil || msg.Message == nil {
721 
+		return nil, grpc.Errorf(codes.InvalidArgument, "no message provided")
722 
+	}
723 
+
720 724 
 	// Don't process the message if this comes from
721 725 
 	// a node in the remove set
722 726 
 	if n.cluster.IsIDRemoved(msg.Message.From) {
723 727 
 		return nil, ErrMemberRemoved
724 728 
 	}
725 729
730 
+	if msg.Message.Type == raftpb.MsgProp {
731 
+		// We don't accepted forwarded proposals. Our
732 
+		// current architecture depends on only the leader
733 
+		// making proposals, so in-flight proposals can be
734 
+		// guaranteed not to conflict.
735 
+		return nil, grpc.Errorf(codes.InvalidArgument, "proposals not accepted")
736 
+	}
737 
+
726 738 
 	// can't stop the raft node while an async RPC is in progress
727 739 
 	n.stopMu.RLock()
728 740 
 	defer n.stopMu.RUnlock()
... ... 
@@ -731,10 +773,6 @@ func (n *Node) ProcessRaftMessage(ctx context.Context, msg *api.ProcessRaftMessa
731 731 
 		return nil, ErrNoRaftMember
732 732 
 	}
733 733
734 
-	if n.IsStopped() {
735 
-		return nil, ErrStopped
736 
-	}
737 
-
738 734 
 	if err := n.Step(n.Ctx, *msg.Message); err != nil {
739 735 
 		return nil, err
740 736 
 	}
... ... 
@@ -772,21 +810,16 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques
772 772 
 // LeaderAddr returns address of current cluster leader.
773 773 
 // With this method Node satisfies raftpicker.AddrSelector interface.
774 774 
 func (n *Node) LeaderAddr() (string, error) {
775 
-	n.stopMu.RLock()
776 
-	defer n.stopMu.RUnlock()
777 
-	if n.isShutdown() {
778 
-		return "", fmt.Errorf("raft node is shut down")
779 
-	}
780 775 
 	ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second)
781 776 
 	defer cancel()
782 777 
 	if err := WaitForLeader(ctx, n); err != nil {
783 778 
 		return "", ErrNoClusterLeader
784 779 
 	}
785 
-	if n.IsStopped() {
786 
-		return "", ErrStopped
780 
+	if !n.IsMember() {
781 
+		return "", ErrNoRaftMember
787 782 
 	}
788 783 
 	ms := n.cluster.Members()
789 
-	l := ms[n.Leader()]
784 
+	l := ms[n.leader()]
790 785 
 	if l == nil {
791 786 
 		return "", ErrNoClusterLeader
792 787 
 	}
... ... 
@@ -864,6 +897,13 @@ func (n *Node) ProposeValue(ctx context.Context, storeAction []*api.StoreAction,
864 864
865 865 
 // GetVersion returns the sequence information for the current raft round.
866 866 
 func (n *Node) GetVersion() *api.Version {
867 
+	n.stopMu.RLock()
868 
+	defer n.stopMu.RUnlock()
869 
+
870 
+	if !n.IsMember() {
871 
+		return nil
872 
+	}
873 
+
867 874 
 	status := n.Node.Status()
868 875 
 	return &api.Version{Index: status.Commit}
869 876 
 }
... ... 
@@ -921,14 +961,6 @@ func (n *Node) IsMember() bool {
921 921 
 	return atomic.LoadUint32(&n.isMember) == 1
922 922 
 }
923 923
924 
-// IsStopped checks if the raft node is stopped or not
925 
-func (n *Node) IsStopped() bool {
926 
-	if n.Node == nil {
927 
-		return true
928 
-	}
929 
-	return false
930 
-}
931 
-
932 924 
 // canSubmitProposal defines if any more proposals
933 925 
 // could be submitted and processed.
934 926 
 func (n *Node) canSubmitProposal() bool {
... ... 
@@ -980,6 +1012,14 @@ func (n *Node) send(messages []raftpb.Message) error {
980 980 
 			continue
981 981 
 		}
982 982
983 
+		if m.Type == raftpb.MsgProp {
984 
+			// We don't forward proposals to the leader. Our
985 
+			// current architecture depends on only the leader
986 
+			// making proposals, so in-flight proposals can be
987 
+			// guaranteed not to conflict.
988 
+			continue
989 
+		}
990 
+
983 991 
 		n.asyncTasks.Add(1)
984 992 
 		go n.sendToMember(members, m)
985 993 
 	}
... ... 
@@ -1044,15 +1084,14 @@ func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Mess
1044 1044 
 	_, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
1045 1045 
 	if err != nil {
1046 1046 
 		if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() {
1047 
-			n.removeRaftOnce.Do(func() {
1048 
-				close(n.removeRaftCh)
1049 
-			})
1047 
+			n.removeRaftFunc()
1050 1048 
 		}
1051 1049 
 		if m.Type == raftpb.MsgSnap {
1052 1050 
 			n.ReportSnapshot(m.To, raft.SnapshotFailure)
1053 1051 
 		}
1054 
-		if n.IsStopped() {
1055 
-			panic("node is nil")
1052 
+		if !n.IsMember() {
1053 
+			// node is removed from cluster or stopped
1054 
+			return
1056 1055 
 		}
1057 1056 
 		n.ReportUnreachable(m.To)
1058 1057
... ... 
@@ -1091,7 +1130,7 @@ func (n *Node) processInternalRaftRequest(ctx context.Context, r *api.InternalRa
1091 1091 
 	ch := n.wait.register(r.ID, cb)
1092 1092
1093 1093 
 	// Do this check after calling register to avoid a race.
1094 
-	if !n.IsLeader() {
1094 
+	if atomic.LoadUint32(&n.signalledLeadership) != 1 {
1095 1095 
 		n.wait.cancel(r.ID)
1096 1096 
 		return nil, ErrLostLeadership
1097 1097 
 	}
... ... 
@@ -1262,13 +1301,15 @@ func (n *Node) applyRemoveNode(cc raftpb.ConfChange) (err error) {
1262 1262 
 	// a follower and the leader steps down, Campaign
1263 1263 
 	// to be the leader.
1264 1264
1265 
-	if cc.NodeID == n.Leader() && !n.IsLeader() {
1265 
+	if cc.NodeID == n.leader() && !n.isLeader() {
1266 1266 
 		if err = n.Campaign(n.Ctx); err != nil {
1267 1267 
 			return err
1268 1268 
 		}
1269 1269 
 	}
1270 1270
1271 1271 
 	if cc.NodeID == n.Config.ID {
1272 
+		n.removeRaftFunc()
1273 
+
1272 1274 
 		// wait the commit ack to be sent before closing connection
1273 1275 
 		n.asyncTasks.Wait()
1274 1276
vendor/src/github.com/docker/swarmkit/picker/picker.go
History View file @ 3b555a5
... ... 
@@ -185,7 +185,7 @@ func clip(x float64) float64 {
185 185 
 func (mwr *remotesWeightedRandom) observe(peer api.Peer, weight float64) {
186 186
187 187 
 	// While we have a decent, ad-hoc approach here to weight subsequent
188 
-	// observerations, we may want to look into applying forward decay:
188 
+	// observations, we may want to look into applying forward decay:
189 189 
 	//
190 190 
 	//  http://dimacs.rutgers.edu/~graham/pubs/papers/fwddecay.pdf
191 191 
 	//