Browse code

Add --force to node removal

Signed-off-by: Diogo Monica <diogo.monica@gmail.com>

Diogo Monica authored on 2016/07/28 13:17:00
Showing 27 changed files
... ...
@@ -7,26 +7,36 @@ import (
7 7
 
8 8
 	"github.com/docker/docker/api/client"
9 9
 	"github.com/docker/docker/cli"
10
+	"github.com/docker/engine-api/types"
10 11
 	"github.com/spf13/cobra"
11 12
 )
12 13
 
14
+type removeOptions struct {
15
+	force bool
16
+}
17
+
13 18
 func newRemoveCommand(dockerCli *client.DockerCli) *cobra.Command {
14
-	return &cobra.Command{
15
-		Use:     "rm NODE [NODE...]",
19
+	opts := removeOptions{}
20
+
21
+	cmd := &cobra.Command{
22
+		Use:     "rm [OPTIONS] NODE [NODE...]",
16 23
 		Aliases: []string{"remove"},
17 24
 		Short:   "Remove one or more nodes from the swarm",
18 25
 		Args:    cli.RequiresMinArgs(1),
19 26
 		RunE: func(cmd *cobra.Command, args []string) error {
20
-			return runRemove(dockerCli, args)
27
+			return runRemove(dockerCli, args, opts)
21 28
 		},
22 29
 	}
30
+	flags := cmd.Flags()
31
+	flags.BoolVar(&opts.force, "force", false, "Force remove an active node")
32
+	return cmd
23 33
 }
24 34
 
25
-func runRemove(dockerCli *client.DockerCli, args []string) error {
35
+func runRemove(dockerCli *client.DockerCli, args []string, opts removeOptions) error {
26 36
 	client := dockerCli.Client()
27 37
 	ctx := context.Background()
28 38
 	for _, nodeID := range args {
29
-		err := client.NodeRemove(ctx, nodeID)
39
+		err := client.NodeRemove(ctx, nodeID, types.NodeRemoveOptions{Force: opts.force})
30 40
 		if err != nil {
31 41
 			return err
32 42
 		}
... ...
@@ -20,7 +20,7 @@ type Backend interface {
20 20
 	GetNodes(basictypes.NodeListOptions) ([]types.Node, error)
21 21
 	GetNode(string) (types.Node, error)
22 22
 	UpdateNode(string, uint64, types.NodeSpec) error
23
-	RemoveNode(string) error
23
+	RemoveNode(string, bool) error
24 24
 	GetTasks(basictypes.TaskListOptions) ([]types.Task, error)
25 25
 	GetTask(string) (types.Task, error)
26 26
 }
... ...
@@ -219,7 +219,13 @@ func (sr *swarmRouter) updateNode(ctx context.Context, w http.ResponseWriter, r
219 219
 }
220 220
 
221 221
 func (sr *swarmRouter) removeNode(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
222
-	if err := sr.backend.RemoveNode(vars["id"]); err != nil {
222
+	if err := httputils.ParseForm(r); err != nil {
223
+		return err
224
+	}
225
+
226
+	force := httputils.BoolValue(r, "force")
227
+
228
+	if err := sr.backend.RemoveNode(vars["id"], force); err != nil {
223 229
 		logrus.Errorf("Error removing node %s: %v", vars["id"], err)
224 230
 		return err
225 231
 	}
... ...
@@ -1023,7 +1023,7 @@ func (c *Cluster) UpdateNode(nodeID string, version uint64, spec types.NodeSpec)
1023 1023
 }
1024 1024
 
1025 1025
 // RemoveNode removes a node from a cluster
1026
-func (c *Cluster) RemoveNode(input string) error {
1026
+func (c *Cluster) RemoveNode(input string, force bool) error {
1027 1027
 	c.RLock()
1028 1028
 	defer c.RUnlock()
1029 1029
 
... ...
@@ -1039,7 +1039,7 @@ func (c *Cluster) RemoveNode(input string) error {
1039 1039
 		return err
1040 1040
 	}
1041 1041
 
1042
-	if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID}); err != nil {
1042
+	if _, err := c.client.RemoveNode(ctx, &swarmapi.RemoveNodeRequest{NodeID: node.ID, Force: force}); err != nil {
1043 1043
 		return err
1044 1044
 	}
1045 1045
 	return nil
... ...
@@ -11,7 +11,7 @@ parent = "smn_cli"
11 11
 # node rm
12 12
 
13 13
 ```markdown
14
-Usage:  docker node rm NODE [NODE...]
14
+Usage:  docker node rm [OPTIONS] NODE [NODE...]
15 15
 
16 16
 Remove one or more nodes from the swarm
17 17
 
... ...
@@ -19,6 +19,7 @@ Aliases:
19 19
   rm, remove
20 20
 
21 21
 Options:
22
+      --force  Force remove an active node
22 23
       --help   Print usage
23 24
 ```
24 25
 
... ...
@@ -30,6 +31,24 @@ Example output:
30 30
     $ docker node rm swarm-node-02
31 31
     Node swarm-node-02 removed from swarm
32 32
 
33
+Removes nodes from the swarm that are in the down state. Attempting to remove
34
+an active node will result in an error:
35
+
36
+```bash
37
+$ docker node rm swarm-node-03
38
+Error response from daemon: rpc error: code = 9 desc = node swarm-node-03 is not down and can't be removed
39
+```
40
+
41
+If a worker node becomes compromised, exhibits unexpected or unwanted behavior, or if you lose access to it so
42
+that a clean shutdown is impossible, you can use the force option.
43
+
44
+```bash
45
+$ docker node rm --force swarm-node-03
46
+Node swarm-node-03 removed from swarm
47
+```
48
+
49
+Note that manager nodes have to be demoted to worker nodes before they can be removed
50
+from the cluster.
33 51
 
34 52
 ## Related information
35 53
 
... ...
@@ -60,7 +60,8 @@ clone git golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://gith
60 60
 clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git
61 61
 clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
62 62
 clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d
63
-clone git github.com/docker/engine-api 3d1601b9d2436a70b0dfc045a23f6503d19195df
63
+
64
+clone git github.com/docker/engine-api 228c7390a733320d48697cb41ae8cde4942cd3e5
64 65
 clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
65 66
 clone git github.com/imdario/mergo 0.2.1
66 67
 
... ...
@@ -139,7 +140,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0
139 139
 clone git github.com/docker/containerd 0ac3cd1be170d180b2baed755e8f0da547ceb267
140 140
 
141 141
 # cluster
142
-clone git github.com/docker/swarmkit 9d4c2f73124e70f8fa85f9076635b827d17b109f
142
+clone git github.com/docker/swarmkit e1c0d64515d839b76e2ef33d396c74933753ffaf
143 143
 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
144 144
 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028
145 145
 clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b
... ...
@@ -208,6 +208,17 @@ func (d *SwarmDaemon) getNode(c *check.C, id string) *swarm.Node {
208 208
 	return &node
209 209
 }
210 210
 
211
+func (d *SwarmDaemon) removeNode(c *check.C, id string, force bool) {
212
+	url := "/nodes/" + id
213
+	if force {
214
+		url += "?force=1"
215
+	}
216
+
217
+	status, out, err := d.SockRequest("DELETE", url, nil)
218
+	c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out)))
219
+	c.Assert(err, checker.IsNil)
220
+}
221
+
211 222
 func (d *SwarmDaemon) updateNode(c *check.C, id string, f ...nodeConstructor) {
212 223
 	for i := 0; ; i++ {
213 224
 		node := d.getNode(c, id)
... ...
@@ -544,6 +544,37 @@ func (s *DockerSwarmSuite) TestApiSwarmNodeUpdate(c *check.C) {
544 544
 	c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause)
545 545
 }
546 546
 
547
+func (s *DockerSwarmSuite) TestApiSwarmNodeRemove(c *check.C) {
548
+	testRequires(c, Network)
549
+	d1 := s.AddDaemon(c, true, true)
550
+	d2 := s.AddDaemon(c, true, false)
551
+	_ = s.AddDaemon(c, true, false)
552
+
553
+	nodes := d1.listNodes(c)
554
+	c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes))
555
+
556
+	// Getting the info so we can take the NodeID
557
+	d2Info, err := d2.info()
558
+	c.Assert(err, checker.IsNil)
559
+
560
+	// forceful removal of d2 should work
561
+	d1.removeNode(c, d2Info.NodeID, true)
562
+
563
+	nodes = d1.listNodes(c)
564
+	c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
565
+
566
+	// Restart the node that was removed
567
+	err = d2.Restart()
568
+	c.Assert(err, checker.IsNil)
569
+
570
+	// Give some time for the node to rejoin
571
+	time.Sleep(1 * time.Second)
572
+
573
+	// Make sure the node didn't rejoin
574
+	nodes = d1.listNodes(c)
575
+	c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
576
+}
577
+
547 578
 func (s *DockerSwarmSuite) TestApiSwarmNodeDrainPause(c *check.C) {
548 579
 	d1 := s.AddDaemon(c, true, true)
549 580
 	d2 := s.AddDaemon(c, true, false)
550 581
deleted file mode 100644
... ...
@@ -1,4 +0,0 @@
1
-package client
2
-
3
-// DefaultDockerHost defines os specific default if DOCKER_HOST is unset
4
-const DefaultDockerHost = "tcp://127.0.0.1:2375"
... ...
@@ -1,4 +1,4 @@
1
-// +build linux freebsd solaris openbsd
1
+// +build linux freebsd solaris openbsd darwin
2 2
 
3 3
 package client
4 4
 
... ...
@@ -94,7 +94,7 @@ type NetworkAPIClient interface {
94 94
 type NodeAPIClient interface {
95 95
 	NodeInspectWithRaw(ctx context.Context, nodeID string) (swarm.Node, []byte, error)
96 96
 	NodeList(ctx context.Context, options types.NodeListOptions) ([]swarm.Node, error)
97
-	NodeRemove(ctx context.Context, nodeID string) error
97
+	NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error
98 98
 	NodeUpdate(ctx context.Context, nodeID string, version swarm.Version, node swarm.NodeSpec) error
99 99
 }
100 100
 
... ...
@@ -1,10 +1,21 @@
1 1
 package client
2 2
 
3
-import "golang.org/x/net/context"
3
+import (
4
+	"net/url"
5
+
6
+	"github.com/docker/engine-api/types"
7
+
8
+	"golang.org/x/net/context"
9
+)
4 10
 
5 11
 // NodeRemove removes a Node.
6
-func (cli *Client) NodeRemove(ctx context.Context, nodeID string) error {
7
-	resp, err := cli.delete(ctx, "/nodes/"+nodeID, nil, nil)
12
+func (cli *Client) NodeRemove(ctx context.Context, nodeID string, options types.NodeRemoveOptions) error {
13
+	query := url.Values{}
14
+	if options.Force {
15
+		query.Set("force", "1")
16
+	}
17
+
18
+	resp, err := cli.delete(ctx, "/nodes/"+nodeID, query, nil)
8 19
 	ensureReaderClosed(resp)
9 20
 	return err
10 21
 }
... ...
@@ -241,11 +241,16 @@ func (v VersionResponse) ServerOK() bool {
241 241
 	return v.Server != nil
242 242
 }
243 243
 
244
-// NodeListOptions holds parameters to list  nodes with.
244
+// NodeListOptions holds parameters to list nodes with.
245 245
 type NodeListOptions struct {
246 246
 	Filter filters.Args
247 247
 }
248 248
 
249
+// NodeRemoveOptions holds parameters to remove nodes with.
250
+type NodeRemoveOptions struct {
251
+	Force bool
252
+}
253
+
249 254
 // ServiceCreateOptions contains the options to use when creating a service.
250 255
 type ServiceCreateOptions struct {
251 256
 	// EncodedRegistryAuth is the encoded registry authorization credentials to
... ...
@@ -12,12 +12,12 @@ import (
12 12
 )
13 13
 
14 14
 const (
15
-	initialSessionFailureBackoff = time.Second
15
+	initialSessionFailureBackoff = 100 * time.Millisecond
16 16
 	maxSessionFailureBackoff     = 8 * time.Second
17 17
 )
18 18
 
19 19
 // Agent implements the primary node functionality for a member of a swarm
20
-// cluster. The primary functionality id to run and report on the status of
20
+// cluster. The primary functionality is to run and report on the status of
21 21
 // tasks assigned to the node.
22 22
 type Agent struct {
23 23
 	config *Config
... ...
@@ -187,7 +187,7 @@ func (n *Node) run(ctx context.Context) (err error) {
187 187
 	if n.config.JoinAddr != "" || n.config.ForceNewCluster {
188 188
 		n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
189 189
 		if n.config.JoinAddr != "" {
190
-			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, 1)
190
+			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, picker.DefaultObservationWeight)
191 191
 		}
192 192
 	}
193 193
 
... ...
@@ -647,7 +647,7 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig
647 647
 			go func(ready chan struct{}) {
648 648
 				select {
649 649
 				case <-ready:
650
-					n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, 5)
650
+					n.remotes.Observe(api.Peer{NodeID: n.nodeID, Addr: n.config.ListenRemoteAPI}, picker.DefaultObservationWeight)
651 651
 				case <-connCtx.Done():
652 652
 				}
653 653
 			}(ready)
... ...
@@ -200,7 +200,7 @@ func (tm *taskManager) run(ctx context.Context) {
200 200
 				cancel() // cancel outstanding if necessary.
201 201
 			} else {
202 202
 				// If this channel op fails, it means there is already a
203
-				// message un the run queue.
203
+				// message on the run queue.
204 204
 				select {
205 205
 				case run <- struct{}{}:
206 206
 				default:
... ...
@@ -106,6 +106,7 @@ func (*UpdateNodeResponse) Descriptor() ([]byte, []int) { return fileDescriptorC
106 106
 // RemoveNodeRequest requests to delete the specified node from store.
107 107
 type RemoveNodeRequest struct {
108 108
 	NodeID string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
109
+	Force  bool   `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"`
109 110
 }
110 111
 
111 112
 func (m *RemoveNodeRequest) Reset()                    { *m = RemoveNodeRequest{} }
... ...
@@ -786,6 +787,7 @@ func (m *RemoveNodeRequest) Copy() *RemoveNodeRequest {
786 786
 
787 787
 	o := &RemoveNodeRequest{
788 788
 		NodeID: m.NodeID,
789
+		Force:  m.Force,
789 790
 	}
790 791
 
791 792
 	return o
... ...
@@ -1473,9 +1475,10 @@ func (this *RemoveNodeRequest) GoString() string {
1473 1473
 	if this == nil {
1474 1474
 		return "nil"
1475 1475
 	}
1476
-	s := make([]string, 0, 5)
1476
+	s := make([]string, 0, 6)
1477 1477
 	s = append(s, "&api.RemoveNodeRequest{")
1478 1478
 	s = append(s, "NodeID: "+fmt.Sprintf("%#v", this.NodeID)+",\n")
1479
+	s = append(s, "Force: "+fmt.Sprintf("%#v", this.Force)+",\n")
1479 1480
 	s = append(s, "}")
1480 1481
 	return strings.Join(s, "")
1481 1482
 }
... ...
@@ -2938,6 +2941,16 @@ func (m *RemoveNodeRequest) MarshalTo(data []byte) (int, error) {
2938 2938
 		i = encodeVarintControl(data, i, uint64(len(m.NodeID)))
2939 2939
 		i += copy(data[i:], m.NodeID)
2940 2940
 	}
2941
+	if m.Force {
2942
+		data[i] = 0x10
2943
+		i++
2944
+		if m.Force {
2945
+			data[i] = 1
2946
+		} else {
2947
+			data[i] = 0
2948
+		}
2949
+		i++
2950
+	}
2941 2951
 	return i, nil
2942 2952
 }
2943 2953
 
... ...
@@ -4692,6 +4705,9 @@ func (m *RemoveNodeRequest) Size() (n int) {
4692 4692
 	if l > 0 {
4693 4693
 		n += 1 + l + sovControl(uint64(l))
4694 4694
 	}
4695
+	if m.Force {
4696
+		n += 2
4697
+	}
4695 4698
 	return n
4696 4699
 }
4697 4700
 
... ...
@@ -5286,6 +5302,7 @@ func (this *RemoveNodeRequest) String() string {
5286 5286
 	}
5287 5287
 	s := strings.Join([]string{`&RemoveNodeRequest{`,
5288 5288
 		`NodeID:` + fmt.Sprintf("%v", this.NodeID) + `,`,
5289
+		`Force:` + fmt.Sprintf("%v", this.Force) + `,`,
5289 5290
 		`}`,
5290 5291
 	}, "")
5291 5292
 	return s
... ...
@@ -6617,6 +6634,26 @@ func (m *RemoveNodeRequest) Unmarshal(data []byte) error {
6617 6617
 			}
6618 6618
 			m.NodeID = string(data[iNdEx:postIndex])
6619 6619
 			iNdEx = postIndex
6620
+		case 2:
6621
+			if wireType != 0 {
6622
+				return fmt.Errorf("proto: wrong wireType = %d for field Force", wireType)
6623
+			}
6624
+			var v int
6625
+			for shift := uint(0); ; shift += 7 {
6626
+				if shift >= 64 {
6627
+					return ErrIntOverflowControl
6628
+				}
6629
+				if iNdEx >= l {
6630
+					return io.ErrUnexpectedEOF
6631
+				}
6632
+				b := data[iNdEx]
6633
+				iNdEx++
6634
+				v |= (int(b) & 0x7F) << shift
6635
+				if b < 0x80 {
6636
+					break
6637
+				}
6638
+			}
6639
+			m.Force = bool(v != 0)
6620 6640
 		default:
6621 6641
 			iNdEx = preIndex
6622 6642
 			skippy, err := skipControl(data[iNdEx:])
... ...
@@ -10521,99 +10558,100 @@ var (
10521 10521
 )
10522 10522
 
10523 10523
 var fileDescriptorControl = []byte{
10524
-	// 1498 bytes of a gzipped FileDescriptorProto
10524
+	// 1512 bytes of a gzipped FileDescriptorProto
10525 10525
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x59, 0xcf, 0x6f, 0x1b, 0xc5,
10526
-	0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x09, 0xda,
10526
+	0x17, 0xaf, 0x9d, 0x34, 0x8e, 0x9f, 0x6b, 0xb7, 0x9e, 0xba, 0xfa, 0x46, 0x6e, 0xbf, 0x0d, 0xda,
10527 10527
 	0xd2, 0x34, 0x91, 0x82, 0x03, 0x8e, 0x2a, 0x02, 0x48, 0x20, 0x9c, 0xd0, 0xca, 0xd0, 0x86, 0x6a,
10528
-	0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38,
10528
+	0xd3, 0x02, 0xb7, 0xc8, 0xb1, 0x27, 0x61, 0xf1, 0x8f, 0x35, 0xbb, 0x9b, 0xb4, 0x11, 0x17, 0x38,
10529 10529
 	0x20, 0xf1, 0x27, 0x70, 0xe5, 0xca, 0x81, 0x7f, 0x81, 0x6b, 0xc4, 0x89, 0x0b, 0x12, 0xa7, 0x88,
10530 10530
 	0xf6, 0xc4, 0x09, 0xf1, 0x17, 0x20, 0xe6, 0xc7, 0x9b, 0xdd, 0xf5, 0x7a, 0x76, 0x6d, 0x27, 0x41,
10531
-	0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xfd, 0x98, 0x79, 0x9f, 0xf7, 0xf6, 0xcd, 0x04, 0xf2, 0x4d,
10532
-	0xbb, 0xe7, 0x39, 0x76, 0xa7, 0xd2, 0x77, 0x6c, 0xcf, 0x26, 0xa4, 0x65, 0x37, 0xdb, 0xd4, 0xa9,
10533
-	0xb8, 0x4f, 0x1b, 0x4e, 0xb7, 0x6d, 0x79, 0x95, 0x83, 0x37, 0xca, 0x39, 0xb7, 0x4f, 0x9b, 0xae,
10534
-	0x04, 0x94, 0xf3, 0xf6, 0xee, 0x17, 0xb4, 0xe9, 0xa9, 0xd7, 0x9c, 0x77, 0xd8, 0xa7, 0xea, 0xa5,
10535
-	0xb4, 0x67, 0xef, 0xd9, 0xe2, 0x71, 0x95, 0x3f, 0xe1, 0xe8, 0xd5, 0x7e, 0x67, 0x7f, 0xcf, 0xea,
10536
-	0xad, 0xca, 0x3f, 0x72, 0xd0, 0xb8, 0x03, 0x85, 0x7b, 0xd4, 0xdb, 0xb2, 0x5b, 0xd4, 0xa4, 0x5f,
10537
-	0xee, 0x53, 0xd7, 0x23, 0x37, 0x21, 0xd3, 0x63, 0xaf, 0x3b, 0x56, 0x6b, 0x2e, 0xf5, 0x4a, 0x6a,
10538
-	0x29, 0x5b, 0x83, 0x17, 0xc7, 0x0b, 0x33, 0x1c, 0x51, 0xdf, 0x34, 0x67, 0xf8, 0x54, 0xbd, 0x65,
10539
-	0xbc, 0x07, 0x97, 0x7d, 0x31, 0xb7, 0x6f, 0xf7, 0x5c, 0x4a, 0x56, 0x60, 0x9a, 0x4f, 0x0a, 0xa1,
10540
-	0x5c, 0x75, 0xae, 0x32, 0xbc, 0x80, 0x8a, 0xc0, 0x0b, 0x94, 0x71, 0x3c, 0x05, 0x57, 0xee, 0x5b,
10541
-	0xae, 0x50, 0xe1, 0x2a, 0xd3, 0x77, 0x21, 0xf3, 0xc4, 0xea, 0x78, 0xd4, 0x71, 0x51, 0xcb, 0x8a,
10542
-	0x4e, 0x4b, 0x54, 0xac, 0x72, 0x57, 0xca, 0x98, 0x4a, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0x70, 0x90,
10543
-	0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x5c, 0xe3, 0xd4, 0x52, 0xd6, 0x94, 0x2f, 0x64, 0x15, 0x72,
10544
-	0x56, 0x6b, 0xa7, 0xef, 0xd0, 0x27, 0xd6, 0x33, 0x36, 0x97, 0xe6, 0x73, 0xb5, 0x02, 0x5b, 0x28,
10545
-	0xd4, 0x37, 0x1f, 0xe2, 0xa8, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x87, 0x30, 0xd3, 0x69, 0xec, 0xd2,
10546
-	0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xfa, 0x24, 0x9e, 0x55, 0xee, 0x0b, 0xd1, 0x0f, 0x58,
10547
-	0x80, 0x0f, 0x4d, 0xd4, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x2e, 0x9b, 0xfe, 0xdc, 0xea, 0xbb,
10548
-	0x73, 0xd3, 0x4c, 0x6d, 0xa1, 0x7a, 0x3b, 0x6e, 0xdb, 0xb6, 0x59, 0xe8, 0x2b, 0x0f, 0x7c, 0xbc,
10549
-	0x19, 0x96, 0x25, 0x55, 0xb8, 0xc8, 0x98, 0xc3, 0xd6, 0x71, 0x51, 0x28, 0xb9, 0x11, 0xbb, 0xf7,
10550
-	0x0c, 0x64, 0x4a, 0x28, 0x0b, 0x73, 0x9e, 0x6f, 0x45, 0xb0, 0x07, 0x33, 0x62, 0x7f, 0x2e, 0xf1,
10551
-	0x41, 0xb5, 0xea, 0xf2, 0x5b, 0x90, 0x0b, 0xb9, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a, 0x5a,
10552
-	0x98, 0xfc, 0x91, 0xef, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0e, 0xf2, 0x31, 0xf9, 0xf2, 0x76,
10553
-	0x7a, 0x3d, 0x65, 0x6c, 0x40, 0x31, 0xb4, 0x1d, 0xc8, 0x91, 0x0a, 0x0b, 0x06, 0x1f, 0x10, 0xc1,
10554
-	0x48, 0x22, 0x89, 0x84, 0x19, 0x3f, 0xa6, 0xa0, 0xf8, 0xb8, 0xdf, 0x6a, 0x78, 0x74, 0x52, 0x86,
10555
-	0x92, 0x77, 0xe1, 0x92, 0x00, 0x1d, 0xb0, 0x4d, 0xb2, 0xec, 0x9e, 0x70, 0x30, 0x57, 0xbd, 0xae,
10556
-	0xb3, 0xf8, 0x89, 0x84, 0x98, 0x39, 0x2e, 0x80, 0x2f, 0xe4, 0x75, 0x98, 0xe6, 0xe9, 0xc6, 0xc2,
10557
-	0xcd, 0xe5, 0x6e, 0x24, 0xc5, 0xc5, 0x14, 0x48, 0xa3, 0x06, 0x24, 0xec, 0xeb, 0x89, 0xd2, 0x62,
10558
-	0x1d, 0x8a, 0x26, 0xed, 0xda, 0x07, 0x13, 0xaf, 0xd7, 0x28, 0x01, 0x09, 0x4b, 0x4a, 0xeb, 0x98,
10559
-	0xde, 0x8f, 0x1a, 0x6e, 0x3b, 0xa4, 0xcc, 0x63, 0xaf, 0x11, 0x65, 0x1c, 0xc1, 0x95, 0xf1, 0x29,
10560
-	0x3f, 0xbd, 0xa5, 0x58, 0xb0, 0x0e, 0x3e, 0x99, 0xb4, 0x0e, 0x81, 0x17, 0xa8, 0x60, 0x1d, 0x13,
10561
-	0x9b, 0xf6, 0xd7, 0x11, 0xb6, 0x6e, 0xfc, 0x83, 0xe5, 0x82, 0x0f, 0x9e, 0xa0, 0x5c, 0x84, 0xc5,
10562
-	0x86, 0xcb, 0xc5, 0x0f, 0xe7, 0x58, 0x2e, 0x74, 0x9e, 0x69, 0xcb, 0x05, 0x73, 0xc1, 0xa5, 0xce,
10563
-	0x81, 0xd5, 0xe4, 0x3c, 0x90, 0xe5, 0x02, 0x5d, 0xd8, 0x96, 0xc3, 0xf5, 0x4d, 0xe6, 0x02, 0x42,
10564
-	0xea, 0x2d, 0x97, 0x2c, 0xc2, 0x2c, 0xb2, 0x46, 0xd6, 0x85, 0x6c, 0x2d, 0xc7, 0xd0, 0x19, 0x49,
10565
-	0x1b, 0xb6, 0x7a, 0xc9, 0x1b, 0x97, 0x6c, 0x42, 0x81, 0xa5, 0x9a, 0xe5, 0xd0, 0xd6, 0x8e, 0xeb,
10566
-	0x31, 0xf6, 0xca, 0x4a, 0x50, 0xa8, 0xfe, 0x3f, 0x2e, 0xc4, 0xdb, 0x1c, 0x65, 0xe6, 0x51, 0x48,
10567
-	0xbc, 0x69, 0xca, 0x49, 0xe6, 0x3f, 0x29, 0x27, 0xb8, 0x5d, 0x41, 0x39, 0xe1, 0xac, 0x49, 0x2c,
10568
-	0x27, 0x82, 0x46, 0x12, 0x66, 0x7c, 0x04, 0xa5, 0x0d, 0x87, 0x32, 0x7f, 0x71, 0xcb, 0x14, 0x91,
10569
-	0xd6, 0x30, 0xd7, 0x25, 0x8b, 0x16, 0x74, 0x6a, 0x50, 0x22, 0x94, 0xee, 0x5b, 0x70, 0x2d, 0xa2,
10570
-	0x0c, 0xbd, 0xba, 0x03, 0x19, 0x0c, 0x03, 0x2a, 0xbc, 0x9e, 0xa0, 0xd0, 0x54, 0x58, 0xe3, 0x7d,
10571
-	0x28, 0xb2, 0x9c, 0x8b, 0x78, 0xb6, 0x02, 0x10, 0x44, 0x1d, 0xb3, 0x26, 0xcf, 0xc2, 0x98, 0xf5,
10572
-	0x83, 0x6e, 0x66, 0xfd, 0x98, 0xb3, 0xf5, 0x91, 0xb0, 0x8a, 0xd3, 0xf9, 0xf3, 0x73, 0x0a, 0x4a,
10573
-	0xb2, 0x9e, 0x9d, 0xc6, 0x27, 0x46, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x29, 0x2e, 0xa0, 0x8c, 0xaa,
10574
-	0xc6, 0x6b, 0x03, 0xd5, 0x78, 0xfc, 0x08, 0x45, 0x16, 0x70, 0xba, 0x1d, 0xd9, 0x84, 0x92, 0x2c,
10575
-	0x4d, 0xa7, 0x0a, 0xd2, 0xff, 0xe0, 0x5a, 0x44, 0x0b, 0xd6, 0xb8, 0x3f, 0xd3, 0x70, 0x95, 0x73,
10576
-	0x1c, 0xc7, 0xfd, 0x32, 0x57, 0x8f, 0x96, 0xb9, 0xd5, 0xb8, 0x62, 0x12, 0x91, 0x1c, 0xae, 0x74,
10577
-	0xdf, 0xa6, 0xcf, 0xbc, 0xd2, 0x6d, 0x47, 0x2a, 0xdd, 0x3b, 0x13, 0x3a, 0xa7, 0x2d, 0x76, 0x43,
10578
-	0xd5, 0x64, 0xfa, 0x6c, 0xab, 0xc9, 0xc7, 0x50, 0x1a, 0x74, 0x09, 0x89, 0xf1, 0x26, 0xcc, 0x62,
10579
-	0xa0, 0x54, 0x4d, 0x49, 0x64, 0x86, 0x0f, 0x0e, 0x2a, 0xcb, 0x16, 0xf5, 0x9e, 0xda, 0x4e, 0x7b,
10580
-	0x82, 0xca, 0x82, 0x12, 0xba, 0xca, 0xe2, 0x2b, 0x0b, 0x78, 0xdb, 0x93, 0x43, 0x49, 0xbc, 0x55,
10581
-	0x52, 0x0a, 0x6b, 0x3c, 0x16, 0x95, 0x25, 0xe2, 0x19, 0x61, 0x7d, 0x09, 0xdb, 0x4d, 0xdc, 0x2f,
10582
-	0xf1, 0xcc, 0x89, 0x8c, 0x32, 0x9c, 0xc8, 0xe9, 0x80, 0xc8, 0x28, 0xcb, 0x89, 0x8c, 0x00, 0xbf,
10583
-	0xda, 0x9c, 0x91, 0x8f, 0x9f, 0xa9, 0xdc, 0x3a, 0x73, 0x37, 0xfd, 0x7c, 0x8b, 0x78, 0xea, 0xe7,
10584
-	0x1b, 0x8e, 0x9f, 0x20, 0xdf, 0x22, 0x92, 0x2f, 0x57, 0xbe, 0xc5, 0x38, 0x77, 0x9e, 0xf9, 0x16,
10585
-	0xb8, 0x14, 0xe4, 0x1b, 0x06, 0x2a, 0x31, 0xdf, 0x54, 0xe4, 0x7c, 0x30, 0x7e, 0x2c, 0x37, 0x3a,
10586
-	0xfb, 0x2e, 0x5b, 0x53, 0xa8, 0x0e, 0x37, 0xe5, 0x48, 0xa4, 0x0e, 0x23, 0x8e, 0xf3, 0x02, 0x01,
10587
-	0x3e, 0x7d, 0x7d, 0x15, 0x01, 0x7d, 0x11, 0x92, 0x44, 0x5f, 0x25, 0xa5, 0xb0, 0x3e, 0x97, 0x70,
10588
-	0xe2, 0x04, 0x5c, 0x8a, 0x48, 0xbe, 0x5c, 0x5c, 0x8a, 0x71, 0xee, 0x3c, 0xb9, 0x14, 0xb8, 0x14,
10589
-	0x70, 0x09, 0xa3, 0x91, 0xc8, 0x25, 0x15, 0x3a, 0x1f, 0x6c, 0xec, 0x43, 0xf1, 0x43, 0xdb, 0xea,
10590
-	0x3d, 0xb2, 0xdb, 0xb4, 0x67, 0xda, 0xac, 0x9d, 0xe5, 0x0d, 0x47, 0x05, 0xae, 0x3a, 0xfc, 0x99,
10591
-	0xee, 0x70, 0xc2, 0x31, 0x46, 0x79, 0x7c, 0x5a, 0x78, 0x38, 0x6b, 0x16, 0xe5, 0xd4, 0xa7, 0x62,
10592
-	0x46, 0xc8, 0xb1, 0xe3, 0x62, 0x09, 0xf1, 0xdd, 0x46, 0xaf, 0xb1, 0xe7, 0x0b, 0xa4, 0x85, 0x00,
10593
-	0x91, 0x73, 0x0f, 0xe4, 0x94, 0x90, 0x30, 0xbe, 0x4b, 0xab, 0xfe, 0xea, 0x34, 0x34, 0xe6, 0xfd,
10594
-	0x95, 0x42, 0x4f, 0xd2, 0x5f, 0xa1, 0xcc, 0x04, 0xfd, 0x15, 0x5a, 0x0f, 0xbe, 0x53, 0xe4, 0x1e,
10595
-	0xcc, 0x3a, 0xb8, 0x5f, 0x2c, 0xc8, 0x5c, 0xf0, 0x96, 0x4e, 0x70, 0x68, 0x73, 0x6b, 0xd3, 0x47,
10596
-	0xc7, 0x0b, 0x17, 0x4c, 0x5f, 0x38, 0x68, 0xd4, 0xce, 0x26, 0x1b, 0xab, 0xbf, 0x15, 0x21, 0xb3,
10597
-	0x21, 0xaf, 0xd3, 0x88, 0x05, 0x19, 0xbc, 0xa9, 0x22, 0x86, 0x4e, 0x78, 0xf0, 0xf6, 0xab, 0x7c,
10598
-	0x33, 0x11, 0x83, 0x5f, 0x8e, 0x6b, 0xbf, 0xfc, 0xf4, 0xd7, 0xf7, 0xe9, 0xcb, 0x90, 0x17, 0xa0,
10599
-	0xd7, 0x30, 0xe2, 0xc4, 0x86, 0xac, 0x7f, 0xe5, 0x41, 0x5e, 0x1d, 0xe7, 0x82, 0xa8, 0x7c, 0x6b,
10600
-	0x04, 0x2a, 0xd9, 0xa0, 0x03, 0x10, 0xdc, 0x38, 0x10, 0xad, 0xae, 0xa1, 0xdb, 0x93, 0xf2, 0xe2,
10601
-	0x28, 0xd8, 0x48, 0x9b, 0xc1, 0x3d, 0x83, 0xde, 0xe6, 0xd0, 0x0d, 0x86, 0xde, 0xa6, 0xe6, 0xba,
10602
-	0x22, 0xc6, 0xa6, 0x8c, 0x21, 0x3f, 0xc9, 0xc5, 0xc6, 0x30, 0x74, 0xcf, 0x10, 0x1b, 0xc3, 0x81,
10603
-	0x1b, 0x85, 0xe4, 0x18, 0x8a, 0x73, 0x66, 0x7c, 0x0c, 0xc3, 0xa7, 0xf6, 0xf8, 0x18, 0x0e, 0x1c,
10604
-	0x56, 0x47, 0xee, 0xa7, 0x58, 0x5e, 0xc2, 0x7e, 0x86, 0x57, 0xb8, 0x38, 0x0a, 0x36, 0xd2, 0x66,
10605
-	0x70, 0x4e, 0xd4, 0xdb, 0x1c, 0x3a, 0x8a, 0xea, 0x6d, 0x0e, 0x1f, 0x37, 0xe3, 0x6c, 0x3e, 0x83,
10606
-	0x4b, 0xe1, 0x96, 0x9b, 0xdc, 0x1e, 0xf3, 0x9c, 0x50, 0x5e, 0x1a, 0x0d, 0x4c, 0xb6, 0xfc, 0x15,
10607
-	0xe4, 0x07, 0x0e, 0xea, 0x44, 0xab, 0x51, 0x77, 0x31, 0x50, 0x5e, 0x1e, 0x03, 0x39, 0xd2, 0xf8,
10608
-	0xc0, 0x19, 0x54, 0x6f, 0x5c, 0x77, 0xce, 0xd6, 0x1b, 0xd7, 0x1e, 0x68, 0x13, 0x8c, 0x0f, 0x1c,
10609
-	0x35, 0xf5, 0xc6, 0x75, 0x67, 0x5a, 0xbd, 0x71, 0xfd, 0xb9, 0x35, 0x91, 0x64, 0xd8, 0xba, 0xc5,
10610
-	0x92, 0x6c, 0xb0, 0xdd, 0x8f, 0x25, 0x59, 0xb4, 0x77, 0x4f, 0x26, 0x99, 0xea, 0x33, 0xe3, 0x49,
10611
-	0x16, 0x69, 0x8e, 0xe3, 0x49, 0x16, 0x6d, 0x59, 0x47, 0x92, 0x4c, 0x2d, 0x38, 0x81, 0x64, 0x91,
10612
-	0x35, 0x2f, 0x8f, 0x81, 0x1c, 0x33, 0xce, 0x89, 0xc6, 0x75, 0xe7, 0xab, 0xa4, 0x38, 0x8f, 0x69,
10613
-	0x5c, 0xc6, 0x19, 0xbf, 0xc1, 0xb1, 0x71, 0x1e, 0xec, 0x71, 0x62, 0xe3, 0x1c, 0x69, 0x00, 0x46,
10614
-	0xc4, 0x59, 0xf5, 0x80, 0xf1, 0x71, 0x8e, 0x34, 0xae, 0xf1, 0x71, 0x8e, 0xb6, 0x93, 0x23, 0xf3,
10615
-	0x59, 0x2d, 0x38, 0x21, 0x9f, 0x23, 0x6b, 0x5e, 0x1e, 0x03, 0x99, 0x68, 0xbc, 0x76, 0xe3, 0xe8,
10616
-	0xf9, 0xfc, 0x85, 0xdf, 0xd9, 0xef, 0xef, 0xe7, 0xf3, 0xa9, 0xaf, 0x5f, 0xcc, 0xa7, 0x8e, 0xd8,
10617
-	0xef, 0x57, 0xf6, 0xfb, 0x83, 0xfd, 0x76, 0x67, 0xc4, 0x7f, 0xf4, 0xd6, 0xfe, 0x0d, 0x00, 0x00,
10618
-	0xff, 0xff, 0xf3, 0xcc, 0x22, 0xcd, 0x4a, 0x1c, 0x00, 0x00,
10531
+	0xe9, 0xc1, 0xca, 0xee, 0xcc, 0xe7, 0xcd, 0x7b, 0x33, 0x9f, 0xcf, 0xbc, 0x7d, 0x33, 0x81, 0x7c,
10532
+	0xd3, 0xee, 0x79, 0x8e, 0xdd, 0xa9, 0xf4, 0x1d, 0xdb, 0xb3, 0x09, 0x69, 0xd9, 0xcd, 0x36, 0x75,
10533
+	0x2a, 0xee, 0xd3, 0x86, 0xd3, 0x6d, 0x5b, 0x5e, 0xe5, 0xe0, 0x8d, 0x72, 0xce, 0xed, 0xd3, 0xa6,
10534
+	0x2b, 0x01, 0xe5, 0xbc, 0xbd, 0xf3, 0x05, 0x6d, 0x7a, 0xea, 0x35, 0xe7, 0x1d, 0xf6, 0xa9, 0x7a,
10535
+	0x29, 0xed, 0xd9, 0x7b, 0xb6, 0x78, 0x5c, 0xe1, 0x4f, 0xd8, 0x7a, 0xb5, 0xdf, 0xd9, 0xdf, 0xb3,
10536
+	0x7a, 0x2b, 0xf2, 0x8f, 0x6c, 0x34, 0xee, 0x42, 0xe1, 0x3e, 0xf5, 0x36, 0xed, 0x16, 0x35, 0xe9,
10537
+	0x97, 0xfb, 0xd4, 0xf5, 0xc8, 0x2d, 0xc8, 0xf4, 0xd8, 0xeb, 0xb6, 0xd5, 0x9a, 0x4b, 0xbd, 0x92,
10538
+	0x5a, 0xcc, 0xd6, 0xe0, 0xc5, 0xf1, 0xfc, 0x0c, 0x47, 0xd4, 0x37, 0xcc, 0x19, 0xde, 0x55, 0x6f,
10539
+	0x19, 0xef, 0xc1, 0x65, 0xdf, 0xcc, 0xed, 0xdb, 0x3d, 0x97, 0x92, 0x65, 0x98, 0xe6, 0x9d, 0xc2,
10540
+	0x28, 0x57, 0x9d, 0xab, 0x0c, 0x4f, 0xa0, 0x22, 0xf0, 0x02, 0x65, 0x1c, 0x4f, 0xc1, 0x95, 0x07,
10541
+	0x96, 0x2b, 0x86, 0x70, 0x95, 0xeb, 0x7b, 0x90, 0xd9, 0xb5, 0x3a, 0x1e, 0x75, 0x5c, 0x1c, 0x65,
10542
+	0x59, 0x37, 0x4a, 0xd4, 0xac, 0x72, 0x4f, 0xda, 0x98, 0xca, 0xb8, 0xfc, 0xcd, 0x14, 0x64, 0xb0,
10543
+	0x91, 0x94, 0xe0, 0x62, 0xaf, 0xd1, 0xa5, 0x7c, 0xc4, 0xa9, 0xc5, 0xac, 0x29, 0x5f, 0xc8, 0x0a,
10544
+	0xe4, 0xac, 0xd6, 0x76, 0xdf, 0xa1, 0xbb, 0xd6, 0x33, 0xd6, 0x97, 0xe6, 0x7d, 0xb5, 0x02, 0x9b,
10545
+	0x28, 0xd4, 0x37, 0x1e, 0x61, 0xab, 0x09, 0x56, 0x4b, 0x3d, 0x93, 0x47, 0x30, 0xd3, 0x69, 0xec,
10546
+	0xd0, 0x8e, 0x3b, 0x37, 0xc5, 0xb0, 0xb9, 0xea, 0xda, 0x24, 0x91, 0x55, 0x1e, 0x08, 0xd3, 0x0f,
10547
+	0x18, 0xc1, 0x87, 0x26, 0x8e, 0x43, 0xea, 0x90, 0xeb, 0xd2, 0xee, 0x0e, 0xeb, 0xfe, 0xdc, 0xea,
10548
+	0xbb, 0x73, 0xd3, 0x6c, 0xd8, 0x42, 0xf5, 0x4e, 0xdc, 0xb2, 0x6d, 0x31, 0xea, 0x2b, 0x0f, 0x7d,
10549
+	0xbc, 0x19, 0xb6, 0x25, 0x55, 0xb8, 0xc8, 0x94, 0xc3, 0xe6, 0x71, 0x51, 0x0c, 0x72, 0x23, 0x76,
10550
+	0xed, 0x19, 0xc8, 0x94, 0x50, 0x46, 0x73, 0x9e, 0x2f, 0x45, 0xb0, 0x06, 0x33, 0x62, 0x7d, 0x2e,
10551
+	0xf1, 0x46, 0x35, 0xeb, 0xf2, 0x5b, 0x90, 0x0b, 0x85, 0x4e, 0xae, 0xc0, 0x54, 0x9b, 0x1e, 0x4a,
10552
+	0x59, 0x98, 0xfc, 0x91, 0xaf, 0xee, 0x41, 0xa3, 0xb3, 0x4f, 0xd9, 0x0a, 0xf2, 0x36, 0xf9, 0xf2,
10553
+	0x76, 0x7a, 0x2d, 0x65, 0xac, 0x43, 0x31, 0xb4, 0x1c, 0xa8, 0x91, 0x0a, 0x23, 0x83, 0x37, 0x08,
10554
+	0x32, 0x92, 0x44, 0x22, 0x61, 0xc6, 0x8f, 0x29, 0x28, 0x3e, 0xe9, 0xb7, 0x1a, 0x1e, 0x9d, 0x54,
10555
+	0xa1, 0xe4, 0x5d, 0xb8, 0x24, 0x40, 0x07, 0x6c, 0x91, 0x2c, 0xbb, 0x27, 0x02, 0xcc, 0x55, 0xaf,
10556
+	0xeb, 0x3c, 0x7e, 0x22, 0x21, 0x66, 0x8e, 0x1b, 0xe0, 0x0b, 0x79, 0x1d, 0xa6, 0xf9, 0x76, 0x63,
10557
+	0x74, 0x73, 0xbb, 0x1b, 0x49, 0xbc, 0x98, 0x02, 0x69, 0xd4, 0x80, 0x84, 0x63, 0x3d, 0xd1, 0xb6,
10558
+	0xd8, 0x84, 0xa2, 0x49, 0xbb, 0xf6, 0xc1, 0xe4, 0xf3, 0x65, 0x4c, 0xec, 0xda, 0x4e, 0x53, 0x32,
10559
+	0x31, 0x6b, 0xca, 0x17, 0xa3, 0x04, 0x24, 0x3c, 0x9e, 0x8c, 0x09, 0x37, 0xfd, 0xe3, 0x86, 0xdb,
10560
+	0x0e, 0xb9, 0xf0, 0xd8, 0x6b, 0xc4, 0x05, 0x47, 0x70, 0x17, 0xbc, 0xcb, 0xdf, 0xf4, 0xd2, 0x2c,
10561
+	0x98, 0x1d, 0xef, 0x4c, 0x9a, 0x9d, 0xc0, 0x0b, 0x94, 0xb1, 0xa6, 0x66, 0x37, 0xb1, 0x6b, 0x7f,
10562
+	0x1e, 0x61, 0xef, 0xc6, 0x3f, 0x98, 0x44, 0x78, 0xe3, 0x09, 0x92, 0x48, 0xd8, 0x6c, 0x38, 0x89,
10563
+	0xfc, 0x70, 0x8e, 0x49, 0x44, 0x17, 0x99, 0x36, 0x89, 0xb0, 0x10, 0x5c, 0xea, 0x1c, 0x58, 0x4d,
10564
+	0xae, 0x0e, 0x99, 0x44, 0x30, 0x84, 0x2d, 0xd9, 0x5c, 0xdf, 0x60, 0x21, 0x20, 0xa4, 0xde, 0x72,
10565
+	0xc9, 0x02, 0xcc, 0xa2, 0x96, 0x64, 0xb6, 0xc8, 0xd6, 0x72, 0x0c, 0x9d, 0x91, 0x62, 0x62, 0xb3,
10566
+	0x97, 0x6a, 0x72, 0xc9, 0x06, 0x14, 0xd8, 0x06, 0xb4, 0x1c, 0xda, 0xda, 0x76, 0x3d, 0xa6, 0x69,
10567
+	0x99, 0x1f, 0x0a, 0xd5, 0xff, 0xc7, 0x51, 0xbc, 0xc5, 0x51, 0x66, 0x1e, 0x8d, 0xc4, 0x9b, 0x26,
10568
+	0xc9, 0x64, 0xfe, 0x93, 0x24, 0x83, 0xcb, 0x15, 0x24, 0x19, 0xae, 0x9a, 0xc4, 0x24, 0x23, 0x64,
10569
+	0x24, 0x61, 0xc6, 0x47, 0x50, 0x5a, 0x77, 0x28, 0x8b, 0x17, 0x97, 0x4c, 0x09, 0x69, 0x15, 0x33,
10570
+	0x80, 0x54, 0xd1, 0xbc, 0x6e, 0x18, 0xb4, 0x08, 0x25, 0x81, 0x4d, 0xb8, 0x16, 0x19, 0x0c, 0xa3,
10571
+	0xba, 0x0b, 0x19, 0xa4, 0x01, 0x07, 0xbc, 0x9e, 0x30, 0xa0, 0xa9, 0xb0, 0xc6, 0xfb, 0x50, 0x64,
10572
+	0x7b, 0x2e, 0x12, 0xd9, 0x32, 0x40, 0xc0, 0x3a, 0xee, 0x9a, 0x3c, 0xa3, 0x31, 0xeb, 0x93, 0x6e,
10573
+	0x66, 0x7d, 0xce, 0xd9, 0xfc, 0x48, 0x78, 0x88, 0xd3, 0xc5, 0xf3, 0x73, 0x0a, 0x4a, 0x32, 0xcb,
10574
+	0x9d, 0x26, 0x26, 0x26, 0xaf, 0xcb, 0x0a, 0x3d, 0x41, 0x82, 0x2e, 0xa0, 0x8d, 0xca, 0xd1, 0xab,
10575
+	0x03, 0x39, 0x7a, 0x7c, 0x86, 0x22, 0x13, 0x38, 0xdd, 0x8a, 0x6c, 0x40, 0x49, 0xa6, 0xa6, 0x53,
10576
+	0x91, 0xf4, 0x3f, 0xb8, 0x16, 0x19, 0x05, 0x73, 0xdc, 0x9f, 0x69, 0xb8, 0xca, 0x35, 0x8e, 0xed,
10577
+	0x7e, 0x9a, 0xab, 0x47, 0xd3, 0xdc, 0x4a, 0x5c, 0x32, 0x89, 0x58, 0x0e, 0x67, 0xba, 0x6f, 0xd3,
10578
+	0x67, 0x9e, 0xe9, 0xb6, 0x22, 0x99, 0xee, 0x9d, 0x09, 0x83, 0xd3, 0x26, 0xbb, 0xa1, 0x6c, 0x32,
10579
+	0x7d, 0xb6, 0xd9, 0xe4, 0x63, 0x28, 0x0d, 0x86, 0x84, 0xc2, 0x78, 0x13, 0x66, 0x91, 0x28, 0x95,
10580
+	0x53, 0x12, 0x95, 0xe1, 0x83, 0x83, 0xcc, 0xb2, 0x49, 0xbd, 0xa7, 0xb6, 0xd3, 0x9e, 0x20, 0xb3,
10581
+	0xa0, 0x85, 0x2e, 0xb3, 0xf8, 0x83, 0x05, 0xba, 0xed, 0xc9, 0xa6, 0x24, 0xdd, 0x2a, 0x2b, 0x85,
10582
+	0x35, 0x9e, 0x88, 0xcc, 0x12, 0x89, 0x8c, 0xb0, 0x6a, 0x85, 0xad, 0x26, 0xae, 0x97, 0x78, 0xe6,
10583
+	0x42, 0x46, 0x1b, 0x2e, 0xe4, 0x74, 0x20, 0x64, 0xb4, 0xe5, 0x42, 0x46, 0x80, 0x9f, 0x6d, 0xce,
10584
+	0x28, 0xc6, 0xcf, 0xd4, 0xde, 0x3a, 0xf3, 0x30, 0xfd, 0xfd, 0x16, 0x89, 0xd4, 0xdf, 0x6f, 0xd8,
10585
+	0x7e, 0x82, 0xfd, 0x16, 0xb1, 0x7c, 0xb9, 0xf6, 0x5b, 0x4c, 0x70, 0xe7, 0xb9, 0xdf, 0x82, 0x90,
10586
+	0x82, 0xfd, 0x86, 0x44, 0x25, 0xee, 0x37, 0xc5, 0x9c, 0x0f, 0xc6, 0x8f, 0xe5, 0x7a, 0x67, 0xdf,
10587
+	0x65, 0x73, 0x0a, 0xe5, 0xe1, 0xa6, 0x6c, 0x89, 0xe4, 0x61, 0xc4, 0x71, 0x5d, 0x20, 0xc0, 0x97,
10588
+	0xaf, 0x3f, 0x44, 0x20, 0x5f, 0x84, 0x24, 0xc9, 0x57, 0x59, 0x29, 0xac, 0xaf, 0x25, 0xec, 0x38,
10589
+	0x81, 0x96, 0x22, 0x96, 0x2f, 0x97, 0x96, 0x62, 0x82, 0x3b, 0x4f, 0x2d, 0x05, 0x21, 0x05, 0x5a,
10590
+	0x42, 0x36, 0x12, 0xb5, 0xa4, 0xa8, 0xf3, 0xc1, 0xc6, 0x3e, 0x14, 0x3f, 0xb4, 0xad, 0xde, 0x63,
10591
+	0xbb, 0x4d, 0x7b, 0xa6, 0xcd, 0xca, 0x59, 0x5e, 0x70, 0x54, 0xe0, 0xaa, 0xc3, 0x9f, 0xe9, 0x36,
10592
+	0x17, 0x1c, 0x53, 0x94, 0xc7, 0xbb, 0x45, 0x84, 0xb3, 0x66, 0x51, 0x76, 0x7d, 0x2a, 0x7a, 0x84,
10593
+	0x1d, 0x3b, 0x44, 0x96, 0x10, 0xdf, 0x6d, 0xf4, 0x1a, 0x7b, 0xbe, 0x81, 0x3c, 0xa3, 0x11, 0xd9,
10594
+	0xf7, 0x50, 0x76, 0x09, 0x0b, 0xe3, 0xbb, 0xb4, 0xaa, 0xaf, 0x4e, 0x23, 0x63, 0x5e, 0x5f, 0x29,
10595
+	0xf4, 0x24, 0xf5, 0x15, 0xda, 0x4c, 0x50, 0x5f, 0xa1, 0xf7, 0xe0, 0x3b, 0x45, 0xee, 0xc3, 0xac,
10596
+	0x83, 0xeb, 0xc5, 0x48, 0xe6, 0x86, 0xb7, 0x75, 0x86, 0x43, 0x8b, 0x5b, 0x9b, 0x3e, 0x3a, 0x9e,
10597
+	0xbf, 0x60, 0xfa, 0xc6, 0x41, 0xa1, 0x76, 0x36, 0xbb, 0xb1, 0xfa, 0x5b, 0x11, 0x32, 0xeb, 0xf2,
10598
+	0x92, 0x8d, 0x58, 0x90, 0xc1, 0xfb, 0x2b, 0x62, 0xe8, 0x8c, 0x07, 0xef, 0xc4, 0xca, 0xb7, 0x12,
10599
+	0x31, 0xf8, 0xe5, 0xb8, 0xf6, 0xcb, 0x4f, 0x7f, 0x7d, 0x9f, 0xbe, 0x0c, 0x79, 0x01, 0x7a, 0x0d,
10600
+	0x19, 0x27, 0x36, 0x64, 0xfd, 0x8b, 0x10, 0xf2, 0xea, 0x38, 0xd7, 0x46, 0xe5, 0xdb, 0x23, 0x50,
10601
+	0xc9, 0x0e, 0x1d, 0x80, 0xe0, 0x1e, 0x82, 0x68, 0xc7, 0x1a, 0xba, 0x53, 0x29, 0x2f, 0x8c, 0x82,
10602
+	0x8d, 0xf4, 0x19, 0xdc, 0x33, 0xe8, 0x7d, 0x0e, 0xdd, 0x6b, 0xe8, 0x7d, 0x6a, 0xae, 0x2b, 0x62,
10603
+	0x7c, 0x4a, 0x0e, 0xf9, 0x49, 0x2e, 0x96, 0xc3, 0xd0, 0x3d, 0x43, 0x2c, 0x87, 0x03, 0x37, 0x0a,
10604
+	0xc9, 0x1c, 0x8a, 0x73, 0x66, 0x3c, 0x87, 0xe1, 0x53, 0x7b, 0x3c, 0x87, 0x03, 0x87, 0xd5, 0x91,
10605
+	0xeb, 0x29, 0xa6, 0x97, 0xb0, 0x9e, 0xe1, 0x19, 0x2e, 0x8c, 0x82, 0x8d, 0xf4, 0x19, 0x9c, 0x13,
10606
+	0xf5, 0x3e, 0x87, 0x8e, 0xa2, 0x7a, 0x9f, 0xc3, 0xc7, 0xcd, 0x38, 0x9f, 0xcf, 0xe0, 0x52, 0xb8,
10607
+	0xe4, 0x26, 0x77, 0xc6, 0x3c, 0x27, 0x94, 0x17, 0x47, 0x03, 0x93, 0x3d, 0x7f, 0x05, 0xf9, 0x81,
10608
+	0x83, 0x3a, 0xd1, 0x8e, 0xa8, 0xbb, 0x18, 0x28, 0x2f, 0x8d, 0x81, 0x1c, 0xe9, 0x7c, 0xe0, 0x0c,
10609
+	0xaa, 0x77, 0xae, 0x3b, 0x67, 0xeb, 0x9d, 0x6b, 0x0f, 0xb4, 0x09, 0xce, 0x07, 0x8e, 0x9a, 0x7a,
10610
+	0xe7, 0xba, 0x33, 0xad, 0xde, 0xb9, 0xfe, 0xdc, 0x9a, 0x28, 0x32, 0x2c, 0xdd, 0x62, 0x45, 0x36,
10611
+	0x58, 0xee, 0xc7, 0x8a, 0x2c, 0x5a, 0xbb, 0x27, 0x8b, 0x4c, 0xd5, 0x99, 0xf1, 0x22, 0x8b, 0x14,
10612
+	0xc7, 0xf1, 0x22, 0x8b, 0x96, 0xac, 0x23, 0x45, 0xa6, 0x26, 0x9c, 0x20, 0xb2, 0xc8, 0x9c, 0x97,
10613
+	0xc6, 0x40, 0x8e, 0xc9, 0x73, 0xa2, 0x73, 0xdd, 0xf9, 0x2a, 0x89, 0xe7, 0x31, 0x9d, 0x4b, 0x9e,
10614
+	0xf1, 0x1b, 0x1c, 0xcb, 0xf3, 0x60, 0x8d, 0x13, 0xcb, 0x73, 0xa4, 0x00, 0x18, 0xc1, 0xb3, 0xaa,
10615
+	0x01, 0xe3, 0x79, 0x8e, 0x14, 0xae, 0xf1, 0x3c, 0x47, 0xcb, 0xc9, 0x91, 0xfb, 0x59, 0x4d, 0x38,
10616
+	0x61, 0x3f, 0x47, 0xe6, 0xbc, 0x34, 0x06, 0x32, 0xd1, 0x79, 0xed, 0xc6, 0xd1, 0xf3, 0x9b, 0x17,
10617
+	0x7e, 0x67, 0xbf, 0xbf, 0x9f, 0xdf, 0x4c, 0x7d, 0xfd, 0xe2, 0x66, 0xea, 0x88, 0xfd, 0x7e, 0x65,
10618
+	0xbf, 0x3f, 0xd8, 0x6f, 0x67, 0x46, 0xfc, 0x9f, 0x6f, 0xf5, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff,
10619
+	0xbb, 0x97, 0x43, 0x36, 0x60, 0x1c, 0x00, 0x00,
10619 10620
 }
... ...
@@ -115,6 +115,7 @@ message UpdateNodeResponse {
115 115
 // RemoveNodeRequest requests to delete the specified node from store.
116 116
 message RemoveNodeRequest {
117 117
 	string node_id = 1 [(gogoproto.customname) = "NodeID"];
118
+	bool force = 2;
118 119
 }
119 120
 
120 121
 message RemoveNodeResponse {
... ...
@@ -309,15 +309,6 @@ func (s *Server) Run(ctx context.Context) error {
309 309
 	logger := log.G(ctx).WithField("module", "ca")
310 310
 	ctx = log.WithLogger(ctx, logger)
311 311
 
312
-	// Run() should never be called twice, but just in case, we're
313
-	// attempting to close the started channel in a safe way
314
-	select {
315
-	case <-s.started:
316
-		return fmt.Errorf("CA server cannot be started more than once")
317
-	default:
318
-		close(s.started)
319
-	}
320
-
321 312
 	// Retrieve the channels to keep track of changes in the cluster
322 313
 	// Retrieve all the currently registered nodes
323 314
 	var nodes []*api.Node
... ...
@@ -346,6 +337,7 @@ func (s *Server) Run(ctx context.Context) error {
346 346
 	s.mu.Lock()
347 347
 	s.ctx, s.cancel = context.WithCancel(ctx)
348 348
 	s.mu.Unlock()
349
+	close(s.started)
349 350
 
350 351
 	if err != nil {
351 352
 		log.G(ctx).WithFields(logrus.Fields{
... ...
@@ -377,7 +377,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, nc *networkContext, ev even
377 377
 
378 378
 		node.Attachment.Network = nc.ingressNetwork.Copy()
379 379
 		if err := a.allocateNode(ctx, nc, node); err != nil {
380
-			log.G(ctx).Errorf("Fauled to allocate network resources for node %s: %v", node.ID, err)
380
+			log.G(ctx).Errorf("Failed to allocate network resources for node %s: %v", node.ID, err)
381 381
 		}
382 382
 	}
383 383
 }
... ...
@@ -283,7 +283,7 @@ func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest)
283 283
 				return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID)
284 284
 			}
285 285
 		}
286
-		if node.Status.State == api.NodeStatus_READY {
286
+		if !request.Force && node.Status.State == api.NodeStatus_READY {
287 287
 			return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
288 288
 		}
289 289
 		return store.DeleteNode(tx, request.NodeID)
... ...
@@ -20,6 +20,7 @@ import (
20 20
 	"github.com/docker/swarmkit/manager/state"
21 21
 	"github.com/docker/swarmkit/manager/state/store"
22 22
 	"github.com/docker/swarmkit/manager/state/watch"
23
+	"github.com/docker/swarmkit/picker"
23 24
 	"github.com/docker/swarmkit/protobuf/ptypes"
24 25
 	"golang.org/x/net/context"
25 26
 )
... ...
@@ -60,8 +61,6 @@ var (
60 60
 // Config is configuration for Dispatcher. For default you should use
61 61
 // DefautConfig.
62 62
 type Config struct {
63
-	// Addr configures the address the dispatcher reports to agents.
64
-	Addr             string
65 63
 	HeartbeatPeriod  time.Duration
66 64
 	HeartbeatEpsilon time.Duration
67 65
 	// RateLimitPeriod specifies how often node with same ID can try to register
... ...
@@ -90,7 +89,6 @@ type Cluster interface {
90 90
 // Dispatcher is responsible for dispatching tasks and tracking agent health.
91 91
 type Dispatcher struct {
92 92
 	mu                   sync.Mutex
93
-	addr                 string
94 93
 	nodes                *nodeStore
95 94
 	store                *store.MemoryStore
96 95
 	mgrQueue             *watch.Queue
... ...
@@ -121,7 +119,6 @@ func (b weightedPeerByNodeID) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
121 121
 // NOTE: each handler which does something with raft must add to Dispatcher.wg
122 122
 func New(cluster Cluster, c *Config) *Dispatcher {
123 123
 	return &Dispatcher{
124
-		addr:                      c.Addr,
125 124
 		nodes:                     newNodeStore(c.HeartbeatPeriod, c.HeartbeatEpsilon, c.GracePeriodMultiplier, c.RateLimitPeriod),
126 125
 		store:                     cluster.MemoryStore(),
127 126
 		cluster:                   cluster,
... ...
@@ -142,7 +139,11 @@ func getWeightedPeers(cluster Cluster) []*api.WeightedPeer {
142 142
 				NodeID: m.NodeID,
143 143
 				Addr:   m.Addr,
144 144
 			},
145
-			Weight: 1,
145
+
146
+			// TODO(stevvooe): Calculate weight of manager selection based on
147
+			// cluster-level observations, such as number of connections and
148
+			// load.
149
+			Weight: picker.DefaultObservationWeight,
146 150
 		})
147 151
 	}
148 152
 	return mgrs
... ...
@@ -574,14 +575,18 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
574 574
 		}
575 575
 
576 576
 		// bursty events should be processed in batches and sent out snapshot
577
-		const modificationBatchLimit = 200
578
-		const eventPausedGap = 50 * time.Millisecond
579
-		var modificationCnt int
580
-		// eventPaused is true when there have been modifications
581
-		// but next event has not arrived within eventPausedGap
582
-		eventPaused := false
583
-
584
-		for modificationCnt < modificationBatchLimit && !eventPaused {
577
+		const (
578
+			modificationBatchLimit = 200
579
+			eventPausedGap         = 50 * time.Millisecond
580
+		)
581
+		var (
582
+			modificationCnt    int
583
+			eventPausedTimer   *time.Timer
584
+			eventPausedTimeout <-chan time.Time
585
+		)
586
+
587
+	batchingLoop:
588
+		for modificationCnt < modificationBatchLimit {
585 589
 			select {
586 590
 			case event := <-nodeTasks:
587 591
 				switch v := event.(type) {
... ...
@@ -602,16 +607,24 @@ func (d *Dispatcher) Tasks(r *api.TasksRequest, stream api.Dispatcher_TasksServe
602 602
 					delete(tasksMap, v.Task.ID)
603 603
 					modificationCnt++
604 604
 				}
605
-			case <-time.After(eventPausedGap):
606
-				if modificationCnt > 0 {
607
-					eventPaused = true
605
+				if eventPausedTimer != nil {
606
+					eventPausedTimer.Reset(eventPausedGap)
607
+				} else {
608
+					eventPausedTimer = time.NewTimer(eventPausedGap)
609
+					eventPausedTimeout = eventPausedTimer.C
608 610
 				}
611
+			case <-eventPausedTimeout:
612
+				break batchingLoop
609 613
 			case <-stream.Context().Done():
610 614
 				return stream.Context().Err()
611 615
 			case <-d.ctx.Done():
612 616
 				return d.ctx.Err()
613 617
 			}
614 618
 		}
619
+
620
+		if eventPausedTimer != nil {
621
+			eventPausedTimer.Stop()
622
+		}
615 623
 	}
616 624
 }
617 625
 
... ...
@@ -43,26 +43,29 @@ func (rn *registeredNode) checkSessionID(sessionID string) error {
43 43
 }
44 44
 
45 45
 type nodeStore struct {
46
-	periodChooser         *periodChooser
47
-	gracePeriodMultiplier time.Duration
48
-	rateLimitPeriod       time.Duration
49
-	nodes                 map[string]*registeredNode
50
-	mu                    sync.RWMutex
46
+	periodChooser                *periodChooser
47
+	gracePeriodMultiplierNormal  time.Duration
48
+	gracePeriodMultiplierUnknown time.Duration
49
+	rateLimitPeriod              time.Duration
50
+	nodes                        map[string]*registeredNode
51
+	mu                           sync.RWMutex
51 52
 }
52 53
 
53 54
 func newNodeStore(hbPeriod, hbEpsilon time.Duration, graceMultiplier int, rateLimitPeriod time.Duration) *nodeStore {
54 55
 	return &nodeStore{
55
-		nodes:                 make(map[string]*registeredNode),
56
-		periodChooser:         newPeriodChooser(hbPeriod, hbEpsilon),
57
-		gracePeriodMultiplier: time.Duration(graceMultiplier),
58
-		rateLimitPeriod:       rateLimitPeriod,
56
+		nodes:                        make(map[string]*registeredNode),
57
+		periodChooser:                newPeriodChooser(hbPeriod, hbEpsilon),
58
+		gracePeriodMultiplierNormal:  time.Duration(graceMultiplier),
59
+		gracePeriodMultiplierUnknown: time.Duration(graceMultiplier) * 2,
60
+		rateLimitPeriod:              rateLimitPeriod,
59 61
 	}
60 62
 }
61 63
 
62 64
 func (s *nodeStore) updatePeriod(hbPeriod, hbEpsilon time.Duration, gracePeriodMultiplier int) {
63 65
 	s.mu.Lock()
64 66
 	s.periodChooser = newPeriodChooser(hbPeriod, hbEpsilon)
65
-	s.gracePeriodMultiplier = time.Duration(gracePeriodMultiplier)
67
+	s.gracePeriodMultiplierNormal = time.Duration(gracePeriodMultiplier)
68
+	s.gracePeriodMultiplierUnknown = s.gracePeriodMultiplierNormal * 2
66 69
 	s.mu.Unlock()
67 70
 }
68 71
 
... ...
@@ -79,7 +82,7 @@ func (s *nodeStore) AddUnknown(n *api.Node, expireFunc func()) error {
79 79
 		Node: n,
80 80
 	}
81 81
 	s.nodes[n.ID] = rn
82
-	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc)
82
+	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierUnknown, expireFunc)
83 83
 	return nil
84 84
 }
85 85
 
... ...
@@ -124,7 +127,7 @@ func (s *nodeStore) Add(n *api.Node, expireFunc func()) *registeredNode {
124 124
 		Disconnect: make(chan struct{}),
125 125
 	}
126 126
 	s.nodes[n.ID] = rn
127
-	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplier, expireFunc)
127
+	rn.Heartbeat = heartbeat.New(s.periodChooser.Choose()*s.gracePeriodMultiplierNormal, expireFunc)
128 128
 	return rn
129 129
 }
130 130
 
... ...
@@ -154,7 +157,7 @@ func (s *nodeStore) Heartbeat(id, sid string) (time.Duration, error) {
154 154
 		return 0, err
155 155
 	}
156 156
 	period := s.periodChooser.Choose() // base period for node
157
-	grace := period * time.Duration(s.gracePeriodMultiplier)
157
+	grace := period * time.Duration(s.gracePeriodMultiplierNormal)
158 158
 	rn.mu.Lock()
159 159
 	rn.Heartbeat.Update(grace)
160 160
 	rn.Heartbeat.Beat()
... ...
@@ -89,9 +89,11 @@ type Manager struct {
89 89
 	server                 *grpc.Server
90 90
 	localserver            *grpc.Server
91 91
 	RaftNode               *raft.Node
92
+	connSelector           *raftpicker.ConnSelector
92 93
 
93 94
 	mu sync.Mutex
94 95
 
96
+	started chan struct{}
95 97
 	stopped chan struct{}
96 98
 }
97 99
 
... ...
@@ -139,9 +141,6 @@ func New(config *Config) (*Manager, error) {
139 139
 		tcpAddr = net.JoinHostPort("0.0.0.0", tcpAddrPort)
140 140
 	}
141 141
 
142
-	// FIXME(aaronl): Remove this. It appears to be unused.
143
-	dispatcherConfig.Addr = tcpAddr
144
-
145 142
 	err := os.MkdirAll(filepath.Dir(config.ProtoAddr["unix"]), 0700)
146 143
 	if err != nil {
147 144
 		return nil, fmt.Errorf("failed to create socket directory: %v", err)
... ...
@@ -220,6 +219,7 @@ func New(config *Config) (*Manager, error) {
220 220
 		server:      grpc.NewServer(opts...),
221 221
 		localserver: grpc.NewServer(opts...),
222 222
 		RaftNode:    RaftNode,
223
+		started:     make(chan struct{}),
223 224
 		stopped:     make(chan struct{}),
224 225
 	}
225 226
 
... ...
@@ -428,11 +428,12 @@ func (m *Manager) Run(parent context.Context) error {
428 428
 	}()
429 429
 
430 430
 	proxyOpts := []grpc.DialOption{
431
-		grpc.WithBackoffMaxDelay(2 * time.Second),
431
+		grpc.WithBackoffMaxDelay(time.Second),
432 432
 		grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds),
433 433
 	}
434 434
 
435 435
 	cs := raftpicker.NewConnSelector(m.RaftNode, proxyOpts...)
436
+	m.connSelector = cs
436 437
 
437 438
 	authorize := func(ctx context.Context, roles []string) error {
438 439
 		// Authorize the remote roles, ensure they can only be forwarded by managers
... ...
@@ -506,6 +507,8 @@ func (m *Manager) Run(parent context.Context) error {
506 506
 		return fmt.Errorf("can't initialize raft node: %v", err)
507 507
 	}
508 508
 
509
+	close(m.started)
510
+
509 511
 	go func() {
510 512
 		err := m.RaftNode.Run(ctx)
511 513
 		if err != nil {
... ...
@@ -560,12 +563,15 @@ func (m *Manager) Run(parent context.Context) error {
560 560
 func (m *Manager) Stop(ctx context.Context) {
561 561
 	log.G(ctx).Info("Stopping manager")
562 562
 
563
+	// It's not safe to start shutting down while the manager is still
564
+	// starting up.
565
+	<-m.started
566
+
563 567
 	// the mutex stops us from trying to stop while we're alrady stopping, or
564 568
 	// from returning before we've finished stopping.
565 569
 	m.mu.Lock()
566 570
 	defer m.mu.Unlock()
567 571
 	select {
568
-
569 572
 	// check to see that we've already stopped
570 573
 	case <-m.stopped:
571 574
 		return
... ...
@@ -600,6 +606,9 @@ func (m *Manager) Stop(ctx context.Context) {
600 600
 		m.keyManager.Stop()
601 601
 	}
602 602
 
603
+	if m.connSelector != nil {
604
+		m.connSelector.Stop()
605
+	}
603 606
 	m.RaftNode.Shutdown()
604 607
 	// some time after this point, Run will receive an error from one of these
605 608
 	m.server.Stop()
... ...
@@ -2,6 +2,7 @@ package raftpicker
2 2
 
3 3
 import (
4 4
 	"sync"
5
+	"time"
5 6
 
6 7
 	"golang.org/x/net/context"
7 8
 	"google.golang.org/grpc"
... ...
@@ -14,46 +15,37 @@ type picker struct {
14 14
 	addr string
15 15
 	raft AddrSelector
16 16
 	conn *grpc.Conn
17
-	cc   *grpc.ClientConn
17
+
18
+	stop chan struct{}
19
+	done chan struct{}
18 20
 }
19 21
 
20
-// Init does initial processing for the Picker, e.g., initiate some connections.
21
-func (p *picker) Init(cc *grpc.ClientConn) error {
22
-	p.cc = cc
23
-	return nil
22
+func newPicker(raft AddrSelector, addr string) *picker {
23
+	return &picker{
24
+		raft: raft,
25
+		addr: addr,
26
+
27
+		stop: make(chan struct{}),
28
+		done: make(chan struct{}),
29
+	}
24 30
 }
25 31
 
26
-func (p *picker) initConn() error {
27
-	if p.conn == nil {
28
-		conn, err := grpc.NewConn(p.cc)
29
-		if err != nil {
30
-			return err
31
-		}
32
-		p.conn = conn
32
+// Init does initial processing for the Picker, e.g., initiate some connections.
33
+func (p *picker) Init(cc *grpc.ClientConn) error {
34
+	conn, err := grpc.NewConn(cc)
35
+	if err != nil {
36
+		return err
33 37
 	}
38
+	p.conn = conn
34 39
 	return nil
35 40
 }
36 41
 
37 42
 // Pick blocks until either a transport.ClientTransport is ready for the upcoming RPC
38 43
 // or some error happens.
39 44
 func (p *picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
40
-	p.mu.Lock()
41
-	if err := p.initConn(); err != nil {
42
-		p.mu.Unlock()
45
+	if err := p.updateConn(); err != nil {
43 46
 		return nil, err
44 47
 	}
45
-	p.mu.Unlock()
46
-
47
-	addr, err := p.raft.LeaderAddr()
48
-	if err != nil {
49
-		return nil, err
50
-	}
51
-	p.mu.Lock()
52
-	if p.addr != addr {
53
-		p.addr = addr
54
-		p.conn.NotifyReset()
55
-	}
56
-	p.mu.Unlock()
57 48
 	return p.conn.Wait(ctx)
58 49
 }
59 50
 
... ...
@@ -89,15 +81,46 @@ func (p *picker) Reset() error {
89 89
 
90 90
 // Close closes all the Conn's owned by this Picker.
91 91
 func (p *picker) Close() error {
92
+	close(p.stop)
93
+	<-p.done
92 94
 	return p.conn.Close()
93 95
 }
94 96
 
97
+func (p *picker) updateConn() error {
98
+	addr, err := p.raft.LeaderAddr()
99
+	if err != nil {
100
+		return err
101
+	}
102
+	p.mu.Lock()
103
+	if p.addr != addr {
104
+		p.addr = addr
105
+		p.Reset()
106
+	}
107
+	p.mu.Unlock()
108
+	return nil
109
+}
110
+
111
+func (p *picker) updateLoop() {
112
+	defer close(p.done)
113
+	ticker := time.NewTicker(1 * time.Second)
114
+	defer ticker.Stop()
115
+	for {
116
+		select {
117
+		case <-ticker.C:
118
+			p.updateConn()
119
+		case <-p.stop:
120
+			return
121
+		}
122
+	}
123
+}
124
+
95 125
 // ConnSelector is struct for obtaining connection with raftpicker.
96 126
 type ConnSelector struct {
97 127
 	mu      sync.Mutex
98 128
 	cc      *grpc.ClientConn
99 129
 	cluster RaftCluster
100 130
 	opts    []grpc.DialOption
131
+	picker  *picker
101 132
 }
102 133
 
103 134
 // NewConnSelector returns new ConnSelector with cluster and grpc.DialOpts which
... ...
@@ -122,8 +145,9 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
122 122
 	if err != nil {
123 123
 		return nil, err
124 124
 	}
125
-	picker := &picker{raft: c.cluster, addr: addr}
126
-	opts := append(c.opts, grpc.WithPicker(picker))
125
+	c.picker = newPicker(c.cluster, addr)
126
+	go c.picker.updateLoop()
127
+	opts := append(c.opts, grpc.WithPicker(c.picker))
127 128
 	cc, err := grpc.Dial(addr, opts...)
128 129
 	if err != nil {
129 130
 		return nil, err
... ...
@@ -131,3 +155,13 @@ func (c *ConnSelector) Conn() (*grpc.ClientConn, error) {
131 131
 	c.cc = cc
132 132
 	return c.cc, nil
133 133
 }
134
+
135
+// Stop cancels tracking loop for raftpicker and closes it.
136
+func (c *ConnSelector) Stop() {
137
+	c.mu.Lock()
138
+	defer c.mu.Unlock()
139
+	if c.picker == nil {
140
+		return
141
+	}
142
+	c.picker.Close()
143
+}
... ...
@@ -451,6 +451,17 @@ func (n *Node) Shutdown() {
451 451
 	}
452 452
 }
453 453
 
454
+// isShutdown indicates if node was shut down.
455
+// This method should be called under n.stopMu to avoid races with n.stop().
456
+func (n *Node) isShutdown() bool {
457
+	select {
458
+	case <-n.Ctx.Done():
459
+		return true
460
+	default:
461
+		return false
462
+	}
463
+}
464
+
454 465
 func (n *Node) stop() {
455 466
 	n.stopMu.Lock()
456 467
 	defer n.stopMu.Unlock()
... ...
@@ -763,7 +774,10 @@ func (n *Node) ResolveAddress(ctx context.Context, msg *api.ResolveAddressReques
763 763
 func (n *Node) LeaderAddr() (string, error) {
764 764
 	n.stopMu.RLock()
765 765
 	defer n.stopMu.RUnlock()
766
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
766
+	if n.isShutdown() {
767
+		return "", fmt.Errorf("raft node is shut down")
768
+	}
769
+	ctx, cancel := context.WithTimeout(n.Ctx, 10*time.Second)
767 770
 	defer cancel()
768 771
 	if err := WaitForLeader(ctx, n); err != nil {
769 772
 		return "", ErrNoClusterLeader
... ...
@@ -1288,7 +1302,7 @@ func (n *Node) ConnectToMember(addr string, timeout time.Duration) (*membership.
1288 1288
 
1289 1289
 // SubscribeLeadership returns channel to which events about leadership change
1290 1290
 // will be sent in form of raft.LeadershipState. Also cancel func is returned -
1291
-// it should be called when listener is not longer interested in events.
1291
+// it should be called when listener is no longer interested in events.
1292 1292
 func (n *Node) SubscribeLeadership() (q chan events.Event, cancel func()) {
1293 1293
 	ch := events.NewChannel(0)
1294 1294
 	sink := events.Sink(events.NewQueue(ch))
... ...
@@ -15,6 +15,10 @@ import (
15 15
 
16 16
 var errRemotesUnavailable = fmt.Errorf("no remote hosts provided")
17 17
 
18
+// DefaultObservationWeight provides a weight to use for positive observations
19
+// that will balance well under repeated observations.
20
+const DefaultObservationWeight = 10
21
+
18 22
 // Remotes keeps track of remote addresses by weight, informed by
19 23
 // observations.
20 24
 type Remotes interface {
... ...
@@ -49,7 +53,7 @@ func NewRemotes(peers ...api.Peer) Remotes {
49 49
 	}
50 50
 
51 51
 	for _, peer := range peers {
52
-		mwr.Observe(peer, 1)
52
+		mwr.Observe(peer, DefaultObservationWeight)
53 53
 	}
54 54
 
55 55
 	return mwr
... ...
@@ -96,7 +100,7 @@ func (mwr *remotesWeightedRandom) Select(excludes ...string) (api.Peer, error) {
96 96
 
97 97
 	// bias to zero-weighted remotes have same probability. otherwise, we
98 98
 	// always select first entry when all are zero.
99
-	const bias = 0.1
99
+	const bias = 0.001
100 100
 
101 101
 	// clear out workspace
102 102
 	mwr.cdf = mwr.cdf[:0]
... ...
@@ -165,7 +169,7 @@ const (
165 165
 	// See
166 166
 	// https://en.wikipedia.org/wiki/Exponential_smoothing#Basic_exponential_smoothing
167 167
 	// for details.
168
-	remoteWeightSmoothingFactor = 0.7
168
+	remoteWeightSmoothingFactor = 0.5
169 169
 	remoteWeightMax             = 1 << 8
170 170
 )
171 171
 
... ...
@@ -228,7 +232,7 @@ func (p *Picker) Init(cc *grpc.ClientConn) error {
228 228
 	peer := p.peer
229 229
 	p.mu.Unlock()
230 230
 
231
-	p.r.ObserveIfExists(peer, 1)
231
+	p.r.ObserveIfExists(peer, DefaultObservationWeight)
232 232
 	c, err := grpc.NewConn(cc)
233 233
 	if err != nil {
234 234
 		return err
... ...
@@ -248,7 +252,7 @@ func (p *Picker) Pick(ctx context.Context) (transport.ClientTransport, error) {
248 248
 	p.mu.Unlock()
249 249
 	transport, err := p.conn.Wait(ctx)
250 250
 	if err != nil {
251
-		p.r.ObserveIfExists(peer, -1)
251
+		p.r.ObserveIfExists(peer, -DefaultObservationWeight)
252 252
 	}
253 253
 
254 254
 	return transport, err
... ...
@@ -261,7 +265,7 @@ func (p *Picker) PickAddr() (string, error) {
261 261
 	peer := p.peer
262 262
 	p.mu.Unlock()
263 263
 
264
-	p.r.ObserveIfExists(peer, -1) // downweight the current addr
264
+	p.r.ObserveIfExists(peer, -DefaultObservationWeight) // downweight the current addr
265 265
 
266 266
 	var err error
267 267
 	peer, err = p.r.Select()
... ...
@@ -299,15 +303,15 @@ func (p *Picker) WaitForStateChange(ctx context.Context, sourceState grpc.Connec
299 299
 	// TODO(stevvooe): This is questionable, but we'll see how it works.
300 300
 	switch state {
301 301
 	case grpc.Idle:
302
-		p.r.ObserveIfExists(peer, 1)
302
+		p.r.ObserveIfExists(peer, DefaultObservationWeight)
303 303
 	case grpc.Connecting:
304
-		p.r.ObserveIfExists(peer, 1)
304
+		p.r.ObserveIfExists(peer, DefaultObservationWeight)
305 305
 	case grpc.Ready:
306
-		p.r.ObserveIfExists(peer, 1)
306
+		p.r.ObserveIfExists(peer, DefaultObservationWeight)
307 307
 	case grpc.TransientFailure:
308
-		p.r.ObserveIfExists(peer, -1)
308
+		p.r.ObserveIfExists(peer, -DefaultObservationWeight)
309 309
 	case grpc.Shutdown:
310
-		p.r.ObserveIfExists(peer, -1)
310
+		p.r.ObserveIfExists(peer, -DefaultObservationWeight)
311 311
 	}
312 312
 
313 313
 	return state, err