Browse code

Merge pull request #23312 from justincormack/proxy

Make the docker proxy a standalone binary not a re-exec

Tibor Vass authored on 2016/07/08 09:14:40
Showing 49 changed files
... ...
@@ -136,13 +136,13 @@ func SwarmSpecUpdateAcceptancePolicy(spec *swarmapi.ClusterSpec, acceptancePolic
136 136
 					hashPwd, _ = bcrypt.GenerateFromPassword([]byte(*p.Secret), 0)
137 137
 					hashs[*p.Secret] = hashPwd
138 138
 				}
139
-				policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{
139
+				policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_Secret{
140 140
 					Data: hashPwd,
141 141
 					Alg:  "bcrypt",
142 142
 				}
143 143
 			}
144 144
 		} else if oldSecret := getOldSecret(oldSpec, policy.Role); oldSecret != nil { // else use the old one.
145
-			policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{
145
+			policy.Secret = &swarmapi.AcceptancePolicy_RoleAdmissionPolicy_Secret{
146 146
 				Data: oldSecret.Data,
147 147
 				Alg:  oldSecret.Alg,
148 148
 			}
... ...
@@ -153,7 +153,7 @@ func SwarmSpecUpdateAcceptancePolicy(spec *swarmapi.ClusterSpec, acceptancePolic
153 153
 	return nil
154 154
 }
155 155
 
156
-func getOldSecret(oldSpec *swarmapi.ClusterSpec, role swarmapi.NodeRole) *swarmapi.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret {
156
+func getOldSecret(oldSpec *swarmapi.ClusterSpec, role swarmapi.NodeRole) *swarmapi.AcceptancePolicy_RoleAdmissionPolicy_Secret {
157 157
 	if oldSpec == nil {
158 158
 		return nil
159 159
 	}
... ...
@@ -131,6 +131,9 @@ clean() {
131 131
 		findArgs+=( -path "vendor/src/$import" )
132 132
 	done
133 133
 
134
+	# The docker proxy command is built from libnetwork
135
+	findArgs+=( -or -path vendor/src/github.com/docker/libnetwork/cmd/proxy )
136
+
134 137
 	local IFS=$'\n'
135 138
 	local prune=( $($find vendor -depth -type d -not '(' "${findArgs[@]}" ')') )
136 139
 	unset IFS
... ...
@@ -2,3 +2,4 @@
2 2
 
3 3
 DOCKER_CLIENT_BINARY_NAME='docker'
4 4
 DOCKER_DAEMON_BINARY_NAME='dockerd'
5
+DOCKER_PROXY_BINARY_NAME='docker-proxy'
... ...
@@ -22,6 +22,7 @@ override_dh_auto_install:
22 22
 	mkdir -p debian/docker-engine/usr/bin
23 23
 	cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary-client/docker)" debian/docker-engine/usr/bin/docker
24 24
 	cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary-daemon/dockerd)" debian/docker-engine/usr/bin/dockerd
25
+	cp -aT "$$(readlink -f bundles/$(VERSION)/dynbinary-daemon/docker-proxy)" debian/docker-engine/usr/bin/docker-proxy
25 26
 	cp -aT /usr/local/bin/containerd debian/docker-engine/usr/bin/docker-containerd
26 27
 	cp -aT /usr/local/bin/containerd-shim debian/docker-engine/usr/bin/docker-containerd-shim
27 28
 	cp -aT /usr/local/bin/ctr debian/docker-engine/usr/bin/docker-containerd-ctr
... ...
@@ -126,6 +126,7 @@ export DOCKER_GITCOMMIT=%{_gitcommit}
126 126
 install -d $RPM_BUILD_ROOT/%{_bindir}
127 127
 install -p -m 755 bundles/%{_origversion}/dynbinary-client/docker-%{_origversion} $RPM_BUILD_ROOT/%{_bindir}/docker
128 128
 install -p -m 755 bundles/%{_origversion}/dynbinary-daemon/dockerd-%{_origversion} $RPM_BUILD_ROOT/%{_bindir}/dockerd
129
+install -p -m 755 bundles/%{_origversion}/dynbinary-daemon/docker-proxy-%{_origversion} $RPM_BUILD_ROOT/%{_bindir}/docker-proxy
129 130
 
130 131
 # install containerd
131 132
 install -p -m 755 /usr/local/bin/containerd $RPM_BUILD_ROOT/%{_bindir}/docker-containerd
... ...
@@ -9,5 +9,8 @@ set -e
9 9
 	export BINARY_SHORT_NAME="$DOCKER_DAEMON_BINARY_NAME"
10 10
 	export SOURCE_PATH='./cmd/dockerd'
11 11
 	source "${MAKEDIR}/.binary"
12
+	export BINARY_SHORT_NAME="$DOCKER_PROXY_BINARY_NAME"
13
+	export SOURCE_PATH='./vendor/src/github.com/docker/libnetwork/cmd/proxy'
14
+	source "${MAKEDIR}/.binary"
12 15
 	copy_containerd "$DEST" 'hash'
13 16
 )
... ...
@@ -9,4 +9,7 @@ set -e
9 9
 	export BUILDFLAGS=( "${BUILDFLAGS[@]/netgo /}" ) # disable netgo, since we don't need it for a dynamic binary
10 10
 	export BUILDFLAGS=( "${BUILDFLAGS[@]/static_build /}" ) # we're not building a "static" binary here
11 11
 	source "${MAKEDIR}/.binary"
12
+	export BINARY_SHORT_NAME='docker-proxy'
13
+	export SOURCE_PATH='./vendor/src/github.com/docker/libnetwork/cmd/proxy'
14
+	source "${MAKEDIR}/.binary"
12 15
 )
... ...
@@ -5,6 +5,9 @@ BINARY_NAME="dockerd-$VERSION"
5 5
 BINARY_EXTENSION="$(binary_extension)"
6 6
 BINARY_FULLNAME="$BINARY_NAME$BINARY_EXTENSION"
7 7
 
8
+PROXY_NAME="docker-proxy-$VERSION"
9
+PROXY_FULLNAME="$PROXY_NAME$BINARY_EXTENSION"
10
+
8 11
 CLIENTBIN_NAME="docker-$VERSION"
9 12
 CLIENTBIN_FULLNAME="$CLIENTBIN_NAME$BINARY_EXTENSION"
10 13
 
... ...
@@ -29,6 +32,21 @@ go build -compiler=gccgo \
29 29
 echo "Created binary: $DEST/$BINARY_FULLNAME"
30 30
 ln -sf "$BINARY_FULLNAME" "$DEST/dockerd$BINARY_EXTENSION"
31 31
 
32
+go build -compiler=gccgo \
33
+	-o "$DEST/$PROXY_FULLNAME" \
34
+	"${BUILDFLAGS[@]}" \
35
+	-gccgoflags "
36
+		-g
37
+		$EXTLDFLAGS_STATIC
38
+		-Wl,--no-export-dynamic
39
+		-ldl
40
+		-pthread
41
+	" \
42
+	./vendor/src/github.com/docker/libnetwork/cmd/proxy
43
+
44
+echo "Created binary: $DEST/$PROXY_FULLNAME"
45
+ln -sf "$PROXY_FULLNAME" "$DEST/docker-proxy$BINARY_EXTENSION"
46
+
32 47
 copy_containerd "$DEST" "hash"
33 48
 hash_files "$DEST/$BINARY_FULLNAME"
34 49
 
... ...
@@ -7,4 +7,5 @@ rm -rf "$DEST"
7 7
 	DEST="$(dirname $DEST)/binary-daemon"
8 8
 	source "${MAKEDIR}/.binary-setup"
9 9
 	install_binary "${DEST}/${DOCKER_DAEMON_BINARY_NAME}"
10
+	install_binary "${DEST}/${DOCKER_PROXY_BINARY_NAME}"
10 11
 )
... ...
@@ -18,6 +18,7 @@ for d in "$CROSS/"*/*; do
18 18
 
19 19
 	BINARY_NAME="${DOCKER_CLIENT_BINARY_NAME}-$VERSION"
20 20
 	DAEMON_BINARY_NAME="${DOCKER_DAEMON_BINARY_NAME}-$VERSION"
21
+	PROXY_BINARY_NAME="${DOCKER_PROXY_BINARY_NAME}-$VERSION"
21 22
 	BINARY_EXTENSION="$(export GOOS && binary_extension)"
22 23
 	if [ "$GOOS" = 'windows' ]; then
23 24
 		# if windows use a zip, not tgz
... ...
@@ -29,6 +30,7 @@ for d in "$CROSS/"*/*; do
29 29
 	fi
30 30
 	BINARY_FULLNAME="$BINARY_NAME$BINARY_EXTENSION"
31 31
 	DAEMON_BINARY_FULLNAME="$DAEMON_BINARY_NAME$BINARY_EXTENSION"
32
+	PROXY_BINARY_FULLNAME="$PROXY_BINARY_NAME$BINARY_EXTENSION"
32 33
 	mkdir -p "$DEST/$GOOS/$GOARCH"
33 34
 	TGZ="$DEST/$GOOS/$GOARCH/$BINARY_NAME$BUNDLE_EXTENSION"
34 35
 
... ...
@@ -47,6 +49,9 @@ for d in "$CROSS/"*/*; do
47 47
 	if [ -f "$d/$DAEMON_BINARY_FULLNAME" ]; then
48 48
 		cp -L "$d/$DAEMON_BINARY_FULLNAME" "$TAR_PATH/${DOCKER_DAEMON_BINARY_NAME}${BINARY_EXTENSION}"
49 49
 	fi
50
+	if [ -f "$d/$PROXY_BINARY_FULLNAME" ]; then
51
+		cp -L "$d/$PROXY_BINARY_FULLNAME" "$TAR_PATH/${DOCKER_PROXY_BINARY_NAME}${BINARY_EXTENSION}"
52
+	fi
50 53
 
51 54
 	# copy over all the containerd binaries
52 55
 	copy_containerd $TAR_PATH
... ...
@@ -65,7 +65,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
65 65
 clone git github.com/imdario/mergo 0.2.1
66 66
 
67 67
 #get libnetwork packages
68
-clone git github.com/docker/libnetwork 377a7337f2387cce3be1df7a4503446147b68ff1
68
+clone git github.com/docker/libnetwork 6eece7dcc21dcd34d907f3e91dd71cb8640b661c
69 69
 clone git github.com/docker/go-events 39718a26497694185f8fb58a7d6f31947f3dc42d
70 70
 clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
71 71
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
... ...
@@ -139,7 +139,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0
139 139
 clone git github.com/docker/containerd 1b3a81545ca79456086dc2aa424357be98b962ee
140 140
 
141 141
 # cluster
142
-clone git github.com/docker/swarmkit 24eaf0021a2eea7938fce7493ce4731f53c2b87c
142
+clone git github.com/docker/swarmkit 16fa595d3b6fec012830179dc8e9b2d90335527d
143 143
 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
144 144
 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028
145 145
 clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b
... ...
@@ -277,8 +277,6 @@ func (s *DockerSwarmSuite) TestApiSwarmPromoteDemote(c *check.C) {
277 277
 		}
278 278
 		if i > 100 {
279 279
 			c.Errorf("node did not turn into manager")
280
-		} else {
281
-			break
282 280
 		}
283 281
 		time.Sleep(100 * time.Millisecond)
284 282
 	}
... ...
@@ -296,8 +294,6 @@ func (s *DockerSwarmSuite) TestApiSwarmPromoteDemote(c *check.C) {
296 296
 		}
297 297
 		if i > 100 {
298 298
 			c.Errorf("node did not turn into worker")
299
-		} else {
300
-			break
301 299
 		}
302 300
 		time.Sleep(100 * time.Millisecond)
303 301
 	}
304 302
deleted file mode 100644
... ...
@@ -1,216 +0,0 @@
1
-package proxy
2
-
3
-import (
4
-	"bytes"
5
-	"fmt"
6
-	"io"
7
-	"net"
8
-	"strings"
9
-	"testing"
10
-	"time"
11
-)
12
-
13
-var testBuf = []byte("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo")
14
-var testBufSize = len(testBuf)
15
-
16
-type EchoServer interface {
17
-	Run()
18
-	Close()
19
-	LocalAddr() net.Addr
20
-}
21
-
22
-type TCPEchoServer struct {
23
-	listener net.Listener
24
-	testCtx  *testing.T
25
-}
26
-
27
-type UDPEchoServer struct {
28
-	conn    net.PacketConn
29
-	testCtx *testing.T
30
-}
31
-
32
-func NewEchoServer(t *testing.T, proto, address string) EchoServer {
33
-	var server EchoServer
34
-	if strings.HasPrefix(proto, "tcp") {
35
-		listener, err := net.Listen(proto, address)
36
-		if err != nil {
37
-			t.Fatal(err)
38
-		}
39
-		server = &TCPEchoServer{listener: listener, testCtx: t}
40
-	} else {
41
-		socket, err := net.ListenPacket(proto, address)
42
-		if err != nil {
43
-			t.Fatal(err)
44
-		}
45
-		server = &UDPEchoServer{conn: socket, testCtx: t}
46
-	}
47
-	return server
48
-}
49
-
50
-func (server *TCPEchoServer) Run() {
51
-	go func() {
52
-		for {
53
-			client, err := server.listener.Accept()
54
-			if err != nil {
55
-				return
56
-			}
57
-			go func(client net.Conn) {
58
-				if _, err := io.Copy(client, client); err != nil {
59
-					server.testCtx.Logf("can't echo to the client: %v\n", err.Error())
60
-				}
61
-				client.Close()
62
-			}(client)
63
-		}
64
-	}()
65
-}
66
-
67
-func (server *TCPEchoServer) LocalAddr() net.Addr { return server.listener.Addr() }
68
-func (server *TCPEchoServer) Close()              { server.listener.Close() }
69
-
70
-func (server *UDPEchoServer) Run() {
71
-	go func() {
72
-		readBuf := make([]byte, 1024)
73
-		for {
74
-			read, from, err := server.conn.ReadFrom(readBuf)
75
-			if err != nil {
76
-				return
77
-			}
78
-			for i := 0; i != read; {
79
-				written, err := server.conn.WriteTo(readBuf[i:read], from)
80
-				if err != nil {
81
-					break
82
-				}
83
-				i += written
84
-			}
85
-		}
86
-	}()
87
-}
88
-
89
-func (server *UDPEchoServer) LocalAddr() net.Addr { return server.conn.LocalAddr() }
90
-func (server *UDPEchoServer) Close()              { server.conn.Close() }
91
-
92
-func testProxyAt(t *testing.T, proto string, proxy Proxy, addr string) {
93
-	defer proxy.Close()
94
-	go proxy.Run()
95
-	client, err := net.Dial(proto, addr)
96
-	if err != nil {
97
-		t.Fatalf("Can't connect to the proxy: %v", err)
98
-	}
99
-	defer client.Close()
100
-	client.SetDeadline(time.Now().Add(10 * time.Second))
101
-	if _, err = client.Write(testBuf); err != nil {
102
-		t.Fatal(err)
103
-	}
104
-	recvBuf := make([]byte, testBufSize)
105
-	if _, err = client.Read(recvBuf); err != nil {
106
-		t.Fatal(err)
107
-	}
108
-	if !bytes.Equal(testBuf, recvBuf) {
109
-		t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf))
110
-	}
111
-}
112
-
113
-func testProxy(t *testing.T, proto string, proxy Proxy) {
114
-	testProxyAt(t, proto, proxy, proxy.FrontendAddr().String())
115
-}
116
-
117
-func TestTCP4Proxy(t *testing.T) {
118
-	backend := NewEchoServer(t, "tcp", "127.0.0.1:0")
119
-	defer backend.Close()
120
-	backend.Run()
121
-	frontendAddr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
122
-	proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
123
-	if err != nil {
124
-		t.Fatal(err)
125
-	}
126
-	testProxy(t, "tcp", proxy)
127
-}
128
-
129
-func TestTCP6Proxy(t *testing.T) {
130
-	backend := NewEchoServer(t, "tcp", "[::1]:0")
131
-	defer backend.Close()
132
-	backend.Run()
133
-	frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0}
134
-	proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
135
-	if err != nil {
136
-		t.Fatal(err)
137
-	}
138
-	testProxy(t, "tcp", proxy)
139
-}
140
-
141
-func TestTCPDualStackProxy(t *testing.T) {
142
-	// If I understand `godoc -src net favoriteAddrFamily` (used by the
143
-	// net.Listen* functions) correctly this should work, but it doesn't.
144
-	t.Skip("No support for dual stack yet")
145
-	backend := NewEchoServer(t, "tcp", "[::1]:0")
146
-	defer backend.Close()
147
-	backend.Run()
148
-	frontendAddr := &net.TCPAddr{IP: net.IPv6loopback, Port: 0}
149
-	proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
150
-	if err != nil {
151
-		t.Fatal(err)
152
-	}
153
-	ipv4ProxyAddr := &net.TCPAddr{
154
-		IP:   net.IPv4(127, 0, 0, 1),
155
-		Port: proxy.FrontendAddr().(*net.TCPAddr).Port,
156
-	}
157
-	testProxyAt(t, "tcp", proxy, ipv4ProxyAddr.String())
158
-}
159
-
160
-func TestUDP4Proxy(t *testing.T) {
161
-	backend := NewEchoServer(t, "udp", "127.0.0.1:0")
162
-	defer backend.Close()
163
-	backend.Run()
164
-	frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
165
-	proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
166
-	if err != nil {
167
-		t.Fatal(err)
168
-	}
169
-	testProxy(t, "udp", proxy)
170
-}
171
-
172
-func TestUDP6Proxy(t *testing.T) {
173
-	backend := NewEchoServer(t, "udp", "[::1]:0")
174
-	defer backend.Close()
175
-	backend.Run()
176
-	frontendAddr := &net.UDPAddr{IP: net.IPv6loopback, Port: 0}
177
-	proxy, err := NewProxy(frontendAddr, backend.LocalAddr())
178
-	if err != nil {
179
-		t.Fatal(err)
180
-	}
181
-	testProxy(t, "udp", proxy)
182
-}
183
-
184
-func TestUDPWriteError(t *testing.T) {
185
-	frontendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0}
186
-	// Hopefully, this port will be free: */
187
-	backendAddr := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 25587}
188
-	proxy, err := NewProxy(frontendAddr, backendAddr)
189
-	if err != nil {
190
-		t.Fatal(err)
191
-	}
192
-	defer proxy.Close()
193
-	go proxy.Run()
194
-	client, err := net.Dial("udp", "127.0.0.1:25587")
195
-	if err != nil {
196
-		t.Fatalf("Can't connect to the proxy: %v", err)
197
-	}
198
-	defer client.Close()
199
-	// Make sure the proxy doesn't stop when there is no actual backend:
200
-	client.Write(testBuf)
201
-	client.Write(testBuf)
202
-	backend := NewEchoServer(t, "udp", "127.0.0.1:25587")
203
-	defer backend.Close()
204
-	backend.Run()
205
-	client.SetDeadline(time.Now().Add(10 * time.Second))
206
-	if _, err = client.Write(testBuf); err != nil {
207
-		t.Fatal(err)
208
-	}
209
-	recvBuf := make([]byte, testBufSize)
210
-	if _, err = client.Read(recvBuf); err != nil {
211
-		t.Fatal(err)
212
-	}
213
-	if !bytes.Equal(testBuf, recvBuf) {
214
-		t.Fatal(fmt.Errorf("Expected [%v] but got [%v]", testBuf, recvBuf))
215
-	}
216
-}
217 1
deleted file mode 100644
... ...
@@ -1,37 +0,0 @@
1
-// Package proxy provides a network Proxy interface and implementations for TCP
2
-// and UDP.
3
-package proxy
4
-
5
-import (
6
-	"fmt"
7
-	"net"
8
-)
9
-
10
-// Proxy defines the behavior of a proxy. It forwards traffic back and forth
11
-// between two endpoints : the frontend and the backend.
12
-// It can be used to do software port-mapping between two addresses.
13
-// e.g. forward all traffic between the frontend (host) 127.0.0.1:3000
14
-// to the backend (container) at 172.17.42.108:4000.
15
-type Proxy interface {
16
-	// Run starts forwarding traffic back and forth between the front
17
-	// and back-end addresses.
18
-	Run()
19
-	// Close stops forwarding traffic and close both ends of the Proxy.
20
-	Close()
21
-	// FrontendAddr returns the address on which the proxy is listening.
22
-	FrontendAddr() net.Addr
23
-	// BackendAddr returns the proxied address.
24
-	BackendAddr() net.Addr
25
-}
26
-
27
-// NewProxy creates a Proxy according to the specified frontendAddr and backendAddr.
28
-func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
29
-	switch frontendAddr.(type) {
30
-	case *net.UDPAddr:
31
-		return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr))
32
-	case *net.TCPAddr:
33
-		return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr))
34
-	default:
35
-		panic(fmt.Errorf("Unsupported protocol"))
36
-	}
37
-}
38 1
deleted file mode 100644
... ...
@@ -1,31 +0,0 @@
1
-package proxy
2
-
3
-import (
4
-	"net"
5
-)
6
-
7
-// StubProxy is a proxy that is a stub (does nothing).
8
-type StubProxy struct {
9
-	frontendAddr net.Addr
10
-	backendAddr  net.Addr
11
-}
12
-
13
-// Run does nothing.
14
-func (p *StubProxy) Run() {}
15
-
16
-// Close does nothing.
17
-func (p *StubProxy) Close() {}
18
-
19
-// FrontendAddr returns the frontend address.
20
-func (p *StubProxy) FrontendAddr() net.Addr { return p.frontendAddr }
21
-
22
-// BackendAddr returns the backend address.
23
-func (p *StubProxy) BackendAddr() net.Addr { return p.backendAddr }
24
-
25
-// NewStubProxy creates a new StubProxy
26
-func NewStubProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
27
-	return &StubProxy{
28
-		frontendAddr: frontendAddr,
29
-		backendAddr:  backendAddr,
30
-	}, nil
31
-}
32 1
deleted file mode 100644
... ...
@@ -1,96 +0,0 @@
1
-package proxy
2
-
3
-import (
4
-	"io"
5
-	"net"
6
-	"sync"
7
-	"syscall"
8
-
9
-	"github.com/Sirupsen/logrus"
10
-)
11
-
12
-// TCPProxy is a proxy for TCP connections. It implements the Proxy interface to
13
-// handle TCP traffic forwarding between the frontend and backend addresses.
14
-type TCPProxy struct {
15
-	listener     *net.TCPListener
16
-	frontendAddr *net.TCPAddr
17
-	backendAddr  *net.TCPAddr
18
-}
19
-
20
-// NewTCPProxy creates a new TCPProxy.
21
-func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
22
-	listener, err := net.ListenTCP("tcp", frontendAddr)
23
-	if err != nil {
24
-		return nil, err
25
-	}
26
-	// If the port in frontendAddr was 0 then ListenTCP will have a picked
27
-	// a port to listen on, hence the call to Addr to get that actual port:
28
-	return &TCPProxy{
29
-		listener:     listener,
30
-		frontendAddr: listener.Addr().(*net.TCPAddr),
31
-		backendAddr:  backendAddr,
32
-	}, nil
33
-}
34
-
35
-func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
36
-	backend, err := net.DialTCP("tcp", nil, proxy.backendAddr)
37
-	if err != nil {
38
-		logrus.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err)
39
-		client.Close()
40
-		return
41
-	}
42
-
43
-	var wg sync.WaitGroup
44
-	var broker = func(to, from *net.TCPConn) {
45
-		if _, err := io.Copy(to, from); err != nil {
46
-			// If the socket we are writing to is shutdown with
47
-			// SHUT_WR, forward it to the other end of the pipe:
48
-			if err, ok := err.(*net.OpError); ok && err.Err == syscall.EPIPE {
49
-				from.CloseWrite()
50
-			}
51
-		}
52
-		to.CloseRead()
53
-		wg.Done()
54
-	}
55
-
56
-	wg.Add(2)
57
-	go broker(client, backend)
58
-	go broker(backend, client)
59
-
60
-	finish := make(chan struct{})
61
-	go func() {
62
-		wg.Wait()
63
-		close(finish)
64
-	}()
65
-
66
-	select {
67
-	case <-quit:
68
-	case <-finish:
69
-	}
70
-	client.Close()
71
-	backend.Close()
72
-	<-finish
73
-}
74
-
75
-// Run starts forwarding the traffic using TCP.
76
-func (proxy *TCPProxy) Run() {
77
-	quit := make(chan bool)
78
-	defer close(quit)
79
-	for {
80
-		client, err := proxy.listener.Accept()
81
-		if err != nil {
82
-			logrus.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
83
-			return
84
-		}
85
-		go proxy.clientLoop(client.(*net.TCPConn), quit)
86
-	}
87
-}
88
-
89
-// Close stops forwarding the traffic.
90
-func (proxy *TCPProxy) Close() { proxy.listener.Close() }
91
-
92
-// FrontendAddr returns the TCP address on which the proxy is listening.
93
-func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
94
-
95
-// BackendAddr returns the TCP proxied address.
96
-func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
97 1
deleted file mode 100644
... ...
@@ -1,169 +0,0 @@
1
-package proxy
2
-
3
-import (
4
-	"encoding/binary"
5
-	"net"
6
-	"strings"
7
-	"sync"
8
-	"syscall"
9
-	"time"
10
-
11
-	"github.com/Sirupsen/logrus"
12
-)
13
-
14
-const (
15
-	// UDPConnTrackTimeout is the timeout used for UDP connection tracking
16
-	UDPConnTrackTimeout = 90 * time.Second
17
-	// UDPBufSize is the buffer size for the UDP proxy
18
-	UDPBufSize = 65507
19
-)
20
-
21
-// A net.Addr where the IP is split into two fields so you can use it as a key
22
-// in a map:
23
-type connTrackKey struct {
24
-	IPHigh uint64
25
-	IPLow  uint64
26
-	Port   int
27
-}
28
-
29
-func newConnTrackKey(addr *net.UDPAddr) *connTrackKey {
30
-	if len(addr.IP) == net.IPv4len {
31
-		return &connTrackKey{
32
-			IPHigh: 0,
33
-			IPLow:  uint64(binary.BigEndian.Uint32(addr.IP)),
34
-			Port:   addr.Port,
35
-		}
36
-	}
37
-	return &connTrackKey{
38
-		IPHigh: binary.BigEndian.Uint64(addr.IP[:8]),
39
-		IPLow:  binary.BigEndian.Uint64(addr.IP[8:]),
40
-		Port:   addr.Port,
41
-	}
42
-}
43
-
44
-type connTrackMap map[connTrackKey]*net.UDPConn
45
-
46
-// UDPProxy is proxy for which handles UDP datagrams. It implements the Proxy
47
-// interface to handle UDP traffic forwarding between the frontend and backend
48
-// addresses.
49
-type UDPProxy struct {
50
-	listener       *net.UDPConn
51
-	frontendAddr   *net.UDPAddr
52
-	backendAddr    *net.UDPAddr
53
-	connTrackTable connTrackMap
54
-	connTrackLock  sync.Mutex
55
-}
56
-
57
-// NewUDPProxy creates a new UDPProxy.
58
-func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) {
59
-	listener, err := net.ListenUDP("udp", frontendAddr)
60
-	if err != nil {
61
-		return nil, err
62
-	}
63
-	return &UDPProxy{
64
-		listener:       listener,
65
-		frontendAddr:   listener.LocalAddr().(*net.UDPAddr),
66
-		backendAddr:    backendAddr,
67
-		connTrackTable: make(connTrackMap),
68
-	}, nil
69
-}
70
-
71
-func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) {
72
-	defer func() {
73
-		proxy.connTrackLock.Lock()
74
-		delete(proxy.connTrackTable, *clientKey)
75
-		proxy.connTrackLock.Unlock()
76
-		proxyConn.Close()
77
-	}()
78
-
79
-	readBuf := make([]byte, UDPBufSize)
80
-	for {
81
-		proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout))
82
-	again:
83
-		read, err := proxyConn.Read(readBuf)
84
-		if err != nil {
85
-			if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED {
86
-				// This will happen if the last write failed
87
-				// (e.g: nothing is actually listening on the
88
-				// proxied port on the container), ignore it
89
-				// and continue until UDPConnTrackTimeout
90
-				// expires:
91
-				goto again
92
-			}
93
-			return
94
-		}
95
-		for i := 0; i != read; {
96
-			written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr)
97
-			if err != nil {
98
-				return
99
-			}
100
-			i += written
101
-		}
102
-	}
103
-}
104
-
105
-// Run starts forwarding the traffic using UDP.
106
-func (proxy *UDPProxy) Run() {
107
-	readBuf := make([]byte, UDPBufSize)
108
-	for {
109
-		read, from, err := proxy.listener.ReadFromUDP(readBuf)
110
-		if err != nil {
111
-			// NOTE: Apparently ReadFrom doesn't return
112
-			// ECONNREFUSED like Read do (see comment in
113
-			// UDPProxy.replyLoop)
114
-			if !isClosedError(err) {
115
-				logrus.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
116
-			}
117
-			break
118
-		}
119
-
120
-		fromKey := newConnTrackKey(from)
121
-		proxy.connTrackLock.Lock()
122
-		proxyConn, hit := proxy.connTrackTable[*fromKey]
123
-		if !hit {
124
-			proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr)
125
-			if err != nil {
126
-				logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
127
-				proxy.connTrackLock.Unlock()
128
-				continue
129
-			}
130
-			proxy.connTrackTable[*fromKey] = proxyConn
131
-			go proxy.replyLoop(proxyConn, from, fromKey)
132
-		}
133
-		proxy.connTrackLock.Unlock()
134
-		for i := 0; i != read; {
135
-			written, err := proxyConn.Write(readBuf[i:read])
136
-			if err != nil {
137
-				logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
138
-				break
139
-			}
140
-			i += written
141
-		}
142
-	}
143
-}
144
-
145
-// Close stops forwarding the traffic.
146
-func (proxy *UDPProxy) Close() {
147
-	proxy.listener.Close()
148
-	proxy.connTrackLock.Lock()
149
-	defer proxy.connTrackLock.Unlock()
150
-	for _, conn := range proxy.connTrackTable {
151
-		conn.Close()
152
-	}
153
-}
154
-
155
-// FrontendAddr returns the UDP address on which the proxy is listening.
156
-func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
157
-
158
-// BackendAddr returns the proxied UDP address.
159
-func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
160
-
161
-func isClosedError(err error) bool {
162
-	/* This comparison is ugly, but unfortunately, net.go doesn't export errClosing.
163
-	 * See:
164
-	 * http://golang.org/src/pkg/net/net.go
165
-	 * https://code.google.com/p/go/issues/detail?id=4337
166
-	 * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ
167
-	 */
168
-	return strings.HasSuffix(err.Error(), "use of closed network connection")
169
-}
... ...
@@ -7,6 +7,7 @@ docker = docker run --rm -it ${dockerargs} $$EXTRA_ARGS ${container_env} ${build
7 7
 ciargs = -e CIRCLECI -e "COVERALLS_TOKEN=$$COVERALLS_TOKEN" -e "INSIDECONTAINER=-incontainer=true"
8 8
 cidocker = docker run ${dockerargs} ${ciargs} $$EXTRA_ARGS ${container_env} ${build_image}
9 9
 CROSS_PLATFORMS = linux/amd64 linux/386 linux/arm windows/amd64
10
+export PATH := $(CURDIR)/bin:$(PATH)
10 11
 
11 12
 all: ${build_image}.created build check integration-tests clean
12 13
 
... ...
@@ -24,10 +25,11 @@ build: ${build_image}.created
24 24
 build-local:
25 25
 	@mkdir -p "bin"
26 26
 	$(shell which godep) go build -tags experimental -o "bin/dnet" ./cmd/dnet
27
+	$(shell which godep) go build -o "bin/docker-proxy" ./cmd/proxy
27 28
 
28 29
 clean:
29 30
 	@if [ -d bin ]; then \
30
-		echo "Removing dnet binaries"; \
31
+		echo "Removing dnet and proxy binaries"; \
31 32
 		rm -rf bin; \
32 33
 	fi
33 34
 
... ...
@@ -41,6 +43,7 @@ cross: ${build_image}.created
41 41
 
42 42
 cross-local:
43 43
 	$(shell which godep) go build -o "bin/dnet-$$GOOS-$$GOARCH" ./cmd/dnet
44
+	$(shell which godep) go build -o "bin/docker-proxy-$$GOOS-$$GOARCH" ./cmd/proxy
44 45
 
45 46
 check: ${build_image}.created
46 47
 	@${docker} ./wrapmake.sh check-local
... ...
@@ -102,4 +105,4 @@ circle-ci-check: ${build_image}.created
102 102
 circle-ci-build: ${build_image}.created
103 103
 	@${cidocker} make build-local
104 104
 
105
-circle-ci: circle-ci-check circle-ci-cross circle-ci-build integration-tests
105
+circle-ci: circle-ci-build circle-ci-check circle-ci-cross integration-tests
106 106
new file mode 100644
... ...
@@ -0,0 +1,67 @@
0
+package main
1
+
2
+import (
3
+	"flag"
4
+	"fmt"
5
+	"log"
6
+	"net"
7
+	"os"
8
+	"os/signal"
9
+	"syscall"
10
+)
11
+
12
+func main() {
13
+	f := os.NewFile(3, "signal-parent")
14
+	host, container := parseHostContainerAddrs()
15
+
16
+	p, err := NewProxy(host, container)
17
+	if err != nil {
18
+		fmt.Fprintf(f, "1\n%s", err)
19
+		f.Close()
20
+		os.Exit(1)
21
+	}
22
+	go handleStopSignals(p)
23
+	fmt.Fprint(f, "0\n")
24
+	f.Close()
25
+
26
+	// Run will block until the proxy stops
27
+	p.Run()
28
+}
29
+
30
+// parseHostContainerAddrs parses the flags passed on reexec to create the TCP or UDP
31
+// net.Addrs to map the host and container ports
32
+func parseHostContainerAddrs() (host net.Addr, container net.Addr) {
33
+	var (
34
+		proto         = flag.String("proto", "tcp", "proxy protocol")
35
+		hostIP        = flag.String("host-ip", "", "host ip")
36
+		hostPort      = flag.Int("host-port", -1, "host port")
37
+		containerIP   = flag.String("container-ip", "", "container ip")
38
+		containerPort = flag.Int("container-port", -1, "container port")
39
+	)
40
+
41
+	flag.Parse()
42
+
43
+	switch *proto {
44
+	case "tcp":
45
+		host = &net.TCPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort}
46
+		container = &net.TCPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort}
47
+	case "udp":
48
+		host = &net.UDPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort}
49
+		container = &net.UDPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort}
50
+	default:
51
+		log.Fatalf("unsupported protocol %s", *proto)
52
+	}
53
+
54
+	return host, container
55
+}
56
+
57
+func handleStopSignals(p Proxy) {
58
+	s := make(chan os.Signal, 10)
59
+	signal.Notify(s, os.Interrupt, syscall.SIGTERM)
60
+
61
+	for range s {
62
+		p.Close()
63
+
64
+		os.Exit(0)
65
+	}
66
+}
0 67
new file mode 100644
... ...
@@ -0,0 +1,37 @@
0
+// docker-proxy provides a network Proxy interface and implementations for TCP
1
+// and UDP.
2
+package main
3
+
4
+import (
5
+	"fmt"
6
+	"net"
7
+)
8
+
9
+// Proxy defines the behavior of a proxy. It forwards traffic back and forth
10
+// between two endpoints : the frontend and the backend.
11
+// It can be used to do software port-mapping between two addresses.
12
+// e.g. forward all traffic between the frontend (host) 127.0.0.1:3000
13
+// to the backend (container) at 172.17.42.108:4000.
14
+type Proxy interface {
15
+	// Run starts forwarding traffic back and forth between the front
16
+	// and back-end addresses.
17
+	Run()
18
+	// Close stops forwarding traffic and close both ends of the Proxy.
19
+	Close()
20
+	// FrontendAddr returns the address on which the proxy is listening.
21
+	FrontendAddr() net.Addr
22
+	// BackendAddr returns the proxied address.
23
+	BackendAddr() net.Addr
24
+}
25
+
26
+// NewProxy creates a Proxy according to the specified frontendAddr and backendAddr.
27
+func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
28
+	switch frontendAddr.(type) {
29
+	case *net.UDPAddr:
30
+		return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr))
31
+	case *net.TCPAddr:
32
+		return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr))
33
+	default:
34
+		panic(fmt.Errorf("Unsupported protocol"))
35
+	}
36
+}
0 37
new file mode 100644
... ...
@@ -0,0 +1,31 @@
0
+package main
1
+
2
+import (
3
+	"net"
4
+)
5
+
6
+// StubProxy is a proxy that is a stub (does nothing).
7
+type StubProxy struct {
8
+	frontendAddr net.Addr
9
+	backendAddr  net.Addr
10
+}
11
+
12
+// Run does nothing.
13
+func (p *StubProxy) Run() {}
14
+
15
+// Close does nothing.
16
+func (p *StubProxy) Close() {}
17
+
18
+// FrontendAddr returns the frontend address.
19
+func (p *StubProxy) FrontendAddr() net.Addr { return p.frontendAddr }
20
+
21
+// BackendAddr returns the backend address.
22
+func (p *StubProxy) BackendAddr() net.Addr { return p.backendAddr }
23
+
24
+// NewStubProxy creates a new StubProxy
25
+func NewStubProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
26
+	return &StubProxy{
27
+		frontendAddr: frontendAddr,
28
+		backendAddr:  backendAddr,
29
+	}, nil
30
+}
0 31
new file mode 100644
... ...
@@ -0,0 +1,96 @@
0
+package main
1
+
2
+import (
3
+	"io"
4
+	"net"
5
+	"sync"
6
+	"syscall"
7
+
8
+	"github.com/Sirupsen/logrus"
9
+)
10
+
11
+// TCPProxy is a proxy for TCP connections. It implements the Proxy interface to
12
+// handle TCP traffic forwarding between the frontend and backend addresses.
13
+type TCPProxy struct {
14
+	listener     *net.TCPListener
15
+	frontendAddr *net.TCPAddr
16
+	backendAddr  *net.TCPAddr
17
+}
18
+
19
+// NewTCPProxy creates a new TCPProxy.
20
+func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
21
+	listener, err := net.ListenTCP("tcp", frontendAddr)
22
+	if err != nil {
23
+		return nil, err
24
+	}
25
+	// If the port in frontendAddr was 0 then ListenTCP will have a picked
26
+	// a port to listen on, hence the call to Addr to get that actual port:
27
+	return &TCPProxy{
28
+		listener:     listener,
29
+		frontendAddr: listener.Addr().(*net.TCPAddr),
30
+		backendAddr:  backendAddr,
31
+	}, nil
32
+}
33
+
34
+func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
35
+	backend, err := net.DialTCP("tcp", nil, proxy.backendAddr)
36
+	if err != nil {
37
+		logrus.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err)
38
+		client.Close()
39
+		return
40
+	}
41
+
42
+	var wg sync.WaitGroup
43
+	var broker = func(to, from *net.TCPConn) {
44
+		if _, err := io.Copy(to, from); err != nil {
45
+			// If the socket we are writing to is shutdown with
46
+			// SHUT_WR, forward it to the other end of the pipe:
47
+			if err, ok := err.(*net.OpError); ok && err.Err == syscall.EPIPE {
48
+				from.CloseWrite()
49
+			}
50
+		}
51
+		to.CloseRead()
52
+		wg.Done()
53
+	}
54
+
55
+	wg.Add(2)
56
+	go broker(client, backend)
57
+	go broker(backend, client)
58
+
59
+	finish := make(chan struct{})
60
+	go func() {
61
+		wg.Wait()
62
+		close(finish)
63
+	}()
64
+
65
+	select {
66
+	case <-quit:
67
+	case <-finish:
68
+	}
69
+	client.Close()
70
+	backend.Close()
71
+	<-finish
72
+}
73
+
74
+// Run starts forwarding the traffic using TCP.
75
+func (proxy *TCPProxy) Run() {
76
+	quit := make(chan bool)
77
+	defer close(quit)
78
+	for {
79
+		client, err := proxy.listener.Accept()
80
+		if err != nil {
81
+			logrus.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
82
+			return
83
+		}
84
+		go proxy.clientLoop(client.(*net.TCPConn), quit)
85
+	}
86
+}
87
+
88
+// Close stops forwarding the traffic.
89
+func (proxy *TCPProxy) Close() { proxy.listener.Close() }
90
+
91
+// FrontendAddr returns the TCP address on which the proxy is listening.
92
+func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
93
+
94
+// BackendAddr returns the TCP proxied address.
95
+func (proxy *TCPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
0 96
new file mode 100644
... ...
@@ -0,0 +1,169 @@
0
+package main
1
+
2
+import (
3
+	"encoding/binary"
4
+	"net"
5
+	"strings"
6
+	"sync"
7
+	"syscall"
8
+	"time"
9
+
10
+	"github.com/Sirupsen/logrus"
11
+)
12
+
13
+const (
14
+	// UDPConnTrackTimeout is the timeout used for UDP connection tracking
15
+	UDPConnTrackTimeout = 90 * time.Second
16
+	// UDPBufSize is the buffer size for the UDP proxy
17
+	UDPBufSize = 65507
18
+)
19
+
20
+// A net.Addr where the IP is split into two fields so you can use it as a key
21
+// in a map:
22
+type connTrackKey struct {
23
+	IPHigh uint64
24
+	IPLow  uint64
25
+	Port   int
26
+}
27
+
28
+func newConnTrackKey(addr *net.UDPAddr) *connTrackKey {
29
+	if len(addr.IP) == net.IPv4len {
30
+		return &connTrackKey{
31
+			IPHigh: 0,
32
+			IPLow:  uint64(binary.BigEndian.Uint32(addr.IP)),
33
+			Port:   addr.Port,
34
+		}
35
+	}
36
+	return &connTrackKey{
37
+		IPHigh: binary.BigEndian.Uint64(addr.IP[:8]),
38
+		IPLow:  binary.BigEndian.Uint64(addr.IP[8:]),
39
+		Port:   addr.Port,
40
+	}
41
+}
42
+
43
+type connTrackMap map[connTrackKey]*net.UDPConn
44
+
45
+// UDPProxy is proxy for which handles UDP datagrams. It implements the Proxy
46
+// interface to handle UDP traffic forwarding between the frontend and backend
47
+// addresses.
48
+type UDPProxy struct {
49
+	listener       *net.UDPConn
50
+	frontendAddr   *net.UDPAddr
51
+	backendAddr    *net.UDPAddr
52
+	connTrackTable connTrackMap
53
+	connTrackLock  sync.Mutex
54
+}
55
+
56
+// NewUDPProxy creates a new UDPProxy.
57
+func NewUDPProxy(frontendAddr, backendAddr *net.UDPAddr) (*UDPProxy, error) {
58
+	listener, err := net.ListenUDP("udp", frontendAddr)
59
+	if err != nil {
60
+		return nil, err
61
+	}
62
+	return &UDPProxy{
63
+		listener:       listener,
64
+		frontendAddr:   listener.LocalAddr().(*net.UDPAddr),
65
+		backendAddr:    backendAddr,
66
+		connTrackTable: make(connTrackMap),
67
+	}, nil
68
+}
69
+
70
+func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr, clientKey *connTrackKey) {
71
+	defer func() {
72
+		proxy.connTrackLock.Lock()
73
+		delete(proxy.connTrackTable, *clientKey)
74
+		proxy.connTrackLock.Unlock()
75
+		proxyConn.Close()
76
+	}()
77
+
78
+	readBuf := make([]byte, UDPBufSize)
79
+	for {
80
+		proxyConn.SetReadDeadline(time.Now().Add(UDPConnTrackTimeout))
81
+	again:
82
+		read, err := proxyConn.Read(readBuf)
83
+		if err != nil {
84
+			if err, ok := err.(*net.OpError); ok && err.Err == syscall.ECONNREFUSED {
85
+				// This will happen if the last write failed
86
+				// (e.g: nothing is actually listening on the
87
+				// proxied port on the container), ignore it
88
+				// and continue until UDPConnTrackTimeout
89
+				// expires:
90
+				goto again
91
+			}
92
+			return
93
+		}
94
+		for i := 0; i != read; {
95
+			written, err := proxy.listener.WriteToUDP(readBuf[i:read], clientAddr)
96
+			if err != nil {
97
+				return
98
+			}
99
+			i += written
100
+		}
101
+	}
102
+}
103
+
104
+// Run starts forwarding the traffic using UDP.
105
+func (proxy *UDPProxy) Run() {
106
+	readBuf := make([]byte, UDPBufSize)
107
+	for {
108
+		read, from, err := proxy.listener.ReadFromUDP(readBuf)
109
+		if err != nil {
110
+			// NOTE: Apparently ReadFrom doesn't return
111
+			// ECONNREFUSED like Read do (see comment in
112
+			// UDPProxy.replyLoop)
113
+			if !isClosedError(err) {
114
+				logrus.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err)
115
+			}
116
+			break
117
+		}
118
+
119
+		fromKey := newConnTrackKey(from)
120
+		proxy.connTrackLock.Lock()
121
+		proxyConn, hit := proxy.connTrackTable[*fromKey]
122
+		if !hit {
123
+			proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr)
124
+			if err != nil {
125
+				logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
126
+				proxy.connTrackLock.Unlock()
127
+				continue
128
+			}
129
+			proxy.connTrackTable[*fromKey] = proxyConn
130
+			go proxy.replyLoop(proxyConn, from, fromKey)
131
+		}
132
+		proxy.connTrackLock.Unlock()
133
+		for i := 0; i != read; {
134
+			written, err := proxyConn.Write(readBuf[i:read])
135
+			if err != nil {
136
+				logrus.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err)
137
+				break
138
+			}
139
+			i += written
140
+		}
141
+	}
142
+}
143
+
144
+// Close stops forwarding the traffic.
145
+func (proxy *UDPProxy) Close() {
146
+	proxy.listener.Close()
147
+	proxy.connTrackLock.Lock()
148
+	defer proxy.connTrackLock.Unlock()
149
+	for _, conn := range proxy.connTrackTable {
150
+		conn.Close()
151
+	}
152
+}
153
+
154
+// FrontendAddr returns the UDP address on which the proxy is listening.
155
+func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
156
+
157
+// BackendAddr returns the proxied UDP address.
158
+func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr }
159
+
160
+func isClosedError(err error) bool {
161
+	/* This comparison is ugly, but unfortunately, net.go doesn't export errClosing.
162
+	 * See:
163
+	 * http://golang.org/src/pkg/net/net.go
164
+	 * https://code.google.com/p/go/issues/detail?id=4337
165
+	 * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ
166
+	 */
167
+	return strings.HasSuffix(err.Error(), "use of closed network connection")
168
+}
... ...
@@ -193,6 +193,11 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
193 193
 			return nil, err
194 194
 		}
195 195
 	}
196
+
197
+	if err = initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope)); err != nil {
198
+		return nil, err
199
+	}
200
+
196 201
 	c.drvRegistry = drvRegistry
197 202
 
198 203
 	if c.cfg != nil && c.cfg.Cluster.Watcher != nil {
... ...
@@ -330,7 +330,7 @@ func (c *networkConfiguration) conflictsWithNetworks(id string, others []*bridge
330 330
 		// bridges. This could not be completely caught by the config conflict
331 331
 		// check, because networks which config does not specify the AddressIPv4
332 332
 		// get their address and subnet selected by the driver (see electBridgeIPv4())
333
-		if c.AddressIPv4 != nil {
333
+		if c.AddressIPv4 != nil && nwBridge.bridgeIPv4 != nil {
334 334
 			if nwBridge.bridgeIPv4.Contains(c.AddressIPv4.IP) ||
335 335
 				c.AddressIPv4.Contains(nwBridge.bridgeIPv4.IP) {
336 336
 				return types.ForbiddenErrorf("conflicts with network %s (%s) by ip network", nwID, nwConfig.BridgeName)
337 337
new file mode 100644
... ...
@@ -0,0 +1,23 @@
0
+package libnetwork
1
+
2
+import (
3
+	"github.com/docker/libnetwork/drvregistry"
4
+	"github.com/docker/libnetwork/ipamapi"
5
+	builtinIpam "github.com/docker/libnetwork/ipams/builtin"
6
+	nullIpam "github.com/docker/libnetwork/ipams/null"
7
+	remoteIpam "github.com/docker/libnetwork/ipams/remote"
8
+)
9
+
10
+func initIPAMDrivers(r *drvregistry.DrvRegistry, lDs, gDs interface{}) error {
11
+	for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){
12
+		builtinIpam.Init,
13
+		remoteIpam.Init,
14
+		nullIpam.Init,
15
+	} {
16
+		if err := fn(r, lDs, gDs); err != nil {
17
+			return err
18
+		}
19
+	}
20
+
21
+	return nil
22
+}
... ...
@@ -8,10 +8,6 @@ import (
8 8
 	"github.com/docker/libnetwork/driverapi"
9 9
 	"github.com/docker/libnetwork/ipamapi"
10 10
 	"github.com/docker/libnetwork/types"
11
-
12
-	builtinIpam "github.com/docker/libnetwork/ipams/builtin"
13
-	nullIpam "github.com/docker/libnetwork/ipams/null"
14
-	remoteIpam "github.com/docker/libnetwork/ipams/remote"
15 11
 )
16 12
 
17 13
 type driverData struct {
... ...
@@ -64,10 +60,6 @@ func New(lDs, gDs interface{}, dfn DriverNotifyFunc, ifn IPAMNotifyFunc) (*DrvRe
64 64
 		ifn:         ifn,
65 65
 	}
66 66
 
67
-	if err := r.initIPAMs(lDs, gDs); err != nil {
68
-		return nil, err
69
-	}
70
-
71 67
 	return r, nil
72 68
 }
73 69
 
... ...
@@ -157,20 +149,6 @@ func (r *DrvRegistry) IPAMDefaultAddressSpaces(name string) (string, string, err
157 157
 	return i.defaultLocalAddressSpace, i.defaultGlobalAddressSpace, nil
158 158
 }
159 159
 
160
-func (r *DrvRegistry) initIPAMs(lDs, gDs interface{}) error {
161
-	for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){
162
-		builtinIpam.Init,
163
-		remoteIpam.Init,
164
-		nullIpam.Init,
165
-	} {
166
-		if err := fn(r, nil, gDs); err != nil {
167
-			return err
168
-		}
169
-	}
170
-
171
-	return nil
172
-}
173
-
174 160
 // RegisterDriver registers the network driver when it gets discovered.
175 161
 func (r *DrvRegistry) RegisterDriver(ntype string, driver driverapi.Driver, capability driverapi.Capability) error {
176 162
 	if strings.TrimSpace(ntype) == "" {
... ...
@@ -119,25 +119,34 @@ func Add(path string, recs []Record) error {
119 119
 		return nil
120 120
 	}
121 121
 
122
-	f, err := os.Open(path)
122
+	b, err := mergeRecords(path, recs)
123 123
 	if err != nil {
124 124
 		return err
125 125
 	}
126 126
 
127
-	content := bytes.NewBuffer(nil)
127
+	return ioutil.WriteFile(path, b, 0644)
128
+}
128 129
 
129
-	_, err = content.ReadFrom(f)
130
+func mergeRecords(path string, recs []Record) ([]byte, error) {
131
+	f, err := os.Open(path)
130 132
 	if err != nil {
131
-		return err
133
+		return nil, err
134
+	}
135
+	defer f.Close()
136
+
137
+	content := bytes.NewBuffer(nil)
138
+
139
+	if _, err := content.ReadFrom(f); err != nil {
140
+		return nil, err
132 141
 	}
133 142
 
134 143
 	for _, r := range recs {
135 144
 		if _, err := r.WriteTo(content); err != nil {
136
-			return err
145
+			return nil, err
137 146
 		}
138 147
 	}
139 148
 
140
-	return ioutil.WriteFile(path, content.Bytes(), 0644)
149
+	return content.Bytes(), nil
141 150
 }
142 151
 
143 152
 // Delete deletes an arbitrary number of Records already existing in /etc/hosts file
... ...
@@ -14,7 +14,7 @@ import (
14 14
 	"github.com/hashicorp/memberlist"
15 15
 )
16 16
 
17
-const reapInterval = 2 * time.Second
17
+const reapInterval = 30 * time.Second
18 18
 
19 19
 type logWriter struct{}
20 20
 
... ...
@@ -130,6 +130,11 @@ func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
130 130
 		return
131 131
 	}
132 132
 
133
+	// Ignore messages that this node generated.
134
+	if tEvent.NodeName == nDB.config.NodeName {
135
+		return
136
+	}
137
+
133 138
 	// Do not rebroadcast a bulk sync
134 139
 	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync {
135 140
 		var err error
... ...
@@ -90,7 +90,10 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart,
90 90
 		}
91 91
 
92 92
 		if useProxy {
93
-			m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
93
+			m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
94
+			if err != nil {
95
+				return nil, err
96
+			}
94 97
 		} else {
95 98
 			m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
96 99
 		}
... ...
@@ -107,7 +110,10 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart,
107 107
 		}
108 108
 
109 109
 		if useProxy {
110
-			m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
110
+			m.userlandProxy, err = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
111
+			if err != nil {
112
+				return nil, err
113
+			}
111 114
 		} else {
112 115
 			m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
113 116
 		}
... ...
@@ -2,8 +2,8 @@ package portmapper
2 2
 
3 3
 import "net"
4 4
 
5
-func newMockProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) userlandProxy {
6
-	return &mockProxyCommand{}
5
+func newMockProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) (userlandProxy, error) {
6
+	return &mockProxyCommand{}, nil
7 7
 }
8 8
 
9 9
 type mockProxyCommand struct {
... ...
@@ -1,29 +1,19 @@
1 1
 package portmapper
2 2
 
3 3
 import (
4
-	"flag"
5 4
 	"fmt"
6 5
 	"io"
7 6
 	"io/ioutil"
8
-	"log"
9 7
 	"net"
10 8
 	"os"
11 9
 	"os/exec"
12
-	"os/signal"
13 10
 	"strconv"
14 11
 	"syscall"
15 12
 	"time"
16
-
17
-	"github.com/docker/docker/pkg/proxy"
18
-	"github.com/docker/docker/pkg/reexec"
19 13
 )
20 14
 
21 15
 const userlandProxyCommandName = "docker-proxy"
22 16
 
23
-func init() {
24
-	reexec.Register(userlandProxyCommandName, execProxy)
25
-}
26
-
27 17
 type userlandProxy interface {
28 18
 	Start() error
29 19
 	Stop() error
... ...
@@ -35,66 +25,15 @@ type proxyCommand struct {
35 35
 	cmd *exec.Cmd
36 36
 }
37 37
 
38
-// execProxy is the reexec function that is registered to start the userland proxies
39
-func execProxy() {
40
-	f := os.NewFile(3, "signal-parent")
41
-	host, container := parseHostContainerAddrs()
38
+func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) (userlandProxy, error) {
39
+	cmd, err := exec.LookPath(userlandProxyCommandName)
42 40
 
43
-	p, err := proxy.NewProxy(host, container)
44 41
 	if err != nil {
45
-		fmt.Fprintf(f, "1\n%s", err)
46
-		f.Close()
47
-		os.Exit(1)
48
-	}
49
-	go handleStopSignals(p)
50
-	fmt.Fprint(f, "0\n")
51
-	f.Close()
52
-
53
-	// Run will block until the proxy stops
54
-	p.Run()
55
-}
56
-
57
-// parseHostContainerAddrs parses the flags passed on reexec to create the TCP or UDP
58
-// net.Addrs to map the host and container ports
59
-func parseHostContainerAddrs() (host net.Addr, container net.Addr) {
60
-	var (
61
-		proto         = flag.String("proto", "tcp", "proxy protocol")
62
-		hostIP        = flag.String("host-ip", "", "host ip")
63
-		hostPort      = flag.Int("host-port", -1, "host port")
64
-		containerIP   = flag.String("container-ip", "", "container ip")
65
-		containerPort = flag.Int("container-port", -1, "container port")
66
-	)
67
-
68
-	flag.Parse()
69
-
70
-	switch *proto {
71
-	case "tcp":
72
-		host = &net.TCPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort}
73
-		container = &net.TCPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort}
74
-	case "udp":
75
-		host = &net.UDPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort}
76
-		container = &net.UDPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort}
77
-	default:
78
-		log.Fatalf("unsupported protocol %s", *proto)
79
-	}
80
-
81
-	return host, container
82
-}
83
-
84
-func handleStopSignals(p proxy.Proxy) {
85
-	s := make(chan os.Signal, 10)
86
-	signal.Notify(s, os.Interrupt, syscall.SIGTERM, syscall.SIGSTOP)
87
-
88
-	for range s {
89
-		p.Close()
90
-
91
-		os.Exit(0)
42
+		return nil, err
92 43
 	}
93
-}
94 44
 
95
-func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) userlandProxy {
96 45
 	args := []string{
97
-		userlandProxyCommandName,
46
+		cmd,
98 47
 		"-proto", proto,
99 48
 		"-host-ip", hostIP.String(),
100 49
 		"-host-port", strconv.Itoa(hostPort),
... ...
@@ -104,13 +43,13 @@ func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.
104 104
 
105 105
 	return &proxyCommand{
106 106
 		cmd: &exec.Cmd{
107
-			Path: reexec.Self(),
107
+			Path: cmd,
108 108
 			Args: args,
109 109
 			SysProcAttr: &syscall.SysProcAttr{
110 110
 				Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies
111 111
 			},
112 112
 		},
113
-	}
113
+	}, nil
114 114
 }
115 115
 
116 116
 func (p *proxyCommand) Start() error {
... ...
@@ -275,7 +275,15 @@ func (sb *sandbox) updateDNS(ipv6Enabled bool) error {
275 275
 	if err != nil {
276 276
 		return err
277 277
 	}
278
-	if err = ioutil.WriteFile(tmpHashFile.Name(), []byte(newRC.Hash), filePerm); err != nil {
278
+	if err = tmpHashFile.Chmod(filePerm); err != nil {
279
+		tmpHashFile.Close()
280
+		return err
281
+	}
282
+	_, err = tmpHashFile.Write([]byte(newRC.Hash))
283
+	if err1 := tmpHashFile.Close(); err == nil {
284
+		err = err1
285
+	}
286
+	if err != nil {
279 287
 		return err
280 288
 	}
281 289
 	return os.Rename(tmpHashFile.Name(), hashFile)
... ...
@@ -135,6 +135,8 @@ func (c *controller) acceptClientConnections(sock string, l net.Listener) {
135 135
 			continue
136 136
 		}
137 137
 		go func() {
138
+			defer conn.Close()
139
+
138 140
 			err := c.processExternalKey(conn)
139 141
 			ret := success
140 142
 			if err != nil {
... ...
@@ -195,6 +195,8 @@ func (a *Agent) run(ctx context.Context) {
195 195
 				log.G(ctx).WithError(err).Error("agent: closing session failed")
196 196
 			}
197 197
 			sessionq = nil
198
+			// if we're here before <-registered, do nothing for that event
199
+			registered = nil
198 200
 		case <-session.closed:
199 201
 			log.G(ctx).Debugf("agent: rebuild session")
200 202
 
... ...
@@ -954,7 +954,7 @@ type AcceptancePolicy_RoleAdmissionPolicy struct {
954 954
 	Autoaccept bool `protobuf:"varint,2,opt,name=autoaccept,proto3" json:"autoaccept,omitempty"`
955 955
 	// Secret represents a user-provided string that is necessary for new
956 956
 	// nodes to join the cluster
957
-	Secret *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret `protobuf:"bytes,3,opt,name=secret" json:"secret,omitempty"`
957
+	Secret *AcceptancePolicy_RoleAdmissionPolicy_Secret `protobuf:"bytes,3,opt,name=secret" json:"secret,omitempty"`
958 958
 }
959 959
 
960 960
 func (m *AcceptancePolicy_RoleAdmissionPolicy) Reset()      { *m = AcceptancePolicy_RoleAdmissionPolicy{} }
... ...
@@ -963,18 +963,18 @@ func (*AcceptancePolicy_RoleAdmissionPolicy) Descriptor() ([]byte, []int) {
963 963
 	return fileDescriptorTypes, []int{23, 0}
964 964
 }
965 965
 
966
-type AcceptancePolicy_RoleAdmissionPolicy_HashedSecret struct {
967
-	// The actual hashed content
966
+type AcceptancePolicy_RoleAdmissionPolicy_Secret struct {
967
+	// The actual content (possibly hashed)
968 968
 	Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
969
-	// The type of hash we are using
969
+	// The type of hash we are using, or "plaintext"
970 970
 	Alg string `protobuf:"bytes,2,opt,name=alg,proto3" json:"alg,omitempty"`
971 971
 }
972 972
 
973
-func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Reset() {
974
-	*m = AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{}
973
+func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Reset() {
974
+	*m = AcceptancePolicy_RoleAdmissionPolicy_Secret{}
975 975
 }
976
-func (*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) ProtoMessage() {}
977
-func (*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Descriptor() ([]byte, []int) {
976
+func (*AcceptancePolicy_RoleAdmissionPolicy_Secret) ProtoMessage() {}
977
+func (*AcceptancePolicy_RoleAdmissionPolicy_Secret) Descriptor() ([]byte, []int) {
978 978
 	return fileDescriptorTypes, []int{23, 0, 0}
979 979
 }
980 980
 
... ...
@@ -1145,7 +1145,7 @@ func init() {
1145 1145
 	proto.RegisterType((*IssuanceStatus)(nil), "docker.swarmkit.v1.IssuanceStatus")
1146 1146
 	proto.RegisterType((*AcceptancePolicy)(nil), "docker.swarmkit.v1.AcceptancePolicy")
1147 1147
 	proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy")
1148
-	proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy_HashedSecret)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy.HashedSecret")
1148
+	proto.RegisterType((*AcceptancePolicy_RoleAdmissionPolicy_Secret)(nil), "docker.swarmkit.v1.AcceptancePolicy.RoleAdmissionPolicy.Secret")
1149 1149
 	proto.RegisterType((*ExternalCA)(nil), "docker.swarmkit.v1.ExternalCA")
1150 1150
 	proto.RegisterType((*CAConfig)(nil), "docker.swarmkit.v1.CAConfig")
1151 1151
 	proto.RegisterType((*OrchestrationConfig)(nil), "docker.swarmkit.v1.OrchestrationConfig")
... ...
@@ -1593,12 +1593,12 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Copy() *AcceptancePolicy_RoleAdmi
1593 1593
 	return o
1594 1594
 }
1595 1595
 
1596
-func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Copy() *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret {
1596
+func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Copy() *AcceptancePolicy_RoleAdmissionPolicy_Secret {
1597 1597
 	if m == nil {
1598 1598
 		return nil
1599 1599
 	}
1600 1600
 
1601
-	o := &AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{
1601
+	o := &AcceptancePolicy_RoleAdmissionPolicy_Secret{
1602 1602
 		Data: m.Data,
1603 1603
 		Alg:  m.Alg,
1604 1604
 	}
... ...
@@ -2180,12 +2180,12 @@ func (this *AcceptancePolicy_RoleAdmissionPolicy) GoString() string {
2180 2180
 	s = append(s, "}")
2181 2181
 	return strings.Join(s, "")
2182 2182
 }
2183
-func (this *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) GoString() string {
2183
+func (this *AcceptancePolicy_RoleAdmissionPolicy_Secret) GoString() string {
2184 2184
 	if this == nil {
2185 2185
 		return "nil"
2186 2186
 	}
2187 2187
 	s := make([]string, 0, 6)
2188
-	s = append(s, "&api.AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{")
2188
+	s = append(s, "&api.AcceptancePolicy_RoleAdmissionPolicy_Secret{")
2189 2189
 	s = append(s, "Data: "+fmt.Sprintf("%#v", this.Data)+",\n")
2190 2190
 	s = append(s, "Alg: "+fmt.Sprintf("%#v", this.Alg)+",\n")
2191 2191
 	s = append(s, "}")
... ...
@@ -3407,7 +3407,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) MarshalTo(data []byte) (int, erro
3407 3407
 	return i, nil
3408 3408
 }
3409 3409
 
3410
-func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Marshal() (data []byte, err error) {
3410
+func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Marshal() (data []byte, err error) {
3411 3411
 	size := m.Size()
3412 3412
 	data = make([]byte, size)
3413 3413
 	n, err := m.MarshalTo(data)
... ...
@@ -3417,7 +3417,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Marshal() (data []by
3417 3417
 	return data[:n], nil
3418 3418
 }
3419 3419
 
3420
-func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) MarshalTo(data []byte) (int, error) {
3420
+func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) MarshalTo(data []byte) (int, error) {
3421 3421
 	var i int
3422 3422
 	_ = i
3423 3423
 	var l int
... ...
@@ -4301,7 +4301,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Size() (n int) {
4301 4301
 	return n
4302 4302
 }
4303 4303
 
4304
-func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Size() (n int) {
4304
+func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Size() (n int) {
4305 4305
 	var l int
4306 4306
 	_ = l
4307 4307
 	l = len(m.Data)
... ...
@@ -4866,16 +4866,16 @@ func (this *AcceptancePolicy_RoleAdmissionPolicy) String() string {
4866 4866
 	s := strings.Join([]string{`&AcceptancePolicy_RoleAdmissionPolicy{`,
4867 4867
 		`Role:` + fmt.Sprintf("%v", this.Role) + `,`,
4868 4868
 		`Autoaccept:` + fmt.Sprintf("%v", this.Autoaccept) + `,`,
4869
-		`Secret:` + strings.Replace(fmt.Sprintf("%v", this.Secret), "AcceptancePolicy_RoleAdmissionPolicy_HashedSecret", "AcceptancePolicy_RoleAdmissionPolicy_HashedSecret", 1) + `,`,
4869
+		`Secret:` + strings.Replace(fmt.Sprintf("%v", this.Secret), "AcceptancePolicy_RoleAdmissionPolicy_Secret", "AcceptancePolicy_RoleAdmissionPolicy_Secret", 1) + `,`,
4870 4870
 		`}`,
4871 4871
 	}, "")
4872 4872
 	return s
4873 4873
 }
4874
-func (this *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) String() string {
4874
+func (this *AcceptancePolicy_RoleAdmissionPolicy_Secret) String() string {
4875 4875
 	if this == nil {
4876 4876
 		return "nil"
4877 4877
 	}
4878
-	s := strings.Join([]string{`&AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{`,
4878
+	s := strings.Join([]string{`&AcceptancePolicy_RoleAdmissionPolicy_Secret{`,
4879 4879
 		`Data:` + fmt.Sprintf("%v", this.Data) + `,`,
4880 4880
 		`Alg:` + fmt.Sprintf("%v", this.Alg) + `,`,
4881 4881
 		`}`,
... ...
@@ -8641,7 +8641,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Unmarshal(data []byte) error {
8641 8641
 				return io.ErrUnexpectedEOF
8642 8642
 			}
8643 8643
 			if m.Secret == nil {
8644
-				m.Secret = &AcceptancePolicy_RoleAdmissionPolicy_HashedSecret{}
8644
+				m.Secret = &AcceptancePolicy_RoleAdmissionPolicy_Secret{}
8645 8645
 			}
8646 8646
 			if err := m.Secret.Unmarshal(data[iNdEx:postIndex]); err != nil {
8647 8647
 				return err
... ...
@@ -8668,7 +8668,7 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy) Unmarshal(data []byte) error {
8668 8668
 	}
8669 8669
 	return nil
8670 8670
 }
8671
-func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Unmarshal(data []byte) error {
8671
+func (m *AcceptancePolicy_RoleAdmissionPolicy_Secret) Unmarshal(data []byte) error {
8672 8672
 	l := len(data)
8673 8673
 	iNdEx := 0
8674 8674
 	for iNdEx < l {
... ...
@@ -8691,10 +8691,10 @@ func (m *AcceptancePolicy_RoleAdmissionPolicy_HashedSecret) Unmarshal(data []byt
8691 8691
 		fieldNum := int32(wire >> 3)
8692 8692
 		wireType := int(wire & 0x7)
8693 8693
 		if wireType == 4 {
8694
-			return fmt.Errorf("proto: HashedSecret: wiretype end group for non-group")
8694
+			return fmt.Errorf("proto: Secret: wiretype end group for non-group")
8695 8695
 		}
8696 8696
 		if fieldNum <= 0 {
8697
-			return fmt.Errorf("proto: HashedSecret: illegal tag %d (wire type %d)", fieldNum, wire)
8697
+			return fmt.Errorf("proto: Secret: illegal tag %d (wire type %d)", fieldNum, wire)
8698 8698
 		}
8699 8699
 		switch fieldNum {
8700 8700
 		case 1:
... ...
@@ -10199,196 +10199,196 @@ var (
10199 10199
 )
10200 10200
 
10201 10201
 var fileDescriptorTypes = []byte{
10202
-	// 3043 bytes of a gzipped FileDescriptorProto
10203
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0x4d, 0x6c, 0x1b, 0xc7,
10204
-	0x15, 0x16, 0x7f, 0x45, 0x0e, 0x29, 0x89, 0x5e, 0x3b, 0x8e, 0xcc, 0xa8, 0xb2, 0xbb, 0x89, 0x1b,
10205
-	0xe7, 0xa7, 0x4c, 0xac, 0xa4, 0x85, 0x9b, 0xa0, 0x4d, 0x96, 0x3f, 0xb2, 0x58, 0x4b, 0x24, 0x31,
10206
-	0x94, 0x64, 0x04, 0x45, 0x4b, 0xac, 0x96, 0x23, 0x69, 0xa3, 0xe5, 0x2e, 0xbb, 0xbb, 0x94, 0x4c,
10202
+	// 3042 bytes of a gzipped FileDescriptorProto
10203
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0xcd, 0x6f, 0x1b, 0xc7,
10204
+	0x15, 0x17, 0x3f, 0x45, 0x3e, 0x52, 0x12, 0xbd, 0x76, 0x1c, 0x99, 0x51, 0x65, 0x77, 0x13, 0x37,
10205
+	0xce, 0x47, 0x99, 0x58, 0x49, 0x0b, 0x37, 0x41, 0xeb, 0x2c, 0x3f, 0x64, 0xb1, 0x96, 0x48, 0x62,
10206
+	0xa8, 0x0f, 0x04, 0x05, 0x4a, 0xac, 0x96, 0x23, 0x71, 0xa3, 0xe5, 0x2e, 0xbb, 0xbb, 0x94, 0x4c,
10207 10207
 	0x14, 0x05, 0x9c, 0x5e, 0x5a, 0x04, 0x3d, 0xf4, 0x5e, 0x04, 0x41, 0xd1, 0xa2, 0xb7, 0x9e, 0x0b,
10208
-	0xf4, 0xe4, 0xa3, 0x8f, 0x2d, 0x0a, 0x14, 0x39, 0x05, 0x4d, 0x7a, 0xe8, 0xb5, 0x40, 0x0f, 0x39,
10209
-	0xb4, 0x87, 0xbe, 0x37, 0x3f, 0xcb, 0x1f, 0xaf, 0x15, 0xa7, 0xc9, 0x81, 0xe0, 0xce, 0x7b, 0xdf,
10210
-	0x7b, 0x33, 0x6f, 0xe6, 0xcd, 0xfb, 0x19, 0x52, 0x08, 0xc7, 0x43, 0x16, 0x54, 0x86, 0xbe, 0x17,
10211
-	0x7a, 0x9a, 0xd6, 0xf7, 0xac, 0x13, 0xe6, 0x57, 0x82, 0x33, 0xd3, 0x1f, 0x9c, 0xd8, 0x61, 0xe5,
10212
-	0xf4, 0x66, 0xf9, 0x4a, 0x68, 0x0f, 0x58, 0x10, 0x9a, 0x83, 0xe1, 0x2b, 0xd1, 0x97, 0x80, 0x97,
10213
-	0x9f, 0xee, 0x8f, 0x7c, 0x33, 0xb4, 0x3d, 0xf7, 0x15, 0xf5, 0x21, 0x19, 0x97, 0x8e, 0xbc, 0x23,
10214
-	0x8f, 0x7f, 0xbe, 0x82, 0x5f, 0x82, 0xaa, 0x5f, 0x25, 0x8b, 0xfb, 0xcc, 0x0f, 0x00, 0xa6, 0x5d,
10215
-	0x22, 0x19, 0xdb, 0xed, 0xb3, 0x7b, 0xab, 0x89, 0x6b, 0x89, 0x1b, 0x69, 0x2a, 0x06, 0xfa, 0x6f,
10216
-	0x12, 0xa4, 0x60, 0xb8, 0xae, 0x17, 0x72, 0x5d, 0x81, 0xa6, 0x91, 0xb4, 0x6b, 0x0e, 0x18, 0x07,
10217
-	0xe5, 0x29, 0xff, 0xd6, 0x6a, 0x24, 0xeb, 0x98, 0x07, 0xcc, 0x09, 0x56, 0x93, 0xd7, 0x52, 0x37,
10218
-	0x0a, 0x1b, 0x2f, 0x55, 0x1e, 0x5d, 0x73, 0x65, 0x4a, 0x49, 0x65, 0x9b, 0xa3, 0x1b, 0x6e, 0xe8,
10219
-	0x8f, 0xa9, 0x14, 0x2d, 0x7f, 0x87, 0x14, 0xa6, 0xc8, 0x5a, 0x89, 0xa4, 0x4e, 0xd8, 0x58, 0x4e,
10220
-	0x83, 0x9f, 0xb8, 0xbe, 0x53, 0xd3, 0x19, 0x31, 0x98, 0x04, 0x69, 0x62, 0xf0, 0x46, 0xf2, 0x56,
10221
-	0x42, 0x7f, 0x87, 0xe4, 0x29, 0x0b, 0xbc, 0x91, 0x6f, 0xb1, 0x40, 0x7b, 0x81, 0xe4, 0x5d, 0xd3,
10222
-	0xf5, 0x7a, 0xd6, 0x70, 0x14, 0x70, 0xf1, 0x54, 0xb5, 0xf8, 0xe9, 0xc7, 0x57, 0x73, 0x2d, 0x20,
10223
-	0xd6, 0x3a, 0x7b, 0x01, 0xcd, 0x21, 0xbb, 0x06, 0x5c, 0xed, 0xeb, 0xa4, 0x38, 0x60, 0x03, 0xcf,
10224
-	0x1f, 0xf7, 0x0e, 0xc6, 0x21, 0x0b, 0xb8, 0xe2, 0x14, 0x2d, 0x08, 0x5a, 0x15, 0x49, 0xfa, 0xaf,
10225
-	0x12, 0xe4, 0x92, 0xd2, 0x4d, 0xd9, 0x8f, 0x47, 0xb6, 0xcf, 0x06, 0xcc, 0x0d, 0x03, 0xed, 0x5b,
10226
-	0x60, 0xb3, 0x3d, 0xb0, 0x43, 0x31, 0x47, 0x61, 0xe3, 0x6b, 0x71, 0x36, 0x47, 0xab, 0xa2, 0x12,
10227
-	0xac, 0x19, 0xa4, 0xe8, 0xb3, 0x80, 0xf9, 0xa7, 0x62, 0x27, 0xf8, 0x94, 0x9f, 0x2b, 0x3c, 0x23,
10228
-	0xa2, 0x6f, 0x92, 0x5c, 0xc7, 0x31, 0xc3, 0x43, 0xcf, 0x1f, 0x68, 0x3a, 0x29, 0x9a, 0xbe, 0x75,
10229
-	0x6c, 0x87, 0xcc, 0x0a, 0x47, 0xbe, 0x3a, 0x95, 0x19, 0x9a, 0x76, 0x99, 0x24, 0x3d, 0x31, 0x51,
10230
-	0xbe, 0x9a, 0x85, 0x9d, 0x48, 0xb6, 0xbb, 0x14, 0x28, 0xfa, 0x9b, 0xe4, 0x42, 0xc7, 0x19, 0x1d,
10231
-	0xd9, 0x6e, 0x9d, 0x05, 0x96, 0x6f, 0x0f, 0x51, 0x3b, 0x1e, 0x2f, 0x3a, 0x9f, 0x3a, 0x5e, 0xfc,
10232
-	0x8e, 0x8e, 0x3c, 0x39, 0x39, 0x72, 0xfd, 0xe7, 0x49, 0x72, 0xa1, 0xe1, 0x82, 0x30, 0x9b, 0x96,
10233
-	0xbe, 0x4e, 0x96, 0x19, 0x27, 0xf6, 0x4e, 0x85, 0x53, 0x49, 0x3d, 0x4b, 0x82, 0xaa, 0x3c, 0xad,
10234
-	0x39, 0xe7, 0x2f, 0x37, 0xe3, 0xcc, 0x7f, 0x44, 0x7b, 0x9c, 0xd7, 0x68, 0x0d, 0xb2, 0x38, 0xe4,
10235
-	0x46, 0x04, 0xab, 0x29, 0xae, 0xeb, 0x7a, 0x9c, 0xae, 0x47, 0xec, 0xac, 0xa6, 0x1f, 0x7e, 0x7c,
10236
-	0x75, 0x81, 0x2a, 0xd9, 0x2f, 0xe3, 0x7c, 0xff, 0x48, 0x90, 0x95, 0x96, 0xd7, 0x9f, 0xd9, 0x87,
10237
-	0x32, 0xc9, 0x1d, 0x7b, 0x41, 0x38, 0x75, 0x51, 0xa2, 0xb1, 0x76, 0x8b, 0xe4, 0x86, 0xf2, 0xf8,
10238
-	0xe4, 0xe9, 0xaf, 0xc5, 0x2f, 0x59, 0x60, 0x68, 0x84, 0xd6, 0xde, 0x24, 0x79, 0x5f, 0xf9, 0x04,
10239
-	0x58, 0xfb, 0x04, 0x8e, 0x33, 0xc1, 0x6b, 0xdf, 0x25, 0x59, 0x71, 0x08, 0xab, 0x69, 0x2e, 0x79,
10240
-	0xfd, 0x89, 0xf6, 0x9c, 0x4a, 0x21, 0xfd, 0xa3, 0x04, 0x29, 0x51, 0xf3, 0x30, 0xdc, 0x61, 0x83,
10241
-	0x03, 0xe6, 0x77, 0xe1, 0x22, 0xc3, 0xfd, 0xb9, 0x0c, 0xe7, 0xc8, 0xcc, 0x3e, 0xf3, 0xb9, 0x91,
10242
-	0x39, 0x2a, 0x47, 0xda, 0x1e, 0x3a, 0xb9, 0x69, 0x1d, 0x9b, 0x07, 0xb6, 0x63, 0x87, 0x63, 0x6e,
10243
-	0xe6, 0x72, 0xfc, 0x29, 0xcf, 0xeb, 0x84, 0xc5, 0x4f, 0x04, 0xe9, 0x8c, 0x1a, 0x6d, 0x95, 0x2c,
10244
-	0x42, 0xac, 0x0b, 0xcc, 0x23, 0xc6, 0xad, 0xcf, 0x53, 0x35, 0x04, 0x57, 0x2e, 0x4e, 0xcb, 0x69,
10245
-	0x05, 0xb2, 0xb8, 0xd7, 0xba, 0xd3, 0x6a, 0xdf, 0x6d, 0x95, 0x16, 0xb4, 0x15, 0x52, 0xd8, 0x6b,
10246
-	0xd1, 0x86, 0x51, 0xdb, 0x32, 0xaa, 0xdb, 0x8d, 0x52, 0x42, 0x5b, 0x82, 0x70, 0x11, 0x0d, 0x93,
10247
-	0xfa, 0x87, 0x09, 0x42, 0xf0, 0x00, 0xa5, 0x51, 0x6f, 0x90, 0x0c, 0xc4, 0xd3, 0x50, 0x1c, 0xdc,
10248
-	0xf2, 0xc6, 0x73, 0x71, 0xab, 0x9e, 0xc0, 0x2b, 0xf8, 0xc7, 0xa8, 0x10, 0x99, 0x5e, 0x61, 0x72,
10249
-	0x7e, 0x85, 0x19, 0x8e, 0x9c, 0x5d, 0x5a, 0x8e, 0xa4, 0xeb, 0xf8, 0x95, 0xd0, 0xf2, 0x24, 0x03,
10250
-	0x6b, 0xaa, 0xbf, 0x53, 0x4a, 0x82, 0xf3, 0x15, 0xeb, 0xcd, 0x6e, 0xad, 0xdd, 0x6a, 0x35, 0x6a,
10251
-	0xbb, 0x8d, 0x7a, 0x29, 0xa5, 0x5f, 0x27, 0x99, 0xe6, 0x00, 0xb4, 0x68, 0x6b, 0xe8, 0x01, 0x87,
10252
-	0xcc, 0x67, 0xae, 0xa5, 0x1c, 0x6b, 0x42, 0xd0, 0x7f, 0x99, 0x23, 0x99, 0x1d, 0x6f, 0xe4, 0x86,
10253
-	0xda, 0xc6, 0xd4, 0x2d, 0x5e, 0xde, 0x58, 0x8f, 0x33, 0x81, 0x03, 0x2b, 0xbb, 0x80, 0x92, 0xb7,
10254
-	0x1c, 0x0e, 0x53, 0xf8, 0x8a, 0x5c, 0xba, 0x1c, 0x21, 0x3d, 0x34, 0xfd, 0x23, 0x16, 0xca, 0x4d,
10255
-	0x97, 0x23, 0xed, 0x06, 0xc9, 0xc1, 0xe9, 0xf4, 0x3d, 0xd7, 0x19, 0x73, 0x97, 0xca, 0x89, 0x30,
10256
-	0x0b, 0xe7, 0xd0, 0x6f, 0x03, 0x8d, 0x46, 0x5c, 0x6d, 0x8b, 0x14, 0x0f, 0x20, 0x99, 0xf4, 0xbc,
10257
-	0xa1, 0x88, 0x79, 0x99, 0xc7, 0x3b, 0xa0, 0x58, 0x55, 0x15, 0xd0, 0x6d, 0x01, 0xa6, 0x85, 0x83,
10258
-	0xc9, 0x40, 0x6b, 0x91, 0xe5, 0x53, 0xcf, 0x19, 0x0d, 0x58, 0xa4, 0x2b, 0xcb, 0x75, 0x3d, 0xff,
10259
-	0x78, 0x5d, 0xfb, 0x1c, 0xaf, 0xb4, 0x2d, 0x9d, 0x4e, 0x0f, 0xcb, 0x3f, 0x4b, 0x91, 0xc2, 0xd4,
10260
-	0x64, 0x5a, 0x97, 0x14, 0x20, 0x2d, 0x0e, 0xcd, 0x23, 0x1e, 0x6a, 0xe5, 0xf6, 0xdd, 0x7c, 0xa2,
10261
-	0x85, 0x56, 0x3a, 0x13, 0x41, 0x3a, 0xad, 0x45, 0xff, 0x20, 0x49, 0x0a, 0x53, 0x4c, 0xed, 0x45,
10262
-	0x92, 0xa3, 0x1d, 0xda, 0xdc, 0x37, 0x76, 0x1b, 0xa5, 0x85, 0xf2, 0xda, 0xfb, 0x1f, 0x5c, 0x5b,
10263
-	0xe5, 0xda, 0xa6, 0x15, 0x74, 0x7c, 0xfb, 0x14, 0xbd, 0xe5, 0x06, 0x59, 0x54, 0xd0, 0x44, 0xf9,
10264
-	0x19, 0x80, 0x3e, 0x3d, 0x0f, 0x9d, 0x42, 0xd2, 0xee, 0x96, 0x41, 0xc1, 0x61, 0x92, 0xf1, 0x48,
10265
-	0xda, 0x3d, 0x36, 0x7d, 0xd6, 0xd7, 0xbe, 0x41, 0xb2, 0x12, 0x98, 0x2a, 0x97, 0x01, 0x78, 0x79,
10266
-	0x1e, 0x38, 0xc1, 0xd1, 0xee, 0xb6, 0xb1, 0xdf, 0x28, 0xa5, 0xe3, 0x71, 0xb4, 0xeb, 0x98, 0xa7,
10267
-	0x4c, 0x7b, 0x0e, 0x5c, 0x9b, 0xc3, 0x32, 0xe5, 0x2b, 0x00, 0x7b, 0xea, 0x11, 0x75, 0x88, 0x2a,
10268
-	0xaf, 0xfe, 0xe2, 0xb7, 0xeb, 0x0b, 0x7f, 0xfa, 0xdd, 0x7a, 0x69, 0x9e, 0x5d, 0xfe, 0x6f, 0x82,
10269
-	0x2c, 0xcd, 0x9c, 0x12, 0x64, 0xb5, 0xac, 0xeb, 0x59, 0xde, 0x50, 0x44, 0xe0, 0x5c, 0x95, 0x80,
10270
-	0x63, 0x65, 0x5b, 0x5e, 0x0d, 0x28, 0x54, 0x72, 0xb4, 0x3b, 0x73, 0x39, 0xe4, 0xb5, 0x27, 0x74,
10271
-	0x81, 0xd8, 0x2c, 0xf2, 0x16, 0x59, 0xea, 0xc3, 0x3e, 0x32, 0xbf, 0x67, 0x79, 0xee, 0xa1, 0x7d,
10272
-	0x24, 0xa3, 0x6b, 0x39, 0x4e, 0x67, 0x9d, 0x03, 0x69, 0x51, 0x08, 0xd4, 0x38, 0xfe, 0xcb, 0xe4,
10273
-	0x8f, 0xbb, 0x24, 0x8d, 0xb7, 0x50, 0x7b, 0x86, 0xa4, 0xab, 0xcd, 0x56, 0x1d, 0x5c, 0xe2, 0x02,
10274
-	0xec, 0xe2, 0x12, 0x5f, 0x3a, 0x32, 0xd0, 0xc7, 0xb4, 0xab, 0x24, 0xbb, 0xdf, 0xde, 0xde, 0xdb,
10275
-	0x41, 0x37, 0xb8, 0x08, 0xec, 0x95, 0x88, 0x2d, 0x8c, 0x2b, 0x5f, 0x90, 0xdb, 0x9b, 0x8f, 0x18,
10276
-	0xfa, 0x7f, 0x92, 0x64, 0x89, 0x62, 0x6d, 0xe8, 0x87, 0x1d, 0xcf, 0xb1, 0xad, 0xb1, 0xd6, 0x21,
10277
-	0x79, 0xb0, 0xaf, 0x6f, 0x4f, 0x39, 0xf7, 0xc6, 0x63, 0x12, 0xc8, 0x44, 0x4a, 0x8d, 0x6a, 0x4a,
10278
-	0x92, 0x4e, 0x94, 0x40, 0xa0, 0xc9, 0xf4, 0x99, 0x63, 0x8e, 0xcf, 0xcb, 0x64, 0x75, 0x59, 0x87,
10279
-	0x52, 0x01, 0xe5, 0x55, 0x97, 0x79, 0xaf, 0x67, 0x86, 0x21, 0x1b, 0x0c, 0x43, 0x91, 0xc9, 0xd2,
10280
-	0x50, 0x75, 0x99, 0xf7, 0x0c, 0x49, 0xd2, 0x5e, 0x27, 0xd9, 0x33, 0x30, 0xdb, 0x3b, 0x93, 0xc9,
10281
-	0xea, 0x7c, 0xbd, 0x12, 0xab, 0xbf, 0x8f, 0x39, 0x6a, 0x6e, 0xb1, 0xb8, 0xad, 0xad, 0x76, 0xab,
10282
-	0xa1, 0xb6, 0x55, 0xf2, 0xdb, 0x6e, 0xcb, 0x73, 0xd1, 0x75, 0x49, 0xbb, 0xd5, 0xdb, 0x34, 0x9a,
10283
-	0xdb, 0x7b, 0x14, 0xb7, 0xf6, 0x12, 0x40, 0x4a, 0x11, 0x64, 0xd3, 0xb4, 0x1d, 0x2c, 0xa0, 0xae,
10284
-	0x90, 0x94, 0xd1, 0x82, 0xc8, 0x5c, 0x2e, 0x01, 0xbb, 0x18, 0xb1, 0x0d, 0x77, 0x3c, 0xf1, 0xea,
10285
-	0xf9, 0x79, 0xf5, 0x77, 0x49, 0x71, 0x6f, 0xd8, 0x87, 0x9b, 0x29, 0x3c, 0x44, 0xbb, 0x06, 0xa1,
10286
-	0xc5, 0xf4, 0x4d, 0xc7, 0x61, 0x8e, 0x1d, 0x0c, 0x64, 0x8d, 0x3d, 0x4d, 0x82, 0xc2, 0xe0, 0xc9,
10287
-	0xf7, 0x52, 0xd6, 0x2f, 0x42, 0x40, 0xff, 0x29, 0x59, 0x81, 0x59, 0x42, 0x13, 0x12, 0xb5, 0x4a,
10288
-	0xcd, 0x1b, 0xa4, 0x68, 0x29, 0x52, 0xcf, 0xee, 0x0b, 0x57, 0xac, 0xae, 0xc0, 0x45, 0x2a, 0x44,
10289
-	0xd0, 0x66, 0x9d, 0x16, 0x22, 0x50, 0xb3, 0x8f, 0x76, 0x0e, 0x01, 0x8a, 0xd3, 0x67, 0xaa, 0x8b,
10290
-	0x00, 0x4d, 0x75, 0x00, 0x82, 0x34, 0xd8, 0xc5, 0x3c, 0xbb, 0x67, 0x87, 0x70, 0x3d, 0xfa, 0x22,
10291
-	0xf9, 0x66, 0x68, 0x0e, 0x09, 0x35, 0x18, 0xeb, 0xef, 0x25, 0x09, 0xd9, 0x35, 0x83, 0x13, 0x39,
10292
-	0x35, 0x94, 0x29, 0x51, 0x53, 0x72, 0x5e, 0x71, 0xbc, 0xab, 0x40, 0x74, 0x82, 0xd7, 0x5e, 0x53,
10293
-	0xd9, 0x57, 0xd4, 0x0c, 0xf1, 0x82, 0x72, 0xae, 0xb8, 0xb4, 0x3b, 0x5b, 0x18, 0xe0, 0x45, 0x64,
10294
-	0xbe, 0xcf, 0xbd, 0x08, 0x2e, 0x22, 0x7c, 0x42, 0xaf, 0x92, 0x8f, 0x6c, 0x96, 0x99, 0xe8, 0xd9,
10295
-	0xb8, 0x49, 0xe6, 0x36, 0x74, 0x6b, 0x81, 0x4e, 0xe4, 0xaa, 0x25, 0xb2, 0xec, 0xc3, 0x35, 0x83,
10296
-	0x55, 0xf7, 0x02, 0xce, 0xd6, 0xff, 0x0a, 0x7b, 0xd0, 0xec, 0x18, 0x3b, 0xf2, 0xb4, 0xeb, 0x24,
10297
-	0x7b, 0x68, 0x0e, 0x6c, 0x67, 0x2c, 0xaf, 0xd9, 0xcb, 0x71, 0x53, 0x4c, 0xf0, 0x15, 0xa3, 0xdf,
10298
-	0x87, 0x52, 0x2d, 0xd8, 0xe4, 0x32, 0x54, 0xca, 0xf2, 0x94, 0x3c, 0x3a, 0x70, 0x21, 0xf5, 0xaa,
10299
-	0x94, 0xcc, 0x47, 0x18, 0x4c, 0x7c, 0xd3, 0x8d, 0xac, 0x15, 0x03, 0xdc, 0x05, 0x88, 0xa8, 0xec,
10300
-	0xcc, 0x1c, 0x4b, 0x7b, 0xd5, 0x10, 0x12, 0x70, 0x4e, 0x74, 0x10, 0xac, 0x0f, 0x26, 0x63, 0xb4,
10301
-	0xfc, 0xbc, 0xf5, 0x50, 0x09, 0x17, 0x61, 0x32, 0x92, 0x2e, 0xbf, 0xc9, 0x43, 0xca, 0x84, 0xf5,
10302
-	0x85, 0x22, 0xdd, 0xab, 0x64, 0x69, 0xc6, 0xce, 0x47, 0x6a, 0xa1, 0x66, 0x67, 0xff, 0xf5, 0x52,
10303
-	0x5a, 0x7e, 0x7d, 0xbb, 0x94, 0xd5, 0xff, 0x0d, 0xa5, 0x59, 0xc7, 0xe3, 0xd7, 0x0a, 0x77, 0x35,
10304
-	0xbe, 0xf7, 0xcc, 0xf1, 0x4e, 0xd6, 0xf2, 0x1c, 0xe9, 0x33, 0xb1, 0xc5, 0xc0, 0x44, 0x0b, 0x26,
10305
-	0x6a, 0x0e, 0xa7, 0x91, 0x20, 0x84, 0xd7, 0x82, 0xa8, 0x6a, 0x7a, 0x43, 0xc0, 0xf1, 0x6d, 0x5d,
10306
-	0xa2, 0x44, 0x90, 0x50, 0x12, 0x1b, 0x9b, 0xe1, 0xe8, 0x00, 0xae, 0xe9, 0x31, 0xeb, 0x0b, 0x4c,
10307
-	0x9a, 0x63, 0x96, 0x22, 0x2a, 0xc2, 0xf4, 0x3a, 0xb4, 0x66, 0x4a, 0xe7, 0x2a, 0x49, 0xed, 0xd6,
10308
-	0x3a, 0x10, 0x77, 0x56, 0x20, 0x6a, 0x14, 0x14, 0x19, 0x48, 0xc8, 0xd9, 0xab, 0x77, 0x20, 0xdc,
10309
-	0xcc, 0x70, 0x80, 0x54, 0x4e, 0x63, 0x38, 0xd1, 0x7f, 0x9d, 0x20, 0x59, 0x91, 0x65, 0x62, 0x2d,
10310
-	0x36, 0xc8, 0xa2, 0xaa, 0x7e, 0x44, 0xea, 0x7b, 0xfe, 0xf1, 0x69, 0xaa, 0x22, 0xb3, 0x9e, 0x38,
10311
-	0x47, 0x25, 0x57, 0x7e, 0x83, 0x14, 0xa7, 0x19, 0x5f, 0xe8, 0x14, 0x7f, 0x42, 0x0a, 0xe8, 0x28,
10312
-	0x2a, 0x57, 0x6f, 0x90, 0xac, 0xc8, 0x84, 0xf2, 0xaa, 0x9f, 0x97, 0x33, 0x25, 0x12, 0x22, 0xdd,
10313
-	0xa2, 0xc8, 0xb3, 0xaa, 0x69, 0x5b, 0x3f, 0xdf, 0x1d, 0xa9, 0x82, 0xeb, 0x6f, 0x91, 0x74, 0x87,
10314
-	0x81, 0x86, 0x67, 0xc9, 0xa2, 0x0b, 0xa1, 0x67, 0x12, 0xd9, 0x64, 0x89, 0xd0, 0x67, 0x10, 0xb1,
10315
-	0xb2, 0xc8, 0x82, 0x78, 0x06, 0x9b, 0x67, 0x82, 0xbf, 0xa9, 0xbe, 0x15, 0xbf, 0xf5, 0x5d, 0x52,
10316
-	0xbc, 0xcb, 0xec, 0xa3, 0xe3, 0x10, 0x4e, 0x0c, 0x15, 0xbd, 0x4c, 0xd2, 0x43, 0x16, 0x2d, 0x7e,
10317
-	0x35, 0xd6, 0x75, 0x80, 0x4f, 0x39, 0x0a, 0x2f, 0xe4, 0x19, 0x97, 0x96, 0x4f, 0x05, 0x72, 0xa4,
10318
-	0xff, 0x21, 0x49, 0x96, 0x9b, 0x41, 0x30, 0x32, 0xa1, 0x0c, 0x97, 0x51, 0xf0, 0x7b, 0xb3, 0x6d,
10319
-	0xc4, 0x8d, 0x58, 0x0b, 0x67, 0x44, 0x66, 0x5b, 0x09, 0x19, 0xb9, 0x92, 0x51, 0xe4, 0xd2, 0x1f,
10320
-	0x26, 0x54, 0x0f, 0x71, 0x7d, 0xea, 0xde, 0x94, 0x57, 0xc1, 0x89, 0x2e, 0x4d, 0x6b, 0x62, 0x7b,
10321
-	0xee, 0x89, 0xeb, 0x9d, 0xb9, 0x90, 0x68, 0xa1, 0xa7, 0x68, 0x35, 0xee, 0x82, 0xa7, 0x5d, 0x06,
10322
-	0x90, 0x36, 0x03, 0xa2, 0xcc, 0x65, 0x67, 0xa8, 0xa9, 0xd3, 0x68, 0xd5, 0x9b, 0xad, 0xdb, 0x90,
10323
-	0xde, 0x1e, 0xd5, 0xd4, 0x61, 0x90, 0xce, 0xdc, 0x23, 0xd8, 0xee, 0x6c, 0xb3, 0xdb, 0xdd, 0xe3,
10324
-	0x25, 0xe3, 0xd3, 0x80, 0xba, 0x38, 0x83, 0xc2, 0x01, 0xd4, 0x8b, 0x00, 0xc2, 0x4c, 0x0a, 0xa0,
10325
-	0x74, 0x0c, 0x08, 0x93, 0x29, 0x04, 0x10, 0xe1, 0xe1, 0xff, 0x4c, 0x92, 0x92, 0x61, 0x59, 0x6c,
10326
-	0x18, 0x22, 0x5f, 0x56, 0x27, 0xbb, 0x70, 0x93, 0xf1, 0xcb, 0x66, 0xf8, 0xa6, 0x82, 0x6e, 0x71,
10327
-	0x2b, 0xf6, 0x1d, 0x69, 0x4e, 0xae, 0x42, 0x3d, 0x87, 0x19, 0xfd, 0x81, 0x1d, 0xe0, 0xdb, 0x82,
10328
-	0xa0, 0xd1, 0x48, 0x13, 0x56, 0x97, 0x17, 0x63, 0x10, 0xda, 0xab, 0x24, 0xed, 0x03, 0x59, 0x1e,
10329
-	0xcf, 0xda, 0xe3, 0xba, 0x3c, 0x14, 0xa5, 0x1c, 0xa9, 0xad, 0x13, 0x62, 0x8e, 0x42, 0xcf, 0xe4,
10330
-	0xf3, 0xf3, 0x83, 0xc9, 0xd1, 0x29, 0x8a, 0xf6, 0x43, 0x88, 0xd6, 0xcc, 0xf2, 0x65, 0xa3, 0x54,
10331
-	0xd8, 0x68, 0xfc, 0xbf, 0xab, 0xaf, 0x6c, 0x99, 0x18, 0x51, 0xba, 0x5c, 0x19, 0x95, 0x4a, 0xcb,
10332
-	0xaf, 0x93, 0xe2, 0x34, 0x1d, 0xbd, 0x1b, 0xca, 0x0b, 0x93, 0x1b, 0x50, 0xa4, 0xfc, 0x1b, 0x9d,
10333
-	0xc6, 0x74, 0x8e, 0x94, 0xd3, 0xc0, 0xa7, 0xfe, 0x21, 0xe4, 0xa5, 0xc6, 0xbd, 0x90, 0xf9, 0xae,
10334
-	0xe9, 0xd4, 0x0c, 0xad, 0x31, 0x15, 0x2d, 0x85, 0xe5, 0x2f, 0xc4, 0xbe, 0x03, 0x44, 0x12, 0x95,
10335
-	0x9a, 0x11, 0x13, 0x2f, 0xa1, 0x52, 0x18, 0xf9, 0x8e, 0x7c, 0x53, 0xe2, 0x95, 0xc2, 0x1e, 0xdd,
10336
-	0xa6, 0x48, 0xc3, 0x07, 0x19, 0x15, 0x9d, 0x52, 0x8f, 0x7f, 0x0c, 0x9c, 0x9a, 0xe0, 0xab, 0x8f,
10337
-	0x50, 0x2f, 0x13, 0x32, 0x59, 0x35, 0x1c, 0x5b, 0xa6, 0xb6, 0xd9, 0xed, 0x6e, 0xc3, 0x55, 0xe1,
10338
-	0x95, 0xf3, 0x84, 0xc5, 0xc9, 0xfa, 0xef, 0x13, 0x24, 0x57, 0x33, 0x64, 0x86, 0xd9, 0x24, 0x25,
10339
-	0x1e, 0x57, 0x2c, 0xe6, 0x87, 0x3d, 0x76, 0x6f, 0x68, 0xfb, 0x63, 0x19, 0x1a, 0xce, 0x2f, 0x41,
10340
-	0x97, 0x51, 0xaa, 0x06, 0x42, 0x0d, 0x2e, 0xa3, 0x51, 0x52, 0x64, 0xd2, 0xc4, 0x9e, 0x65, 0xaa,
10341
-	0x40, 0xbd, 0x7e, 0xfe, 0x56, 0x88, 0xf2, 0x6c, 0x32, 0x86, 0xe6, 0x57, 0x29, 0xa9, 0x99, 0x81,
10342
-	0xbe, 0x4f, 0x2e, 0xb6, 0x7d, 0xeb, 0x18, 0x0a, 0x25, 0x31, 0xa9, 0x5c, 0xf2, 0x5b, 0x64, 0x2d,
10343
-	0x84, 0x82, 0xa8, 0x77, 0x6c, 0x07, 0x21, 0x3e, 0x65, 0x82, 0x6f, 0x30, 0x17, 0xf9, 0x3d, 0xfe,
10344
-	0xe4, 0x28, 0x9e, 0x40, 0xe9, 0x15, 0xc4, 0x6c, 0x09, 0x08, 0x55, 0x88, 0x6d, 0x04, 0xe8, 0x3f,
10345
-	0x20, 0xa5, 0xba, 0x1d, 0x0c, 0xcd, 0x10, 0x74, 0xcb, 0x7e, 0x46, 0xbb, 0x4d, 0x4a, 0xc7, 0x0c,
10346
-	0xea, 0xd9, 0x03, 0x66, 0x42, 0x4e, 0x64, 0xbe, 0xed, 0xf5, 0x9f, 0x68, 0x1f, 0x56, 0x22, 0xa9,
10347
-	0x0e, 0x17, 0xd2, 0x3f, 0x83, 0x0c, 0x8e, 0x6f, 0x3c, 0x52, 0xef, 0x4b, 0xe4, 0x42, 0xe0, 0x9a,
10348
-	0xc3, 0xe0, 0xd8, 0x0b, 0x7b, 0xb6, 0x1b, 0xe2, 0xa3, 0xa6, 0x23, 0x6b, 0xe1, 0x92, 0x62, 0x34,
10349
-	0x25, 0x1d, 0x62, 0xb3, 0x76, 0xc2, 0xd8, 0xb0, 0xe7, 0x39, 0xfd, 0x9e, 0x62, 0x8a, 0x87, 0x4c,
10350
-	0x40, 0x23, 0xa7, 0xed, 0xf4, 0xbb, 0x8a, 0xae, 0x55, 0xc9, 0xba, 0xe3, 0x1d, 0xf5, 0xc0, 0x32,
10351
-	0x1f, 0xee, 0x7d, 0xef, 0xd0, 0xf3, 0x7b, 0x81, 0xe3, 0x9d, 0xc1, 0x87, 0x03, 0x7f, 0xcc, 0x57,
10352
-	0x8d, 0x46, 0x19, 0x50, 0x0d, 0x01, 0xda, 0xf4, 0xfc, 0x2e, 0xf0, 0x36, 0x15, 0x02, 0xd3, 0xfc,
10353
-	0xc4, 0xec, 0xd0, 0xb6, 0x4e, 0x54, 0x9a, 0x8f, 0xa8, 0xbb, 0x40, 0x84, 0x48, 0xb7, 0xc4, 0x1c,
10354
-	0x66, 0xf1, 0x4d, 0xe6, 0xa8, 0x0c, 0x47, 0x15, 0x15, 0x11, 0x41, 0xfa, 0x37, 0x49, 0xbe, 0xe3,
10355
-	0x98, 0x16, 0x7f, 0x2e, 0xc6, 0xea, 0x1f, 0x52, 0x18, 0x9e, 0x1c, 0x58, 0x2d, 0xc2, 0x5b, 0x9e,
10356
-	0x4e, 0x93, 0xf4, 0xf7, 0x20, 0xe9, 0x53, 0xcf, 0x0b, 0xe1, 0x92, 0x5e, 0x23, 0x59, 0xcb, 0xec,
10357
-	0x29, 0x77, 0x2f, 0x56, 0xf3, 0xe0, 0x16, 0x99, 0x9a, 0x71, 0x87, 0x8d, 0x69, 0xc6, 0x32, 0xe1,
10358
-	0x0f, 0xd3, 0x1f, 0x20, 0xd0, 0x49, 0xf9, 0x76, 0x14, 0x45, 0xfa, 0x03, 0x2f, 0x06, 0x0a, 0x05,
10359
-	0x61, 0xfc, 0x87, 0x08, 0x57, 0x94, 0xa0, 0xde, 0x31, 0x04, 0x0e, 0x51, 0x2c, 0x56, 0x97, 0x01,
10360
-	0x49, 0x04, 0x12, 0xc3, 0x09, 0x25, 0x02, 0x8d, 0xdf, 0xfa, 0xdf, 0x12, 0xa4, 0x80, 0x03, 0xfb,
10361
-	0xd0, 0xb6, 0x30, 0xcf, 0x7c, 0xf1, 0x18, 0x09, 0x81, 0xc1, 0x0a, 0x7c, 0xb9, 0x28, 0x1e, 0x18,
10362
-	0x6a, 0x5d, 0x4a, 0x91, 0xa6, 0xbd, 0x0d, 0xe1, 0x91, 0xe7, 0x39, 0x19, 0x1e, 0xf5, 0xcf, 0xcf,
10363
-	0x88, 0xb2, 0xcb, 0x91, 0x72, 0x7c, 0x13, 0x27, 0xab, 0xe3, 0x47, 0x53, 0xa4, 0xd3, 0x24, 0x7c,
10364
-	0xea, 0xb6, 0x5c, 0x7e, 0x1a, 0xf2, 0xa9, 0xbb, 0xd6, 0xa2, 0x40, 0xd1, 0xff, 0x92, 0x20, 0x4b,
10365
-	0x0d, 0xd7, 0xf2, 0xc7, 0x3c, 0xa4, 0xe0, 0x0e, 0xae, 0x91, 0x3c, 0x14, 0xd3, 0xc1, 0x38, 0x80,
10366
-	0x8e, 0x53, 0xbd, 0xa4, 0x45, 0x04, 0xad, 0x49, 0xf2, 0x10, 0x3c, 0x3d, 0xdf, 0x0e, 0x8f, 0x07,
10367
-	0xb2, 0xaa, 0x8c, 0x0f, 0x63, 0xd3, 0x3a, 0x2b, 0x86, 0x12, 0xa1, 0x13, 0x69, 0x15, 0xb8, 0x52,
10368
-	0x7c, 0xb1, 0x3c, 0x70, 0x41, 0xff, 0xeb, 0x40, 0xab, 0x03, 0xf5, 0x62, 0x0f, 0x3b, 0x08, 0x6e,
10369
-	0x07, 0xb4, 0x82, 0x92, 0x86, 0x5d, 0x91, 0xae, 0x93, 0x7c, 0xa4, 0x0c, 0xdf, 0x2f, 0x8d, 0x46,
10370
-	0xb7, 0x77, 0x73, 0xe3, 0x56, 0xef, 0x76, 0x6d, 0x07, 0xc2, 0x98, 0xc8, 0xa1, 0x7f, 0x04, 0x9b,
10371
-	0x76, 0x4c, 0x17, 0xba, 0x1c, 0xd5, 0xf3, 0x81, 0x57, 0xf8, 0x70, 0xd5, 0x54, 0x51, 0x94, 0x16,
10372
-	0x5e, 0x81, 0xb7, 0x0f, 0x8b, 0x22, 0x64, 0xc5, 0x17, 0x45, 0x53, 0xef, 0xb8, 0xa9, 0x73, 0xdf,
10373
-	0x71, 0xd3, 0x5f, 0xc9, 0x3b, 0xee, 0x8b, 0x9f, 0xa5, 0x48, 0x3e, 0xea, 0xe1, 0xd0, 0x65, 0xb0,
10374
-	0x46, 0x59, 0x10, 0xdd, 0x75, 0x44, 0x6f, 0xf1, 0xea, 0x24, 0x6f, 0x6c, 0x6f, 0xb7, 0x6b, 0x06,
10375
-	0x3e, 0x83, 0xbe, 0x2d, 0x8a, 0x98, 0x08, 0x60, 0xc0, 0xa5, 0xc5, 0x43, 0xef, 0x6b, 0xfa, 0xa4,
10376
-	0x88, 0xb9, 0x2f, 0x7b, 0xf8, 0x08, 0xa5, 0x2a, 0x98, 0xe7, 0x48, 0xce, 0xe8, 0x76, 0x9b, 0xb7,
10377
-	0x5b, 0xa0, 0xe9, 0x41, 0xa2, 0xfc, 0x14, 0x80, 0x2e, 0x4c, 0x54, 0x41, 0xf2, 0x3d, 0x72, 0x41,
10378
-	0x13, 0xa2, 0x6a, 0xb5, 0x46, 0x07, 0xe7, 0xbb, 0x9f, 0x9c, 0x47, 0xf1, 0xd4, 0xcd, 0x1f, 0xc6,
10379
-	0xf2, 0x1d, 0xda, 0xe8, 0x18, 0x14, 0x67, 0x7c, 0x90, 0x9c, 0x5b, 0x57, 0xc7, 0x67, 0xd0, 0xd5,
10380
-	0xe3, 0x9c, 0xeb, 0xea, 0x4d, 0xf7, 0x7e, 0xaa, 0xac, 0x01, 0x66, 0x79, 0xd2, 0xb8, 0xc2, 0xfe,
10381
-	0x8e, 0x71, 0xb6, 0xee, 0xae, 0x41, 0x77, 0xb9, 0x9a, 0xd4, 0xdc, 0x6c, 0x5d, 0x7c, 0x4e, 0x40,
10382
-	0x2d, 0x60, 0x1d, 0xdd, 0x6b, 0xb5, 0xb8, 0x75, 0xe9, 0x39, 0xeb, 0xe8, 0xc8, 0x75, 0x11, 0x73,
10383
-	0x1d, 0x52, 0x58, 0x7b, 0xa7, 0xb3, 0xdd, 0xd8, 0x6d, 0x94, 0x1e, 0xa4, 0xe7, 0x16, 0x54, 0xf3,
10384
-	0x06, 0x43, 0x87, 0x85, 0xc2, 0xbc, 0xee, 0xd6, 0xde, 0x2e, 0x7f, 0x72, 0xbe, 0x9f, 0x99, 0x9f,
10385
-	0xf0, 0x78, 0x14, 0xf6, 0xb1, 0x6c, 0xbc, 0x16, 0xd5, 0x71, 0x0f, 0x32, 0x22, 0x65, 0x46, 0x18,
10386
-	0x51, 0xc4, 0xa1, 0x1e, 0xda, 0xf8, 0xbe, 0x78, 0x9d, 0xbe, 0x9f, 0x9d, 0xd3, 0x43, 0xd9, 0xbb,
10387
-	0x10, 0x05, 0xa1, 0xd4, 0x8b, 0x9e, 0xa4, 0x22, 0xd6, 0x8b, 0x3f, 0x22, 0x39, 0x15, 0x30, 0x60,
10388
-	0x77, 0xb2, 0x77, 0xdb, 0xf4, 0x4e, 0x83, 0xc2, 0xd1, 0xf3, 0xdd, 0x51, 0x9c, 0xbb, 0x9e, 0x0f,
10389
-	0xde, 0x05, 0xcb, 0x58, 0xdc, 0x31, 0x5a, 0xc6, 0x6d, 0x00, 0xc8, 0x37, 0x2f, 0x05, 0x90, 0x5e,
10390
-	0x5f, 0x2e, 0xc9, 0x09, 0x22, 0x9d, 0xd5, 0xb5, 0x87, 0x9f, 0xac, 0x2f, 0x7c, 0x04, 0xbf, 0x7f,
10391
-	0x7d, 0xb2, 0x9e, 0xb8, 0xff, 0xe9, 0x7a, 0xe2, 0x21, 0xfc, 0xfe, 0x0c, 0xbf, 0xbf, 0xc3, 0xef,
10392
-	0x20, 0xcb, 0xeb, 0x97, 0xd7, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x61, 0x4f, 0x38, 0x13, 0x5f,
10393
-	0x1d, 0x00, 0x00,
10208
+	0xf4, 0xe4, 0xa3, 0x8f, 0x2d, 0x0a, 0x14, 0x01, 0x0a, 0x04, 0x4d, 0xfa, 0x0f, 0x04, 0xe8, 0x21,
10209
+	0x87, 0xf6, 0xd0, 0x37, 0x1f, 0xbb, 0x4b, 0xd2, 0x6b, 0xc5, 0x6e, 0x72, 0x20, 0xb8, 0xf3, 0xde,
10210
+	0xef, 0xbd, 0x99, 0x37, 0xf3, 0xe6, 0x7d, 0x0c, 0x14, 0xfc, 0xc9, 0x88, 0x7a, 0x95, 0x91, 0xeb,
10211
+	0xf8, 0x8e, 0xa2, 0xf4, 0x1d, 0xe3, 0x84, 0xba, 0x15, 0xef, 0x4c, 0x77, 0x87, 0x27, 0xa6, 0x5f,
10212
+	0x39, 0xbd, 0x59, 0xbe, 0xe2, 0x9b, 0x43, 0xea, 0xf9, 0xfa, 0x70, 0xf4, 0x5a, 0xf8, 0x25, 0xe0,
10213
+	0xe5, 0x67, 0xfb, 0x63, 0x57, 0xf7, 0x4d, 0xc7, 0x7e, 0x2d, 0xf8, 0x90, 0x8c, 0x4b, 0xc7, 0xce,
10214
+	0xb1, 0xc3, 0x3f, 0x5f, 0x63, 0x5f, 0x82, 0xaa, 0x5e, 0x85, 0xc5, 0x7d, 0xea, 0x7a, 0x08, 0x53,
10215
+	0x2e, 0x41, 0xc6, 0xb4, 0xfb, 0xf4, 0xde, 0x6a, 0xe2, 0x5a, 0xe2, 0x46, 0x9a, 0x88, 0x81, 0xfa,
10216
+	0xdb, 0x04, 0x14, 0x34, 0xdb, 0x76, 0x7c, 0xae, 0xcb, 0x53, 0x14, 0x48, 0xdb, 0xfa, 0x90, 0x72,
10217
+	0x50, 0x9e, 0xf0, 0x6f, 0xa5, 0x06, 0x59, 0x4b, 0x3f, 0xa4, 0x96, 0xb7, 0x9a, 0xbc, 0x96, 0xba,
10218
+	0x51, 0xd8, 0x78, 0xa5, 0xf2, 0xe8, 0x9a, 0x2b, 0x53, 0x4a, 0x2a, 0xdb, 0x1c, 0xdd, 0xb0, 0x7d,
10219
+	0x77, 0x42, 0xa4, 0x68, 0xf9, 0x7b, 0x50, 0x98, 0x22, 0x2b, 0x25, 0x48, 0x9d, 0xd0, 0x89, 0x9c,
10220
+	0x86, 0x7d, 0xb2, 0xf5, 0x9d, 0xea, 0xd6, 0x98, 0xe2, 0x24, 0x8c, 0x26, 0x06, 0x6f, 0x25, 0x6f,
10221
+	0x25, 0xd4, 0x77, 0x21, 0x4f, 0xa8, 0xe7, 0x8c, 0x5d, 0x83, 0x7a, 0xca, 0x4b, 0x90, 0xb7, 0x75,
10222
+	0xdb, 0xe9, 0x19, 0xa3, 0xb1, 0xc7, 0xc5, 0x53, 0xd5, 0xe2, 0x67, 0x9f, 0x5c, 0xcd, 0xb5, 0x90,
10223
+	0x58, 0xeb, 0xec, 0x79, 0x24, 0xc7, 0xd8, 0x35, 0xe4, 0x2a, 0xdf, 0x84, 0xe2, 0x90, 0x0e, 0x1d,
10224
+	0x77, 0xd2, 0x3b, 0x9c, 0xf8, 0xd4, 0xe3, 0x8a, 0x53, 0xa4, 0x20, 0x68, 0x55, 0x46, 0x52, 0x7f,
10225
+	0x9d, 0x80, 0x4b, 0x81, 0x6e, 0x42, 0x7f, 0x32, 0x36, 0x5d, 0x3a, 0xa4, 0xb6, 0xef, 0x29, 0xdf,
10226
+	0x41, 0x9b, 0xcd, 0xa1, 0xe9, 0x8b, 0x39, 0x0a, 0x1b, 0xdf, 0x88, 0xb3, 0x39, 0x5c, 0x15, 0x91,
10227
+	0x60, 0x45, 0x83, 0xa2, 0x4b, 0x3d, 0xea, 0x9e, 0x8a, 0x9d, 0xe0, 0x53, 0x7e, 0xa9, 0xf0, 0x8c,
10228
+	0x88, 0xba, 0x09, 0xb9, 0x8e, 0xa5, 0xfb, 0x47, 0x8e, 0x3b, 0x54, 0x54, 0x28, 0xea, 0xae, 0x31,
10229
+	0x30, 0x7d, 0x6a, 0xf8, 0x63, 0x37, 0x38, 0x95, 0x19, 0x9a, 0x72, 0x19, 0x92, 0x8e, 0x98, 0x28,
10230
+	0x5f, 0xcd, 0xe2, 0x4e, 0x24, 0xdb, 0x5d, 0x82, 0x14, 0xf5, 0x6d, 0xb8, 0xd0, 0xb1, 0xc6, 0xc7,
10231
+	0xa6, 0x5d, 0xa7, 0x9e, 0xe1, 0x9a, 0x23, 0xa6, 0x9d, 0x1d, 0x2f, 0x73, 0xbe, 0xe0, 0x78, 0xd9,
10232
+	0x77, 0x78, 0xe4, 0xc9, 0xe8, 0xc8, 0xd5, 0x5f, 0x24, 0xe1, 0x42, 0xc3, 0x46, 0x61, 0x3a, 0x2d,
10233
+	0x7d, 0x1d, 0x96, 0x29, 0x27, 0xf6, 0x4e, 0x85, 0x53, 0x49, 0x3d, 0x4b, 0x82, 0x1a, 0x78, 0x5a,
10234
+	0x73, 0xce, 0x5f, 0x6e, 0xc6, 0x99, 0xff, 0x88, 0xf6, 0x38, 0xaf, 0x51, 0x1a, 0xb0, 0x38, 0xe2,
10235
+	0x46, 0x78, 0xab, 0x29, 0xae, 0xeb, 0x7a, 0x9c, 0xae, 0x47, 0xec, 0xac, 0xa6, 0x1f, 0x7e, 0x72,
10236
+	0x75, 0x81, 0x04, 0xb2, 0x5f, 0xc5, 0xf9, 0xfe, 0x95, 0x80, 0x95, 0x96, 0xd3, 0x9f, 0xd9, 0x87,
10237
+	0x32, 0xe4, 0x06, 0x8e, 0xe7, 0x4f, 0x5d, 0x94, 0x70, 0xac, 0xdc, 0x82, 0xdc, 0x48, 0x1e, 0x9f,
10238
+	0x3c, 0xfd, 0xb5, 0xf8, 0x25, 0x0b, 0x0c, 0x09, 0xd1, 0xca, 0xdb, 0x90, 0x77, 0x03, 0x9f, 0x40,
10239
+	0x6b, 0x9f, 0xc0, 0x71, 0x22, 0xbc, 0xf2, 0x7d, 0xc8, 0x8a, 0x43, 0x58, 0x4d, 0x73, 0xc9, 0xeb,
10240
+	0x4f, 0xb4, 0xe7, 0x44, 0x0a, 0xa9, 0x1f, 0x27, 0xa0, 0x44, 0xf4, 0x23, 0x7f, 0x87, 0x0e, 0x0f,
10241
+	0xa9, 0xdb, 0xc5, 0x8b, 0x8c, 0xf7, 0xe7, 0x32, 0x9e, 0x23, 0xd5, 0xfb, 0xd4, 0xe5, 0x46, 0xe6,
10242
+	0x88, 0x1c, 0x29, 0x7b, 0xcc, 0xc9, 0x75, 0x63, 0xa0, 0x1f, 0x9a, 0x96, 0xe9, 0x4f, 0xb8, 0x99,
10243
+	0xcb, 0xf1, 0xa7, 0x3c, 0xaf, 0x13, 0x17, 0x1f, 0x09, 0x92, 0x19, 0x35, 0xca, 0x2a, 0x2c, 0x62,
10244
+	0xac, 0xf3, 0xf4, 0x63, 0xca, 0xad, 0xcf, 0x93, 0x60, 0x88, 0xae, 0x5c, 0x9c, 0x96, 0x53, 0x0a,
10245
+	0xb0, 0xb8, 0xd7, 0xba, 0xdb, 0x6a, 0x1f, 0xb4, 0x4a, 0x0b, 0xca, 0x0a, 0x14, 0xf6, 0x5a, 0xa4,
10246
+	0xa1, 0xd5, 0xb6, 0xb4, 0xea, 0x76, 0xa3, 0x94, 0x50, 0x96, 0x30, 0x5c, 0x84, 0xc3, 0xa4, 0xfa,
10247
+	0x51, 0x02, 0x80, 0x1d, 0xa0, 0x34, 0xea, 0x2d, 0xc8, 0x60, 0x3c, 0xf5, 0xc5, 0xc1, 0x2d, 0x6f,
10248
+	0xbc, 0x10, 0xb7, 0xea, 0x08, 0x5e, 0x61, 0x7f, 0x94, 0x08, 0x91, 0xe9, 0x15, 0x26, 0xe7, 0x57,
10249
+	0x98, 0xe1, 0xc8, 0xd9, 0xa5, 0xe5, 0x20, 0x5d, 0x67, 0x5f, 0x09, 0x25, 0x0f, 0x19, 0x5c, 0x53,
10250
+	0xfd, 0xdd, 0x52, 0x12, 0x9d, 0xaf, 0x58, 0x6f, 0x76, 0x6b, 0xed, 0x56, 0xab, 0x51, 0xdb, 0x6d,
10251
+	0xd4, 0x4b, 0x29, 0xf5, 0x3a, 0x64, 0x9a, 0x43, 0xd4, 0xa2, 0xac, 0x31, 0x0f, 0x38, 0xa2, 0x2e,
10252
+	0xb5, 0x8d, 0xc0, 0xb1, 0x22, 0x82, 0xfa, 0xab, 0x1c, 0x64, 0x76, 0x9c, 0xb1, 0xed, 0x2b, 0x1b,
10253
+	0x53, 0xb7, 0x78, 0x79, 0x63, 0x3d, 0xce, 0x04, 0x0e, 0xac, 0xec, 0x22, 0x4a, 0xde, 0x72, 0x3c,
10254
+	0x4c, 0xe1, 0x2b, 0x72, 0xe9, 0x72, 0xc4, 0xe8, 0xbe, 0xee, 0x1e, 0x53, 0x5f, 0x6e, 0xba, 0x1c,
10255
+	0x29, 0x37, 0x20, 0x87, 0xa7, 0xd3, 0x77, 0x6c, 0x6b, 0xc2, 0x5d, 0x2a, 0x27, 0xc2, 0x2c, 0x9e,
10256
+	0x43, 0xbf, 0x8d, 0x34, 0x12, 0x72, 0x95, 0x2d, 0x28, 0x1e, 0x62, 0x32, 0xe9, 0x39, 0x23, 0x11,
10257
+	0xf3, 0x32, 0x8f, 0x77, 0x40, 0xb1, 0xaa, 0x2a, 0xa2, 0xdb, 0x02, 0x4c, 0x0a, 0x87, 0xd1, 0x40,
10258
+	0x69, 0xc1, 0xf2, 0xa9, 0x63, 0x8d, 0x87, 0x34, 0xd4, 0x95, 0xe5, 0xba, 0x5e, 0x7c, 0xbc, 0xae,
10259
+	0x7d, 0x8e, 0x0f, 0xb4, 0x2d, 0x9d, 0x4e, 0x0f, 0xcb, 0x3f, 0x4f, 0x41, 0x61, 0x6a, 0x32, 0xa5,
10260
+	0x0b, 0x05, 0x4c, 0x8b, 0x23, 0xfd, 0x98, 0x87, 0x5a, 0xb9, 0x7d, 0x37, 0x9f, 0x68, 0xa1, 0x95,
10261
+	0x4e, 0x24, 0x48, 0xa6, 0xb5, 0xa8, 0x1f, 0x26, 0xa1, 0x30, 0xc5, 0x54, 0x5e, 0x86, 0x1c, 0xe9,
10262
+	0x90, 0xe6, 0xbe, 0xb6, 0xdb, 0x28, 0x2d, 0x94, 0xd7, 0x3e, 0xf8, 0xf0, 0xda, 0x2a, 0xd7, 0x36,
10263
+	0xad, 0xa0, 0xe3, 0x9a, 0xa7, 0xcc, 0x5b, 0x6e, 0xc0, 0x62, 0x00, 0x4d, 0x94, 0x9f, 0x43, 0xe8,
10264
+	0xb3, 0xf3, 0xd0, 0x29, 0x24, 0xe9, 0x6e, 0x69, 0x04, 0x1d, 0x26, 0x19, 0x8f, 0x24, 0xdd, 0x81,
10265
+	0xee, 0xd2, 0xbe, 0xf2, 0x2d, 0xc8, 0x4a, 0x60, 0xaa, 0x5c, 0x46, 0xe0, 0xe5, 0x79, 0x60, 0x84,
10266
+	0x23, 0xdd, 0x6d, 0x6d, 0xbf, 0x51, 0x4a, 0xc7, 0xe3, 0x48, 0xd7, 0xd2, 0x4f, 0xa9, 0xf2, 0x02,
10267
+	0xba, 0x36, 0x87, 0x65, 0xca, 0x57, 0x10, 0xf6, 0xcc, 0x23, 0xea, 0x18, 0xaa, 0xbc, 0xfa, 0xcb,
10268
+	0xdf, 0xad, 0x2f, 0xfc, 0xf9, 0xf7, 0xeb, 0xa5, 0x79, 0x76, 0xf9, 0xbf, 0x09, 0x58, 0x9a, 0x39,
10269
+	0x25, 0xcc, 0x6a, 0x59, 0xdb, 0x31, 0x9c, 0x91, 0x88, 0xc0, 0xb9, 0x2a, 0xa0, 0x63, 0x65, 0x5b,
10270
+	0x4e, 0x0d, 0x29, 0x44, 0x72, 0x94, 0xbb, 0x73, 0x39, 0xe4, 0x8d, 0x27, 0x74, 0x81, 0xd8, 0x2c,
10271
+	0x72, 0x1b, 0x96, 0xfa, 0xb8, 0x8f, 0xd4, 0xed, 0x19, 0x8e, 0x7d, 0x64, 0x1e, 0xcb, 0xe8, 0x5a,
10272
+	0x8e, 0xd3, 0x59, 0xe7, 0x40, 0x52, 0x14, 0x02, 0x35, 0x8e, 0xff, 0x2a, 0xf9, 0xe3, 0x00, 0xd2,
10273
+	0xec, 0x16, 0x2a, 0xcf, 0x41, 0xba, 0xda, 0x6c, 0xd5, 0xd1, 0x25, 0x2e, 0xe0, 0x2e, 0x2e, 0xf1,
10274
+	0xa5, 0x33, 0x06, 0xf3, 0x31, 0xe5, 0x2a, 0x64, 0xf7, 0xdb, 0xdb, 0x7b, 0x3b, 0xcc, 0x0d, 0x2e,
10275
+	0x22, 0x7b, 0x25, 0x64, 0x0b, 0xe3, 0xca, 0x17, 0xe4, 0xf6, 0xe6, 0x43, 0x86, 0xfa, 0x9f, 0x24,
10276
+	0x2c, 0x11, 0x56, 0x1b, 0xba, 0x7e, 0xc7, 0xb1, 0x4c, 0x63, 0xa2, 0x74, 0x20, 0x8f, 0xf6, 0xf5,
10277
+	0xcd, 0x29, 0xe7, 0xde, 0x78, 0x4c, 0x02, 0x89, 0xa4, 0x82, 0x51, 0x2d, 0x90, 0x24, 0x91, 0x12,
10278
+	0x0c, 0x34, 0x99, 0x3e, 0xb5, 0xf4, 0xc9, 0x79, 0x99, 0xac, 0x2e, 0xeb, 0x50, 0x22, 0xa0, 0xbc,
10279
+	0xea, 0xd2, 0xef, 0xf5, 0x74, 0xdf, 0xa7, 0xc3, 0x91, 0x2f, 0x32, 0x59, 0x1a, 0xab, 0x2e, 0xfd,
10280
+	0x9e, 0x26, 0x49, 0xca, 0x9b, 0x90, 0x3d, 0x43, 0xb3, 0x9d, 0x33, 0x99, 0xac, 0xce, 0xd7, 0x2b,
10281
+	0xb1, 0xea, 0x07, 0x2c, 0x47, 0xcd, 0x2d, 0x96, 0x6d, 0x6b, 0xab, 0xdd, 0x6a, 0x04, 0xdb, 0x2a,
10282
+	0xf9, 0x6d, 0xbb, 0xe5, 0xd8, 0xcc, 0x75, 0xa1, 0xdd, 0xea, 0x6d, 0x6a, 0xcd, 0xed, 0x3d, 0xc2,
10283
+	0xb6, 0xf6, 0x12, 0x42, 0x4a, 0x21, 0x64, 0x53, 0x37, 0x2d, 0x56, 0x40, 0x5d, 0x81, 0x94, 0xd6,
10284
+	0xc2, 0xc8, 0x5c, 0x2e, 0x21, 0xbb, 0x18, 0xb2, 0x35, 0x7b, 0x12, 0x79, 0xf5, 0xfc, 0xbc, 0xea,
10285
+	0x7b, 0x50, 0xdc, 0x1b, 0xf5, 0xf1, 0x66, 0x0a, 0x0f, 0x51, 0xae, 0x61, 0x68, 0xd1, 0x5d, 0xdd,
10286
+	0xb2, 0xa8, 0x65, 0x7a, 0x43, 0x59, 0x63, 0x4f, 0x93, 0xb0, 0x30, 0x78, 0xf2, 0xbd, 0x94, 0xf5,
10287
+	0x8b, 0x10, 0x50, 0x7f, 0x06, 0x2b, 0x38, 0x8b, 0xaf, 0x63, 0xa2, 0x0e, 0x52, 0xf3, 0x06, 0x14,
10288
+	0x8d, 0x80, 0xd4, 0x33, 0xfb, 0xc2, 0x15, 0xab, 0x2b, 0x78, 0x91, 0x0a, 0x21, 0xb4, 0x59, 0x27,
10289
+	0x85, 0x10, 0xd4, 0xec, 0x33, 0x3b, 0x47, 0x08, 0x65, 0xd3, 0x67, 0xaa, 0x8b, 0x08, 0x4d, 0x75,
10290
+	0x10, 0xc2, 0x68, 0xb8, 0x8b, 0x79, 0x7a, 0xcf, 0xf4, 0xf1, 0x7a, 0xf4, 0x45, 0xf2, 0xcd, 0x90,
10291
+	0x1c, 0x23, 0xd4, 0x70, 0xac, 0xbe, 0x9f, 0x04, 0xd8, 0xd5, 0xbd, 0x13, 0x39, 0x35, 0x96, 0x29,
10292
+	0x61, 0x53, 0x72, 0x5e, 0x71, 0xbc, 0x1b, 0x80, 0x48, 0x84, 0x57, 0xde, 0x08, 0xb2, 0xaf, 0xa8,
10293
+	0x19, 0xe2, 0x05, 0xe5, 0x5c, 0x71, 0x69, 0x77, 0xb6, 0x30, 0x60, 0x17, 0x91, 0xba, 0x2e, 0xf7,
10294
+	0x22, 0xbc, 0x88, 0xf8, 0x89, 0xbd, 0x4a, 0x3e, 0xb4, 0x59, 0x66, 0xa2, 0xe7, 0xe3, 0x26, 0x99,
10295
+	0xdb, 0xd0, 0xad, 0x05, 0x12, 0xc9, 0x55, 0x4b, 0xb0, 0xec, 0xe2, 0x35, 0xc3, 0x55, 0xf7, 0x3c,
10296
+	0xce, 0x56, 0xff, 0x86, 0x7b, 0xd0, 0xec, 0x68, 0x3b, 0xf2, 0xb4, 0xeb, 0x90, 0x3d, 0xd2, 0x87,
10297
+	0xa6, 0x35, 0x91, 0xd7, 0xec, 0xd5, 0xb8, 0x29, 0x22, 0x7c, 0x45, 0xeb, 0xf7, 0xb1, 0x54, 0xf3,
10298
+	0x36, 0xb9, 0x0c, 0x91, 0xb2, 0x3c, 0x25, 0x8f, 0x0f, 0x6d, 0x4c, 0xbd, 0x41, 0x4a, 0xe6, 0x23,
10299
+	0x16, 0x4c, 0x5c, 0xdd, 0x0e, 0xad, 0x15, 0x03, 0xb6, 0x0b, 0x18, 0x51, 0xe9, 0x99, 0x3e, 0x91,
10300
+	0xf6, 0x06, 0x43, 0x4c, 0xc0, 0x39, 0xd1, 0x41, 0xd0, 0x3e, 0x9a, 0xcc, 0xa2, 0xe5, 0x97, 0xad,
10301
+	0x87, 0x48, 0xb8, 0x08, 0x93, 0xa1, 0x74, 0xf9, 0x6d, 0x1e, 0x52, 0x22, 0xd6, 0x53, 0x45, 0xba,
10302
+	0xd7, 0x61, 0x69, 0xc6, 0xce, 0x47, 0x6a, 0xa1, 0x66, 0x67, 0xff, 0xcd, 0x52, 0x5a, 0x7e, 0x7d,
10303
+	0xb7, 0x94, 0x55, 0xff, 0x8d, 0xa5, 0x59, 0xc7, 0xe1, 0xd7, 0x8a, 0xed, 0x6a, 0x7c, 0xef, 0x99,
10304
+	0xe3, 0x9d, 0xac, 0xe1, 0x58, 0xd2, 0x67, 0x62, 0x8b, 0x81, 0x48, 0x0b, 0x4b, 0xd4, 0x1c, 0x4e,
10305
+	0x42, 0x41, 0x0c, 0xaf, 0x05, 0x51, 0xd5, 0xf4, 0x46, 0x88, 0xe3, 0xdb, 0xba, 0x44, 0x40, 0x90,
10306
+	0x98, 0x24, 0x6b, 0x6c, 0x46, 0xe3, 0x43, 0xbc, 0xa6, 0x03, 0xda, 0x17, 0x98, 0x34, 0xc7, 0x2c,
10307
+	0x85, 0x54, 0x06, 0x53, 0xeb, 0xd8, 0x9a, 0x05, 0x3a, 0x57, 0x21, 0xb5, 0x5b, 0xeb, 0x60, 0xdc,
10308
+	0x59, 0xc1, 0xa8, 0x51, 0x08, 0xc8, 0x48, 0x62, 0x9c, 0xbd, 0x7a, 0x07, 0xc3, 0xcd, 0x0c, 0x07,
10309
+	0x49, 0xe5, 0x34, 0x0b, 0x27, 0xea, 0x6f, 0x12, 0x90, 0x15, 0x59, 0x26, 0xd6, 0x62, 0x0d, 0x16,
10310
+	0x83, 0xea, 0x47, 0xa4, 0xbe, 0x17, 0x1f, 0x9f, 0xa6, 0x2a, 0x32, 0xeb, 0x89, 0x73, 0x0c, 0xe4,
10311
+	0xca, 0x6f, 0x41, 0x71, 0x9a, 0xf1, 0x54, 0xa7, 0xf8, 0x53, 0x28, 0x30, 0x47, 0x09, 0x72, 0xf5,
10312
+	0x06, 0x64, 0x45, 0x26, 0x94, 0x57, 0xfd, 0xbc, 0x9c, 0x29, 0x91, 0x18, 0xe9, 0x16, 0x45, 0x9e,
10313
+	0x0d, 0x9a, 0xb6, 0xf5, 0xf3, 0xdd, 0x91, 0x04, 0x70, 0xf5, 0x36, 0xa4, 0x3b, 0x14, 0x35, 0x3c,
10314
+	0x0f, 0x8b, 0x36, 0x86, 0x9e, 0x28, 0xb2, 0xc9, 0x12, 0xa1, 0x4f, 0x31, 0x62, 0x65, 0x19, 0x0b,
10315
+	0xe3, 0x19, 0x6e, 0x9e, 0x8e, 0xfe, 0x16, 0xf4, 0xad, 0xec, 0x5b, 0xdd, 0x85, 0xe2, 0x01, 0x35,
10316
+	0x8f, 0x07, 0x3e, 0x9e, 0x18, 0x53, 0xf4, 0x2a, 0xa4, 0x47, 0x34, 0x5c, 0xfc, 0x6a, 0xac, 0xeb,
10317
+	0x20, 0x9f, 0x70, 0x14, 0xbb, 0x90, 0x67, 0x5c, 0x5a, 0x3e, 0x15, 0xc8, 0x91, 0xfa, 0xc7, 0x24,
10318
+	0x2c, 0x37, 0x3d, 0x6f, 0xac, 0x63, 0x19, 0x2e, 0xa3, 0xe0, 0x0f, 0x66, 0xdb, 0x88, 0x1b, 0xb1,
10319
+	0x16, 0xce, 0x88, 0xcc, 0xb6, 0x12, 0x32, 0x72, 0x25, 0xc3, 0xc8, 0xa5, 0x3e, 0x4c, 0x04, 0x3d,
10320
+	0xc4, 0xf5, 0xa9, 0x7b, 0x53, 0x5e, 0x45, 0x27, 0xba, 0x34, 0xad, 0x89, 0xee, 0xd9, 0x27, 0xb6,
10321
+	0x73, 0x66, 0x63, 0xa2, 0xc5, 0x9e, 0xa2, 0xd5, 0x38, 0x40, 0x4f, 0xbb, 0x8c, 0x20, 0x65, 0x06,
10322
+	0x44, 0xa8, 0x4d, 0xcf, 0x98, 0xa6, 0x4e, 0xa3, 0x55, 0x6f, 0xb6, 0xee, 0x60, 0x7a, 0x7b, 0x54,
10323
+	0x53, 0x87, 0x62, 0x3a, 0xb3, 0x8f, 0x71, 0xbb, 0xb3, 0xcd, 0x6e, 0x77, 0x8f, 0x97, 0x8c, 0xcf,
10324
+	0x22, 0xea, 0xe2, 0x0c, 0x8a, 0x0d, 0xb0, 0x5e, 0x44, 0x10, 0xcb, 0xa4, 0x08, 0x4a, 0xc7, 0x80,
10325
+	0x58, 0x32, 0xc5, 0x00, 0x22, 0x3c, 0xfc, 0x1f, 0x49, 0x28, 0x69, 0x86, 0x41, 0x47, 0x3e, 0xe3,
10326
+	0xcb, 0xea, 0x64, 0x17, 0x6f, 0x32, 0xfb, 0x32, 0x29, 0x7b, 0x53, 0x61, 0x6e, 0x71, 0x2b, 0xf6,
10327
+	0x1d, 0x69, 0x4e, 0xae, 0x42, 0x1c, 0x8b, 0x6a, 0xfd, 0xa1, 0xe9, 0xb1, 0xb7, 0x05, 0x41, 0x23,
10328
+	0xa1, 0xa6, 0xf2, 0xe7, 0x09, 0xb8, 0x18, 0x83, 0x50, 0x5e, 0x87, 0xb4, 0x8b, 0x64, 0x79, 0x3c,
10329
+	0x6b, 0x8f, 0xeb, 0xf2, 0x98, 0x28, 0xe1, 0x48, 0x65, 0x1d, 0x40, 0x1f, 0xfb, 0x8e, 0xce, 0xe7,
10330
+	0xe7, 0x07, 0x93, 0x23, 0x53, 0x14, 0xe5, 0x00, 0xa3, 0x35, 0x35, 0x5c, 0xd9, 0x28, 0x15, 0x36,
10331
+	0x6e, 0xff, 0xbf, 0xab, 0xaf, 0x74, 0xb9, 0x1a, 0x22, 0xd5, 0x95, 0x2b, 0x58, 0xb0, 0xf3, 0x2f,
10332
+	0xe6, 0xd1, 0x58, 0x52, 0xe8, 0x7c, 0xd1, 0x45, 0xc2, 0xbf, 0x99, 0xa3, 0xe8, 0xd6, 0x71, 0xe0,
10333
+	0x28, 0xf8, 0xa9, 0x7e, 0x84, 0xb9, 0xa8, 0x71, 0xcf, 0xa7, 0xae, 0xad, 0x5b, 0x35, 0x4d, 0x69,
10334
+	0x4c, 0x45, 0x48, 0x61, 0xed, 0x4b, 0xb1, 0xbd, 0x7f, 0x28, 0x51, 0xa9, 0x69, 0x31, 0x31, 0x12,
10335
+	0xab, 0x83, 0xb1, 0x6b, 0xc9, 0x77, 0x24, 0x5e, 0x1d, 0xec, 0x91, 0x6d, 0xc2, 0x68, 0xec, 0x11,
10336
+	0x26, 0x88, 0x48, 0xa9, 0xc7, 0x3f, 0x00, 0x4e, 0x4d, 0xf0, 0xf5, 0x47, 0xa5, 0x57, 0x01, 0xa2,
10337
+	0x55, 0xe3, 0x51, 0x65, 0x6a, 0x9b, 0xdd, 0xee, 0x36, 0x5e, 0x0f, 0x5e, 0x2d, 0x47, 0x2c, 0x4e,
10338
+	0x56, 0xff, 0x90, 0x80, 0x5c, 0x4d, 0x93, 0x59, 0x65, 0x13, 0x4a, 0x3c, 0x96, 0x18, 0xd4, 0xf5,
10339
+	0x7b, 0xf4, 0xde, 0xc8, 0x74, 0x27, 0x32, 0x1c, 0x9c, 0x5f, 0x76, 0x2e, 0x33, 0xa9, 0x1a, 0x0a,
10340
+	0x35, 0xb8, 0x8c, 0x42, 0xa0, 0x48, 0xa5, 0x89, 0x3d, 0x43, 0x0f, 0x82, 0xf3, 0xfa, 0xf9, 0x5b,
10341
+	0x21, 0x4a, 0xb2, 0x68, 0x8c, 0x0d, 0x6f, 0xa0, 0xa4, 0xa6, 0x7b, 0xea, 0x3e, 0x5c, 0x6c, 0xbb,
10342
+	0xc6, 0x00, 0x8b, 0x23, 0x31, 0xa9, 0x5c, 0xf2, 0x6d, 0x58, 0xf3, 0xb1, 0x08, 0xea, 0x0d, 0x4c,
10343
+	0xcf, 0x67, 0xcf, 0x97, 0xe8, 0x1b, 0xd4, 0x66, 0xfc, 0x1e, 0x7f, 0x66, 0x14, 0xcf, 0x9e, 0xe4,
10344
+	0x0a, 0xc3, 0x6c, 0x09, 0x08, 0x09, 0x10, 0xdb, 0x0c, 0xa0, 0xfe, 0x08, 0x4a, 0x75, 0xd3, 0x1b,
10345
+	0xe9, 0x3e, 0xea, 0x96, 0x3d, 0x8c, 0x72, 0x07, 0x4a, 0x03, 0x8a, 0x35, 0xec, 0x21, 0xd5, 0x31,
10346
+	0x0f, 0x52, 0xd7, 0x74, 0xfa, 0x4f, 0xb4, 0x0f, 0x2b, 0xa1, 0x54, 0x87, 0x0b, 0xa9, 0x5f, 0x60,
10347
+	0xd6, 0x66, 0xef, 0x3a, 0x52, 0xef, 0x2b, 0x70, 0xc1, 0xb3, 0xf5, 0x91, 0x37, 0x70, 0xfc, 0x9e,
10348
+	0x69, 0xfb, 0xec, 0x21, 0xd3, 0x92, 0xf5, 0x6f, 0x29, 0x60, 0x34, 0x25, 0x1d, 0xe3, 0xb1, 0x72,
10349
+	0x42, 0xe9, 0xa8, 0xe7, 0x58, 0xfd, 0x5e, 0xc0, 0x14, 0x8f, 0x97, 0x88, 0x66, 0x9c, 0xb6, 0xd5,
10350
+	0xef, 0x06, 0x74, 0xa5, 0x0a, 0xeb, 0x96, 0x73, 0xdc, 0x43, 0xcb, 0x5c, 0xbc, 0xeb, 0xbd, 0x23,
10351
+	0xc7, 0xed, 0x79, 0x96, 0x73, 0x86, 0x1f, 0x16, 0xfe, 0x51, 0x37, 0x68, 0x2e, 0xca, 0x88, 0x6a,
10352
+	0x08, 0xd0, 0xa6, 0xe3, 0x76, 0x91, 0xb7, 0x19, 0x20, 0x58, 0x6a, 0x8f, 0xcc, 0xf6, 0x4d, 0xe3,
10353
+	0x24, 0x48, 0xed, 0x21, 0x75, 0x17, 0x89, 0x18, 0xdd, 0x96, 0xa8, 0x45, 0x0d, 0xbe, 0xc9, 0x1c,
10354
+	0x95, 0xe1, 0xa8, 0x62, 0x40, 0x64, 0x20, 0xf5, 0xdb, 0x90, 0xef, 0x58, 0xba, 0xc1, 0x9f, 0x88,
10355
+	0x59, 0xc5, 0x8f, 0x69, 0x8b, 0x9d, 0x1c, 0x5a, 0x2d, 0x42, 0x5a, 0x9e, 0x4c, 0x93, 0xd4, 0xf7,
10356
+	0x31, 0xd1, 0x13, 0xc7, 0xf1, 0xf1, 0x92, 0x5e, 0x83, 0xac, 0xa1, 0xf7, 0x02, 0x77, 0x2f, 0x56,
10357
+	0xf3, 0xe8, 0x16, 0x99, 0x9a, 0x76, 0x97, 0x4e, 0x48, 0xc6, 0xd0, 0xf1, 0x8f, 0xa5, 0x3c, 0x44,
10358
+	0x30, 0x27, 0xe5, 0xdb, 0x51, 0x14, 0x29, 0x0f, 0xbd, 0x18, 0x29, 0x04, 0x85, 0xd9, 0x3f, 0x46,
10359
+	0xb5, 0xa2, 0x04, 0xf5, 0x06, 0xba, 0x37, 0x10, 0x05, 0x62, 0x75, 0x19, 0x91, 0x20, 0x90, 0x5b,
10360
+	0x48, 0x25, 0x20, 0xd0, 0xec, 0x5b, 0xfd, 0x7b, 0x02, 0x0a, 0x6c, 0x60, 0x1e, 0x99, 0x06, 0xcb,
10361
+	0x2d, 0x4f, 0x1f, 0x17, 0x31, 0x30, 0x18, 0x9e, 0x2b, 0x17, 0xc5, 0x03, 0x43, 0xad, 0x4b, 0x08,
10362
+	0xa3, 0x29, 0xef, 0x60, 0x48, 0xe4, 0xb9, 0x4d, 0x86, 0x44, 0xf5, 0xcb, 0xb3, 0xa0, 0xec, 0x6c,
10363
+	0xa4, 0x1c, 0xdf, 0xc4, 0x68, 0x75, 0xfc, 0x68, 0x8a, 0x64, 0x9a, 0xc4, 0x9e, 0xb7, 0x0d, 0x9b,
10364
+	0x9f, 0x86, 0x7c, 0xde, 0xae, 0xb5, 0x08, 0x52, 0xd4, 0xbf, 0x26, 0x60, 0xa9, 0x61, 0x1b, 0xee,
10365
+	0x84, 0x87, 0x14, 0xb6, 0x83, 0x6b, 0x90, 0xc7, 0x02, 0xda, 0x9b, 0x78, 0xd8, 0x65, 0x06, 0xaf,
10366
+	0x67, 0x21, 0x41, 0x69, 0x42, 0x1e, 0x83, 0xa7, 0xe3, 0x9a, 0xfe, 0x60, 0x28, 0x2b, 0xc9, 0xf8,
10367
+	0x30, 0x36, 0xad, 0xb3, 0xa2, 0x05, 0x22, 0x24, 0x92, 0x0e, 0x02, 0x57, 0x8a, 0x2f, 0x96, 0x07,
10368
+	0x2e, 0xec, 0x79, 0x2d, 0x6c, 0x6f, 0xb0, 0x46, 0xec, 0xb1, 0xae, 0x81, 0xdb, 0x81, 0xed, 0x9f,
10369
+	0xa4, 0xb1, 0x4e, 0x48, 0x55, 0x21, 0x1f, 0x2a, 0x63, 0x6f, 0x96, 0x5a, 0xa3, 0xdb, 0xbb, 0xb9,
10370
+	0x71, 0xab, 0x77, 0xa7, 0xb6, 0x83, 0x61, 0x4c, 0xe4, 0xcd, 0x3f, 0xa1, 0x4d, 0x3b, 0xba, 0x8d,
10371
+	0x9d, 0x4d, 0xd0, 0xe7, 0xa1, 0x57, 0xb8, 0x78, 0xd5, 0x82, 0x42, 0x28, 0x2d, 0xbc, 0x82, 0xdd,
10372
+	0x3e, 0x56, 0x08, 0x31, 0x56, 0x7c, 0x21, 0x34, 0xf5, 0x76, 0x9b, 0x3a, 0xf7, 0xed, 0x36, 0xfd,
10373
+	0xb5, 0xbc, 0xdd, 0xbe, 0xfc, 0x45, 0x0a, 0xf2, 0x61, 0xdf, 0xc6, 0x5c, 0x86, 0xd5, 0x25, 0x0b,
10374
+	0xa2, 0xa3, 0x0e, 0xe9, 0x2d, 0x5e, 0x91, 0xe4, 0xb5, 0xed, 0xed, 0x76, 0x4d, 0x63, 0x4f, 0x9f,
10375
+	0xef, 0x88, 0xc2, 0x25, 0x04, 0x68, 0x78, 0x69, 0xd9, 0xa1, 0xf7, 0x15, 0x35, 0x2a, 0x5c, 0xee,
10376
+	0xcb, 0xbe, 0x3d, 0x44, 0x05, 0x55, 0xcb, 0x0b, 0x90, 0xd3, 0xba, 0xdd, 0xe6, 0x9d, 0x16, 0x6a,
10377
+	0x7a, 0x90, 0x28, 0x3f, 0x83, 0xa0, 0x0b, 0x91, 0x2a, 0x4c, 0xb8, 0xc7, 0x36, 0x6a, 0x62, 0xa8,
10378
+	0x5a, 0xad, 0xd1, 0x61, 0xf3, 0xdd, 0x4f, 0xce, 0xa3, 0x78, 0xba, 0xe6, 0x8f, 0x61, 0xf9, 0x0e,
10379
+	0x69, 0x74, 0x34, 0xc2, 0x66, 0x7c, 0x90, 0x9c, 0x5b, 0x57, 0xc7, 0xa5, 0xd8, 0xc9, 0xb3, 0x39,
10380
+	0xd7, 0x83, 0x77, 0xdc, 0xfb, 0xa9, 0xb2, 0x82, 0x98, 0xe5, 0xa8, 0x59, 0xc5, 0xfd, 0x9d, 0xb0,
10381
+	0xd9, 0xba, 0xbb, 0x1a, 0xd9, 0xe5, 0x6a, 0x52, 0x73, 0xb3, 0x75, 0xd9, 0x13, 0x02, 0xd3, 0x82,
10382
+	0xd6, 0x91, 0xbd, 0x56, 0x8b, 0x5b, 0x97, 0x9e, 0xb3, 0x8e, 0x8c, 0x6d, 0x9b, 0x61, 0xae, 0x63,
10383
+	0x0a, 0x6b, 0xef, 0x74, 0xb6, 0x1b, 0xbb, 0x8d, 0xd2, 0x83, 0xf4, 0xdc, 0x82, 0x6a, 0xce, 0x70,
10384
+	0x64, 0x51, 0x5f, 0x98, 0xd7, 0xdd, 0xda, 0xdb, 0xe5, 0xcf, 0xcc, 0xf7, 0x33, 0xf3, 0x13, 0x0e,
10385
+	0xc6, 0x7e, 0x9f, 0x95, 0x8a, 0xd7, 0xc2, 0xda, 0xed, 0x41, 0x46, 0xa4, 0xcc, 0x10, 0x23, 0x0a,
10386
+	0x37, 0xa6, 0x87, 0x34, 0x7e, 0x28, 0x5e, 0xa4, 0xef, 0x67, 0xe7, 0xf4, 0x10, 0xfa, 0x1e, 0x46,
10387
+	0x41, 0x2c, 0xef, 0xc2, 0x67, 0xa8, 0x90, 0xf5, 0xf2, 0x8f, 0x21, 0x17, 0x04, 0x0c, 0xdc, 0x9d,
10388
+	0xec, 0x41, 0x9b, 0xdc, 0x6d, 0x10, 0x3c, 0x7a, 0xbe, 0x3b, 0x01, 0xe7, 0xc0, 0x71, 0xd1, 0xbb,
10389
+	0x70, 0x19, 0x8b, 0x3b, 0x5a, 0x4b, 0xbb, 0x83, 0x00, 0xf9, 0xce, 0x15, 0x00, 0xa4, 0xd7, 0x97,
10390
+	0x4b, 0x72, 0x82, 0x50, 0x67, 0x75, 0xed, 0xe1, 0xa7, 0xeb, 0x0b, 0x1f, 0xe3, 0xef, 0xf3, 0x4f,
10391
+	0xd7, 0x13, 0xf7, 0x3f, 0x5b, 0x4f, 0x3c, 0xc4, 0xdf, 0x5f, 0xf0, 0xf7, 0x4f, 0xfc, 0x1d, 0x66,
10392
+	0x79, 0xfd, 0xf2, 0xc6, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x79, 0xcb, 0x88, 0x96, 0x53, 0x1d,
10393
+	0x00, 0x00,
10394 10394
 }
... ...
@@ -426,10 +426,10 @@ message IssuanceStatus {
426 426
 
427 427
 message AcceptancePolicy {
428 428
 	message RoleAdmissionPolicy {
429
-		message HashedSecret {
430
-			// The actual hashed content
429
+		message Secret {
430
+			// The actual content (possibly hashed)
431 431
 			bytes data = 1;
432
-			// The type of hash we are using
432
+			// The type of hash we are using, or "plaintext"
433 433
 			string alg = 2;
434 434
 		}
435 435
 
... ...
@@ -439,13 +439,12 @@ message AcceptancePolicy {
439 439
 		bool autoaccept = 2;
440 440
 		// Secret represents a user-provided string that is necessary for new
441 441
 		// nodes to join the cluster
442
-		HashedSecret secret = 3;
442
+		Secret secret = 3;
443 443
 	}
444 444
 
445 445
 	repeated RoleAdmissionPolicy policies = 1;
446 446
 }
447 447
 
448
-
449 448
 message ExternalCA {
450 449
 	enum CAProtocol {
451 450
 		CFSSL = 0 [(gogoproto.enumvalue_customname) = "CAProtocolCFSSL"];
... ...
@@ -1,6 +1,8 @@
1 1
 package ca
2 2
 
3 3
 import (
4
+	"crypto/subtle"
5
+	"errors"
4 6
 	"fmt"
5 7
 	"strings"
6 8
 	"sync"
... ...
@@ -255,7 +257,7 @@ func (s *Server) IssueNodeCertificate(ctx context.Context, request *api.IssueNod
255 255
 }
256 256
 
257 257
 // checkSecretValidity verifies if a secret string matches the secret hash stored in the
258
-// Acceptance Policy. It currently only supports bcrypted hashes.
258
+// Acceptance Policy. It currently only supports bcrypted hashes and plaintext.
259 259
 func checkSecretValidity(policy *api.AcceptancePolicy_RoleAdmissionPolicy, secret string) error {
260 260
 	if policy == nil || secret == "" {
261 261
 		return fmt.Errorf("invalid policy or secret")
... ...
@@ -264,6 +266,11 @@ func checkSecretValidity(policy *api.AcceptancePolicy_RoleAdmissionPolicy, secre
264 264
 	switch strings.ToLower(policy.Secret.Alg) {
265 265
 	case "bcrypt":
266 266
 		return bcrypt.CompareHashAndPassword(policy.Secret.Data, []byte(secret))
267
+	case "plaintext":
268
+		if subtle.ConstantTimeCompare(policy.Secret.Data, []byte(secret)) == 1 {
269
+			return nil
270
+		}
271
+		return errors.New("incorrect secret")
267 272
 	}
268 273
 
269 274
 	return fmt.Errorf("hash algorithm not supported: %s", policy.Secret.Alg)
... ...
@@ -16,9 +16,8 @@ var (
16 16
 // parameters for random identifier generation. We can tweak this when there is
17 17
 // time for further analysis.
18 18
 const (
19
-	randomIDEntropyBytes     = 16
20
-	randomNodeIDEntropyBytes = 8
21
-	randomIDBase             = 36
19
+	randomIDEntropyBytes = 16
20
+	randomIDBase         = 36
22 21
 
23 22
 	// To ensure that all identifiers are fixed length, we make sure they
24 23
 	// get padded out to 25 characters, which is the maximum for the base36
... ...
@@ -28,8 +27,7 @@ const (
28 28
 	// was calculated from floor(log(2^128-1, 36)) + 1.
29 29
 	//
30 30
 	// See http://mathworld.wolfram.com/NumberLength.html for more information.
31
-	maxRandomIDLength     = 25
32
-	maxRandomNodeIDLength = 13
31
+	maxRandomIDLength = 25
33 32
 )
34 33
 
35 34
 // NewID generates a new identifier for use where random identifiers with low
... ...
@@ -8,6 +8,8 @@ import (
8 8
 	"github.com/docker/libnetwork/drivers/overlay/ovmanager"
9 9
 	"github.com/docker/libnetwork/drvregistry"
10 10
 	"github.com/docker/libnetwork/ipamapi"
11
+	builtinIpam "github.com/docker/libnetwork/ipams/builtin"
12
+	nullIpam "github.com/docker/libnetwork/ipams/null"
11 13
 	"github.com/docker/swarmkit/api"
12 14
 	"github.com/docker/swarmkit/log"
13 15
 	"golang.org/x/net/context"
... ...
@@ -76,6 +78,15 @@ func New() (*NetworkAllocator, error) {
76 76
 		return nil, err
77 77
 	}
78 78
 
79
+	for _, fn := range [](func(ipamapi.Callback, interface{}, interface{}) error){
80
+		builtinIpam.Init,
81
+		nullIpam.Init,
82
+	} {
83
+		if err := fn(reg, nil, nil); err != nil {
84
+			return nil, err
85
+		}
86
+	}
87
+
79 88
 	pa, err := newPortAllocator()
80 89
 	if err != nil {
81 90
 		return nil, err
... ...
@@ -2,6 +2,7 @@ package controlapi
2 2
 
3 3
 import (
4 4
 	"github.com/docker/swarmkit/api"
5
+	"github.com/docker/swarmkit/manager/state/raft/membership"
5 6
 	"github.com/docker/swarmkit/manager/state/store"
6 7
 	"golang.org/x/net/context"
7 8
 	"google.golang.org/grpc"
... ...
@@ -188,8 +189,10 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest)
188 188
 
189 189
 	var (
190 190
 		node   *api.Node
191
+		member *membership.Member
191 192
 		demote bool
192 193
 	)
194
+
193 195
 	err := s.store.Update(func(tx store.Tx) error {
194 196
 		node = store.GetNode(tx, request.NodeID)
195 197
 		if node == nil {
... ...
@@ -199,6 +202,8 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest)
199 199
 		// Demotion sanity checks.
200 200
 		if node.Spec.Role == api.NodeRoleManager && request.Spec.Role == api.NodeRoleWorker {
201 201
 			demote = true
202
+
203
+			// Check for manager entries in Store.
202 204
 			managers, err := store.FindNodes(tx, store.ByRole(api.NodeRoleManager))
203 205
 			if err != nil {
204 206
 				return grpc.Errorf(codes.Internal, "internal store error: %v", err)
... ...
@@ -206,6 +211,16 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest)
206 206
 			if len(managers) == 1 && managers[0].ID == node.ID {
207 207
 				return grpc.Errorf(codes.FailedPrecondition, "attempting to demote the last manager of the swarm")
208 208
 			}
209
+
210
+			// Check for node in memberlist
211
+			if member = s.raft.GetMemberByNodeID(request.NodeID); member == nil {
212
+				return grpc.Errorf(codes.NotFound, "can't find manager in raft memberlist")
213
+			}
214
+
215
+			// Quorum safeguard
216
+			if !s.raft.CanRemoveMember(member.RaftID) {
217
+				return grpc.Errorf(codes.FailedPrecondition, "can't remove member from the raft: this would result in a loss of quorum")
218
+			}
209 219
 		}
210 220
 
211 221
 		node.Meta.Version = *request.NodeVersion
... ...
@@ -220,14 +235,12 @@ func (s *Server) UpdateNode(ctx context.Context, request *api.UpdateNodeRequest)
220 220
 	}
221 221
 
222 222
 	if demote && s.raft != nil {
223
-		memberlist := s.raft.GetMemberlist()
224
-		for raftID, member := range memberlist {
225
-			if member.NodeID == request.NodeID {
226
-				if err := s.raft.RemoveMember(ctx, raftID); err != nil {
227
-					return nil, err
228
-				}
229
-				break
230
-			}
223
+		// TODO(abronan): the remove can potentially fail and leave the node with
224
+		// an incorrect role (worker rather than manager), we need to reconcile the
225
+		// memberlist with the desired state rather than attempting to remove the
226
+		// member once.
227
+		if err := s.raft.RemoveMember(ctx, member.RaftID); err != nil {
228
+			return nil, grpc.Errorf(codes.Internal, "cannot demote manager to worker: %v", err)
231 229
 		}
232 230
 	}
233 231
 
... ...
@@ -7,6 +7,7 @@ import (
7 7
 	"github.com/docker/engine-api/types/reference"
8 8
 	"github.com/docker/swarmkit/api"
9 9
 	"github.com/docker/swarmkit/identity"
10
+	"github.com/docker/swarmkit/manager/scheduler"
10 11
 	"github.com/docker/swarmkit/manager/state/store"
11 12
 	"github.com/docker/swarmkit/protobuf/ptypes"
12 13
 	"golang.org/x/net/context"
... ...
@@ -75,6 +76,14 @@ func validateRestartPolicy(rp *api.RestartPolicy) error {
75 75
 	return nil
76 76
 }
77 77
 
78
+func validatePlacement(placement *api.Placement) error {
79
+	if placement == nil {
80
+		return nil
81
+	}
82
+	_, err := scheduler.ParseExprs(placement.Constraints)
83
+	return err
84
+}
85
+
78 86
 func validateUpdate(uc *api.UpdateConfig) error {
79 87
 	if uc == nil {
80 88
 		return nil
... ...
@@ -101,6 +110,10 @@ func validateTask(taskSpec api.TaskSpec) error {
101 101
 		return err
102 102
 	}
103 103
 
104
+	if err := validatePlacement(taskSpec.Placement); err != nil {
105
+		return err
106
+	}
107
+
104 108
 	if taskSpec.GetRuntime() == nil {
105 109
 		return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime")
106 110
 	}
... ...
@@ -328,14 +328,14 @@ func (d *Dispatcher) isRunning() bool {
328 328
 }
329 329
 
330 330
 // register is used for registration of node with particular dispatcher.
331
-func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, string, error) {
331
+func (d *Dispatcher) register(ctx context.Context, nodeID string, description *api.NodeDescription) (string, error) {
332 332
 	// prevent register until we're ready to accept it
333 333
 	if err := d.isRunningLocked(); err != nil {
334
-		return "", "", err
334
+		return "", err
335 335
 	}
336 336
 
337 337
 	if err := d.nodes.CheckRateLimit(nodeID); err != nil {
338
-		return "", "", err
338
+		return "", err
339 339
 	}
340 340
 
341 341
 	// create or update node in store
... ...
@@ -355,7 +355,7 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a
355 355
 
356 356
 	})
357 357
 	if err != nil {
358
-		return "", "", err
358
+		return "", err
359 359
 	}
360 360
 
361 361
 	expireFunc := func() {
... ...
@@ -377,7 +377,7 @@ func (d *Dispatcher) register(ctx context.Context, nodeID string, description *a
377 377
 	// time a node registers, we invalidate the session and issue a new
378 378
 	// session, once identity is proven. This will cause misbehaved agents to
379 379
 	// be kicked when multiple connections are made.
380
-	return rn.Node.ID, rn.SessionID, nil
380
+	return rn.SessionID, nil
381 381
 }
382 382
 
383 383
 // UpdateTaskStatus updates status of task. Node should send such updates
... ...
@@ -650,7 +650,7 @@ func (d *Dispatcher) Session(r *api.SessionRequest, stream api.Dispatcher_Sessio
650 650
 	}
651 651
 
652 652
 	// register the node.
653
-	nodeID, sessionID, err := d.register(stream.Context(), nodeID, r.Description)
653
+	sessionID, err := d.register(stream.Context(), nodeID, r.Description)
654 654
 	if err != nil {
655 655
 		return err
656 656
 	}
... ...
@@ -70,6 +70,15 @@ func (r *ReplicatedOrchestrator) resolveService(ctx context.Context, task *api.T
70 70
 	return service
71 71
 }
72 72
 
73
+type tasksByRunningState []*api.Task
74
+
75
+func (ts tasksByRunningState) Len() int      { return len(ts) }
76
+func (ts tasksByRunningState) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] }
77
+
78
+func (ts tasksByRunningState) Less(i, j int) bool {
79
+	return ts[i].Status.State == api.TaskStateRunning && ts[j].Status.State != api.TaskStateRunning
80
+}
81
+
73 82
 type taskWithIndex struct {
74 83
 	task *api.Task
75 84
 
... ...
@@ -139,6 +148,14 @@ func (r *ReplicatedOrchestrator) reconcile(ctx context.Context, service *api.Ser
139 139
 
140 140
 		// Preferentially remove tasks on the nodes that have the most
141 141
 		// copies of this service, to leave a more balanced result.
142
+
143
+		// First sort tasks such that tasks which are currently running
144
+		// (in terms of observed state) appear before non-running tasks.
145
+		// This will cause us to prefer to remove non-running tasks, all
146
+		// other things being equal in terms of node balance.
147
+
148
+		sort.Sort(tasksByRunningState(runningTasks))
149
+
142 150
 		// Assign each task an index that counts it as the nth copy of
143 151
 		// of the service on its node (1, 2, 3, ...), and sort the
144 152
 		// tasks by this counter value.
... ...
@@ -18,14 +18,19 @@ type ConstraintFilter struct {
18 18
 
19 19
 // SetTask returns true when the filter is enable for a given task.
20 20
 func (f *ConstraintFilter) SetTask(t *api.Task) bool {
21
-	if t.Spec.Placement != nil && len(t.Spec.Placement.Constraints) > 0 {
22
-		constraints, err := ParseExprs(t.Spec.Placement.Constraints)
23
-		if err == nil {
24
-			f.constraints = constraints
25
-			return true
26
-		}
21
+	if t.Spec.Placement == nil || len(t.Spec.Placement.Constraints) == 0 {
22
+		return false
23
+	}
24
+
25
+	constraints, err := ParseExprs(t.Spec.Placement.Constraints)
26
+	if err != nil {
27
+		// constraints have been validated at controlapi
28
+		// if in any case it finds an error here, treat this task
29
+		// as constraint filter disabled.
30
+		return false
27 31
 	}
28
-	return false
32
+	f.constraints = constraints
33
+	return true
29 34
 }
30 35
 
31 36
 // Check returns true if the task's constraint is supported by the given node.
... ...
@@ -165,19 +165,11 @@ func (c *Cluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
165 165
 // that might block or harm the Cluster on Member recovery
166 166
 func (c *Cluster) CanRemoveMember(from uint64, id uint64) bool {
167 167
 	members := c.Members()
168
-
169
-	nmembers := 0
170 168
 	nreachable := 0
171 169
 
172 170
 	for _, m := range members {
173
-		// Skip the node that is going to be deleted
174
-		if m.RaftID == id {
175
-			continue
176
-		}
177
-
178 171
 		// Local node from where the remove is issued
179 172
 		if m.RaftID == from {
180
-			nmembers++
181 173
 			nreachable++
182 174
 			continue
183 175
 		}
... ...
@@ -186,8 +178,6 @@ func (c *Cluster) CanRemoveMember(from uint64, id uint64) bool {
186 186
 		if err == nil && connState == grpc.Ready {
187 187
 			nreachable++
188 188
 		}
189
-
190
-		nmembers++
191 189
 	}
192 190
 
193 191
 	// Special case of 2 managers
... ...
@@ -195,7 +185,7 @@ func (c *Cluster) CanRemoveMember(from uint64, id uint64) bool {
195 195
 		return false
196 196
 	}
197 197
 
198
-	nquorum := nmembers/2 + 1
198
+	nquorum := (len(members)+1)/2 + 1
199 199
 	if nreachable < nquorum {
200 200
 		return false
201 201
 	}
... ...
@@ -94,6 +94,7 @@ type Node struct {
94 94
 	wal         *wal.WAL
95 95
 	snapshotter *snap.Snapshotter
96 96
 	wasLeader   bool
97
+	restored    bool
97 98
 	isMember    uint32
98 99
 	joinAddr    string
99 100
 
... ...
@@ -394,6 +395,18 @@ func (n *Node) Run(ctx context.Context) error {
394 394
 				}
395 395
 			}
396 396
 
397
+			// If we are the only registered member after
398
+			// restoring from the state, campaign to be the
399
+			// leader.
400
+			if !n.restored {
401
+				if len(n.cluster.Members()) <= 1 {
402
+					if err := n.Campaign(n.Ctx); err != nil {
403
+						panic("raft: cannot campaign to be the leader on node restore")
404
+					}
405
+				}
406
+				n.restored = true
407
+			}
408
+
397 409
 			// Advance the state machine
398 410
 			n.Advance()
399 411
 
... ...
@@ -638,7 +651,14 @@ func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResp
638 638
 	return &api.LeaveResponse{}, nil
639 639
 }
640 640
 
641
-// RemoveMember submits a configuration change to remove a member from the raft cluster.
641
+// CanRemoveMember checks if a member can be removed from
642
+// the context of the current node.
643
+func (n *Node) CanRemoveMember(id uint64) bool {
644
+	return n.cluster.CanRemoveMember(n.Config.ID, id)
645
+}
646
+
647
+// RemoveMember submits a configuration change to remove a member from the raft cluster
648
+// after checking if the operation would not result in a loss of quorum.
642 649
 func (n *Node) RemoveMember(ctx context.Context, id uint64) error {
643 650
 	n.membershipLock.Lock()
644 651
 	defer n.membershipLock.Unlock()
... ...
@@ -828,6 +848,18 @@ func (n *Node) GetMemberlist() map[uint64]*api.RaftMember {
828 828
 	return memberlist
829 829
 }
830 830
 
831
+// GetMemberByNodeID returns member information based
832
+// on its generic Node ID.
833
+func (n *Node) GetMemberByNodeID(nodeID string) *membership.Member {
834
+	members := n.cluster.Members()
835
+	for _, member := range members {
836
+		if member.NodeID == nodeID {
837
+			return member
838
+		}
839
+	}
840
+	return nil
841
+}
842
+
831 843
 // IsMember checks if the raft node has effectively joined
832 844
 // a cluster of existing members.
833 845
 func (n *Node) IsMember() bool {
... ...
@@ -1172,11 +1204,6 @@ func (n *Node) applyRemoveNode(cc raftpb.ConfChange) (err error) {
1172 1172
 		}
1173 1173
 	}
1174 1174
 
1175
-	// Do not unregister yourself
1176
-	if n.Config.ID == cc.NodeID {
1177
-		return nil
1178
-	}
1179
-
1180 1175
 	return n.cluster.RemoveMember(cc.NodeID)
1181 1176
 }
1182 1177
 
... ...
@@ -65,7 +65,6 @@ func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
65 65
 		return err
66 66
 	}
67 67
 
68
-	n.Node = raft.RestartNode(n.Config)
69 68
 	return nil
70 69
 }
71 70