Browse code

vendor: update swarmkit to bddd3f0

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

Tonis Tiigi authored on 2016/10/22 04:53:24
Showing 26 changed files
... ...
@@ -100,7 +100,7 @@ github.com/docker/containerd 8517738ba4b82aff5662c97ca4627e7e4d03b531
100 100
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
101 101
 
102 102
 # cluster
103
-github.com/docker/swarmkit 00890359d8bfba630824b66b848dbf7851149fef
103
+github.com/docker/swarmkit bddd3f0fb45491987d3dec5fb48311d289d21393
104 104
 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
105 105
 github.com/gogo/protobuf v0.3
106 106
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
... ...
@@ -89,6 +89,22 @@ func (m *GetRootCACertificateResponse) Reset()                    { *m = GetRoot
89 89
 func (*GetRootCACertificateResponse) ProtoMessage()               {}
90 90
 func (*GetRootCACertificateResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{5} }
91 91
 
92
+type GetUnlockKeyRequest struct {
93
+}
94
+
95
+func (m *GetUnlockKeyRequest) Reset()                    { *m = GetUnlockKeyRequest{} }
96
+func (*GetUnlockKeyRequest) ProtoMessage()               {}
97
+func (*GetUnlockKeyRequest) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{6} }
98
+
99
+type GetUnlockKeyResponse struct {
100
+	UnlockKey []byte  `protobuf:"bytes,1,opt,name=unlock_key,json=unlockKey,proto3" json:"unlock_key,omitempty"`
101
+	Version   Version `protobuf:"bytes,2,opt,name=version" json:"version"`
102
+}
103
+
104
+func (m *GetUnlockKeyResponse) Reset()                    { *m = GetUnlockKeyResponse{} }
105
+func (*GetUnlockKeyResponse) ProtoMessage()               {}
106
+func (*GetUnlockKeyResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{7} }
107
+
92 108
 func init() {
93 109
 	proto.RegisterType((*NodeCertificateStatusRequest)(nil), "docker.swarmkit.v1.NodeCertificateStatusRequest")
94 110
 	proto.RegisterType((*NodeCertificateStatusResponse)(nil), "docker.swarmkit.v1.NodeCertificateStatusResponse")
... ...
@@ -96,6 +112,8 @@ func init() {
96 96
 	proto.RegisterType((*IssueNodeCertificateResponse)(nil), "docker.swarmkit.v1.IssueNodeCertificateResponse")
97 97
 	proto.RegisterType((*GetRootCACertificateRequest)(nil), "docker.swarmkit.v1.GetRootCACertificateRequest")
98 98
 	proto.RegisterType((*GetRootCACertificateResponse)(nil), "docker.swarmkit.v1.GetRootCACertificateResponse")
99
+	proto.RegisterType((*GetUnlockKeyRequest)(nil), "docker.swarmkit.v1.GetUnlockKeyRequest")
100
+	proto.RegisterType((*GetUnlockKeyResponse)(nil), "docker.swarmkit.v1.GetUnlockKeyResponse")
99 101
 }
100 102
 
101 103
 type authenticatedWrapperCAServer struct {
... ...
@@ -115,6 +133,14 @@ func (p *authenticatedWrapperCAServer) GetRootCACertificate(ctx context.Context,
115 115
 	return p.local.GetRootCACertificate(ctx, r)
116 116
 }
117 117
 
118
+func (p *authenticatedWrapperCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) {
119
+
120
+	if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil {
121
+		return nil, err
122
+	}
123
+	return p.local.GetUnlockKey(ctx, r)
124
+}
125
+
118 126
 type authenticatedWrapperNodeCAServer struct {
119 127
 	local     NodeCAServer
120 128
 	authorize func(context.Context, []string) error
... ...
@@ -211,6 +237,29 @@ func (m *GetRootCACertificateResponse) Copy() *GetRootCACertificateResponse {
211 211
 	return o
212 212
 }
213 213
 
214
+func (m *GetUnlockKeyRequest) Copy() *GetUnlockKeyRequest {
215
+	if m == nil {
216
+		return nil
217
+	}
218
+
219
+	o := &GetUnlockKeyRequest{}
220
+
221
+	return o
222
+}
223
+
224
+func (m *GetUnlockKeyResponse) Copy() *GetUnlockKeyResponse {
225
+	if m == nil {
226
+		return nil
227
+	}
228
+
229
+	o := &GetUnlockKeyResponse{
230
+		UnlockKey: m.UnlockKey,
231
+		Version:   *m.Version.Copy(),
232
+	}
233
+
234
+	return o
235
+}
236
+
214 237
 func (this *NodeCertificateStatusRequest) GoString() string {
215 238
 	if this == nil {
216 239
 		return "nil"
... ...
@@ -278,6 +327,26 @@ func (this *GetRootCACertificateResponse) GoString() string {
278 278
 	s = append(s, "}")
279 279
 	return strings.Join(s, "")
280 280
 }
281
+func (this *GetUnlockKeyRequest) GoString() string {
282
+	if this == nil {
283
+		return "nil"
284
+	}
285
+	s := make([]string, 0, 4)
286
+	s = append(s, "&api.GetUnlockKeyRequest{")
287
+	s = append(s, "}")
288
+	return strings.Join(s, "")
289
+}
290
+func (this *GetUnlockKeyResponse) GoString() string {
291
+	if this == nil {
292
+		return "nil"
293
+	}
294
+	s := make([]string, 0, 6)
295
+	s = append(s, "&api.GetUnlockKeyResponse{")
296
+	s = append(s, "UnlockKey: "+fmt.Sprintf("%#v", this.UnlockKey)+",\n")
297
+	s = append(s, "Version: "+strings.Replace(this.Version.GoString(), `&`, ``, 1)+",\n")
298
+	s = append(s, "}")
299
+	return strings.Join(s, "")
300
+}
281 301
 func valueToGoStringCa(v interface{}, typ string) string {
282 302
 	rv := reflect.ValueOf(v)
283 303
 	if rv.IsNil() {
... ...
@@ -317,6 +386,9 @@ const _ = grpc.SupportPackageIsVersion3
317 317
 
318 318
 type CAClient interface {
319 319
 	GetRootCACertificate(ctx context.Context, in *GetRootCACertificateRequest, opts ...grpc.CallOption) (*GetRootCACertificateResponse, error)
320
+	// GetUnlockKey returns the current unlock key for the cluster for the role of the client
321
+	// asking.
322
+	GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error)
320 323
 }
321 324
 
322 325
 type cAClient struct {
... ...
@@ -336,10 +408,22 @@ func (c *cAClient) GetRootCACertificate(ctx context.Context, in *GetRootCACertif
336 336
 	return out, nil
337 337
 }
338 338
 
339
+func (c *cAClient) GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error) {
340
+	out := new(GetUnlockKeyResponse)
341
+	err := grpc.Invoke(ctx, "/docker.swarmkit.v1.CA/GetUnlockKey", in, out, c.cc, opts...)
342
+	if err != nil {
343
+		return nil, err
344
+	}
345
+	return out, nil
346
+}
347
+
339 348
 // Server API for CA service
340 349
 
341 350
 type CAServer interface {
342 351
 	GetRootCACertificate(context.Context, *GetRootCACertificateRequest) (*GetRootCACertificateResponse, error)
352
+	// GetUnlockKey returns the current unlock key for the cluster for the role of the client
353
+	// asking.
354
+	GetUnlockKey(context.Context, *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error)
343 355
 }
344 356
 
345 357
 func RegisterCAServer(s *grpc.Server, srv CAServer) {
... ...
@@ -364,6 +448,24 @@ func _CA_GetRootCACertificate_Handler(srv interface{}, ctx context.Context, dec
364 364
 	return interceptor(ctx, in, info, handler)
365 365
 }
366 366
 
367
+func _CA_GetUnlockKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
368
+	in := new(GetUnlockKeyRequest)
369
+	if err := dec(in); err != nil {
370
+		return nil, err
371
+	}
372
+	if interceptor == nil {
373
+		return srv.(CAServer).GetUnlockKey(ctx, in)
374
+	}
375
+	info := &grpc.UnaryServerInfo{
376
+		Server:     srv,
377
+		FullMethod: "/docker.swarmkit.v1.CA/GetUnlockKey",
378
+	}
379
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
380
+		return srv.(CAServer).GetUnlockKey(ctx, req.(*GetUnlockKeyRequest))
381
+	}
382
+	return interceptor(ctx, in, info, handler)
383
+}
384
+
367 385
 var _CA_serviceDesc = grpc.ServiceDesc{
368 386
 	ServiceName: "docker.swarmkit.v1.CA",
369 387
 	HandlerType: (*CAServer)(nil),
... ...
@@ -372,6 +474,10 @@ var _CA_serviceDesc = grpc.ServiceDesc{
372 372
 			MethodName: "GetRootCACertificate",
373 373
 			Handler:    _CA_GetRootCACertificate_Handler,
374 374
 		},
375
+		{
376
+			MethodName: "GetUnlockKey",
377
+			Handler:    _CA_GetUnlockKey_Handler,
378
+		},
375 379
 	},
376 380
 	Streams:  []grpc.StreamDesc{},
377 381
 	Metadata: fileDescriptorCa,
... ...
@@ -642,6 +748,56 @@ func (m *GetRootCACertificateResponse) MarshalTo(data []byte) (int, error) {
642 642
 	return i, nil
643 643
 }
644 644
 
645
+func (m *GetUnlockKeyRequest) Marshal() (data []byte, err error) {
646
+	size := m.Size()
647
+	data = make([]byte, size)
648
+	n, err := m.MarshalTo(data)
649
+	if err != nil {
650
+		return nil, err
651
+	}
652
+	return data[:n], nil
653
+}
654
+
655
+func (m *GetUnlockKeyRequest) MarshalTo(data []byte) (int, error) {
656
+	var i int
657
+	_ = i
658
+	var l int
659
+	_ = l
660
+	return i, nil
661
+}
662
+
663
+func (m *GetUnlockKeyResponse) Marshal() (data []byte, err error) {
664
+	size := m.Size()
665
+	data = make([]byte, size)
666
+	n, err := m.MarshalTo(data)
667
+	if err != nil {
668
+		return nil, err
669
+	}
670
+	return data[:n], nil
671
+}
672
+
673
+func (m *GetUnlockKeyResponse) MarshalTo(data []byte) (int, error) {
674
+	var i int
675
+	_ = i
676
+	var l int
677
+	_ = l
678
+	if len(m.UnlockKey) > 0 {
679
+		data[i] = 0xa
680
+		i++
681
+		i = encodeVarintCa(data, i, uint64(len(m.UnlockKey)))
682
+		i += copy(data[i:], m.UnlockKey)
683
+	}
684
+	data[i] = 0x12
685
+	i++
686
+	i = encodeVarintCa(data, i, uint64(m.Version.Size()))
687
+	n3, err := m.Version.MarshalTo(data[i:])
688
+	if err != nil {
689
+		return 0, err
690
+	}
691
+	i += n3
692
+	return i, nil
693
+}
694
+
645 695
 func encodeFixed64Ca(data []byte, offset int, v uint64) int {
646 696
 	data[offset] = uint8(v)
647 697
 	data[offset+1] = uint8(v >> 8)
... ...
@@ -767,6 +923,37 @@ func (p *raftProxyCAServer) GetRootCACertificate(ctx context.Context, r *GetRoot
767 767
 	return resp, err
768 768
 }
769 769
 
770
+func (p *raftProxyCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) {
771
+
772
+	conn, err := p.connSelector.LeaderConn(ctx)
773
+	if err != nil {
774
+		if err == raftselector.ErrIsLeader {
775
+			return p.local.GetUnlockKey(ctx, r)
776
+		}
777
+		return nil, err
778
+	}
779
+	modCtx, err := p.runCtxMods(ctx)
780
+	if err != nil {
781
+		return nil, err
782
+	}
783
+
784
+	resp, err := NewCAClient(conn).GetUnlockKey(modCtx, r)
785
+	if err != nil {
786
+		if !strings.Contains(err.Error(), "is closing") && !strings.Contains(err.Error(), "the connection is unavailable") && !strings.Contains(err.Error(), "connection error") {
787
+			return resp, err
788
+		}
789
+		conn, err := p.pollNewLeaderConn(ctx)
790
+		if err != nil {
791
+			if err == raftselector.ErrIsLeader {
792
+				return p.local.GetUnlockKey(ctx, r)
793
+			}
794
+			return nil, err
795
+		}
796
+		return NewCAClient(conn).GetUnlockKey(modCtx, r)
797
+	}
798
+	return resp, err
799
+}
800
+
770 801
 type raftProxyNodeCAServer struct {
771 802
 	local        NodeCAServer
772 803
 	connSelector raftselector.ConnProvider
... ...
@@ -965,6 +1152,24 @@ func (m *GetRootCACertificateResponse) Size() (n int) {
965 965
 	return n
966 966
 }
967 967
 
968
+func (m *GetUnlockKeyRequest) Size() (n int) {
969
+	var l int
970
+	_ = l
971
+	return n
972
+}
973
+
974
+func (m *GetUnlockKeyResponse) Size() (n int) {
975
+	var l int
976
+	_ = l
977
+	l = len(m.UnlockKey)
978
+	if l > 0 {
979
+		n += 1 + l + sovCa(uint64(l))
980
+	}
981
+	l = m.Version.Size()
982
+	n += 1 + l + sovCa(uint64(l))
983
+	return n
984
+}
985
+
968 986
 func sovCa(x uint64) (n int) {
969 987
 	for {
970 988
 		n++
... ...
@@ -1041,6 +1246,26 @@ func (this *GetRootCACertificateResponse) String() string {
1041 1041
 	}, "")
1042 1042
 	return s
1043 1043
 }
1044
+func (this *GetUnlockKeyRequest) String() string {
1045
+	if this == nil {
1046
+		return "nil"
1047
+	}
1048
+	s := strings.Join([]string{`&GetUnlockKeyRequest{`,
1049
+		`}`,
1050
+	}, "")
1051
+	return s
1052
+}
1053
+func (this *GetUnlockKeyResponse) String() string {
1054
+	if this == nil {
1055
+		return "nil"
1056
+	}
1057
+	s := strings.Join([]string{`&GetUnlockKeyResponse{`,
1058
+		`UnlockKey:` + fmt.Sprintf("%v", this.UnlockKey) + `,`,
1059
+		`Version:` + strings.Replace(strings.Replace(this.Version.String(), "Version", "Version", 1), `&`, ``, 1) + `,`,
1060
+		`}`,
1061
+	}, "")
1062
+	return s
1063
+}
1044 1064
 func valueToStringCa(v interface{}) string {
1045 1065
 	rv := reflect.ValueOf(v)
1046 1066
 	if rv.IsNil() {
... ...
@@ -1602,6 +1827,167 @@ func (m *GetRootCACertificateResponse) Unmarshal(data []byte) error {
1602 1602
 	}
1603 1603
 	return nil
1604 1604
 }
1605
+func (m *GetUnlockKeyRequest) Unmarshal(data []byte) error {
1606
+	l := len(data)
1607
+	iNdEx := 0
1608
+	for iNdEx < l {
1609
+		preIndex := iNdEx
1610
+		var wire uint64
1611
+		for shift := uint(0); ; shift += 7 {
1612
+			if shift >= 64 {
1613
+				return ErrIntOverflowCa
1614
+			}
1615
+			if iNdEx >= l {
1616
+				return io.ErrUnexpectedEOF
1617
+			}
1618
+			b := data[iNdEx]
1619
+			iNdEx++
1620
+			wire |= (uint64(b) & 0x7F) << shift
1621
+			if b < 0x80 {
1622
+				break
1623
+			}
1624
+		}
1625
+		fieldNum := int32(wire >> 3)
1626
+		wireType := int(wire & 0x7)
1627
+		if wireType == 4 {
1628
+			return fmt.Errorf("proto: GetUnlockKeyRequest: wiretype end group for non-group")
1629
+		}
1630
+		if fieldNum <= 0 {
1631
+			return fmt.Errorf("proto: GetUnlockKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire)
1632
+		}
1633
+		switch fieldNum {
1634
+		default:
1635
+			iNdEx = preIndex
1636
+			skippy, err := skipCa(data[iNdEx:])
1637
+			if err != nil {
1638
+				return err
1639
+			}
1640
+			if skippy < 0 {
1641
+				return ErrInvalidLengthCa
1642
+			}
1643
+			if (iNdEx + skippy) > l {
1644
+				return io.ErrUnexpectedEOF
1645
+			}
1646
+			iNdEx += skippy
1647
+		}
1648
+	}
1649
+
1650
+	if iNdEx > l {
1651
+		return io.ErrUnexpectedEOF
1652
+	}
1653
+	return nil
1654
+}
1655
+func (m *GetUnlockKeyResponse) Unmarshal(data []byte) error {
1656
+	l := len(data)
1657
+	iNdEx := 0
1658
+	for iNdEx < l {
1659
+		preIndex := iNdEx
1660
+		var wire uint64
1661
+		for shift := uint(0); ; shift += 7 {
1662
+			if shift >= 64 {
1663
+				return ErrIntOverflowCa
1664
+			}
1665
+			if iNdEx >= l {
1666
+				return io.ErrUnexpectedEOF
1667
+			}
1668
+			b := data[iNdEx]
1669
+			iNdEx++
1670
+			wire |= (uint64(b) & 0x7F) << shift
1671
+			if b < 0x80 {
1672
+				break
1673
+			}
1674
+		}
1675
+		fieldNum := int32(wire >> 3)
1676
+		wireType := int(wire & 0x7)
1677
+		if wireType == 4 {
1678
+			return fmt.Errorf("proto: GetUnlockKeyResponse: wiretype end group for non-group")
1679
+		}
1680
+		if fieldNum <= 0 {
1681
+			return fmt.Errorf("proto: GetUnlockKeyResponse: illegal tag %d (wire type %d)", fieldNum, wire)
1682
+		}
1683
+		switch fieldNum {
1684
+		case 1:
1685
+			if wireType != 2 {
1686
+				return fmt.Errorf("proto: wrong wireType = %d for field UnlockKey", wireType)
1687
+			}
1688
+			var byteLen int
1689
+			for shift := uint(0); ; shift += 7 {
1690
+				if shift >= 64 {
1691
+					return ErrIntOverflowCa
1692
+				}
1693
+				if iNdEx >= l {
1694
+					return io.ErrUnexpectedEOF
1695
+				}
1696
+				b := data[iNdEx]
1697
+				iNdEx++
1698
+				byteLen |= (int(b) & 0x7F) << shift
1699
+				if b < 0x80 {
1700
+					break
1701
+				}
1702
+			}
1703
+			if byteLen < 0 {
1704
+				return ErrInvalidLengthCa
1705
+			}
1706
+			postIndex := iNdEx + byteLen
1707
+			if postIndex > l {
1708
+				return io.ErrUnexpectedEOF
1709
+			}
1710
+			m.UnlockKey = append(m.UnlockKey[:0], data[iNdEx:postIndex]...)
1711
+			if m.UnlockKey == nil {
1712
+				m.UnlockKey = []byte{}
1713
+			}
1714
+			iNdEx = postIndex
1715
+		case 2:
1716
+			if wireType != 2 {
1717
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
1718
+			}
1719
+			var msglen int
1720
+			for shift := uint(0); ; shift += 7 {
1721
+				if shift >= 64 {
1722
+					return ErrIntOverflowCa
1723
+				}
1724
+				if iNdEx >= l {
1725
+					return io.ErrUnexpectedEOF
1726
+				}
1727
+				b := data[iNdEx]
1728
+				iNdEx++
1729
+				msglen |= (int(b) & 0x7F) << shift
1730
+				if b < 0x80 {
1731
+					break
1732
+				}
1733
+			}
1734
+			if msglen < 0 {
1735
+				return ErrInvalidLengthCa
1736
+			}
1737
+			postIndex := iNdEx + msglen
1738
+			if postIndex > l {
1739
+				return io.ErrUnexpectedEOF
1740
+			}
1741
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
1742
+				return err
1743
+			}
1744
+			iNdEx = postIndex
1745
+		default:
1746
+			iNdEx = preIndex
1747
+			skippy, err := skipCa(data[iNdEx:])
1748
+			if err != nil {
1749
+				return err
1750
+			}
1751
+			if skippy < 0 {
1752
+				return ErrInvalidLengthCa
1753
+			}
1754
+			if (iNdEx + skippy) > l {
1755
+				return io.ErrUnexpectedEOF
1756
+			}
1757
+			iNdEx += skippy
1758
+		}
1759
+	}
1760
+
1761
+	if iNdEx > l {
1762
+		return io.ErrUnexpectedEOF
1763
+	}
1764
+	return nil
1765
+}
1605 1766
 func skipCa(data []byte) (n int, err error) {
1606 1767
 	l := len(data)
1607 1768
 	iNdEx := 0
... ...
@@ -1710,36 +2096,42 @@ var (
1710 1710
 func init() { proto.RegisterFile("ca.proto", fileDescriptorCa) }
1711 1711
 
1712 1712
 var fileDescriptorCa = []byte{
1713
-	// 493 bytes of a gzipped FileDescriptorProto
1714
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x94, 0xcf, 0x6e, 0xd3, 0x40,
1715
-	0x10, 0xc6, 0xbb, 0x0e, 0xa4, 0x65, 0x52, 0x05, 0xb4, 0x04, 0x29, 0xa4, 0xa9, 0x53, 0x99, 0x03,
1716
-	0x9c, 0x9c, 0xd6, 0x70, 0xe2, 0x44, 0x62, 0x24, 0x94, 0x03, 0x08, 0x6d, 0x1e, 0x00, 0xb9, 0xf6,
1717
-	0x10, 0xac, 0x24, 0x5e, 0xe3, 0xdd, 0x80, 0xb8, 0x21, 0x81, 0x38, 0x70, 0x47, 0x70, 0xe2, 0x11,
1718
-	0x78, 0x8e, 0x8a, 0x13, 0x47, 0x4e, 0x15, 0xf1, 0x03, 0x20, 0x1e, 0x01, 0xed, 0xda, 0x21, 0xfd,
1719
-	0xb3, 0x89, 0xca, 0xc9, 0x3b, 0xb3, 0xf3, 0x7d, 0xfe, 0xed, 0x8c, 0xd7, 0xb0, 0x15, 0x06, 0x6e,
1720
-	0x9a, 0x71, 0xc9, 0x29, 0x8d, 0x78, 0x38, 0xc6, 0xcc, 0x15, 0xaf, 0x83, 0x6c, 0x3a, 0x8e, 0xa5,
1721
-	0xfb, 0xea, 0xa0, 0x55, 0x93, 0x6f, 0x52, 0x14, 0x45, 0x41, 0xab, 0x26, 0x52, 0x0c, 0x17, 0x41,
1722
-	0x63, 0xc4, 0x47, 0x5c, 0x2f, 0xbb, 0x6a, 0x55, 0x66, 0xaf, 0xa7, 0x93, 0xd9, 0x28, 0x4e, 0xba,
1723
-	0xc5, 0xa3, 0x48, 0x3a, 0x3e, 0xb4, 0x9f, 0xf0, 0x08, 0x7d, 0xcc, 0x64, 0xfc, 0x3c, 0x0e, 0x03,
1724
-	0x89, 0x43, 0x19, 0xc8, 0x99, 0x60, 0xf8, 0x72, 0x86, 0x42, 0xd2, 0x5b, 0xb0, 0x99, 0xf0, 0x08,
1725
-	0x9f, 0xc5, 0x51, 0x93, 0xec, 0x91, 0x3b, 0x57, 0xfa, 0x90, 0x1f, 0x77, 0xaa, 0x4a, 0x32, 0x78,
1726
-	0xc8, 0xaa, 0x6a, 0x6b, 0x10, 0x39, 0x5f, 0x09, 0xec, 0xae, 0x70, 0x11, 0x29, 0x4f, 0x04, 0xd2,
1727
-	0xfb, 0x50, 0x15, 0x3a, 0xa3, 0x5d, 0x6a, 0x9e, 0xe3, 0x9e, 0x3f, 0x90, 0x3b, 0x10, 0x62, 0x16,
1728
-	0x24, 0xe1, 0x42, 0x5b, 0x2a, 0x68, 0x0f, 0x6a, 0xe1, 0xd2, 0xb8, 0x69, 0x69, 0x83, 0x8e, 0xc9,
1729
-	0xe0, 0xc4, 0xfb, 0xd9, 0x49, 0x8d, 0xf3, 0x9e, 0xc0, 0x8e, 0x72, 0xc7, 0x33, 0x94, 0x8b, 0x53,
1730
-	0xde, 0x83, 0x4b, 0x19, 0x9f, 0xa0, 0x86, 0xab, 0x7b, 0x6d, 0x93, 0xb7, 0x52, 0x32, 0x3e, 0xc1,
1731
-	0xbe, 0xd5, 0x24, 0x4c, 0x57, 0xd3, 0x9b, 0x50, 0x09, 0x45, 0xa6, 0x81, 0xb6, 0xfb, 0x9b, 0xf9,
1732
-	0x71, 0xa7, 0xe2, 0x0f, 0x19, 0x53, 0x39, 0xda, 0x80, 0xcb, 0x92, 0x8f, 0x31, 0x69, 0x56, 0x54,
1733
-	0xd3, 0x58, 0x11, 0x38, 0x9f, 0x08, 0xb4, 0xcd, 0x18, 0x65, 0x9b, 0x2e, 0xd2, 0x6d, 0xfa, 0x14,
1734
-	0xae, 0xea, 0xa2, 0x29, 0x4e, 0x0f, 0x31, 0x13, 0x2f, 0xe2, 0x54, 0x23, 0xd4, 0xbd, 0xdb, 0xab,
1735
-	0xb8, 0x87, 0x29, 0x86, 0xee, 0xe3, 0x7f, 0xe5, 0xac, 0xae, 0xf4, 0xcb, 0xd8, 0xd9, 0x85, 0x9d,
1736
-	0x47, 0x28, 0x19, 0xe7, 0xd2, 0xef, 0x9d, 0xef, 0x8e, 0xf3, 0x00, 0xda, 0xe6, 0xed, 0x92, 0x7a,
1737
-	0xef, 0xf4, 0x80, 0x14, 0xf9, 0xf6, 0xa9, 0xfe, 0x7b, 0x1f, 0x09, 0x58, 0x7e, 0x8f, 0xbe, 0x23,
1738
-	0xd0, 0x30, 0x39, 0xd1, 0xae, 0x89, 0x7c, 0x0d, 0x52, 0x6b, 0xff, 0xe2, 0x82, 0x02, 0xd2, 0xd9,
1739
-	0xfa, 0xfe, 0xed, 0xf7, 0x17, 0xcb, 0xba, 0x46, 0xbc, 0xcf, 0x16, 0xe8, 0x96, 0x96, 0x40, 0xa6,
1740
-	0x81, 0x98, 0x81, 0xd6, 0x7c, 0x41, 0x66, 0xa0, 0x75, 0xb3, 0x5e, 0x02, 0xd1, 0x0f, 0x04, 0x6e,
1741
-	0x18, 0xaf, 0x0f, 0xdd, 0x5f, 0x35, 0xd1, 0x55, 0xf7, 0xb5, 0x75, 0xf0, 0x1f, 0x8a, 0xb3, 0x20,
1742
-	0xfd, 0xf6, 0xd1, 0xdc, 0xde, 0xf8, 0x39, 0xb7, 0x37, 0xfe, 0xcc, 0x6d, 0xf2, 0x36, 0xb7, 0xc9,
1743
-	0x51, 0x6e, 0x93, 0x1f, 0xb9, 0x4d, 0x7e, 0xe5, 0x36, 0x39, 0xac, 0xea, 0x3f, 0xc6, 0xdd, 0xbf,
1744
-	0x01, 0x00, 0x00, 0xff, 0xff, 0xb3, 0xf8, 0x41, 0xef, 0x96, 0x04, 0x00, 0x00,
1713
+	// 586 bytes of a gzipped FileDescriptorProto
1714
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcb, 0x6e, 0xd3, 0x40,
1715
+	0x14, 0xcd, 0x38, 0x25, 0x69, 0x6f, 0x42, 0x8a, 0xa6, 0x89, 0x14, 0xf2, 0x70, 0x2a, 0xb3, 0x68,
1716
+	0x37, 0x38, 0x6d, 0x60, 0x05, 0x1b, 0x92, 0x20, 0x55, 0x11, 0x02, 0x21, 0x47, 0xb0, 0xad, 0x5c,
1717
+	0x67, 0x08, 0x56, 0x12, 0x8f, 0xf1, 0x8c, 0x0b, 0xd9, 0x21, 0x51, 0xf1, 0x07, 0x08, 0x56, 0x7c,
1718
+	0x02, 0xdf, 0x11, 0xb1, 0x62, 0xc9, 0x2a, 0x22, 0xfe, 0x00, 0xc4, 0x27, 0x20, 0x8f, 0x6d, 0x9a,
1719
+	0x87, 0x13, 0xda, 0x55, 0x3c, 0xd7, 0xe7, 0x9c, 0x7b, 0xee, 0xc9, 0xf5, 0xc0, 0xb6, 0xa1, 0xab,
1720
+	0xb6, 0x43, 0x39, 0xc5, 0xb8, 0x47, 0x8d, 0x01, 0x71, 0x54, 0xf6, 0x56, 0x77, 0x46, 0x03, 0x93,
1721
+	0xab, 0xe7, 0xc7, 0xa5, 0x0c, 0x1f, 0xdb, 0x84, 0x05, 0x80, 0x52, 0x86, 0xd9, 0xc4, 0x88, 0x0e,
1722
+	0xf9, 0x3e, 0xed, 0x53, 0xf1, 0x58, 0xf7, 0x9f, 0xc2, 0xea, 0x9e, 0x3d, 0x74, 0xfb, 0xa6, 0x55,
1723
+	0x0f, 0x7e, 0x82, 0xa2, 0xd2, 0x86, 0xca, 0x33, 0xda, 0x23, 0x6d, 0xe2, 0x70, 0xf3, 0x95, 0x69,
1724
+	0xe8, 0x9c, 0x74, 0xb9, 0xce, 0x5d, 0xa6, 0x91, 0x37, 0x2e, 0x61, 0x1c, 0xdf, 0x81, 0xb4, 0x45,
1725
+	0x7b, 0xe4, 0xd4, 0xec, 0x15, 0xd1, 0x3e, 0x3a, 0xdc, 0x69, 0x81, 0x37, 0xad, 0xa5, 0x7c, 0x4a,
1726
+	0xe7, 0xb1, 0x96, 0xf2, 0x5f, 0x75, 0x7a, 0xca, 0x57, 0x04, 0xd5, 0x35, 0x2a, 0xcc, 0xa6, 0x16,
1727
+	0x23, 0xf8, 0x01, 0xa4, 0x98, 0xa8, 0x08, 0x95, 0x4c, 0x43, 0x51, 0x57, 0x07, 0x52, 0x3b, 0x8c,
1728
+	0xb9, 0xba, 0x65, 0x44, 0xdc, 0x90, 0x81, 0x9b, 0x90, 0x31, 0x2e, 0x85, 0x8b, 0x92, 0x10, 0xa8,
1729
+	0xc5, 0x09, 0xcc, 0xf5, 0xd7, 0xe6, 0x39, 0xca, 0x05, 0x82, 0xb2, 0xaf, 0x4e, 0x96, 0x5c, 0x46,
1730
+	0x53, 0xde, 0x87, 0x2d, 0x87, 0x0e, 0x89, 0x30, 0x97, 0x6b, 0x54, 0xe2, 0xb4, 0x7d, 0xa6, 0x46,
1731
+	0x87, 0xa4, 0x25, 0x15, 0x91, 0x26, 0xd0, 0xf8, 0x36, 0x24, 0x0d, 0xe6, 0x08, 0x43, 0xd9, 0x56,
1732
+	0xda, 0x9b, 0xd6, 0x92, 0xed, 0xae, 0xa6, 0xf9, 0x35, 0x9c, 0x87, 0x1b, 0x9c, 0x0e, 0x88, 0x55,
1733
+	0x4c, 0xfa, 0xa1, 0x69, 0xc1, 0x41, 0xf9, 0x84, 0xa0, 0x12, 0x6f, 0x23, 0x8c, 0xe9, 0x2a, 0x69,
1734
+	0xe3, 0xe7, 0xb0, 0x2b, 0x40, 0x23, 0x32, 0x3a, 0x23, 0x0e, 0x7b, 0x6d, 0xda, 0xc2, 0x42, 0xae,
1735
+	0x71, 0xb0, 0xce, 0x77, 0xd7, 0x26, 0x86, 0xfa, 0xf4, 0x1f, 0x5c, 0xcb, 0xf9, 0xfc, 0xcb, 0xb3,
1736
+	0x52, 0x85, 0xf2, 0x09, 0xe1, 0x1a, 0xa5, 0xbc, 0xdd, 0x5c, 0x4d, 0x47, 0x79, 0x04, 0x95, 0xf8,
1737
+	0xd7, 0xa1, 0xeb, 0xfd, 0xc5, 0x3f, 0xc8, 0x77, 0x9e, 0x5d, 0xcc, 0xbf, 0x00, 0x7b, 0x27, 0x84,
1738
+	0xbf, 0xb0, 0x86, 0xd4, 0x18, 0x3c, 0x21, 0xe3, 0x48, 0xd8, 0x81, 0xfc, 0x62, 0x39, 0x14, 0xac,
1739
+	0x02, 0xb8, 0xa2, 0x78, 0x3a, 0x20, 0xe3, 0x50, 0x6f, 0xc7, 0x8d, 0x60, 0xf8, 0x21, 0xa4, 0xcf,
1740
+	0x89, 0xc3, 0x4c, 0x6a, 0x85, 0xcb, 0x50, 0x8e, 0x1b, 0xfc, 0x65, 0x00, 0x69, 0x6d, 0x4d, 0xa6,
1741
+	0xb5, 0x84, 0x16, 0x31, 0x1a, 0x17, 0x12, 0x48, 0xed, 0x26, 0xfe, 0x80, 0x44, 0xef, 0x95, 0xa1,
1742
+	0x70, 0x3d, 0x4e, 0x6b, 0x43, 0x3a, 0xa5, 0xa3, 0xab, 0x13, 0x82, 0xf1, 0x94, 0xed, 0xef, 0xdf,
1743
+	0x7e, 0x7f, 0x91, 0xa4, 0x5b, 0x08, 0xbf, 0x83, 0xec, 0x7c, 0x00, 0xf8, 0x60, 0x8d, 0xd6, 0x72,
1744
+	0x72, 0xa5, 0xc3, 0xff, 0x03, 0xc3, 0x66, 0x05, 0xd1, 0x6c, 0x17, 0x6e, 0x0a, 0xe4, 0xdd, 0x91,
1745
+	0x6e, 0xe9, 0x7d, 0xe2, 0x34, 0x3e, 0x4b, 0x20, 0xf6, 0x2a, 0x8c, 0x22, 0x6e, 0x2b, 0xe3, 0xa3,
1746
+	0xd8, 0xf0, 0x19, 0xc5, 0x47, 0xb1, 0x69, 0xe1, 0xe7, 0xa2, 0xf8, 0x88, 0xa0, 0x10, 0x7b, 0x87,
1747
+	0xe0, 0xa3, 0x75, 0x6b, 0xbd, 0xee, 0xd2, 0x2a, 0x1d, 0x5f, 0x83, 0xb1, 0x6c, 0xa4, 0x55, 0x99,
1748
+	0xcc, 0xe4, 0xc4, 0xcf, 0x99, 0x9c, 0xf8, 0x33, 0x93, 0xd1, 0x7b, 0x4f, 0x46, 0x13, 0x4f, 0x46,
1749
+	0x3f, 0x3c, 0x19, 0xfd, 0xf2, 0x64, 0x74, 0x96, 0x12, 0xd7, 0xe6, 0xbd, 0xbf, 0x01, 0x00, 0x00,
1750
+	0xff, 0xff, 0xe7, 0x80, 0x3b, 0x00, 0x9b, 0x05, 0x00, 0x00,
1745 1751
 }
... ...
@@ -13,6 +13,11 @@ service CA {
13 13
 	rpc GetRootCACertificate(GetRootCACertificateRequest) returns (GetRootCACertificateResponse) {
14 14
 		option (docker.protobuf.plugin.tls_authorization) = { insecure: true };
15 15
 	};
16
+	// GetUnlockKey returns the current unlock key for the cluster for the role of the client
17
+	// asking.
18
+	rpc GetUnlockKey(GetUnlockKeyRequest) returns (GetUnlockKeyResponse) {
19
+		option (docker.protobuf.plugin.tls_authorization) = { roles: ["swarm-manager"] };
20
+	};
16 21
 }
17 22
 
18 23
 service NodeCA {
... ...
@@ -55,3 +60,10 @@ message GetRootCACertificateRequest {}
55 55
 message GetRootCACertificateResponse {
56 56
 	bytes certificate = 1;
57 57
 }
58
+
59
+message GetUnlockKeyRequest {}
60
+
61
+message GetUnlockKeyResponse {
62
+	bytes unlock_key = 1;
63
+	Version version = 2 [(gogoproto.nullable) = false];
64
+}
... ...
@@ -405,16 +405,19 @@ func (m *ListClustersResponse) Reset()                    { *m = ListClustersRes
405 405
 func (*ListClustersResponse) ProtoMessage()               {}
406 406
 func (*ListClustersResponse) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{35} }
407 407
 
408
-type JoinTokenRotation struct {
409
-	// RotateWorkerToken tells UpdateCluster to rotate the worker secret.
410
-	RotateWorkerToken bool `protobuf:"varint,1,opt,name=rotate_worker_token,json=rotateWorkerToken,proto3" json:"rotate_worker_token,omitempty"`
411
-	// RotateManagerSecret tells UpdateCluster to rotate the manager secret.
412
-	RotateManagerToken bool `protobuf:"varint,2,opt,name=rotate_manager_token,json=rotateManagerToken,proto3" json:"rotate_manager_token,omitempty"`
408
+// KeyRotation tells UpdateCluster what items to rotate
409
+type KeyRotation struct {
410
+	// WorkerJoinToken tells UpdateCluster to rotate the worker secret token.
411
+	WorkerJoinToken bool `protobuf:"varint,1,opt,name=worker_join_token,json=workerJoinToken,proto3" json:"worker_join_token,omitempty"`
412
+	// ManagerJoinToken tells UpdateCluster to rotate the manager secret token.
413
+	ManagerJoinToken bool `protobuf:"varint,2,opt,name=manager_join_token,json=managerJoinToken,proto3" json:"manager_join_token,omitempty"`
414
+	// ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key
415
+	ManagerUnlockKey bool `protobuf:"varint,3,opt,name=manager_unlock_key,json=managerUnlockKey,proto3" json:"manager_unlock_key,omitempty"`
413 416
 }
414 417
 
415
-func (m *JoinTokenRotation) Reset()                    { *m = JoinTokenRotation{} }
416
-func (*JoinTokenRotation) ProtoMessage()               {}
417
-func (*JoinTokenRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} }
418
+func (m *KeyRotation) Reset()                    { *m = KeyRotation{} }
419
+func (*KeyRotation) ProtoMessage()               {}
420
+func (*KeyRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} }
418 421
 
419 422
 type UpdateClusterRequest struct {
420 423
 	// ClusterID is the cluster ID to update.
... ...
@@ -423,8 +426,8 @@ type UpdateClusterRequest struct {
423 423
 	ClusterVersion *Version `protobuf:"bytes,2,opt,name=cluster_version,json=clusterVersion" json:"cluster_version,omitempty"`
424 424
 	// Spec is the new spec to apply to the cluster.
425 425
 	Spec *ClusterSpec `protobuf:"bytes,3,opt,name=spec" json:"spec,omitempty"`
426
-	// Rotation contains flags for join token rotation
427
-	Rotation JoinTokenRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"`
426
+	// Rotation contains flags for join token and unlock key rotation
427
+	Rotation KeyRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"`
428 428
 }
429 429
 
430 430
 func (m *UpdateClusterRequest) Reset()                    { *m = UpdateClusterRequest{} }
... ...
@@ -598,7 +601,7 @@ func init() {
598 598
 	proto.RegisterType((*ListClustersRequest)(nil), "docker.swarmkit.v1.ListClustersRequest")
599 599
 	proto.RegisterType((*ListClustersRequest_Filters)(nil), "docker.swarmkit.v1.ListClustersRequest.Filters")
600 600
 	proto.RegisterType((*ListClustersResponse)(nil), "docker.swarmkit.v1.ListClustersResponse")
601
-	proto.RegisterType((*JoinTokenRotation)(nil), "docker.swarmkit.v1.JoinTokenRotation")
601
+	proto.RegisterType((*KeyRotation)(nil), "docker.swarmkit.v1.KeyRotation")
602 602
 	proto.RegisterType((*UpdateClusterRequest)(nil), "docker.swarmkit.v1.UpdateClusterRequest")
603 603
 	proto.RegisterType((*UpdateClusterResponse)(nil), "docker.swarmkit.v1.UpdateClusterResponse")
604 604
 	proto.RegisterType((*GetSecretRequest)(nil), "docker.swarmkit.v1.GetSecretRequest")
... ...
@@ -1459,14 +1462,15 @@ func (m *ListClustersResponse) Copy() *ListClustersResponse {
1459 1459
 	return o
1460 1460
 }
1461 1461
 
1462
-func (m *JoinTokenRotation) Copy() *JoinTokenRotation {
1462
+func (m *KeyRotation) Copy() *KeyRotation {
1463 1463
 	if m == nil {
1464 1464
 		return nil
1465 1465
 	}
1466 1466
 
1467
-	o := &JoinTokenRotation{
1468
-		RotateWorkerToken:  m.RotateWorkerToken,
1469
-		RotateManagerToken: m.RotateManagerToken,
1467
+	o := &KeyRotation{
1468
+		WorkerJoinToken:  m.WorkerJoinToken,
1469
+		ManagerJoinToken: m.ManagerJoinToken,
1470
+		ManagerUnlockKey: m.ManagerUnlockKey,
1470 1471
 	}
1471 1472
 
1472 1473
 	return o
... ...
@@ -2199,14 +2203,15 @@ func (this *ListClustersResponse) GoString() string {
2199 2199
 	s = append(s, "}")
2200 2200
 	return strings.Join(s, "")
2201 2201
 }
2202
-func (this *JoinTokenRotation) GoString() string {
2202
+func (this *KeyRotation) GoString() string {
2203 2203
 	if this == nil {
2204 2204
 		return "nil"
2205 2205
 	}
2206
-	s := make([]string, 0, 6)
2207
-	s = append(s, "&api.JoinTokenRotation{")
2208
-	s = append(s, "RotateWorkerToken: "+fmt.Sprintf("%#v", this.RotateWorkerToken)+",\n")
2209
-	s = append(s, "RotateManagerToken: "+fmt.Sprintf("%#v", this.RotateManagerToken)+",\n")
2206
+	s := make([]string, 0, 7)
2207
+	s = append(s, "&api.KeyRotation{")
2208
+	s = append(s, "WorkerJoinToken: "+fmt.Sprintf("%#v", this.WorkerJoinToken)+",\n")
2209
+	s = append(s, "ManagerJoinToken: "+fmt.Sprintf("%#v", this.ManagerJoinToken)+",\n")
2210
+	s = append(s, "ManagerUnlockKey: "+fmt.Sprintf("%#v", this.ManagerUnlockKey)+",\n")
2210 2211
 	s = append(s, "}")
2211 2212
 	return strings.Join(s, "")
2212 2213
 }
... ...
@@ -4734,7 +4739,7 @@ func (m *ListClustersResponse) MarshalTo(data []byte) (int, error) {
4734 4734
 	return i, nil
4735 4735
 }
4736 4736
 
4737
-func (m *JoinTokenRotation) Marshal() (data []byte, err error) {
4737
+func (m *KeyRotation) Marshal() (data []byte, err error) {
4738 4738
 	size := m.Size()
4739 4739
 	data = make([]byte, size)
4740 4740
 	n, err := m.MarshalTo(data)
... ...
@@ -4744,25 +4749,35 @@ func (m *JoinTokenRotation) Marshal() (data []byte, err error) {
4744 4744
 	return data[:n], nil
4745 4745
 }
4746 4746
 
4747
-func (m *JoinTokenRotation) MarshalTo(data []byte) (int, error) {
4747
+func (m *KeyRotation) MarshalTo(data []byte) (int, error) {
4748 4748
 	var i int
4749 4749
 	_ = i
4750 4750
 	var l int
4751 4751
 	_ = l
4752
-	if m.RotateWorkerToken {
4752
+	if m.WorkerJoinToken {
4753 4753
 		data[i] = 0x8
4754 4754
 		i++
4755
-		if m.RotateWorkerToken {
4755
+		if m.WorkerJoinToken {
4756 4756
 			data[i] = 1
4757 4757
 		} else {
4758 4758
 			data[i] = 0
4759 4759
 		}
4760 4760
 		i++
4761 4761
 	}
4762
-	if m.RotateManagerToken {
4762
+	if m.ManagerJoinToken {
4763 4763
 		data[i] = 0x10
4764 4764
 		i++
4765
-		if m.RotateManagerToken {
4765
+		if m.ManagerJoinToken {
4766
+			data[i] = 1
4767
+		} else {
4768
+			data[i] = 0
4769
+		}
4770
+		i++
4771
+	}
4772
+	if m.ManagerUnlockKey {
4773
+		data[i] = 0x18
4774
+		i++
4775
+		if m.ManagerUnlockKey {
4766 4776
 			data[i] = 1
4767 4777
 		} else {
4768 4778
 			data[i] = 0
... ...
@@ -6618,13 +6633,16 @@ func (m *ListClustersResponse) Size() (n int) {
6618 6618
 	return n
6619 6619
 }
6620 6620
 
6621
-func (m *JoinTokenRotation) Size() (n int) {
6621
+func (m *KeyRotation) Size() (n int) {
6622 6622
 	var l int
6623 6623
 	_ = l
6624
-	if m.RotateWorkerToken {
6624
+	if m.WorkerJoinToken {
6625
+		n += 2
6626
+	}
6627
+	if m.ManagerJoinToken {
6625 6628
 		n += 2
6626 6629
 	}
6627
-	if m.RotateManagerToken {
6630
+	if m.ManagerUnlockKey {
6628 6631
 		n += 2
6629 6632
 	}
6630 6633
 	return n
... ...
@@ -7294,13 +7312,14 @@ func (this *ListClustersResponse) String() string {
7294 7294
 	}, "")
7295 7295
 	return s
7296 7296
 }
7297
-func (this *JoinTokenRotation) String() string {
7297
+func (this *KeyRotation) String() string {
7298 7298
 	if this == nil {
7299 7299
 		return "nil"
7300 7300
 	}
7301
-	s := strings.Join([]string{`&JoinTokenRotation{`,
7302
-		`RotateWorkerToken:` + fmt.Sprintf("%v", this.RotateWorkerToken) + `,`,
7303
-		`RotateManagerToken:` + fmt.Sprintf("%v", this.RotateManagerToken) + `,`,
7301
+	s := strings.Join([]string{`&KeyRotation{`,
7302
+		`WorkerJoinToken:` + fmt.Sprintf("%v", this.WorkerJoinToken) + `,`,
7303
+		`ManagerJoinToken:` + fmt.Sprintf("%v", this.ManagerJoinToken) + `,`,
7304
+		`ManagerUnlockKey:` + fmt.Sprintf("%v", this.ManagerUnlockKey) + `,`,
7304 7305
 		`}`,
7305 7306
 	}, "")
7306 7307
 	return s
... ...
@@ -7313,7 +7332,7 @@ func (this *UpdateClusterRequest) String() string {
7313 7313
 		`ClusterID:` + fmt.Sprintf("%v", this.ClusterID) + `,`,
7314 7314
 		`ClusterVersion:` + strings.Replace(fmt.Sprintf("%v", this.ClusterVersion), "Version", "Version", 1) + `,`,
7315 7315
 		`Spec:` + strings.Replace(fmt.Sprintf("%v", this.Spec), "ClusterSpec", "ClusterSpec", 1) + `,`,
7316
-		`Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "JoinTokenRotation", "JoinTokenRotation", 1), `&`, ``, 1) + `,`,
7316
+		`Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "KeyRotation", "KeyRotation", 1), `&`, ``, 1) + `,`,
7317 7317
 		`}`,
7318 7318
 	}, "")
7319 7319
 	return s
... ...
@@ -11855,7 +11874,7 @@ func (m *ListClustersResponse) Unmarshal(data []byte) error {
11855 11855
 	}
11856 11856
 	return nil
11857 11857
 }
11858
-func (m *JoinTokenRotation) Unmarshal(data []byte) error {
11858
+func (m *KeyRotation) Unmarshal(data []byte) error {
11859 11859
 	l := len(data)
11860 11860
 	iNdEx := 0
11861 11861
 	for iNdEx < l {
... ...
@@ -11878,15 +11897,15 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error {
11878 11878
 		fieldNum := int32(wire >> 3)
11879 11879
 		wireType := int(wire & 0x7)
11880 11880
 		if wireType == 4 {
11881
-			return fmt.Errorf("proto: JoinTokenRotation: wiretype end group for non-group")
11881
+			return fmt.Errorf("proto: KeyRotation: wiretype end group for non-group")
11882 11882
 		}
11883 11883
 		if fieldNum <= 0 {
11884
-			return fmt.Errorf("proto: JoinTokenRotation: illegal tag %d (wire type %d)", fieldNum, wire)
11884
+			return fmt.Errorf("proto: KeyRotation: illegal tag %d (wire type %d)", fieldNum, wire)
11885 11885
 		}
11886 11886
 		switch fieldNum {
11887 11887
 		case 1:
11888 11888
 			if wireType != 0 {
11889
-				return fmt.Errorf("proto: wrong wireType = %d for field RotateWorkerToken", wireType)
11889
+				return fmt.Errorf("proto: wrong wireType = %d for field WorkerJoinToken", wireType)
11890 11890
 			}
11891 11891
 			var v int
11892 11892
 			for shift := uint(0); ; shift += 7 {
... ...
@@ -11903,10 +11922,30 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error {
11903 11903
 					break
11904 11904
 				}
11905 11905
 			}
11906
-			m.RotateWorkerToken = bool(v != 0)
11906
+			m.WorkerJoinToken = bool(v != 0)
11907 11907
 		case 2:
11908 11908
 			if wireType != 0 {
11909
-				return fmt.Errorf("proto: wrong wireType = %d for field RotateManagerToken", wireType)
11909
+				return fmt.Errorf("proto: wrong wireType = %d for field ManagerJoinToken", wireType)
11910
+			}
11911
+			var v int
11912
+			for shift := uint(0); ; shift += 7 {
11913
+				if shift >= 64 {
11914
+					return ErrIntOverflowControl
11915
+				}
11916
+				if iNdEx >= l {
11917
+					return io.ErrUnexpectedEOF
11918
+				}
11919
+				b := data[iNdEx]
11920
+				iNdEx++
11921
+				v |= (int(b) & 0x7F) << shift
11922
+				if b < 0x80 {
11923
+					break
11924
+				}
11925
+			}
11926
+			m.ManagerJoinToken = bool(v != 0)
11927
+		case 3:
11928
+			if wireType != 0 {
11929
+				return fmt.Errorf("proto: wrong wireType = %d for field ManagerUnlockKey", wireType)
11910 11930
 			}
11911 11931
 			var v int
11912 11932
 			for shift := uint(0); ; shift += 7 {
... ...
@@ -11923,7 +11962,7 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error {
11923 11923
 					break
11924 11924
 				}
11925 11925
 			}
11926
-			m.RotateManagerToken = bool(v != 0)
11926
+			m.ManagerUnlockKey = bool(v != 0)
11927 11927
 		default:
11928 11928
 			iNdEx = preIndex
11929 11929
 			skippy, err := skipControl(data[iNdEx:])
... ...
@@ -13413,117 +13452,117 @@ var (
13413 13413
 func init() { proto.RegisterFile("control.proto", fileDescriptorControl) }
13414 13414
 
13415 13415
 var fileDescriptorControl = []byte{
13416
-	// 1777 bytes of a gzipped FileDescriptorProto
13417
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcf, 0x6f, 0xdb, 0xc6,
13418
-	0x12, 0x8e, 0x24, 0xdb, 0xb2, 0x47, 0x96, 0x13, 0xaf, 0x95, 0x3c, 0x81, 0xc9, 0x93, 0x03, 0xe6,
13419
-	0xc5, 0x91, 0x81, 0x3c, 0x39, 0x4f, 0x79, 0x41, 0xd3, 0x14, 0xfd, 0x65, 0xbb, 0x71, 0x95, 0x1f,
13420
-	0x6e, 0x40, 0x27, 0x6d, 0x6f, 0x86, 0x2c, 0x6d, 0x5c, 0x56, 0xb2, 0xa8, 0x92, 0xb4, 0x93, 0xa0,
13421
-	0x97, 0x16, 0x68, 0x81, 0xfe, 0x09, 0xbd, 0xf6, 0xda, 0x02, 0x3d, 0xf7, 0xd6, 0x6b, 0xd0, 0x53,
13422
-	0x8f, 0x3d, 0x19, 0x8d, 0x80, 0x02, 0x3d, 0x15, 0xfd, 0x0b, 0x8a, 0x62, 0x77, 0x67, 0x49, 0x8a,
13423
-	0x5a, 0x92, 0x92, 0xe5, 0xc2, 0x39, 0x99, 0x5c, 0x7e, 0xb3, 0x33, 0xbb, 0xf3, 0xed, 0xa7, 0xd9,
13424
-	0x81, 0x21, 0xdf, 0xb0, 0x3a, 0xae, 0x6d, 0xb5, 0x2b, 0x5d, 0xdb, 0x72, 0x2d, 0x42, 0x9a, 0x56,
13425
-	0xa3, 0x45, 0xed, 0x8a, 0xf3, 0xa4, 0x6e, 0xef, 0xb5, 0x4c, 0xb7, 0x72, 0xf0, 0x3f, 0x2d, 0xe7,
13426
-	0x74, 0x69, 0xc3, 0x11, 0x00, 0x2d, 0x6f, 0xed, 0x7c, 0x4c, 0x1b, 0xae, 0x7c, 0xcd, 0xb9, 0xcf,
13427
-	0xba, 0x54, 0xbe, 0x14, 0x76, 0xad, 0x5d, 0x8b, 0x3f, 0xae, 0xb0, 0x27, 0x1c, 0x5d, 0xe8, 0xb6,
13428
-	0xf7, 0x77, 0xcd, 0xce, 0x8a, 0xf8, 0x23, 0x06, 0xf5, 0x1b, 0x30, 0xb7, 0x41, 0xdd, 0x4d, 0xab,
13429
-	0x49, 0x0d, 0xfa, 0xc9, 0x3e, 0x75, 0x5c, 0x72, 0x09, 0xb2, 0x1d, 0xab, 0x49, 0xb7, 0xcd, 0x66,
13430
-	0x31, 0x75, 0x31, 0x55, 0x9e, 0x59, 0x85, 0xde, 0xe1, 0xe2, 0x14, 0x43, 0xd4, 0xd6, 0x8d, 0x29,
13431
-	0xf6, 0xa9, 0xd6, 0xd4, 0xdf, 0x84, 0xd3, 0x9e, 0x99, 0xd3, 0xb5, 0x3a, 0x0e, 0x25, 0x57, 0x61,
13432
-	0x82, 0x7d, 0xe4, 0x46, 0xb9, 0x6a, 0xb1, 0x32, 0xb8, 0x80, 0x0a, 0xc7, 0x73, 0x94, 0x7e, 0x98,
13433
-	0x81, 0x33, 0xf7, 0x4c, 0x87, 0x4f, 0xe1, 0x48, 0xd7, 0xb7, 0x21, 0xfb, 0xd8, 0x6c, 0xbb, 0xd4,
13434
-	0x76, 0x70, 0x96, 0xab, 0xaa, 0x59, 0xc2, 0x66, 0x95, 0xdb, 0xc2, 0xc6, 0x90, 0xc6, 0xda, 0xe7,
13435
-	0x19, 0xc8, 0xe2, 0x20, 0x29, 0xc0, 0x64, 0xa7, 0xbe, 0x47, 0xd9, 0x8c, 0x99, 0xf2, 0x8c, 0x21,
13436
-	0x5e, 0xc8, 0x0a, 0xe4, 0xcc, 0xe6, 0x76, 0xd7, 0xa6, 0x8f, 0xcd, 0xa7, 0xd4, 0x29, 0xa6, 0xd9,
13437
-	0xb7, 0xd5, 0xb9, 0xde, 0xe1, 0x22, 0xd4, 0xd6, 0x1f, 0xe0, 0xa8, 0x01, 0x66, 0x53, 0x3e, 0x93,
13438
-	0x07, 0x30, 0xd5, 0xae, 0xef, 0xd0, 0xb6, 0x53, 0xcc, 0x5c, 0xcc, 0x94, 0x73, 0xd5, 0x9b, 0xa3,
13439
-	0x44, 0x56, 0xb9, 0xc7, 0x4d, 0xdf, 0xe9, 0xb8, 0xf6, 0x33, 0x03, 0xe7, 0x21, 0x35, 0xc8, 0xed,
13440
-	0xd1, 0xbd, 0x1d, 0x6a, 0x3b, 0x1f, 0x99, 0x5d, 0xa7, 0x38, 0x71, 0x31, 0x53, 0x9e, 0xab, 0x5e,
13441
-	0x89, 0xda, 0xb6, 0xad, 0x2e, 0x6d, 0x54, 0xee, 0x7b, 0x78, 0x23, 0x68, 0x4b, 0xaa, 0x30, 0x69,
13442
-	0x5b, 0x6d, 0xea, 0x14, 0x27, 0xf9, 0x24, 0x17, 0x22, 0xf7, 0xde, 0x6a, 0x53, 0x43, 0x40, 0xc9,
13443
-	0x25, 0xc8, 0xb3, 0xad, 0xf0, 0xf7, 0x60, 0x8a, 0xef, 0xcf, 0x2c, 0x1b, 0x94, 0xab, 0xd6, 0x5e,
13444
-	0x85, 0x5c, 0x20, 0x74, 0x72, 0x06, 0x32, 0x2d, 0xfa, 0x4c, 0xd0, 0xc2, 0x60, 0x8f, 0x6c, 0x77,
13445
-	0x0f, 0xea, 0xed, 0x7d, 0x5a, 0x4c, 0xf3, 0x31, 0xf1, 0x72, 0x2b, 0x7d, 0x33, 0xa5, 0xaf, 0xc1,
13446
-	0x7c, 0x60, 0x3b, 0x90, 0x23, 0x15, 0x98, 0x64, 0xd9, 0x17, 0xc9, 0x88, 0x23, 0x89, 0x80, 0xe9,
13447
-	0xdf, 0xa6, 0x60, 0xfe, 0x51, 0xb7, 0x59, 0x77, 0xe9, 0xa8, 0x0c, 0x25, 0x6f, 0xc0, 0x2c, 0x07,
13448
-	0x1d, 0x50, 0xdb, 0x31, 0xad, 0x0e, 0x0f, 0x30, 0x57, 0x3d, 0xaf, 0xf2, 0xf8, 0xbe, 0x80, 0x18,
13449
-	0x39, 0x66, 0x80, 0x2f, 0xe4, 0x1a, 0x4c, 0xb0, 0xe3, 0x56, 0xcc, 0x70, 0xbb, 0x0b, 0x71, 0x79,
13450
-	0x31, 0x38, 0x52, 0x5f, 0x05, 0x12, 0x8c, 0xf5, 0x48, 0xc7, 0x62, 0x13, 0xe6, 0x0d, 0xba, 0x67,
13451
-	0x1d, 0x8c, 0xbe, 0xde, 0x02, 0x4c, 0x3e, 0xb6, 0xec, 0x86, 0xc8, 0xc4, 0xb4, 0x21, 0x5e, 0xf4,
13452
-	0x02, 0x90, 0xe0, 0x7c, 0x22, 0x26, 0x3c, 0xf4, 0x0f, 0xeb, 0x4e, 0x2b, 0xe0, 0xc2, 0xad, 0x3b,
13453
-	0xad, 0x90, 0x0b, 0x86, 0x60, 0x2e, 0xd8, 0x27, 0xef, 0xd0, 0x0b, 0x33, 0x7f, 0x75, 0xec, 0x63,
13454
-	0xdc, 0xea, 0x38, 0x9e, 0xa3, 0xf4, 0x9b, 0x72, 0x75, 0x23, 0xbb, 0xf6, 0xd6, 0x11, 0xf4, 0xae,
13455
-	0xff, 0x85, 0x22, 0xc2, 0x06, 0x8f, 0x20, 0x22, 0x41, 0xb3, 0x41, 0x11, 0xf9, 0xe6, 0x04, 0x45,
13456
-	0x44, 0x15, 0x99, 0x52, 0x44, 0x56, 0x20, 0xe7, 0x50, 0xfb, 0xc0, 0x6c, 0x30, 0x76, 0x08, 0x11,
13457
-	0xc1, 0x10, 0xb6, 0xc4, 0x70, 0x6d, 0xdd, 0x31, 0x00, 0x21, 0xb5, 0xa6, 0x43, 0x96, 0x60, 0x1a,
13458
-	0xb9, 0x24, 0xd4, 0x62, 0x66, 0x35, 0xd7, 0x3b, 0x5c, 0xcc, 0x0a, 0x32, 0x39, 0x46, 0x56, 0xb0,
13459
-	0xc9, 0x21, 0xeb, 0x30, 0xd7, 0xa4, 0x8e, 0x69, 0xd3, 0xe6, 0xb6, 0xe3, 0xd6, 0x5d, 0xd4, 0x87,
13460
-	0xb9, 0xea, 0xbf, 0xa3, 0x52, 0xbc, 0xc5, 0x50, 0x46, 0x1e, 0x8d, 0xf8, 0x9b, 0x42, 0x64, 0xb2,
13461
-	0xff, 0x88, 0xc8, 0xe0, 0x76, 0xf9, 0x22, 0xc3, 0x58, 0x13, 0x2b, 0x32, 0x9c, 0x46, 0x02, 0xa6,
13462
-	0xdf, 0x85, 0xc2, 0x9a, 0x4d, 0xeb, 0x2e, 0xc5, 0x2d, 0x93, 0x44, 0xba, 0x8e, 0x0a, 0x20, 0x58,
13463
-	0xb4, 0xa8, 0x9a, 0x06, 0x2d, 0x02, 0x22, 0xb0, 0x09, 0x67, 0x43, 0x93, 0x61, 0x54, 0x37, 0x20,
13464
-	0x8b, 0x69, 0xc0, 0x09, 0xcf, 0xc7, 0x4c, 0x68, 0x48, 0xac, 0xfe, 0x36, 0xcc, 0x6f, 0x50, 0x37,
13465
-	0x14, 0xd9, 0x55, 0x00, 0x3f, 0xeb, 0x78, 0x6a, 0xf2, 0xbd, 0xc3, 0xc5, 0x19, 0x2f, 0xe9, 0xc6,
13466
-	0x8c, 0x97, 0x73, 0xfd, 0x2e, 0x90, 0xe0, 0x14, 0xe3, 0xc5, 0xf3, 0x63, 0x0a, 0x0a, 0x42, 0xe5,
13467
-	0xc6, 0x89, 0x89, 0xac, 0xc3, 0x69, 0x89, 0x1e, 0x41, 0xa0, 0xe7, 0xd0, 0x46, 0x6a, 0xf4, 0xf5,
13468
-	0x3e, 0x8d, 0x1e, 0x3e, 0x43, 0xa1, 0x05, 0x8c, 0xb7, 0x23, 0xeb, 0x50, 0x10, 0xd2, 0x34, 0x56,
13469
-	0x92, 0xfe, 0x05, 0x67, 0x43, 0xb3, 0xa0, 0xc6, 0xfd, 0x9e, 0x86, 0x05, 0xc6, 0x71, 0x1c, 0xf7,
13470
-	0x64, 0xae, 0x16, 0x96, 0xb9, 0x95, 0x28, 0x31, 0x09, 0x59, 0x0e, 0x2a, 0xdd, 0x97, 0xe9, 0x63,
13471
-	0x57, 0xba, 0xad, 0x90, 0xd2, 0xbd, 0x36, 0x62, 0x70, 0x4a, 0xb1, 0x1b, 0x50, 0x93, 0x89, 0xe3,
13472
-	0x55, 0x93, 0xf7, 0xa0, 0xd0, 0x1f, 0x12, 0x12, 0xe3, 0x15, 0x98, 0xc6, 0x44, 0x49, 0x4d, 0x89,
13473
-	0x65, 0x86, 0x07, 0xf6, 0x95, 0x65, 0x93, 0xba, 0x4f, 0x2c, 0xbb, 0x35, 0x82, 0xb2, 0xa0, 0x85,
13474
-	0x4a, 0x59, 0xbc, 0xc9, 0x7c, 0xde, 0x76, 0xc4, 0x50, 0x1c, 0x6f, 0xa5, 0x95, 0xc4, 0xea, 0x8f,
13475
-	0xb8, 0xb2, 0x84, 0x22, 0x23, 0x30, 0xc1, 0x76, 0x13, 0xf7, 0x8b, 0x3f, 0x33, 0x22, 0xa3, 0x0d,
13476
-	0x23, 0x72, 0xda, 0x27, 0x32, 0xda, 0x32, 0x22, 0x23, 0xc0, 0x53, 0x9b, 0x63, 0x8a, 0xf1, 0x43,
13477
-	0x79, 0xb6, 0x8e, 0x3d, 0x4c, 0xef, 0xbc, 0x85, 0x22, 0xf5, 0xce, 0x1b, 0x8e, 0x1f, 0xe1, 0xbc,
13478
-	0x85, 0x2c, 0x5f, 0xae, 0xf3, 0x16, 0x11, 0xdc, 0x49, 0x9e, 0x37, 0x3f, 0x24, 0xff, 0xbc, 0x61,
13479
-	0xa2, 0x62, 0xcf, 0x9b, 0xcc, 0x9c, 0x07, 0xc6, 0x1f, 0xcb, 0xb5, 0xf6, 0xbe, 0xe3, 0x52, 0x3b,
13480
-	0xa0, 0xc3, 0x0d, 0x31, 0x12, 0xd2, 0x61, 0xc4, 0x31, 0x5e, 0x20, 0xc0, 0xa3, 0xaf, 0x37, 0x85,
13481
-	0x4f, 0x5f, 0x84, 0xc4, 0xd1, 0x57, 0x5a, 0x49, 0xac, 0xc7, 0x25, 0xfc, 0x70, 0x04, 0x2e, 0x85,
13482
-	0x2c, 0x5f, 0x2e, 0x2e, 0x45, 0x04, 0x77, 0x92, 0x5c, 0xf2, 0x43, 0xf2, 0xb9, 0x84, 0xd9, 0x88,
13483
-	0xe5, 0x92, 0x4c, 0x9d, 0x07, 0xd6, 0xf7, 0x61, 0xfe, 0x8e, 0x65, 0x76, 0x1e, 0x5a, 0x2d, 0xda,
13484
-	0x31, 0x2c, 0xb7, 0xee, 0xb2, 0x82, 0xa3, 0x02, 0x0b, 0x36, 0x7b, 0xa6, 0xdb, 0x8c, 0x70, 0xd4,
13485
-	0xde, 0x76, 0xd9, 0x67, 0x1e, 0xe1, 0xb4, 0x31, 0x2f, 0x3e, 0x7d, 0xc0, 0xbf, 0x70, 0x3b, 0x72,
13486
-	0x0d, 0x0a, 0x88, 0xdf, 0xab, 0x77, 0xea, 0xbb, 0x9e, 0x81, 0xb8, 0xa3, 0x11, 0xf1, 0xed, 0xbe,
13487
-	0xf8, 0xc4, 0x2d, 0xf4, 0xaf, 0xd2, 0xb2, 0xbe, 0x1a, 0x87, 0xc6, 0xac, 0xbe, 0x92, 0xe8, 0x51,
13488
-	0xea, 0x2b, 0xb4, 0x19, 0xa1, 0xbe, 0x42, 0xef, 0xfe, 0xef, 0x14, 0xd9, 0x80, 0x69, 0x1b, 0xf7,
13489
-	0xab, 0x38, 0xc1, 0x0d, 0x2f, 0xab, 0x0c, 0x07, 0x36, 0x77, 0x75, 0xe2, 0xf9, 0xe1, 0xe2, 0x29,
13490
-	0xc3, 0x33, 0xf6, 0x0b, 0xb5, 0x63, 0x3a, 0x8d, 0xaf, 0xc3, 0x19, 0x5e, 0x07, 0x37, 0x6c, 0xea,
13491
-	0xca, 0x5d, 0x5d, 0x86, 0x19, 0x87, 0x0f, 0xf8, 0x9b, 0x3a, 0xdb, 0x3b, 0x5c, 0x9c, 0x16, 0xa8,
13492
-	0xda, 0x3a, 0xfb, 0x31, 0xe7, 0x4f, 0x4d, 0x7d, 0x03, 0x2b, 0x71, 0x61, 0x8e, 0xa1, 0x54, 0x61,
13493
-	0x4a, 0x00, 0x30, 0x12, 0x4d, 0x5d, 0x18, 0x70, 0x1b, 0x44, 0xea, 0x3f, 0xa4, 0x60, 0x41, 0x56,
13494
-	0xa0, 0x47, 0x8b, 0x85, 0xac, 0xc2, 0x1c, 0x42, 0x47, 0xc8, 0x6e, 0x5e, 0x98, 0xc8, 0xe4, 0x56,
13495
-	0xfb, 0x92, 0x5b, 0x8a, 0x0e, 0x3c, 0x50, 0x83, 0xdc, 0xf1, 0x8b, 0xff, 0xb1, 0xb7, 0xe1, 0xb7,
13496
-	0x34, 0x10, 0x51, 0x6e, 0xb1, 0x57, 0x4f, 0x1b, 0xdf, 0x0d, 0x6b, 0x63, 0x25, 0xba, 0x74, 0x0c,
13497
-	0x1a, 0x0e, 0x4a, 0xe3, 0x17, 0xc7, 0x2f, 0x8d, 0x46, 0x48, 0x1a, 0x6f, 0x8d, 0x16, 0xdb, 0x89,
13498
-	0x28, 0xe3, 0x5d, 0x79, 0x7f, 0xc0, 0x88, 0x30, 0x65, 0xff, 0x67, 0xb7, 0x1d, 0x3e, 0x84, 0xba,
13499
-	0x18, 0x97, 0x33, 0x09, 0xd5, 0x6b, 0xb0, 0x20, 0xaf, 0xb7, 0x41, 0xea, 0x56, 0xfb, 0x0a, 0xda,
13500
-	0xa1, 0xb9, 0xd4, 0x3f, 0xd5, 0x18, 0x5c, 0x7a, 0x0b, 0x16, 0xe4, 0xed, 0xe9, 0x88, 0xa7, 0xfb,
13501
-	0x9c, 0x7f, 0x8b, 0x0b, 0x46, 0x53, 0xfd, 0xee, 0x1c, 0x64, 0xd7, 0x44, 0x67, 0x9e, 0x98, 0x90,
13502
-	0xc5, 0xa6, 0x37, 0xd1, 0x55, 0x41, 0xf5, 0x37, 0xd2, 0xb5, 0x4b, 0xb1, 0x18, 0x2c, 0x37, 0xcf,
13503
-	0xfe, 0xf4, 0xfd, 0x1f, 0x5f, 0xa7, 0x4f, 0x43, 0x9e, 0x83, 0xfe, 0x8b, 0x3f, 0x13, 0xc4, 0x82,
13504
-	0x19, 0xaf, 0x7b, 0x4a, 0xfe, 0x33, 0x4c, 0xaf, 0x59, 0xbb, 0x9c, 0x80, 0x8a, 0x77, 0x68, 0x03,
13505
-	0xf8, 0xcd, 0x4b, 0xa2, 0x9c, 0x6b, 0xa0, 0x11, 0xab, 0x2d, 0x25, 0xc1, 0x12, 0x7d, 0xfa, 0xcd,
13506
-	0x49, 0xb5, 0xcf, 0x81, 0x66, 0xa8, 0xda, 0xa7, 0xa2, 0xc7, 0x19, 0xe1, 0x53, 0xe4, 0xf0, 0x61,
13507
-	0xdd, 0x69, 0x45, 0xe6, 0x30, 0xd0, 0x9c, 0x8c, 0xcc, 0x61, 0x5f, 0x1b, 0x32, 0x3e, 0x87, 0xbc,
13508
-	0x39, 0x15, 0x9d, 0xc3, 0x60, 0xab, 0x2f, 0x3a, 0x87, 0x7d, 0x1d, 0xae, 0xc4, 0xfd, 0xe4, 0xcb,
13509
-	0x8b, 0xd9, 0xcf, 0xe0, 0x0a, 0x97, 0x92, 0x60, 0x89, 0x3e, 0xfd, 0xe6, 0x92, 0xda, 0xe7, 0x40,
13510
-	0xff, 0x4a, 0xed, 0x73, 0xb0, 0x47, 0x15, 0xe5, 0xf3, 0x29, 0xcc, 0x06, 0xef, 0xe9, 0xe4, 0xca,
13511
-	0x90, 0xcd, 0x05, 0xad, 0x9c, 0x0c, 0x8c, 0xf7, 0xfc, 0x29, 0xe4, 0xfb, 0xba, 0x7b, 0x44, 0x39,
13512
-	0xa3, 0xaa, 0x9b, 0xa8, 0x2d, 0x0f, 0x81, 0x4c, 0x74, 0xde, 0xd7, 0xb8, 0x52, 0x3b, 0x57, 0x35,
13513
-	0xe7, 0xd4, 0xce, 0x95, 0x5d, 0xb0, 0x18, 0xe7, 0x7d, 0xfd, 0x29, 0xb5, 0x73, 0x55, 0x23, 0x4c,
13514
-	0xed, 0x5c, 0xdd, 0xec, 0x8a, 0x25, 0x19, 0xde, 0xf7, 0x22, 0x49, 0xd6, 0xdf, 0x23, 0x88, 0x24,
13515
-	0x59, 0xf8, 0xc2, 0x1f, 0x4f, 0x32, 0x79, 0x39, 0x8d, 0x26, 0x59, 0xe8, 0x46, 0x1d, 0x4d, 0xb2,
13516
-	0xf0, 0x3d, 0x37, 0x91, 0x64, 0x72, 0xc1, 0x31, 0x24, 0x0b, 0xad, 0x79, 0x79, 0x08, 0xe4, 0x90,
13517
-	0x79, 0x8e, 0x75, 0xae, 0x6a, 0xca, 0xc4, 0xe5, 0x79, 0x48, 0xe7, 0x22, 0xcf, 0x58, 0xb8, 0x47,
13518
-	0xe6, 0xb9, 0xff, 0x62, 0x14, 0x99, 0xe7, 0xd0, 0xad, 0x21, 0x21, 0xcf, 0xf2, 0xe2, 0x18, 0x9d,
13519
-	0xe7, 0xd0, 0x6d, 0x37, 0x3a, 0xcf, 0xe1, 0x3b, 0x68, 0xe2, 0x79, 0x96, 0x0b, 0x8e, 0x39, 0xcf,
13520
-	0xa1, 0x35, 0x2f, 0x0f, 0x81, 0x4c, 0xfc, 0x71, 0xf2, 0x6e, 0x33, 0xea, 0x1f, 0xa7, 0xf0, 0x5d,
13521
-	0x49, 0xbb, 0x9c, 0x80, 0x4a, 0xdc, 0xe7, 0xe0, 0xd5, 0x41, 0xbd, 0xcf, 0x8a, 0x6b, 0x91, 0x56,
13522
-	0x4e, 0x06, 0xc6, 0x7b, 0xde, 0x87, 0x5c, 0xa0, 0x00, 0x26, 0x4b, 0xc3, 0xd5, 0xec, 0xda, 0x95,
13523
-	0x44, 0x5c, 0xe2, 0x82, 0x83, 0xf5, 0xad, 0x7a, 0xc1, 0x8a, 0x62, 0x5a, 0x2b, 0x27, 0x03, 0x13,
13524
-	0x3d, 0x07, 0x6b, 0x59, 0xb5, 0x67, 0x45, 0xbd, 0xac, 0x95, 0x93, 0x81, 0xb1, 0x9e, 0x57, 0x2f,
13525
-	0x3c, 0x7f, 0x51, 0x3a, 0xf5, 0xcb, 0x8b, 0xd2, 0xa9, 0x3f, 0x5f, 0x94, 0x52, 0x9f, 0xf5, 0x4a,
13526
-	0xa9, 0xe7, 0xbd, 0x52, 0xea, 0xe7, 0x5e, 0x29, 0xf5, 0x6b, 0xaf, 0x94, 0xda, 0x99, 0xe2, 0xff,
13527
-	0x72, 0x72, 0xfd, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2a, 0x7c, 0x4c, 0x3e, 0xeb, 0x22, 0x00,
13528
-	0x00,
13416
+	// 1781 bytes of a gzipped FileDescriptorProto
13417
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x6f, 0x1b, 0x45,
13418
+	0x14, 0xaf, 0xed, 0x24, 0x4e, 0x9e, 0xe3, 0x7c, 0x4c, 0xdc, 0x62, 0x6d, 0x8b, 0x53, 0x6d, 0x69,
13419
+	0xea, 0xa0, 0xe0, 0x80, 0x4b, 0x45, 0x29, 0xe2, 0xa3, 0x8e, 0x69, 0x71, 0x53, 0x42, 0xb5, 0x69,
13420
+	0x11, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0x6b, 0xc7, 0x6b, 0x76, 0x37, 0x69, 0x23, 0x2e, 0x80, 0xe0,
13421
+	0x4f, 0x40, 0xe2, 0xca, 0x15, 0x24, 0xce, 0xdc, 0xb8, 0x56, 0x9c, 0x38, 0x72, 0xb2, 0xa8, 0x25,
13422
+	0x24, 0x4e, 0x88, 0xbf, 0x00, 0xa1, 0xf9, 0xda, 0x2f, 0xcf, 0xee, 0xda, 0x71, 0x50, 0x7a, 0x8a,
13423
+	0x77, 0xf6, 0xf7, 0xe6, 0xbd, 0x99, 0xf7, 0x9b, 0xdf, 0xbe, 0x79, 0x0a, 0x64, 0x1b, 0x46, 0xc7,
13424
+	0x36, 0x8d, 0x76, 0xa9, 0x6b, 0x1a, 0xb6, 0x81, 0x50, 0xd3, 0x68, 0xb4, 0xb0, 0x59, 0xb2, 0x1e,
13425
+	0xd7, 0xcd, 0xfd, 0x96, 0x6e, 0x97, 0x0e, 0x5f, 0x53, 0x32, 0x56, 0x17, 0x37, 0x2c, 0x06, 0x50,
13426
+	0xb2, 0xc6, 0xee, 0x23, 0xdc, 0xb0, 0xc5, 0x63, 0xc6, 0x3e, 0xea, 0x62, 0xf1, 0x90, 0xdb, 0x33,
13427
+	0xf6, 0x0c, 0xfa, 0x73, 0x9d, 0xfc, 0xe2, 0xa3, 0x4b, 0xdd, 0xf6, 0xc1, 0x9e, 0xde, 0x59, 0x67,
13428
+	0x7f, 0xd8, 0xa0, 0x7a, 0x0d, 0xe6, 0x6e, 0x63, 0x7b, 0xcb, 0x68, 0x62, 0x0d, 0x7f, 0x76, 0x80,
13429
+	0x2d, 0x1b, 0x5d, 0x82, 0x74, 0xc7, 0x68, 0xe2, 0x1d, 0xbd, 0x99, 0x4f, 0x5c, 0x4c, 0x14, 0x67,
13430
+	0x2a, 0xd0, 0xef, 0x2d, 0x4f, 0x11, 0x44, 0xad, 0xaa, 0x4d, 0x91, 0x57, 0xb5, 0xa6, 0xfa, 0x2e,
13431
+	0xcc, 0x3b, 0x66, 0x56, 0xd7, 0xe8, 0x58, 0x18, 0xad, 0xc1, 0x04, 0x79, 0x49, 0x8d, 0x32, 0xe5,
13432
+	0x7c, 0x69, 0x70, 0x01, 0x25, 0x8a, 0xa7, 0x28, 0xb5, 0x97, 0x82, 0x85, 0xbb, 0xba, 0x45, 0xa7,
13433
+	0xb0, 0x84, 0xeb, 0x5b, 0x90, 0x7e, 0xa8, 0xb7, 0x6d, 0x6c, 0x5a, 0x7c, 0x96, 0x35, 0xd9, 0x2c,
13434
+	0x41, 0xb3, 0xd2, 0x2d, 0x66, 0xa3, 0x09, 0x63, 0xe5, 0xcb, 0x14, 0xa4, 0xf9, 0x20, 0xca, 0xc1,
13435
+	0x64, 0xa7, 0xbe, 0x8f, 0xc9, 0x8c, 0xa9, 0xe2, 0x8c, 0xc6, 0x1e, 0xd0, 0x3a, 0x64, 0xf4, 0xe6,
13436
+	0x4e, 0xd7, 0xc4, 0x0f, 0xf5, 0x27, 0xd8, 0xca, 0x27, 0xc9, 0xbb, 0xca, 0x5c, 0xbf, 0xb7, 0x0c,
13437
+	0xb5, 0xea, 0x3d, 0x3e, 0xaa, 0x81, 0xde, 0x14, 0xbf, 0xd1, 0x3d, 0x98, 0x6a, 0xd7, 0x77, 0x71,
13438
+	0xdb, 0xca, 0xa7, 0x2e, 0xa6, 0x8a, 0x99, 0xf2, 0xf5, 0x51, 0x22, 0x2b, 0xdd, 0xa5, 0xa6, 0xef,
13439
+	0x77, 0x6c, 0xf3, 0x48, 0xe3, 0xf3, 0xa0, 0x1a, 0x64, 0xf6, 0xf1, 0xfe, 0x2e, 0x36, 0xad, 0x4f,
13440
+	0xf5, 0xae, 0x95, 0x9f, 0xb8, 0x98, 0x2a, 0xce, 0x95, 0xaf, 0x84, 0x6d, 0xdb, 0x76, 0x17, 0x37,
13441
+	0x4a, 0x1f, 0x3a, 0x78, 0xcd, 0x6b, 0x8b, 0xca, 0x30, 0x69, 0x1a, 0x6d, 0x6c, 0xe5, 0x27, 0xe9,
13442
+	0x24, 0x17, 0x42, 0xf7, 0xde, 0x68, 0x63, 0x8d, 0x41, 0xd1, 0x25, 0xc8, 0x92, 0xad, 0x70, 0xf7,
13443
+	0x60, 0x8a, 0xee, 0xcf, 0x2c, 0x19, 0x14, 0xab, 0x56, 0xde, 0x84, 0x8c, 0x27, 0x74, 0xb4, 0x00,
13444
+	0xa9, 0x16, 0x3e, 0x62, 0xb4, 0xd0, 0xc8, 0x4f, 0xb2, 0xbb, 0x87, 0xf5, 0xf6, 0x01, 0xce, 0x27,
13445
+	0xe9, 0x18, 0x7b, 0xb8, 0x91, 0xbc, 0x9e, 0x50, 0x37, 0x60, 0xd1, 0xb3, 0x1d, 0x9c, 0x23, 0x25,
13446
+	0x98, 0x24, 0xd9, 0x67, 0xc9, 0x88, 0x22, 0x09, 0x83, 0xa9, 0x3f, 0x24, 0x60, 0xf1, 0x41, 0xb7,
13447
+	0x59, 0xb7, 0xf1, 0xa8, 0x0c, 0x45, 0xef, 0xc0, 0x2c, 0x05, 0x1d, 0x62, 0xd3, 0xd2, 0x8d, 0x0e,
13448
+	0x0d, 0x30, 0x53, 0x3e, 0x2f, 0xf3, 0xf8, 0x31, 0x83, 0x68, 0x19, 0x62, 0xc0, 0x1f, 0xd0, 0xab,
13449
+	0x30, 0x41, 0x8e, 0x5b, 0x3e, 0x45, 0xed, 0x2e, 0x44, 0xe5, 0x45, 0xa3, 0x48, 0xb5, 0x02, 0xc8,
13450
+	0x1b, 0xeb, 0xb1, 0x8e, 0xc5, 0x16, 0x2c, 0x6a, 0x78, 0xdf, 0x38, 0x1c, 0x7d, 0xbd, 0x39, 0x98,
13451
+	0x7c, 0x68, 0x98, 0x0d, 0x96, 0x89, 0x69, 0x8d, 0x3d, 0xa8, 0x39, 0x40, 0xde, 0xf9, 0x58, 0x4c,
13452
+	0xfc, 0xd0, 0xdf, 0xaf, 0x5b, 0x2d, 0x8f, 0x0b, 0xbb, 0x6e, 0xb5, 0x02, 0x2e, 0x08, 0x82, 0xb8,
13453
+	0x20, 0xaf, 0x9c, 0x43, 0xcf, 0xcc, 0xdc, 0xd5, 0x91, 0x97, 0x51, 0xab, 0xa3, 0x78, 0x8a, 0x52,
13454
+	0xaf, 0x8b, 0xd5, 0x8d, 0xec, 0xda, 0x59, 0x87, 0xd7, 0xbb, 0xfa, 0x2f, 0x17, 0x11, 0x32, 0x78,
13455
+	0x0c, 0x11, 0xf1, 0x9a, 0x0d, 0x8a, 0xc8, 0xf7, 0xa7, 0x28, 0x22, 0xb2, 0xc8, 0xa4, 0x22, 0xb2,
13456
+	0x0e, 0x19, 0x0b, 0x9b, 0x87, 0x7a, 0x83, 0xb0, 0x83, 0x89, 0x08, 0x0f, 0x61, 0x9b, 0x0d, 0xd7,
13457
+	0xaa, 0x96, 0x06, 0x1c, 0x52, 0x6b, 0x5a, 0x68, 0x05, 0xa6, 0x39, 0x97, 0x98, 0x5a, 0xcc, 0x54,
13458
+	0x32, 0xfd, 0xde, 0x72, 0x9a, 0x91, 0xc9, 0xd2, 0xd2, 0x8c, 0x4d, 0x16, 0xaa, 0xc2, 0x5c, 0x13,
13459
+	0x5b, 0xba, 0x89, 0x9b, 0x3b, 0x96, 0x5d, 0xb7, 0xb9, 0x3e, 0xcc, 0x95, 0x5f, 0x0c, 0x4b, 0xf1,
13460
+	0x36, 0x41, 0x69, 0x59, 0x6e, 0x44, 0x9f, 0x24, 0x22, 0x93, 0xfe, 0x5f, 0x44, 0x86, 0x6f, 0x97,
13461
+	0x2b, 0x32, 0x84, 0x35, 0x91, 0x22, 0x43, 0x69, 0xc4, 0x60, 0xea, 0x26, 0xe4, 0x36, 0x4c, 0x5c,
13462
+	0xb7, 0x31, 0xdf, 0x32, 0x41, 0xa4, 0xab, 0x5c, 0x01, 0x18, 0x8b, 0x96, 0x65, 0xd3, 0x70, 0x0b,
13463
+	0x8f, 0x08, 0x6c, 0xc1, 0xd9, 0xc0, 0x64, 0x3c, 0xaa, 0x6b, 0x90, 0xe6, 0x69, 0xe0, 0x13, 0x9e,
13464
+	0x8f, 0x98, 0x50, 0x13, 0x58, 0xf5, 0x26, 0x2c, 0xde, 0xc6, 0x76, 0x20, 0xb2, 0x35, 0x00, 0x37,
13465
+	0xeb, 0xfc, 0xd4, 0x64, 0xfb, 0xbd, 0xe5, 0x19, 0x27, 0xe9, 0xda, 0x8c, 0x93, 0x73, 0x75, 0x13,
13466
+	0x90, 0x77, 0x8a, 0xf1, 0xe2, 0xf9, 0x25, 0x01, 0x39, 0xa6, 0x72, 0xe3, 0xc4, 0x84, 0xaa, 0x30,
13467
+	0x2f, 0xd0, 0x23, 0x08, 0xf4, 0x1c, 0xb7, 0x11, 0x1a, 0x7d, 0xd5, 0xa7, 0xd1, 0xc3, 0x67, 0x28,
13468
+	0xb0, 0x80, 0xf1, 0x76, 0xa4, 0x0a, 0x39, 0x26, 0x4d, 0x63, 0x25, 0xe9, 0x05, 0x38, 0x1b, 0x98,
13469
+	0x85, 0x6b, 0xdc, 0x5f, 0x49, 0x58, 0x22, 0x1c, 0xe7, 0xe3, 0x8e, 0xcc, 0xd5, 0x82, 0x32, 0xb7,
13470
+	0x1e, 0x26, 0x26, 0x01, 0xcb, 0x41, 0xa5, 0xfb, 0x26, 0x79, 0xe2, 0x4a, 0xb7, 0x1d, 0x50, 0xba,
13471
+	0xb7, 0x46, 0x0c, 0x4e, 0x2a, 0x76, 0x03, 0x6a, 0x32, 0x71, 0xb2, 0x6a, 0xf2, 0x11, 0xe4, 0xfc,
13472
+	0x21, 0x71, 0x62, 0xbc, 0x01, 0xd3, 0x3c, 0x51, 0x42, 0x53, 0x22, 0x99, 0xe1, 0x80, 0x5d, 0x65,
13473
+	0xd9, 0xc2, 0xf6, 0x63, 0xc3, 0x6c, 0x8d, 0xa0, 0x2c, 0xdc, 0x42, 0xa6, 0x2c, 0xce, 0x64, 0x2e,
13474
+	0x6f, 0x3b, 0x6c, 0x28, 0x8a, 0xb7, 0xc2, 0x4a, 0x60, 0xd5, 0x07, 0x54, 0x59, 0x02, 0x91, 0x21,
13475
+	0x98, 0x20, 0xbb, 0xc9, 0xf7, 0x8b, 0xfe, 0x26, 0x44, 0xe6, 0x36, 0x84, 0xc8, 0x49, 0x97, 0xc8,
13476
+	0xdc, 0x96, 0x10, 0x99, 0x03, 0x1c, 0xb5, 0x39, 0xa1, 0x18, 0x3f, 0x11, 0x67, 0xeb, 0xc4, 0xc3,
13477
+	0x74, 0xce, 0x5b, 0x20, 0x52, 0xe7, 0xbc, 0xf1, 0xf1, 0x63, 0x9c, 0xb7, 0x80, 0xe5, 0xf3, 0x75,
13478
+	0xde, 0x42, 0x82, 0x3b, 0xcd, 0xf3, 0xe6, 0x86, 0xe4, 0x9e, 0x37, 0x9e, 0xa8, 0xc8, 0xf3, 0x26,
13479
+	0x32, 0xe7, 0x80, 0xf9, 0xc7, 0x72, 0xa3, 0x7d, 0x60, 0xd9, 0xd8, 0xf4, 0xe8, 0x70, 0x83, 0x8d,
13480
+	0x04, 0x74, 0x98, 0xe3, 0x08, 0x2f, 0x38, 0xc0, 0xa1, 0xaf, 0x33, 0x85, 0x4b, 0x5f, 0x0e, 0x89,
13481
+	0xa2, 0xaf, 0xb0, 0x12, 0x58, 0x87, 0x4b, 0xfc, 0xc5, 0x31, 0xb8, 0x14, 0xb0, 0x7c, 0xbe, 0xb8,
13482
+	0x14, 0x12, 0xdc, 0x69, 0x72, 0xc9, 0x0d, 0xc9, 0xe5, 0x12, 0xcf, 0x46, 0x24, 0x97, 0x44, 0xea,
13483
+	0x1c, 0xb0, 0xfa, 0x6d, 0x02, 0x32, 0x9b, 0xf8, 0x48, 0x33, 0xec, 0xba, 0x4d, 0x6a, 0x8d, 0x97,
13484
+	0x61, 0x91, 0x90, 0x0c, 0x9b, 0x3b, 0x8f, 0x0c, 0xbd, 0xb3, 0x63, 0x1b, 0x2d, 0xdc, 0xa1, 0xa1,
13485
+	0x4d, 0x6b, 0xf3, 0xec, 0xc5, 0x1d, 0x43, 0xef, 0xdc, 0x27, 0xc3, 0x68, 0x0d, 0xd0, 0x7e, 0xbd,
13486
+	0x53, 0xdf, 0xf3, 0x83, 0xd9, 0xc5, 0x6c, 0x81, 0xbf, 0x91, 0xa2, 0x0f, 0x3a, 0x6d, 0xa3, 0xd1,
13487
+	0xda, 0x21, 0xab, 0x4e, 0xf9, 0xd0, 0x0f, 0xe8, 0x8b, 0x4d, 0x7c, 0xa4, 0x7e, 0x95, 0x14, 0x05,
13488
+	0xd8, 0x38, 0x3c, 0x27, 0x05, 0x98, 0x40, 0x8f, 0x52, 0x80, 0x71, 0x9b, 0x11, 0x0a, 0x30, 0xee,
13489
+	0xdd, 0xfd, 0x90, 0xa1, 0x9b, 0x30, 0x6d, 0xf2, 0x5d, 0xcd, 0x4f, 0x84, 0x1b, 0x7a, 0x36, 0xbf,
13490
+	0x32, 0xf1, 0xb4, 0xb7, 0x7c, 0x46, 0x73, 0xcc, 0xdc, 0x1a, 0xee, 0x84, 0x0e, 0xea, 0xdb, 0xb0,
13491
+	0x40, 0x4b, 0xe4, 0x86, 0x89, 0x6d, 0xb1, 0x9f, 0xab, 0x30, 0x63, 0xd1, 0x01, 0x77, 0x3b, 0x67,
13492
+	0xfb, 0xbd, 0xe5, 0x69, 0x86, 0xaa, 0x55, 0xc9, 0x77, 0x9e, 0xfe, 0x6a, 0xaa, 0xb7, 0x79, 0x91,
13493
+	0xce, 0xcc, 0x79, 0x28, 0x65, 0x98, 0x62, 0x00, 0x1e, 0x89, 0x22, 0xaf, 0x19, 0xa8, 0x0d, 0x47,
13494
+	0xaa, 0x3f, 0x27, 0x60, 0x49, 0x14, 0xa7, 0xc7, 0x8b, 0x05, 0x55, 0x60, 0x8e, 0x43, 0x47, 0xc8,
13495
+	0x6b, 0x96, 0x99, 0x88, 0xb4, 0x96, 0x7d, 0x69, 0x2d, 0x84, 0x07, 0xee, 0x29, 0x4f, 0xee, 0xb8,
13496
+	0xf7, 0x82, 0xb1, 0xb7, 0xe1, 0xcf, 0x24, 0x20, 0x56, 0x89, 0x91, 0x47, 0x47, 0x36, 0x3f, 0x08,
13497
+	0xca, 0x66, 0x29, 0xbc, 0xaa, 0xf4, 0x1a, 0x0e, 0xaa, 0xe6, 0xd7, 0x27, 0xaf, 0x9a, 0x5a, 0x40,
13498
+	0x35, 0x6f, 0x8c, 0x16, 0xdb, 0xa9, 0x88, 0xe6, 0xa6, 0xb8, 0x5a, 0xf0, 0x88, 0x78, 0xca, 0x5e,
13499
+	0x27, 0x17, 0x21, 0x3a, 0xc4, 0x25, 0x33, 0x2a, 0x67, 0x02, 0xaa, 0xd6, 0x60, 0x49, 0xdc, 0x7c,
13500
+	0xbd, 0xd4, 0x2d, 0xfb, 0x6a, 0xdd, 0xa1, 0xb9, 0xe4, 0x9f, 0x6a, 0x0c, 0x2e, 0xbd, 0x07, 0x4b,
13501
+	0xe2, 0x62, 0x75, 0xcc, 0xd3, 0x7d, 0xce, 0xbd, 0xe0, 0x79, 0xa3, 0x29, 0xff, 0x78, 0x0e, 0xd2,
13502
+	0x1b, 0xac, 0x69, 0x8f, 0x74, 0x48, 0xf3, 0x7e, 0x38, 0x52, 0x65, 0x41, 0xf9, 0x7b, 0xec, 0xca,
13503
+	0xa5, 0x48, 0x0c, 0xaf, 0x44, 0xcf, 0xfe, 0xfa, 0xd3, 0xdf, 0xdf, 0x25, 0xe7, 0x21, 0x4b, 0x41,
13504
+	0xaf, 0xf0, 0x2f, 0x01, 0x32, 0x60, 0xc6, 0x69, 0xac, 0xa2, 0x97, 0x86, 0x69, 0x43, 0x2b, 0x97,
13505
+	0x63, 0x50, 0xd1, 0x0e, 0x4d, 0x00, 0xb7, 0xaf, 0x89, 0xa4, 0x73, 0x0d, 0xf4, 0x68, 0x95, 0x95,
13506
+	0x38, 0x58, 0xac, 0x4f, 0xb7, 0x6f, 0x29, 0xf7, 0x39, 0xd0, 0x27, 0x95, 0xfb, 0x94, 0xb4, 0x3f,
13507
+	0x43, 0x7c, 0xb2, 0x1c, 0xde, 0xaf, 0x5b, 0xad, 0xd0, 0x1c, 0x7a, 0xfa, 0x96, 0xa1, 0x39, 0xf4,
13508
+	0x75, 0x28, 0xa3, 0x73, 0x48, 0xfb, 0x56, 0xe1, 0x39, 0xf4, 0x76, 0x01, 0xc3, 0x73, 0xe8, 0x6b,
13509
+	0x7e, 0xc5, 0xee, 0x27, 0x5d, 0x5e, 0xc4, 0x7e, 0x7a, 0x57, 0xb8, 0x12, 0x07, 0x8b, 0xf5, 0xe9,
13510
+	0xf6, 0x9d, 0xe4, 0x3e, 0x07, 0x5a, 0x5b, 0x72, 0x9f, 0x83, 0xed, 0xab, 0x30, 0x9f, 0x4f, 0x60,
13511
+	0xd6, 0x7b, 0x85, 0x47, 0x57, 0x86, 0xec, 0x3b, 0x28, 0xc5, 0x78, 0x60, 0xb4, 0xe7, 0xcf, 0x21,
13512
+	0xeb, 0x6b, 0xfc, 0x21, 0xe9, 0x8c, 0xb2, 0x46, 0xa3, 0xb2, 0x3a, 0x04, 0x32, 0xd6, 0xb9, 0xaf,
13513
+	0xa7, 0x25, 0x77, 0x2e, 0xeb, 0xdb, 0xc9, 0x9d, 0x4b, 0x1b, 0x64, 0x11, 0xce, 0x7d, 0xad, 0x2b,
13514
+	0xb9, 0x73, 0x59, 0x8f, 0x4c, 0xee, 0x5c, 0xde, 0x07, 0x8b, 0x24, 0x19, 0xbf, 0x0a, 0x86, 0x92,
13515
+	0xcc, 0xdf, 0x3e, 0x08, 0x25, 0x59, 0xb0, 0x17, 0x10, 0x4d, 0x32, 0x71, 0x6f, 0x0d, 0x27, 0x59,
13516
+	0xe0, 0xb2, 0x1d, 0x4e, 0xb2, 0xe0, 0x15, 0x38, 0x96, 0x64, 0x62, 0xc1, 0x11, 0x24, 0x0b, 0xac,
13517
+	0x79, 0x75, 0x08, 0xe4, 0x90, 0x79, 0x8e, 0x74, 0x2e, 0xeb, 0xd7, 0x44, 0xe5, 0x79, 0x48, 0xe7,
13518
+	0x2c, 0xcf, 0xbc, 0x70, 0x0f, 0xcd, 0xb3, 0xff, 0x4a, 0x14, 0x9a, 0xe7, 0xc0, 0xad, 0x21, 0x26,
13519
+	0xcf, 0xe2, 0x4e, 0x19, 0x9e, 0xe7, 0xc0, 0x45, 0x38, 0x3c, 0xcf, 0xc1, 0xeb, 0x69, 0xec, 0x79,
13520
+	0x16, 0x0b, 0x8e, 0x38, 0xcf, 0x81, 0x35, 0xaf, 0x0e, 0x81, 0x8c, 0xfd, 0x38, 0x39, 0xb7, 0x19,
13521
+	0xf9, 0xc7, 0x29, 0x78, 0x57, 0x52, 0x2e, 0xc7, 0xa0, 0x62, 0xf7, 0xd9, 0x7b, 0x75, 0x90, 0xef,
13522
+	0xb3, 0xe4, 0x5a, 0xa4, 0x14, 0xe3, 0x81, 0xd1, 0x9e, 0x0f, 0x20, 0xe3, 0x29, 0x80, 0xd1, 0xca,
13523
+	0x70, 0x35, 0xbb, 0x72, 0x25, 0x16, 0x17, 0xbb, 0x60, 0x6f, 0x7d, 0x2b, 0x5f, 0xb0, 0xa4, 0x98,
13524
+	0x56, 0x8a, 0xf1, 0xc0, 0x58, 0xcf, 0xde, 0x5a, 0x56, 0xee, 0x59, 0x52, 0x2f, 0x2b, 0xc5, 0x78,
13525
+	0x60, 0xa4, 0xe7, 0xca, 0x85, 0xa7, 0xcf, 0x0a, 0x67, 0x7e, 0x7f, 0x56, 0x38, 0xf3, 0xcf, 0xb3,
13526
+	0x42, 0xe2, 0x8b, 0x7e, 0x21, 0xf1, 0xb4, 0x5f, 0x48, 0xfc, 0xd6, 0x2f, 0x24, 0xfe, 0xe8, 0x17,
13527
+	0x12, 0xbb, 0x53, 0xf4, 0xbf, 0x51, 0xae, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xe8, 0xa4,
13528
+	0xf9, 0x06, 0x23, 0x00, 0x00,
13529 13529
 }
... ...
@@ -313,12 +313,17 @@ message ListClustersResponse {
313 313
 	repeated Cluster clusters = 1;
314 314
 }
315 315
 
316
-message JoinTokenRotation {
317
-	// RotateWorkerToken tells UpdateCluster to rotate the worker secret.
318
-	bool rotate_worker_token = 1;
316
+// KeyRotation tells UpdateCluster what items to rotate
317
+message KeyRotation {
318
+	// WorkerJoinToken tells UpdateCluster to rotate the worker secret token.
319
+	bool worker_join_token = 1;
320
+
321
+	// ManagerJoinToken tells UpdateCluster to rotate the manager secret token.
322
+	bool manager_join_token = 2;
323
+
324
+	// ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key
325
+	bool manager_unlock_key = 3;
319 326
 
320
-	// RotateManagerSecret tells UpdateCluster to rotate the manager secret.
321
-	bool rotate_manager_token = 2;
322 327
 }
323 328
 
324 329
 message UpdateClusterRequest {
... ...
@@ -331,8 +336,8 @@ message UpdateClusterRequest {
331 331
 	// Spec is the new spec to apply to the cluster.
332 332
 	ClusterSpec spec = 3;
333 333
 
334
-	// Rotation contains flags for join token rotation
335
-	JoinTokenRotation rotation = 4 [(gogoproto.nullable) = false];
334
+	// Rotation contains flags for join token and unlock key rotation
335
+	KeyRotation rotation = 4 [(gogoproto.nullable) = false];
336 336
 }
337 337
 
338 338
 message UpdateClusterResponse {
... ...
@@ -232,6 +232,12 @@ type Cluster struct {
232 232
 	// be honored. It's a mapping from CN -> BlacklistedCertificate.
233 233
 	// swarm. Their certificates should effectively be blacklisted.
234 234
 	BlacklistedCertificates map[string]*BlacklistedCertificate `protobuf:"bytes,8,rep,name=blacklisted_certificates,json=blacklistedCertificates" json:"blacklisted_certificates,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"`
235
+	// UnlockKeys defines the keys that lock node data at rest.  For example,
236
+	// this would contain the key encrypting key (KEK) that will encrypt the
237
+	// manager TLS keys at rest and the raft encryption keys at rest.
238
+	// If the key is empty, the node will be unlocked (will not require a key
239
+	// to start up from a shut down state).
240
+	UnlockKeys []*EncryptionKey `protobuf:"bytes,9,rep,name=unlock_keys,json=unlockKeys" json:"unlock_keys,omitempty"`
235 241
 }
236 242
 
237 243
 func (m *Cluster) Reset()                    { *m = Cluster{} }
... ...
@@ -460,6 +466,13 @@ func (m *Cluster) Copy() *Cluster {
460 460
 		}
461 461
 	}
462 462
 
463
+	if m.UnlockKeys != nil {
464
+		o.UnlockKeys = make([]*EncryptionKey, 0, len(m.UnlockKeys))
465
+		for _, v := range m.UnlockKeys {
466
+			o.UnlockKeys = append(o.UnlockKeys, v.Copy())
467
+		}
468
+	}
469
+
463 470
 	return o
464 471
 }
465 472
 
... ...
@@ -633,7 +646,7 @@ func (this *Cluster) GoString() string {
633 633
 	if this == nil {
634 634
 		return "nil"
635 635
 	}
636
-	s := make([]string, 0, 11)
636
+	s := make([]string, 0, 12)
637 637
 	s = append(s, "&api.Cluster{")
638 638
 	s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n")
639 639
 	s = append(s, "Meta: "+strings.Replace(this.Meta.GoString(), `&`, ``, 1)+",\n")
... ...
@@ -656,6 +669,9 @@ func (this *Cluster) GoString() string {
656 656
 	if this.BlacklistedCertificates != nil {
657 657
 		s = append(s, "BlacklistedCertificates: "+mapStringForBlacklistedCertificates+",\n")
658 658
 	}
659
+	if this.UnlockKeys != nil {
660
+		s = append(s, "UnlockKeys: "+fmt.Sprintf("%#v", this.UnlockKeys)+",\n")
661
+	}
659 662
 	s = append(s, "}")
660 663
 	return strings.Join(s, "")
661 664
 }
... ...
@@ -1310,6 +1326,18 @@ func (m *Cluster) MarshalTo(data []byte) (int, error) {
1310 1310
 			}
1311 1311
 		}
1312 1312
 	}
1313
+	if len(m.UnlockKeys) > 0 {
1314
+		for _, msg := range m.UnlockKeys {
1315
+			data[i] = 0x4a
1316
+			i++
1317
+			i = encodeVarintObjects(data, i, uint64(msg.Size()))
1318
+			n, err := msg.MarshalTo(data[i:])
1319
+			if err != nil {
1320
+				return 0, err
1321
+			}
1322
+			i += n
1323
+		}
1324
+	}
1313 1325
 	return i, nil
1314 1326
 }
1315 1327
 
... ...
@@ -1637,6 +1665,12 @@ func (m *Cluster) Size() (n int) {
1637 1637
 			n += mapEntrySize + 1 + sovObjects(uint64(mapEntrySize))
1638 1638
 		}
1639 1639
 	}
1640
+	if len(m.UnlockKeys) > 0 {
1641
+		for _, e := range m.UnlockKeys {
1642
+			l = e.Size()
1643
+			n += 1 + l + sovObjects(uint64(l))
1644
+		}
1645
+	}
1640 1646
 	return n
1641 1647
 }
1642 1648
 
... ...
@@ -1814,6 +1848,7 @@ func (this *Cluster) String() string {
1814 1814
 		`NetworkBootstrapKeys:` + strings.Replace(fmt.Sprintf("%v", this.NetworkBootstrapKeys), "EncryptionKey", "EncryptionKey", 1) + `,`,
1815 1815
 		`EncryptionKeyLamportClock:` + fmt.Sprintf("%v", this.EncryptionKeyLamportClock) + `,`,
1816 1816
 		`BlacklistedCertificates:` + mapStringForBlacklistedCertificates + `,`,
1817
+		`UnlockKeys:` + strings.Replace(fmt.Sprintf("%v", this.UnlockKeys), "EncryptionKey", "EncryptionKey", 1) + `,`,
1817 1818
 		`}`,
1818 1819
 	}, "")
1819 1820
 	return s
... ...
@@ -3863,6 +3898,37 @@ func (m *Cluster) Unmarshal(data []byte) error {
3863 3863
 				m.BlacklistedCertificates[mapkey] = mapvalue
3864 3864
 			}
3865 3865
 			iNdEx = postIndex
3866
+		case 9:
3867
+			if wireType != 2 {
3868
+				return fmt.Errorf("proto: wrong wireType = %d for field UnlockKeys", wireType)
3869
+			}
3870
+			var msglen int
3871
+			for shift := uint(0); ; shift += 7 {
3872
+				if shift >= 64 {
3873
+					return ErrIntOverflowObjects
3874
+				}
3875
+				if iNdEx >= l {
3876
+					return io.ErrUnexpectedEOF
3877
+				}
3878
+				b := data[iNdEx]
3879
+				iNdEx++
3880
+				msglen |= (int(b) & 0x7F) << shift
3881
+				if b < 0x80 {
3882
+					break
3883
+				}
3884
+			}
3885
+			if msglen < 0 {
3886
+				return ErrInvalidLengthObjects
3887
+			}
3888
+			postIndex := iNdEx + msglen
3889
+			if postIndex > l {
3890
+				return io.ErrUnexpectedEOF
3891
+			}
3892
+			m.UnlockKeys = append(m.UnlockKeys, &EncryptionKey{})
3893
+			if err := m.UnlockKeys[len(m.UnlockKeys)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
3894
+				return err
3895
+			}
3896
+			iNdEx = postIndex
3866 3897
 		default:
3867 3898
 			iNdEx = preIndex
3868 3899
 			skippy, err := skipObjects(data[iNdEx:])
... ...
@@ -4199,79 +4265,80 @@ var (
4199 4199
 func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) }
4200 4200
 
4201 4201
 var fileDescriptorObjects = []byte{
4202
-	// 1174 bytes of a gzipped FileDescriptorProto
4203
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x8f, 0x1b, 0x35,
4204
-	0x18, 0xee, 0x24, 0xb3, 0xf9, 0x78, 0xb3, 0x59, 0x81, 0xa9, 0xca, 0x34, 0x2c, 0xc9, 0x92, 0x0a,
4205
-	0x54, 0xa1, 0x2a, 0x15, 0xa5, 0xa0, 0x2d, 0xb4, 0x82, 0x7c, 0x09, 0xa2, 0x52, 0xa8, 0xdc, 0xd2,
4206
-	0x1e, 0x23, 0xef, 0x8c, 0x1b, 0x86, 0x4c, 0xc6, 0x23, 0xdb, 0x49, 0x95, 0x9e, 0x10, 0x3f, 0x80,
4207
-	0x9f, 0xc0, 0x5f, 0xe1, 0xba, 0x07, 0x0e, 0xdc, 0xe0, 0x80, 0x22, 0x36, 0x07, 0x24, 0x6e, 0xfc,
4208
-	0x04, 0x64, 0x8f, 0x27, 0x99, 0x55, 0x26, 0xcb, 0x56, 0xaa, 0xf6, 0xe6, 0x37, 0x7e, 0x9e, 0xc7,
4209
-	0xef, 0x97, 0xdf, 0x71, 0xa0, 0xca, 0x8e, 0xbe, 0xa7, 0xae, 0x14, 0xad, 0x88, 0x33, 0xc9, 0x10,
4210
-	0xf2, 0x98, 0x3b, 0xa6, 0xbc, 0x25, 0x9e, 0x13, 0x3e, 0x19, 0xfb, 0xb2, 0x35, 0xfb, 0xa0, 0x56,
4211
-	0x91, 0xf3, 0x88, 0x1a, 0x40, 0xad, 0x22, 0x22, 0xea, 0x26, 0xc6, 0x55, 0xe9, 0x4f, 0xa8, 0x90,
4212
-	0x64, 0x12, 0xdd, 0x5c, 0xad, 0xcc, 0xd6, 0xe5, 0x11, 0x1b, 0x31, 0xbd, 0xbc, 0xa9, 0x56, 0xf1,
4213
-	0xaf, 0xcd, 0x5f, 0x2c, 0xb0, 0x1f, 0x50, 0x49, 0xd0, 0xa7, 0x50, 0x9c, 0x51, 0x2e, 0x7c, 0x16,
4214
-	0x3a, 0xd6, 0x81, 0x75, 0xbd, 0x72, 0xeb, 0xad, 0xd6, 0xe6, 0xc9, 0xad, 0x27, 0x31, 0xa4, 0x63,
4215
-	0x1f, 0x2f, 0x1a, 0x97, 0x70, 0xc2, 0x40, 0x77, 0x01, 0x5c, 0x4e, 0x89, 0xa4, 0xde, 0x90, 0x48,
4216
-	0x27, 0xa7, 0xf9, 0x6f, 0x67, 0xf1, 0x1f, 0x27, 0x4e, 0xe1, 0xb2, 0x21, 0xb4, 0xa5, 0x62, 0x4f,
4217
-	0x23, 0x2f, 0x61, 0xe7, 0xcf, 0xc5, 0x36, 0x84, 0xb6, 0x6c, 0xfe, 0x93, 0x07, 0xfb, 0x6b, 0xe6,
4218
-	0x51, 0x74, 0x05, 0x72, 0xbe, 0xa7, 0x9d, 0x2f, 0x77, 0x0a, 0xcb, 0x45, 0x23, 0x37, 0xe8, 0xe1,
4219
-	0x9c, 0xef, 0xa1, 0x5b, 0x60, 0x4f, 0xa8, 0x24, 0xc6, 0x2d, 0x27, 0x4b, 0x58, 0x65, 0xc0, 0xc4,
4220
-	0xa4, 0xb1, 0xe8, 0x63, 0xb0, 0x55, 0x5a, 0x8d, 0x33, 0xfb, 0x59, 0x1c, 0x75, 0xe6, 0xa3, 0x88,
4221
-	0xba, 0x09, 0x4f, 0xe1, 0x51, 0x1f, 0x2a, 0x1e, 0x15, 0x2e, 0xf7, 0x23, 0xa9, 0x32, 0x69, 0x6b,
4222
-	0xfa, 0xb5, 0x6d, 0xf4, 0xde, 0x1a, 0x8a, 0xd3, 0x3c, 0x74, 0x17, 0x0a, 0x42, 0x12, 0x39, 0x15,
4223
-	0xce, 0x8e, 0x56, 0xa8, 0x6f, 0x75, 0x40, 0xa3, 0x8c, 0x0b, 0x86, 0x83, 0xbe, 0x84, 0xbd, 0x09,
4224
-	0x09, 0xc9, 0x88, 0xf2, 0xa1, 0x51, 0x29, 0x68, 0x95, 0x77, 0x32, 0x43, 0x8f, 0x91, 0xb1, 0x10,
4225
-	0xae, 0x4e, 0xd2, 0x26, 0xea, 0x03, 0x10, 0x29, 0x89, 0xfb, 0xdd, 0x84, 0x86, 0xd2, 0x29, 0x6a,
4226
-	0x95, 0x77, 0x33, 0x7d, 0xa1, 0xf2, 0x39, 0xe3, 0xe3, 0xf6, 0x0a, 0x8c, 0x53, 0x44, 0xf4, 0x05,
4227
-	0x54, 0x5c, 0xca, 0xa5, 0xff, 0xcc, 0x77, 0x89, 0xa4, 0x4e, 0x49, 0xeb, 0x34, 0xb2, 0x74, 0xba,
4228
-	0x6b, 0x98, 0x09, 0x2a, 0xcd, 0x6c, 0xfe, 0x9e, 0x83, 0xe2, 0x23, 0xca, 0x67, 0xbe, 0xfb, 0x6a,
4229
-	0xcb, 0x7d, 0xe7, 0x54, 0xb9, 0x33, 0x3d, 0x33, 0xc7, 0x6e, 0x54, 0xfc, 0x10, 0x4a, 0x34, 0xf4,
4230
-	0x22, 0xe6, 0x87, 0xd2, 0x94, 0x3b, 0xb3, 0x5b, 0xfa, 0x06, 0x83, 0x57, 0x68, 0xd4, 0x87, 0x6a,
4231
-	0xdc, 0xc5, 0xc3, 0x53, 0xb5, 0x3e, 0xc8, 0xa2, 0x7f, 0xab, 0x81, 0xa6, 0x48, 0xbb, 0xd3, 0x94,
4232
-	0x85, 0x7a, 0x50, 0x8d, 0x38, 0x9d, 0xf9, 0x6c, 0x2a, 0x86, 0x3a, 0x88, 0xc2, 0xb9, 0x82, 0xc0,
4233
-	0xbb, 0x09, 0x4b, 0x59, 0xcd, 0x9f, 0x73, 0x50, 0x4a, 0x7c, 0x44, 0xb7, 0x4d, 0x3a, 0xac, 0xed,
4234
-	0x0e, 0x25, 0x58, 0x2d, 0x15, 0x67, 0xe2, 0x36, 0xec, 0x44, 0x8c, 0x4b, 0xe1, 0xe4, 0x0e, 0xf2,
4235
-	0xdb, 0x7a, 0xf6, 0x21, 0xe3, 0xb2, 0xcb, 0xc2, 0x67, 0xfe, 0x08, 0xc7, 0x60, 0xf4, 0x14, 0x2a,
4236
-	0x33, 0x9f, 0xcb, 0x29, 0x09, 0x86, 0x7e, 0x24, 0x9c, 0xbc, 0xe6, 0xbe, 0x77, 0xd6, 0x91, 0xad,
4237
-	0x27, 0x31, 0x7e, 0xf0, 0xb0, 0xb3, 0xb7, 0x5c, 0x34, 0x60, 0x65, 0x0a, 0x0c, 0x46, 0x6a, 0x10,
4238
-	0x89, 0xda, 0x03, 0x28, 0xaf, 0x76, 0xd0, 0x0d, 0x80, 0x30, 0x6e, 0xd1, 0xe1, 0xaa, 0x69, 0xaa,
4239
-	0xcb, 0x45, 0xa3, 0x6c, 0x1a, 0x77, 0xd0, 0xc3, 0x65, 0x03, 0x18, 0x78, 0x08, 0x81, 0x4d, 0x3c,
4240
-	0x8f, 0xeb, 0x16, 0x2a, 0x63, 0xbd, 0x6e, 0xfe, 0xba, 0x03, 0xf6, 0x63, 0x22, 0xc6, 0x17, 0x3d,
4241
-	0x66, 0xd4, 0x99, 0x1b, 0x4d, 0x77, 0x03, 0x40, 0xc4, 0xa5, 0x54, 0xe1, 0xd8, 0xeb, 0x70, 0x4c,
4242
-	0x81, 0x55, 0x38, 0x06, 0x10, 0x87, 0x23, 0x02, 0x26, 0x75, 0x7f, 0xd9, 0x58, 0xaf, 0xd1, 0x35,
4243
-	0x28, 0x86, 0xcc, 0xd3, 0xf4, 0x82, 0xa6, 0xc3, 0x72, 0xd1, 0x28, 0xa8, 0x91, 0x32, 0xe8, 0xe1,
4244
-	0x82, 0xda, 0x1a, 0x78, 0xea, 0xde, 0x92, 0x30, 0x64, 0x92, 0xa8, 0xa1, 0x24, 0xcc, 0xfd, 0xcf,
4245
-	0x6c, 0xac, 0xf6, 0x1a, 0x96, 0xdc, 0xdb, 0x14, 0x13, 0x3d, 0x81, 0x37, 0x12, 0x7f, 0xd3, 0x82,
4246
-	0xa5, 0x97, 0x11, 0x44, 0x46, 0x21, 0xb5, 0x93, 0x9a, 0x93, 0xe5, 0xed, 0x73, 0x52, 0x67, 0x30,
4247
-	0x6b, 0x4e, 0x76, 0xa0, 0xea, 0x51, 0xe1, 0x73, 0xea, 0xe9, 0x1b, 0x48, 0x1d, 0x38, 0xb0, 0xae,
4248
-	0xef, 0x6d, 0xf9, 0xf4, 0x18, 0x11, 0x8a, 0x77, 0x0d, 0x47, 0x5b, 0xa8, 0x0d, 0x25, 0xd3, 0x37,
4249
-	0xc2, 0xa9, 0xe8, 0xde, 0x3d, 0xe7, 0x7c, 0x5c, 0xd1, 0x4e, 0x4d, 0x90, 0xdd, 0x97, 0x9a, 0x20,
4250
-	0x77, 0x00, 0x02, 0x36, 0x1a, 0x7a, 0xdc, 0x9f, 0x51, 0xee, 0x54, 0x35, 0xb7, 0x96, 0xc5, 0xed,
4251
-	0x69, 0x04, 0x2e, 0x07, 0x6c, 0x14, 0x2f, 0x9b, 0x3f, 0x5a, 0xf0, 0xfa, 0x86, 0x53, 0xe8, 0x23,
4252
-	0x28, 0x1a, 0xb7, 0xce, 0x7a, 0x04, 0x18, 0x1e, 0x4e, 0xb0, 0x68, 0x1f, 0xca, 0xea, 0x8e, 0x50,
4253
-	0x21, 0x68, 0x7c, 0xfb, 0xcb, 0x78, 0xfd, 0x03, 0x72, 0xa0, 0x48, 0x02, 0x9f, 0xa8, 0xbd, 0xbc,
4254
-	0xde, 0x4b, 0xcc, 0xe6, 0x4f, 0x39, 0x28, 0x1a, 0xb1, 0x8b, 0x1e, 0xe7, 0xe6, 0xd8, 0x8d, 0x9b,
4255
-	0x75, 0x0f, 0x76, 0xe3, 0x74, 0x9a, 0x96, 0xb0, 0xff, 0x37, 0xa9, 0x95, 0x18, 0x1f, 0xb7, 0xc3,
4256
-	0x3d, 0xb0, 0xfd, 0x88, 0x4c, 0xcc, 0x28, 0xcf, 0x3c, 0x79, 0xf0, 0xb0, 0xfd, 0xe0, 0x9b, 0x28,
4257
-	0xee, 0xec, 0xd2, 0x72, 0xd1, 0xb0, 0xd5, 0x0f, 0x58, 0xd3, 0x9a, 0x7f, 0xda, 0x50, 0xec, 0x06,
4258
-	0x53, 0x21, 0x29, 0xbf, 0xe8, 0x84, 0x98, 0x63, 0x37, 0x12, 0xd2, 0x85, 0x22, 0x67, 0x4c, 0x0e,
4259
-	0x5d, 0x72, 0x56, 0x2e, 0x30, 0x63, 0xb2, 0xdb, 0xee, 0xec, 0x29, 0xa2, 0x1a, 0x24, 0xb1, 0x8d,
4260
-	0x0b, 0x8a, 0xda, 0x25, 0xe8, 0x29, 0x5c, 0x49, 0xc6, 0xef, 0x11, 0x63, 0x52, 0x48, 0x4e, 0xa2,
4261
-	0xe1, 0x98, 0xce, 0xd5, 0x37, 0x2f, 0xbf, 0xed, 0x65, 0xd2, 0x0f, 0x5d, 0x3e, 0xd7, 0x89, 0xba,
4262
-	0x4f, 0xe7, 0xf8, 0xb2, 0x11, 0xe8, 0x24, 0xfc, 0xfb, 0x74, 0x2e, 0xd0, 0x67, 0xb0, 0x4f, 0x57,
4263
-	0x30, 0xa5, 0x38, 0x0c, 0xc8, 0x44, 0x7d, 0x58, 0x86, 0x6e, 0xc0, 0xdc, 0xb1, 0x9e, 0x6d, 0x36,
4264
-	0xbe, 0x4a, 0xd3, 0x52, 0x5f, 0xc5, 0x88, 0xae, 0x02, 0x20, 0x01, 0xce, 0x51, 0x40, 0xdc, 0x71,
4265
-	0xe0, 0x0b, 0xf5, 0xfe, 0x4c, 0x3d, 0x36, 0xd4, 0x78, 0x52, 0xbe, 0x1d, 0x9e, 0x91, 0xad, 0x56,
4266
-	0x67, 0xcd, 0x4d, 0x3d, 0x5d, 0x44, 0x3f, 0x94, 0x7c, 0x8e, 0xdf, 0x3c, 0xca, 0xde, 0xad, 0xcd,
4267
-	0x60, 0xff, 0x2c, 0x22, 0x7a, 0x0d, 0xf2, 0x63, 0x3a, 0x8f, 0x6b, 0x8f, 0xd5, 0x12, 0x7d, 0x0e,
4268
-	0x3b, 0x33, 0x12, 0x4c, 0xa9, 0xa9, 0xfa, 0xfb, 0x59, 0x3e, 0x65, 0x4b, 0xe2, 0x98, 0xf8, 0x49,
4269
-	0xee, 0xd0, 0x6a, 0xfe, 0x6d, 0x41, 0xe1, 0x11, 0x75, 0x39, 0x95, 0xaf, 0xb4, 0xbb, 0x0e, 0x4f,
4270
-	0x75, 0x57, 0x3d, 0xfb, 0xe1, 0xa1, 0x4e, 0xdd, 0x68, 0xae, 0x2b, 0x50, 0xf0, 0xfc, 0x11, 0x15,
4271
-	0xf1, 0xd3, 0xa9, 0x8c, 0x8d, 0x85, 0x9a, 0x60, 0x0b, 0xff, 0x05, 0xd5, 0xd7, 0x28, 0x1f, 0x7f,
4272
-	0xe5, 0x8d, 0x82, 0xff, 0x82, 0x62, 0xbd, 0x87, 0x6a, 0x50, 0xf2, 0x43, 0x49, 0x79, 0x48, 0x02,
4273
-	0x5d, 0xe6, 0x12, 0x5e, 0xd9, 0x9d, 0xfd, 0xe3, 0x93, 0xfa, 0xa5, 0x3f, 0x4e, 0xea, 0x97, 0xfe,
4274
-	0x3d, 0xa9, 0x5b, 0x3f, 0x2c, 0xeb, 0xd6, 0xf1, 0xb2, 0x6e, 0xfd, 0xb6, 0xac, 0x5b, 0x7f, 0x2d,
4275
-	0xeb, 0xd6, 0x51, 0x41, 0xff, 0xf5, 0xf9, 0xf0, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x40, 0xcf,
4276
-	0x57, 0x63, 0x6a, 0x0d, 0x00, 0x00,
4202
+	// 1192 bytes of a gzipped FileDescriptorProto
4203
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4f, 0x6f, 0x1b, 0x45,
4204
+	0x14, 0xef, 0xda, 0x1b, 0xdb, 0xfb, 0x1c, 0x47, 0x30, 0x54, 0x65, 0x1b, 0x82, 0x1d, 0x5c, 0x81,
4205
+	0x2a, 0x54, 0xb9, 0xa2, 0x14, 0x94, 0x42, 0x2b, 0xb0, 0x9d, 0x08, 0xac, 0x52, 0xa8, 0xa6, 0xa5,
4206
+	0x3d, 0x5a, 0x93, 0xdd, 0xa9, 0x59, 0xbc, 0xde, 0x59, 0xcd, 0x8c, 0x5d, 0xb9, 0x27, 0xc4, 0x07,
4207
+	0xe0, 0x23, 0x20, 0xbe, 0x09, 0xd7, 0x1e, 0x38, 0x70, 0x83, 0x53, 0x44, 0x7d, 0x40, 0xe2, 0xc6,
4208
+	0x47, 0x40, 0xf3, 0x67, 0x9d, 0x8d, 0xbc, 0x0e, 0xa9, 0x54, 0xe5, 0x36, 0xe3, 0xf9, 0xfd, 0x7e,
4209
+	0xef, 0xcd, 0x9b, 0xdf, 0xbc, 0x1d, 0x43, 0x83, 0x1d, 0x7e, 0x4f, 0x03, 0x29, 0x3a, 0x29, 0x67,
4210
+	0x92, 0x21, 0x14, 0xb2, 0x60, 0x4c, 0x79, 0x47, 0x3c, 0x25, 0x7c, 0x32, 0x8e, 0x64, 0x67, 0xf6,
4211
+	0xc1, 0x76, 0x5d, 0xce, 0x53, 0x6a, 0x01, 0xdb, 0x75, 0x91, 0xd2, 0x20, 0x9b, 0x5c, 0x96, 0xd1,
4212
+	0x84, 0x0a, 0x49, 0x26, 0xe9, 0xf5, 0xe5, 0xc8, 0x2e, 0x5d, 0x1c, 0xb1, 0x11, 0xd3, 0xc3, 0xeb,
4213
+	0x6a, 0x64, 0x7e, 0x6d, 0xff, 0xea, 0x80, 0x7b, 0x8f, 0x4a, 0x82, 0x3e, 0x85, 0xea, 0x8c, 0x72,
4214
+	0x11, 0xb1, 0xc4, 0x77, 0x76, 0x9d, 0xab, 0xf5, 0x1b, 0x6f, 0x75, 0x56, 0x23, 0x77, 0x1e, 0x19,
4215
+	0x48, 0xcf, 0x7d, 0x7e, 0xd4, 0xba, 0x80, 0x33, 0x06, 0xba, 0x0d, 0x10, 0x70, 0x4a, 0x24, 0x0d,
4216
+	0x87, 0x44, 0xfa, 0x25, 0xcd, 0x7f, 0xbb, 0x88, 0xff, 0x30, 0x4b, 0x0a, 0x7b, 0x96, 0xd0, 0x95,
4217
+	0x8a, 0x3d, 0x4d, 0xc3, 0x8c, 0x5d, 0x3e, 0x13, 0xdb, 0x12, 0xba, 0xb2, 0xfd, 0x4f, 0x19, 0xdc,
4218
+	0xaf, 0x59, 0x48, 0xd1, 0x25, 0x28, 0x45, 0xa1, 0x4e, 0xde, 0xeb, 0x55, 0x16, 0x47, 0xad, 0xd2,
4219
+	0x60, 0x1f, 0x97, 0xa2, 0x10, 0xdd, 0x00, 0x77, 0x42, 0x25, 0xb1, 0x69, 0xf9, 0x45, 0xc2, 0xaa,
4220
+	0x02, 0x76, 0x4f, 0x1a, 0x8b, 0x3e, 0x06, 0x57, 0x95, 0xd5, 0x26, 0xb3, 0x53, 0xc4, 0x51, 0x31,
4221
+	0x1f, 0xa4, 0x34, 0xc8, 0x78, 0x0a, 0x8f, 0x0e, 0xa0, 0x1e, 0x52, 0x11, 0xf0, 0x28, 0x95, 0xaa,
4222
+	0x92, 0xae, 0xa6, 0x5f, 0x59, 0x47, 0xdf, 0x3f, 0x86, 0xe2, 0x3c, 0x0f, 0xdd, 0x86, 0x8a, 0x90,
4223
+	0x44, 0x4e, 0x85, 0xbf, 0xa1, 0x15, 0x9a, 0x6b, 0x13, 0xd0, 0x28, 0x9b, 0x82, 0xe5, 0xa0, 0x2f,
4224
+	0x61, 0x6b, 0x42, 0x12, 0x32, 0xa2, 0x7c, 0x68, 0x55, 0x2a, 0x5a, 0xe5, 0x9d, 0xc2, 0xad, 0x1b,
4225
+	0xa4, 0x11, 0xc2, 0x8d, 0x49, 0x7e, 0x8a, 0x0e, 0x00, 0x88, 0x94, 0x24, 0xf8, 0x6e, 0x42, 0x13,
4226
+	0xe9, 0x57, 0xb5, 0xca, 0xbb, 0x85, 0xb9, 0x50, 0xf9, 0x94, 0xf1, 0x71, 0x77, 0x09, 0xc6, 0x39,
4227
+	0x22, 0xfa, 0x02, 0xea, 0x01, 0xe5, 0x32, 0x7a, 0x12, 0x05, 0x44, 0x52, 0xbf, 0xa6, 0x75, 0x5a,
4228
+	0x45, 0x3a, 0xfd, 0x63, 0x98, 0xdd, 0x54, 0x9e, 0xd9, 0xfe, 0xa3, 0x04, 0xd5, 0x07, 0x94, 0xcf,
4229
+	0xa2, 0xe0, 0xd5, 0x1e, 0xf7, 0xad, 0x13, 0xc7, 0x5d, 0x98, 0x99, 0x0d, 0xbb, 0x72, 0xe2, 0x7b,
4230
+	0x50, 0xa3, 0x49, 0x98, 0xb2, 0x28, 0x91, 0xf6, 0xb8, 0x0b, 0xdd, 0x72, 0x60, 0x31, 0x78, 0x89,
4231
+	0x46, 0x07, 0xd0, 0x30, 0x2e, 0x1e, 0x9e, 0x38, 0xeb, 0xdd, 0x22, 0xfa, 0xb7, 0x1a, 0x68, 0x0f,
4232
+	0x69, 0x73, 0x9a, 0x9b, 0xa1, 0x7d, 0x68, 0xa4, 0x9c, 0xce, 0x22, 0x36, 0x15, 0x43, 0xbd, 0x89,
4233
+	0xca, 0x99, 0x36, 0x81, 0x37, 0x33, 0x96, 0x9a, 0xb5, 0x7f, 0x2e, 0x41, 0x2d, 0xcb, 0x11, 0xdd,
4234
+	0xb4, 0xe5, 0x70, 0xd6, 0x27, 0x94, 0x61, 0xb5, 0x94, 0xa9, 0xc4, 0x4d, 0xd8, 0x48, 0x19, 0x97,
4235
+	0xc2, 0x2f, 0xed, 0x96, 0xd7, 0x79, 0xf6, 0x3e, 0xe3, 0xb2, 0xcf, 0x92, 0x27, 0xd1, 0x08, 0x1b,
4236
+	0x30, 0x7a, 0x0c, 0xf5, 0x59, 0xc4, 0xe5, 0x94, 0xc4, 0xc3, 0x28, 0x15, 0x7e, 0x59, 0x73, 0xdf,
4237
+	0x3b, 0x2d, 0x64, 0xe7, 0x91, 0xc1, 0x0f, 0xee, 0xf7, 0xb6, 0x16, 0x47, 0x2d, 0x58, 0x4e, 0x05,
4238
+	0x06, 0x2b, 0x35, 0x48, 0xc5, 0xf6, 0x3d, 0xf0, 0x96, 0x2b, 0xe8, 0x1a, 0x40, 0x62, 0x2c, 0x3a,
4239
+	0x5c, 0x9a, 0xa6, 0xb1, 0x38, 0x6a, 0x79, 0xd6, 0xb8, 0x83, 0x7d, 0xec, 0x59, 0xc0, 0x20, 0x44,
4240
+	0x08, 0x5c, 0x12, 0x86, 0x5c, 0x5b, 0xc8, 0xc3, 0x7a, 0xdc, 0xfe, 0x6d, 0x03, 0xdc, 0x87, 0x44,
4241
+	0x8c, 0xcf, 0xbb, 0xcd, 0xa8, 0x98, 0x2b, 0xa6, 0xbb, 0x06, 0x20, 0xcc, 0x51, 0xaa, 0xed, 0xb8,
4242
+	0xc7, 0xdb, 0xb1, 0x07, 0xac, 0xb6, 0x63, 0x01, 0x66, 0x3b, 0x22, 0x66, 0x52, 0xfb, 0xcb, 0xc5,
4243
+	0x7a, 0x8c, 0xae, 0x40, 0x35, 0x61, 0xa1, 0xa6, 0x57, 0x34, 0x1d, 0x16, 0x47, 0xad, 0x8a, 0x6a,
4244
+	0x29, 0x83, 0x7d, 0x5c, 0x51, 0x4b, 0x83, 0x50, 0xdd, 0x5b, 0x92, 0x24, 0x4c, 0x12, 0xd5, 0x94,
4245
+	0x84, 0xbd, 0xff, 0x85, 0xc6, 0xea, 0x1e, 0xc3, 0xb2, 0x7b, 0x9b, 0x63, 0xa2, 0x47, 0xf0, 0x46,
4246
+	0x96, 0x6f, 0x5e, 0xb0, 0xf6, 0x32, 0x82, 0xc8, 0x2a, 0xe4, 0x56, 0x72, 0x7d, 0xd2, 0x5b, 0xdf,
4247
+	0x27, 0x75, 0x05, 0x8b, 0xfa, 0x64, 0x0f, 0x1a, 0x21, 0x15, 0x11, 0xa7, 0xa1, 0xbe, 0x81, 0xd4,
4248
+	0x87, 0x5d, 0xe7, 0xea, 0xd6, 0x9a, 0x4f, 0x8f, 0x15, 0xa1, 0x78, 0xd3, 0x72, 0xf4, 0x0c, 0x75,
4249
+	0xa1, 0x66, 0x7d, 0x23, 0xfc, 0xba, 0xf6, 0xee, 0x19, 0xfb, 0xe3, 0x92, 0x76, 0xa2, 0x83, 0x6c,
4250
+	0xbe, 0x54, 0x07, 0xb9, 0x05, 0x10, 0xb3, 0xd1, 0x30, 0xe4, 0xd1, 0x8c, 0x72, 0xbf, 0xa1, 0xb9,
4251
+	0xdb, 0x45, 0xdc, 0x7d, 0x8d, 0xc0, 0x5e, 0xcc, 0x46, 0x66, 0xd8, 0xfe, 0xd1, 0x81, 0xd7, 0x57,
4252
+	0x92, 0x42, 0x1f, 0x41, 0xd5, 0xa6, 0x75, 0xda, 0x23, 0xc0, 0xf2, 0x70, 0x86, 0x45, 0x3b, 0xe0,
4253
+	0xa9, 0x3b, 0x42, 0x85, 0xa0, 0xe6, 0xf6, 0x7b, 0xf8, 0xf8, 0x07, 0xe4, 0x43, 0x95, 0xc4, 0x11,
4254
+	0x51, 0x6b, 0x65, 0xbd, 0x96, 0x4d, 0xdb, 0x3f, 0x95, 0xa0, 0x6a, 0xc5, 0xce, 0xbb, 0x9d, 0xdb,
4255
+	0xb0, 0x2b, 0x37, 0xeb, 0x0e, 0x6c, 0x9a, 0x72, 0x5a, 0x4b, 0xb8, 0xff, 0x5b, 0xd4, 0xba, 0xc1,
4256
+	0x1b, 0x3b, 0xdc, 0x01, 0x37, 0x4a, 0xc9, 0xc4, 0xb6, 0xf2, 0xc2, 0xc8, 0x83, 0xfb, 0xdd, 0x7b,
4257
+	0xdf, 0xa4, 0xc6, 0xd9, 0xb5, 0xc5, 0x51, 0xcb, 0x55, 0x3f, 0x60, 0x4d, 0x6b, 0xff, 0xb2, 0x01,
4258
+	0xd5, 0x7e, 0x3c, 0x15, 0x92, 0xf2, 0xf3, 0x2e, 0x88, 0x0d, 0xbb, 0x52, 0x90, 0x3e, 0x54, 0x39,
4259
+	0x63, 0x72, 0x18, 0x90, 0xd3, 0x6a, 0x81, 0x19, 0x93, 0xfd, 0x6e, 0x6f, 0x4b, 0x11, 0x55, 0x23,
4260
+	0x31, 0x73, 0x5c, 0x51, 0xd4, 0x3e, 0x41, 0x8f, 0xe1, 0x52, 0xd6, 0x7e, 0x0f, 0x19, 0x93, 0x42,
4261
+	0x72, 0x92, 0x0e, 0xc7, 0x74, 0xae, 0xbe, 0x79, 0xe5, 0x75, 0x2f, 0x93, 0x83, 0x24, 0xe0, 0x73,
4262
+	0x5d, 0xa8, 0xbb, 0x74, 0x8e, 0x2f, 0x5a, 0x81, 0x5e, 0xc6, 0xbf, 0x4b, 0xe7, 0x02, 0x7d, 0x06,
4263
+	0x3b, 0x74, 0x09, 0x53, 0x8a, 0xc3, 0x98, 0x4c, 0xd4, 0x87, 0x65, 0x18, 0xc4, 0x2c, 0x18, 0xeb,
4264
+	0xde, 0xe6, 0xe2, 0xcb, 0x34, 0x2f, 0xf5, 0x95, 0x41, 0xf4, 0x15, 0x00, 0x09, 0xf0, 0x0f, 0x63,
4265
+	0x12, 0x8c, 0xe3, 0x48, 0xa8, 0xf7, 0x67, 0xee, 0xb1, 0xa1, 0xda, 0x93, 0xca, 0x6d, 0xef, 0x94,
4266
+	0x6a, 0x75, 0x7a, 0xc7, 0xdc, 0xdc, 0xd3, 0x45, 0x1c, 0x24, 0x92, 0xcf, 0xf1, 0x9b, 0x87, 0xc5,
4267
+	0xab, 0xa8, 0x07, 0xf5, 0x69, 0xa2, 0xc2, 0x9b, 0x1a, 0x78, 0x67, 0xad, 0x01, 0x18, 0x96, 0xda,
4268
+	0xf9, 0xf6, 0x0c, 0x76, 0x4e, 0x0b, 0x8e, 0x5e, 0x83, 0xf2, 0x98, 0xce, 0x8d, 0x7f, 0xb0, 0x1a,
4269
+	0xa2, 0xcf, 0x61, 0x63, 0x46, 0xe2, 0x29, 0xb5, 0xce, 0x79, 0xbf, 0x28, 0x5e, 0xb1, 0x24, 0x36,
4270
+	0xc4, 0x4f, 0x4a, 0x7b, 0x4e, 0xfb, 0x6f, 0x07, 0x2a, 0x0f, 0x68, 0xc0, 0xa9, 0x7c, 0xa5, 0x0e,
4271
+	0xdd, 0x3b, 0xe1, 0xd0, 0x66, 0xf1, 0xe3, 0x45, 0x45, 0x5d, 0x31, 0xe8, 0x25, 0xa8, 0x84, 0xd1,
4272
+	0x88, 0x0a, 0xf3, 0xfc, 0xf2, 0xb0, 0x9d, 0xa1, 0x36, 0xb8, 0x22, 0x7a, 0x46, 0xf5, 0x55, 0x2c,
4273
+	0x9b, 0x97, 0x82, 0x55, 0x88, 0x9e, 0x51, 0xac, 0xd7, 0xd0, 0x36, 0xd4, 0xa2, 0x44, 0x52, 0x9e,
4274
+	0x90, 0x58, 0x5b, 0xa5, 0x86, 0x97, 0xf3, 0xde, 0xce, 0xf3, 0x17, 0xcd, 0x0b, 0x7f, 0xbe, 0x68,
4275
+	0x5e, 0xf8, 0xf7, 0x45, 0xd3, 0xf9, 0x61, 0xd1, 0x74, 0x9e, 0x2f, 0x9a, 0xce, 0xef, 0x8b, 0xa6,
4276
+	0xf3, 0xd7, 0xa2, 0xe9, 0x1c, 0x56, 0xf4, 0xdf, 0xa7, 0x0f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff,
4277
+	0x49, 0x8f, 0xcd, 0x22, 0xae, 0x0d, 0x00, 0x00,
4277 4278
 }
... ...
@@ -230,6 +230,13 @@ message Cluster {
230 230
 	// be honored. It's a mapping from CN -> BlacklistedCertificate.
231 231
 	// swarm. Their certificates should effectively be blacklisted.
232 232
 	map<string, BlacklistedCertificate> blacklisted_certificates = 8;
233
+
234
+	// UnlockKeys defines the keys that lock node data at rest.  For example,
235
+	// this would contain the key encrypting key (KEK) that will encrypt the
236
+	// manager TLS keys at rest and the raft encryption keys at rest.
237
+	// If the key is empty, the node will be unlocked (will not require a key
238
+	// to start up from a shut down state).
239
+	repeated EncryptionKey unlock_keys = 9;
233 240
 }
234 241
 
235 242
 // Secret represents a secret that should be passed to a container or a node,
... ...
@@ -596,6 +596,8 @@ type ClusterSpec struct {
596 596
 	CAConfig CAConfig `protobuf:"bytes,6,opt,name=ca_config,json=caConfig" json:"ca_config"`
597 597
 	// TaskDefaults specifies the default values to use for task creation.
598 598
 	TaskDefaults TaskDefaults `protobuf:"bytes,7,opt,name=task_defaults,json=taskDefaults" json:"task_defaults"`
599
+	// EncryptionConfig defines the cluster's encryption settings.
600
+	EncryptionConfig EncryptionConfig `protobuf:"bytes,8,opt,name=encryption_config,json=encryptionConfig" json:"encryption_config"`
599 601
 }
600 602
 
601 603
 func (m *ClusterSpec) Reset()                    { *m = ClusterSpec{} }
... ...
@@ -908,6 +910,7 @@ func (m *ClusterSpec) Copy() *ClusterSpec {
908 908
 		Dispatcher:       *m.Dispatcher.Copy(),
909 909
 		CAConfig:         *m.CAConfig.Copy(),
910 910
 		TaskDefaults:     *m.TaskDefaults.Copy(),
911
+		EncryptionConfig: *m.EncryptionConfig.Copy(),
911 912
 	}
912 913
 
913 914
 	return o
... ...
@@ -1159,7 +1162,7 @@ func (this *ClusterSpec) GoString() string {
1159 1159
 	if this == nil {
1160 1160
 		return "nil"
1161 1161
 	}
1162
-	s := make([]string, 0, 11)
1162
+	s := make([]string, 0, 12)
1163 1163
 	s = append(s, "&api.ClusterSpec{")
1164 1164
 	s = append(s, "Annotations: "+strings.Replace(this.Annotations.GoString(), `&`, ``, 1)+",\n")
1165 1165
 	s = append(s, "AcceptancePolicy: "+strings.Replace(this.AcceptancePolicy.GoString(), `&`, ``, 1)+",\n")
... ...
@@ -1168,6 +1171,7 @@ func (this *ClusterSpec) GoString() string {
1168 1168
 	s = append(s, "Dispatcher: "+strings.Replace(this.Dispatcher.GoString(), `&`, ``, 1)+",\n")
1169 1169
 	s = append(s, "CAConfig: "+strings.Replace(this.CAConfig.GoString(), `&`, ``, 1)+",\n")
1170 1170
 	s = append(s, "TaskDefaults: "+strings.Replace(this.TaskDefaults.GoString(), `&`, ``, 1)+",\n")
1171
+	s = append(s, "EncryptionConfig: "+strings.Replace(this.EncryptionConfig.GoString(), `&`, ``, 1)+",\n")
1171 1172
 	s = append(s, "}")
1172 1173
 	return strings.Join(s, "")
1173 1174
 }
... ...
@@ -2008,6 +2012,14 @@ func (m *ClusterSpec) MarshalTo(data []byte) (int, error) {
2008 2008
 		return 0, err
2009 2009
 	}
2010 2010
 	i += n29
2011
+	data[i] = 0x42
2012
+	i++
2013
+	i = encodeVarintSpecs(data, i, uint64(m.EncryptionConfig.Size()))
2014
+	n30, err := m.EncryptionConfig.MarshalTo(data[i:])
2015
+	if err != nil {
2016
+		return 0, err
2017
+	}
2018
+	i += n30
2011 2019
 	return i, nil
2012 2020
 }
2013 2021
 
... ...
@@ -2029,11 +2041,11 @@ func (m *SecretSpec) MarshalTo(data []byte) (int, error) {
2029 2029
 	data[i] = 0xa
2030 2030
 	i++
2031 2031
 	i = encodeVarintSpecs(data, i, uint64(m.Annotations.Size()))
2032
-	n30, err := m.Annotations.MarshalTo(data[i:])
2032
+	n31, err := m.Annotations.MarshalTo(data[i:])
2033 2033
 	if err != nil {
2034 2034
 		return 0, err
2035 2035
 	}
2036
-	i += n30
2036
+	i += n31
2037 2037
 	if len(m.Data) > 0 {
2038 2038
 		data[i] = 0x12
2039 2039
 		i++
... ...
@@ -2392,6 +2404,8 @@ func (m *ClusterSpec) Size() (n int) {
2392 2392
 	n += 1 + l + sovSpecs(uint64(l))
2393 2393
 	l = m.TaskDefaults.Size()
2394 2394
 	n += 1 + l + sovSpecs(uint64(l))
2395
+	l = m.EncryptionConfig.Size()
2396
+	n += 1 + l + sovSpecs(uint64(l))
2395 2397
 	return n
2396 2398
 }
2397 2399
 
... ...
@@ -2629,6 +2643,7 @@ func (this *ClusterSpec) String() string {
2629 2629
 		`Dispatcher:` + strings.Replace(strings.Replace(this.Dispatcher.String(), "DispatcherConfig", "DispatcherConfig", 1), `&`, ``, 1) + `,`,
2630 2630
 		`CAConfig:` + strings.Replace(strings.Replace(this.CAConfig.String(), "CAConfig", "CAConfig", 1), `&`, ``, 1) + `,`,
2631 2631
 		`TaskDefaults:` + strings.Replace(strings.Replace(this.TaskDefaults.String(), "TaskDefaults", "TaskDefaults", 1), `&`, ``, 1) + `,`,
2632
+		`EncryptionConfig:` + strings.Replace(strings.Replace(this.EncryptionConfig.String(), "EncryptionConfig", "EncryptionConfig", 1), `&`, ``, 1) + `,`,
2632 2633
 		`}`,
2633 2634
 	}, "")
2634 2635
 	return s
... ...
@@ -4956,6 +4971,36 @@ func (m *ClusterSpec) Unmarshal(data []byte) error {
4956 4956
 				return err
4957 4957
 			}
4958 4958
 			iNdEx = postIndex
4959
+		case 8:
4960
+			if wireType != 2 {
4961
+				return fmt.Errorf("proto: wrong wireType = %d for field EncryptionConfig", wireType)
4962
+			}
4963
+			var msglen int
4964
+			for shift := uint(0); ; shift += 7 {
4965
+				if shift >= 64 {
4966
+					return ErrIntOverflowSpecs
4967
+				}
4968
+				if iNdEx >= l {
4969
+					return io.ErrUnexpectedEOF
4970
+				}
4971
+				b := data[iNdEx]
4972
+				iNdEx++
4973
+				msglen |= (int(b) & 0x7F) << shift
4974
+				if b < 0x80 {
4975
+					break
4976
+				}
4977
+			}
4978
+			if msglen < 0 {
4979
+				return ErrInvalidLengthSpecs
4980
+			}
4981
+			postIndex := iNdEx + msglen
4982
+			if postIndex > l {
4983
+				return io.ErrUnexpectedEOF
4984
+			}
4985
+			if err := m.EncryptionConfig.Unmarshal(data[iNdEx:postIndex]); err != nil {
4986
+				return err
4987
+			}
4988
+			iNdEx = postIndex
4959 4989
 		default:
4960 4990
 			iNdEx = preIndex
4961 4991
 			skippy, err := skipSpecs(data[iNdEx:])
... ...
@@ -5196,105 +5241,107 @@ var (
5196 5196
 func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
5197 5197
 
5198 5198
 var fileDescriptorSpecs = []byte{
5199
-	// 1597 bytes of a gzipped FileDescriptorProto
5200
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6e, 0xe3, 0xc8,
5201
-	0x11, 0x16, 0x2d, 0x59, 0x3f, 0x45, 0x69, 0x46, 0xd3, 0xd8, 0x1f, 0x8e, 0x76, 0x23, 0x69, 0xb4,
5202
-	0x93, 0x8d, 0x37, 0x8b, 0x78, 0x12, 0x25, 0xd8, 0xcc, 0x66, 0xb2, 0x48, 0xf4, 0x17, 0x8f, 0xe2,
5203
-	0xd8, 0x2b, 0xb4, 0xbd, 0x03, 0xcc, 0x49, 0x68, 0x93, 0x6d, 0x89, 0x30, 0xc5, 0x66, 0x9a, 0x4d,
5204
-	0x2d, 0x7c, 0xcb, 0x71, 0x31, 0x87, 0xbc, 0x81, 0x4f, 0x01, 0xf2, 0x06, 0xb9, 0xe4, 0x09, 0xe6,
5205
-	0x98, 0x63, 0x4e, 0x46, 0xac, 0x27, 0x08, 0x90, 0x17, 0x08, 0xba, 0xd9, 0x94, 0xa8, 0x2c, 0xbd,
5206
-	0x5e, 0x20, 0xbe, 0x75, 0x17, 0xbf, 0xaf, 0xd8, 0x5d, 0xf5, 0xb1, 0xaa, 0x08, 0x66, 0x18, 0x50,
5207
-	0x3b, 0xdc, 0x0f, 0x38, 0x13, 0x0c, 0x21, 0x87, 0xd9, 0x17, 0x94, 0xef, 0x87, 0x5f, 0x13, 0xbe,
5208
-	0xb8, 0x70, 0xc5, 0xfe, 0xf2, 0x67, 0x0d, 0x53, 0x5c, 0x06, 0x54, 0x03, 0x1a, 0xef, 0xcc, 0xd8,
5209
-	0x8c, 0xa9, 0xe5, 0x33, 0xb9, 0xd2, 0xd6, 0xf7, 0x9d, 0x88, 0x13, 0xe1, 0x32, 0xff, 0x59, 0xb2,
5210
-	0x88, 0x1f, 0x74, 0xfe, 0x5c, 0x80, 0xf2, 0x31, 0x73, 0xe8, 0x49, 0x40, 0x6d, 0x74, 0x00, 0x26,
5211
-	0xf1, 0x7d, 0x26, 0x14, 0x20, 0xb4, 0x8c, 0xb6, 0xb1, 0x67, 0x76, 0x5b, 0xfb, 0xdf, 0x7e, 0xe5,
5212
-	0x7e, 0x6f, 0x03, 0xeb, 0x17, 0xde, 0x5e, 0xb7, 0x72, 0x38, 0xcd, 0x44, 0x3f, 0x85, 0x02, 0x67,
5213
-	0x1e, 0xb5, 0x76, 0xda, 0xc6, 0xde, 0x83, 0xee, 0x87, 0x59, 0x1e, 0xe4, 0x4b, 0x31, 0xf3, 0x28,
5214
-	0x56, 0x48, 0x74, 0x00, 0xb0, 0xa0, 0x8b, 0x33, 0xca, 0xc3, 0xb9, 0x1b, 0x58, 0x79, 0xc5, 0xfb,
5215
-	0xd1, 0x6d, 0x3c, 0x79, 0xd8, 0xfd, 0xa3, 0x35, 0x1c, 0xa7, 0xa8, 0xe8, 0x08, 0xaa, 0x64, 0x49,
5216
-	0x5c, 0x8f, 0x9c, 0xb9, 0x9e, 0x2b, 0x2e, 0xad, 0x82, 0x72, 0xf5, 0xc9, 0x77, 0xba, 0xea, 0xa5,
5217
-	0x08, 0x78, 0x8b, 0xde, 0x71, 0x00, 0x36, 0x2f, 0x42, 0x1f, 0x43, 0x69, 0x32, 0x3a, 0x1e, 0x8e,
5218
-	0x8f, 0x0f, 0xea, 0xb9, 0xc6, 0xe3, 0x37, 0x57, 0xed, 0x77, 0xa5, 0x8f, 0x0d, 0x60, 0x42, 0x7d,
5219
-	0xc7, 0xf5, 0x67, 0x68, 0x0f, 0xca, 0xbd, 0xc1, 0x60, 0x34, 0x39, 0x1d, 0x0d, 0xeb, 0x46, 0xa3,
5220
-	0xf1, 0xe6, 0xaa, 0xfd, 0xde, 0x36, 0xb0, 0x67, 0xdb, 0x34, 0x10, 0xd4, 0x69, 0x14, 0xbe, 0xf9,
5221
-	0x4b, 0x33, 0xd7, 0xf9, 0xc6, 0x80, 0x6a, 0xfa, 0x10, 0xe8, 0x63, 0x28, 0xf6, 0x06, 0xa7, 0xe3,
5222
-	0x57, 0xa3, 0x7a, 0x6e, 0x43, 0x4f, 0x23, 0x7a, 0xb6, 0x70, 0x97, 0x14, 0x3d, 0x85, 0xdd, 0x49,
5223
-	0xef, 0xab, 0x93, 0x51, 0xdd, 0xd8, 0x1c, 0x27, 0x0d, 0x9b, 0x90, 0x28, 0x54, 0xa8, 0x21, 0xee,
5224
-	0x8d, 0x8f, 0xeb, 0x3b, 0xd9, 0xa8, 0x21, 0x27, 0xae, 0xaf, 0x8f, 0x72, 0x93, 0x07, 0xf3, 0x84,
5225
-	0xf2, 0xa5, 0x6b, 0xdf, 0xb3, 0x26, 0x3e, 0x83, 0x82, 0x20, 0xe1, 0x85, 0xd2, 0x84, 0x99, 0xad,
5226
-	0x89, 0x53, 0x12, 0x5e, 0xc8, 0x97, 0x6a, 0xba, 0xc2, 0x4b, 0x65, 0x70, 0x1a, 0x78, 0xae, 0x4d,
5227
-	0x04, 0x75, 0x94, 0x32, 0xcc, 0xee, 0x0f, 0xb3, 0xd8, 0x78, 0x8d, 0xd2, 0xe7, 0x7f, 0x99, 0xc3,
5228
-	0x29, 0x2a, 0x7a, 0x01, 0xc5, 0x99, 0xc7, 0xce, 0x88, 0xa7, 0x34, 0x61, 0x76, 0x9f, 0x64, 0x39,
5229
-	0x39, 0x50, 0x88, 0x8d, 0x03, 0x4d, 0x41, 0xcf, 0xa1, 0x18, 0x05, 0x0e, 0x11, 0xd4, 0x2a, 0x2a,
5230
-	0x72, 0x3b, 0x8b, 0xfc, 0x95, 0x42, 0x0c, 0x98, 0x7f, 0xee, 0xce, 0xb0, 0xc6, 0xa3, 0x43, 0x28,
5231
-	0xfb, 0x54, 0x7c, 0xcd, 0xf8, 0x45, 0x68, 0x95, 0xda, 0xf9, 0x3d, 0xb3, 0xfb, 0x69, 0xa6, 0x18,
5232
-	0x63, 0x4c, 0x4f, 0x08, 0x62, 0xcf, 0x17, 0xd4, 0x17, 0xb1, 0x9b, 0xfe, 0x8e, 0x65, 0xe0, 0xb5,
5233
-	0x03, 0xf4, 0x6b, 0x28, 0x53, 0xdf, 0x09, 0x98, 0xeb, 0x0b, 0xab, 0x7c, 0xfb, 0x41, 0x46, 0x1a,
5234
-	0x23, 0x83, 0x89, 0xd7, 0x8c, 0x7e, 0x11, 0x0a, 0x0b, 0xe6, 0xd0, 0xce, 0x33, 0x78, 0xf4, 0xad,
5235
-	0x60, 0xa1, 0x06, 0x94, 0x75, 0xb0, 0xe2, 0x2c, 0x17, 0xf0, 0x7a, 0xdf, 0x79, 0x08, 0xb5, 0xad,
5236
-	0xc0, 0xa8, 0xb2, 0x91, 0x64, 0x0b, 0xf5, 0xa0, 0x62, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b,
5237
-	0x24, 0x33, 0xb6, 0x83, 0x04, 0x24, 0x59, 0x2f, 0x73, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x0a, 0xa7,
5238
-	0x21, 0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xbd, 0xec, 0x1c, 0xc7, 0x20, 0x4c, 0xff, 0x18, 0xb9,
5239
-	0x9c, 0xca, 0x38, 0x85, 0x78, 0x43, 0x45, 0x2f, 0xa0, 0xc4, 0x69, 0x28, 0x08, 0x17, 0xdf, 0x95,
5240
-	0x64, 0x1c, 0x43, 0x26, 0xcc, 0x73, 0xed, 0x4b, 0x9c, 0x30, 0xd0, 0x0b, 0xa8, 0x04, 0x1e, 0xb1,
5241
-	0x95, 0x57, 0x6b, 0x57, 0xd1, 0x7f, 0x90, 0x45, 0x9f, 0x24, 0x20, 0xbc, 0xc1, 0xa3, 0xcf, 0x01,
5242
-	0x3c, 0x36, 0x9b, 0x3a, 0xdc, 0x5d, 0x52, 0xae, 0x45, 0xd2, 0xc8, 0x62, 0x0f, 0x15, 0x02, 0x57,
5243
-	0x3c, 0x36, 0x8b, 0x97, 0xe8, 0xe0, 0xff, 0x52, 0x48, 0x4a, 0x1d, 0x87, 0x00, 0x64, 0xfd, 0x54,
5244
-	0xeb, 0xe3, 0x93, 0xef, 0xe5, 0x4a, 0x67, 0x24, 0x45, 0x47, 0x4f, 0xa0, 0x7a, 0xce, 0xb8, 0x4d,
5245
-	0xa7, 0x5a, 0xf7, 0x15, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0x2b, 0x50, 0xe2, 0x91, 0x2f,
5246
-	0xdc, 0x05, 0xed, 0x1c, 0xc2, 0xbb, 0x99, 0x4e, 0x51, 0x17, 0xaa, 0xeb, 0x34, 0x4f, 0x5d, 0x47,
5247
-	0xe9, 0xa3, 0xd2, 0x7f, 0xb8, 0xba, 0x6e, 0x99, 0x6b, 0x3d, 0x8c, 0x87, 0xd8, 0x5c, 0x83, 0xc6,
5248
-	0x4e, 0xe7, 0xef, 0x25, 0xa8, 0x6d, 0x89, 0x05, 0xbd, 0x03, 0xbb, 0xee, 0x82, 0xcc, 0x68, 0x4c,
5249
-	0xc7, 0xf1, 0x06, 0x8d, 0xa0, 0xe8, 0x91, 0x33, 0xea, 0x49, 0xc9, 0xc8, 0xb0, 0xfd, 0xe4, 0x4e,
5250
-	0xd5, 0xed, 0xff, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x97, 0x58, 0x93, 0x91, 0x05, 0x25, 0x9b, 0x2d,
5251
-	0x16, 0xc4, 0x97, 0xe5, 0x25, 0xbf, 0x57, 0xc1, 0xc9, 0x16, 0x21, 0x28, 0x10, 0x3e, 0x0b, 0xad,
5252
-	0x82, 0x32, 0xab, 0x35, 0xaa, 0x43, 0x9e, 0xfa, 0x4b, 0x6b, 0x57, 0x99, 0xe4, 0x52, 0x5a, 0x1c,
5253
-	0x37, 0xce, 0x79, 0x05, 0xcb, 0xa5, 0xe4, 0x45, 0x21, 0xe5, 0x56, 0x49, 0x99, 0xd4, 0x1a, 0xfd,
5254
-	0x12, 0x8a, 0x0b, 0x16, 0xf9, 0x22, 0xb4, 0xca, 0xea, 0xb0, 0x8f, 0xb3, 0x0e, 0x7b, 0x24, 0x11,
5255
-	0xba, 0xfc, 0x69, 0x38, 0x7a, 0x09, 0x8f, 0x42, 0xc1, 0x82, 0xe9, 0x8c, 0x13, 0x9b, 0x4e, 0x03,
5256
-	0xca, 0x5d, 0xe6, 0xa8, 0x6c, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0x87, 0x92, 0x76, 0x20,
5257
-	0x59, 0x13, 0x45, 0x42, 0x13, 0xa8, 0x06, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e,
5258
-	0xbe, 0x47, 0xd4, 0x26, 0x91, 0xe7, 0x7d, 0x19, 0x93, 0xb0, 0x19, 0x6c, 0x36, 0xe8, 0x3d, 0x28,
5259
-	0xce, 0x38, 0x8b, 0x82, 0xd0, 0x32, 0x55, 0x3c, 0xf4, 0x0e, 0x7d, 0x01, 0xa5, 0x90, 0xda, 0x9c,
5260
-	0x8a, 0xd0, 0xaa, 0xaa, 0xdb, 0x7e, 0x94, 0xf5, 0x92, 0x13, 0x05, 0xc1, 0xf4, 0x9c, 0x72, 0xea,
5261
-	0xdb, 0x14, 0x27, 0x1c, 0xf4, 0x18, 0xf2, 0x42, 0x5c, 0x5a, 0xb5, 0xb6, 0xb1, 0x57, 0xee, 0x97,
5262
-	0x56, 0xd7, 0xad, 0xfc, 0xe9, 0xe9, 0x6b, 0x2c, 0x6d, 0xb2, 0x4c, 0xcd, 0x59, 0x28, 0x7c, 0xb2,
5263
-	0xa0, 0xd6, 0x03, 0x15, 0xde, 0xf5, 0x1e, 0xbd, 0x06, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2,
5264
-	0x7a, 0xa8, 0x6e, 0xf7, 0xe9, 0xdd, 0xb7, 0x1b, 0x1e, 0x9f, 0xe8, 0x62, 0x5b, 0x5b, 0x5d, 0xb7,
5265
-	0x2a, 0xeb, 0x2d, 0xae, 0x38, 0x7e, 0x18, 0x2f, 0x51, 0x1f, 0xcc, 0x39, 0x25, 0x9e, 0x98, 0xdb,
5266
-	0x73, 0x6a, 0x5f, 0x58, 0xf5, 0xdb, 0x6b, 0xef, 0x4b, 0x05, 0xd3, 0x1e, 0xd2, 0x24, 0x29, 0x62,
5267
-	0x79, 0xd4, 0xd0, 0x7a, 0xa4, 0x62, 0x15, 0x6f, 0x1a, 0x9f, 0x83, 0x99, 0x12, 0xa5, 0x14, 0xd3,
5268
-	0x05, 0xbd, 0xd4, 0x3a, 0x97, 0x4b, 0x49, 0x5b, 0x12, 0x2f, 0x8a, 0xa7, 0xa9, 0x0a, 0x8e, 0x37,
5269
-	0xbf, 0xda, 0x79, 0x6e, 0x34, 0xba, 0x60, 0xa6, 0x32, 0x83, 0x3e, 0x82, 0x1a, 0xa7, 0x33, 0x37,
5270
-	0x14, 0xfc, 0x72, 0x4a, 0x22, 0x31, 0xb7, 0x7e, 0xab, 0x08, 0xd5, 0xc4, 0xd8, 0x8b, 0xc4, 0xbc,
5271
-	0x31, 0x85, 0xcd, 0x05, 0x51, 0x1b, 0x4c, 0x19, 0xb8, 0x90, 0xf2, 0x25, 0xe5, 0xb2, 0xec, 0xcb,
5272
-	0x73, 0xa5, 0x4d, 0x32, 0xc1, 0x21, 0x25, 0xdc, 0x9e, 0xab, 0x4f, 0xac, 0x82, 0xf5, 0x4e, 0x7e,
5273
-	0x33, 0x89, 0x8a, 0xf4, 0x37, 0xa3, 0xb7, 0x9d, 0xff, 0x18, 0x50, 0x4d, 0xf7, 0x1f, 0x34, 0x88,
5274
-	0xbb, 0x8e, 0xba, 0xd2, 0x83, 0xee, 0xb3, 0xbb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x1d,
5275
-	0xc9, 0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x37, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb3, 0x2e,
5276
-	0x33, 0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x83, 0x6d, 0x6f, 0xe8, 0x29, 0xe4, 0x5f, 0x8d,
5277
-	0x27, 0xf5, 0x5c, 0xe3, 0x83, 0x37, 0x57, 0xed, 0xf7, 0xb7, 0x1f, 0xbe, 0x72, 0xb9, 0x88, 0x88,
5278
-	0x37, 0x9e, 0xa0, 0x1f, 0xc3, 0xee, 0xf0, 0xf8, 0x04, 0xe3, 0xba, 0xd1, 0x68, 0xbd, 0xb9, 0x6a,
5279
-	0x7f, 0xb0, 0x8d, 0x93, 0x8f, 0x58, 0xe4, 0x3b, 0x98, 0x9d, 0xad, 0xc7, 0xa6, 0xbf, 0xed, 0x80,
5280
-	0xa9, 0x0b, 0xe0, 0xfd, 0x8e, 0x4d, 0xbf, 0x81, 0x5a, 0xdc, 0x53, 0x12, 0x59, 0xef, 0xdc, 0xd9,
5281
-	0x5a, 0xaa, 0x31, 0x41, 0xe7, 0xf8, 0x09, 0x54, 0xdd, 0x60, 0xf9, 0xd9, 0x94, 0xfa, 0xe4, 0xcc,
5282
-	0xd3, 0x13, 0x54, 0x19, 0x9b, 0xd2, 0x36, 0x8a, 0x4d, 0xf2, 0x9b, 0x72, 0x7d, 0x41, 0xb9, 0xaf,
5283
-	0x67, 0xa3, 0x32, 0x5e, 0xef, 0xd1, 0x17, 0x50, 0x70, 0x03, 0xb2, 0xd0, 0xfd, 0x30, 0xf3, 0x06,
5284
-	0xe3, 0x49, 0xef, 0x48, 0x6b, 0xb0, 0x5f, 0x5e, 0x5d, 0xb7, 0x0a, 0xd2, 0x80, 0x15, 0x0d, 0x35,
5285
-	0x93, 0x96, 0x24, 0xdf, 0xa4, 0x4a, 0x64, 0x19, 0xa7, 0x2c, 0x9d, 0xbf, 0x16, 0xc0, 0x1c, 0x78,
5286
-	0x51, 0x28, 0x74, 0xa1, 0xbf, 0xb7, 0xb8, 0xbd, 0x86, 0x47, 0x44, 0x0d, 0xd9, 0xc4, 0x97, 0x55,
5287
-	0x53, 0xb5, 0x7a, 0x1d, 0xbb, 0xa7, 0x99, 0xee, 0xd6, 0xe0, 0x78, 0x2c, 0xe8, 0x17, 0xa5, 0x4f,
5288
-	0xcb, 0xc0, 0x75, 0xf2, 0x3f, 0x4f, 0xd0, 0x09, 0xd4, 0x18, 0xb7, 0xe7, 0x34, 0x14, 0x71, 0xa1,
5289
-	0xd5, 0x43, 0x69, 0xe6, 0xef, 0xca, 0x97, 0x69, 0xa0, 0xae, 0x32, 0xf1, 0x69, 0xb7, 0x7d, 0xa0,
5290
-	0xe7, 0x50, 0xe0, 0xe4, 0x3c, 0x19, 0x5b, 0x32, 0xf5, 0x8d, 0xc9, 0xb9, 0xd8, 0x72, 0xa1, 0x18,
5291
-	0xe8, 0xf7, 0x00, 0x8e, 0x1b, 0x06, 0x44, 0xd8, 0x73, 0xca, 0x75, 0x9e, 0x32, 0xaf, 0x38, 0x5c,
5292
-	0xa3, 0xb6, 0xbc, 0xa4, 0xd8, 0xe8, 0x10, 0x2a, 0x36, 0x49, 0x94, 0x56, 0xbc, 0xbd, 0xc7, 0x0c,
5293
-	0x7a, 0xda, 0x45, 0x5d, 0xba, 0x58, 0x5d, 0xb7, 0xca, 0x89, 0x05, 0x97, 0x6d, 0xa2, 0x95, 0x77,
5294
-	0x08, 0x35, 0x39, 0xc1, 0x4f, 0x1d, 0x7a, 0x4e, 0x22, 0x4f, 0x84, 0xaa, 0x1d, 0xde, 0x52, 0x35,
5295
-	0xe5, 0x30, 0x39, 0xd4, 0x38, 0x7d, 0xae, 0xaa, 0x48, 0xd9, 0x3a, 0x2e, 0x40, 0xdc, 0x2e, 0xee,
5296
-	0x57, 0x26, 0x08, 0x0a, 0x0e, 0x11, 0x44, 0x29, 0xa3, 0x8a, 0xd5, 0xba, 0xff, 0xe1, 0xdb, 0x9b,
5297
-	0x66, 0xee, 0x9f, 0x37, 0xcd, 0xdc, 0xbf, 0x6f, 0x9a, 0xc6, 0x9f, 0x56, 0x4d, 0xe3, 0xed, 0xaa,
5298
-	0x69, 0xfc, 0x63, 0xd5, 0x34, 0xfe, 0xb5, 0x6a, 0x1a, 0x67, 0x45, 0xf5, 0xe3, 0xfc, 0xf3, 0xff,
5299
-	0x06, 0x00, 0x00, 0xff, 0xff, 0x04, 0xd4, 0x09, 0xa4, 0x97, 0x0f, 0x00, 0x00,
5199
+	// 1620 bytes of a gzipped FileDescriptorProto
5200
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0xdb, 0xc8,
5201
+	0x11, 0x26, 0x24, 0x8a, 0x3f, 0x0d, 0xca, 0xa6, 0xa6, 0xf6, 0x07, 0xe6, 0x6e, 0x48, 0x9a, 0xeb,
5202
+	0x6c, 0xb4, 0xd9, 0x8a, 0x9c, 0x30, 0xa9, 0x8d, 0x37, 0xce, 0x56, 0xc2, 0xbf, 0xc8, 0x8c, 0x22,
5203
+	0x2d, 0x6b, 0xa4, 0x75, 0xca, 0x27, 0xd6, 0x08, 0x18, 0x91, 0x28, 0x81, 0x18, 0x64, 0x30, 0xe0,
5204
+	0x16, 0x6f, 0x39, 0x6e, 0xf9, 0x90, 0x37, 0xf0, 0x29, 0xcf, 0x90, 0x4b, 0x9e, 0xc0, 0xc7, 0x1c,
5205
+	0x73, 0x52, 0xc5, 0x7c, 0x82, 0x54, 0xe5, 0x01, 0x92, 0x9a, 0xc1, 0x00, 0x04, 0x77, 0xa1, 0xb5,
5206
+	0xab, 0xe2, 0xdb, 0x4c, 0xe3, 0xfb, 0x1a, 0x3d, 0x3d, 0x1f, 0xba, 0x1b, 0x60, 0x86, 0x01, 0xb5,
5207
+	0xc3, 0xa3, 0x80, 0x33, 0xc1, 0x10, 0x72, 0x98, 0x7d, 0x4d, 0xf9, 0x51, 0xf8, 0x35, 0xe1, 0x8b,
5208
+	0x6b, 0x57, 0x1c, 0x2d, 0x7f, 0xd6, 0x30, 0xc5, 0x2a, 0xa0, 0x1a, 0xd0, 0x78, 0x67, 0xc6, 0x66,
5209
+	0x4c, 0x2d, 0x1f, 0xca, 0x95, 0xb6, 0xbe, 0xef, 0x44, 0x9c, 0x08, 0x97, 0xf9, 0x0f, 0x93, 0x45,
5210
+	0xfc, 0xa0, 0xf3, 0x97, 0x22, 0x54, 0xce, 0x98, 0x43, 0xcf, 0x03, 0x6a, 0xa3, 0x63, 0x30, 0x89,
5211
+	0xef, 0x33, 0xa1, 0x00, 0xa1, 0x65, 0xb4, 0x8d, 0x43, 0xb3, 0xdb, 0x3a, 0xfa, 0xee, 0x2b, 0x8f,
5212
+	0x7a, 0x1b, 0x58, 0xbf, 0xf8, 0xf2, 0xa6, 0x55, 0xc0, 0x59, 0x26, 0xfa, 0x29, 0x14, 0x39, 0xf3,
5213
+	0xa8, 0xb5, 0xd3, 0x36, 0x0e, 0xef, 0x74, 0x3f, 0xcc, 0xf3, 0x20, 0x5f, 0x8a, 0x99, 0x47, 0xb1,
5214
+	0x42, 0xa2, 0x63, 0x80, 0x05, 0x5d, 0x5c, 0x52, 0x1e, 0xce, 0xdd, 0xc0, 0xda, 0x55, 0xbc, 0x1f,
5215
+	0xdd, 0xc6, 0x93, 0xc1, 0x1e, 0x9d, 0xa6, 0x70, 0x9c, 0xa1, 0xa2, 0x53, 0xa8, 0x91, 0x25, 0x71,
5216
+	0x3d, 0x72, 0xe9, 0x7a, 0xae, 0x58, 0x59, 0x45, 0xe5, 0xea, 0x93, 0xef, 0x75, 0xd5, 0xcb, 0x10,
5217
+	0xf0, 0x16, 0xbd, 0xe3, 0x00, 0x6c, 0x5e, 0x84, 0x3e, 0x86, 0xf2, 0x64, 0x74, 0x36, 0x1c, 0x9f,
5218
+	0x1d, 0xd7, 0x0b, 0x8d, 0x7b, 0xcf, 0x5f, 0xb4, 0xdf, 0x95, 0x3e, 0x36, 0x80, 0x09, 0xf5, 0x1d,
5219
+	0xd7, 0x9f, 0xa1, 0x43, 0xa8, 0xf4, 0x06, 0x83, 0xd1, 0xe4, 0x62, 0x34, 0xac, 0x1b, 0x8d, 0xc6,
5220
+	0xf3, 0x17, 0xed, 0xf7, 0xb6, 0x81, 0x3d, 0xdb, 0xa6, 0x81, 0xa0, 0x4e, 0xa3, 0xf8, 0xcd, 0x5f,
5221
+	0x9b, 0x85, 0xce, 0x37, 0x06, 0xd4, 0xb2, 0x41, 0xa0, 0x8f, 0xa1, 0xd4, 0x1b, 0x5c, 0x8c, 0x9f,
5222
+	0x8e, 0xea, 0x85, 0x0d, 0x3d, 0x8b, 0xe8, 0xd9, 0xc2, 0x5d, 0x52, 0xf4, 0x00, 0xf6, 0x26, 0xbd,
5223
+	0xaf, 0xce, 0x47, 0x75, 0x63, 0x13, 0x4e, 0x16, 0x36, 0x21, 0x51, 0xa8, 0x50, 0x43, 0xdc, 0x1b,
5224
+	0x9f, 0xd5, 0x77, 0xf2, 0x51, 0x43, 0x4e, 0x5c, 0x5f, 0x87, 0xf2, 0x6a, 0x17, 0xcc, 0x73, 0xca,
5225
+	0x97, 0xae, 0xfd, 0x96, 0x35, 0xf1, 0x19, 0x14, 0x05, 0x09, 0xaf, 0x95, 0x26, 0xcc, 0x7c, 0x4d,
5226
+	0x5c, 0x90, 0xf0, 0x5a, 0xbe, 0x54, 0xd3, 0x15, 0x5e, 0x2a, 0x83, 0xd3, 0xc0, 0x73, 0x6d, 0x22,
5227
+	0xa8, 0xa3, 0x94, 0x61, 0x76, 0x7f, 0x98, 0xc7, 0xc6, 0x29, 0x4a, 0xc7, 0xff, 0xa4, 0x80, 0x33,
5228
+	0x54, 0xf4, 0x18, 0x4a, 0x33, 0x8f, 0x5d, 0x12, 0x4f, 0x69, 0xc2, 0xec, 0xde, 0xcf, 0x73, 0x72,
5229
+	0xac, 0x10, 0x1b, 0x07, 0x9a, 0x82, 0x1e, 0x41, 0x29, 0x0a, 0x1c, 0x22, 0xa8, 0x55, 0x52, 0xe4,
5230
+	0x76, 0x1e, 0xf9, 0x2b, 0x85, 0x18, 0x30, 0xff, 0xca, 0x9d, 0x61, 0x8d, 0x47, 0x27, 0x50, 0xf1,
5231
+	0xa9, 0xf8, 0x9a, 0xf1, 0xeb, 0xd0, 0x2a, 0xb7, 0x77, 0x0f, 0xcd, 0xee, 0xa7, 0xb9, 0x62, 0x8c,
5232
+	0x31, 0x3d, 0x21, 0x88, 0x3d, 0x5f, 0x50, 0x5f, 0xc4, 0x6e, 0xfa, 0x3b, 0x96, 0x81, 0x53, 0x07,
5233
+	0xe8, 0xd7, 0x50, 0xa1, 0xbe, 0x13, 0x30, 0xd7, 0x17, 0x56, 0xe5, 0xf6, 0x40, 0x46, 0x1a, 0x23,
5234
+	0x93, 0x89, 0x53, 0x46, 0xbf, 0x04, 0xc5, 0x05, 0x73, 0x68, 0xe7, 0x21, 0x1c, 0x7c, 0x27, 0x59,
5235
+	0xa8, 0x01, 0x15, 0x9d, 0xac, 0xf8, 0x96, 0x8b, 0x38, 0xdd, 0x77, 0xee, 0xc2, 0xfe, 0x56, 0x62,
5236
+	0x54, 0xd9, 0x48, 0x6e, 0x0b, 0xf5, 0xa0, 0x6a, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b, 0x24,
5237
+	0x37, 0xb7, 0x83, 0x04, 0x24, 0x59, 0x4f, 0x0a, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x2a, 0xa7, 0x21,
5238
+	0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xc3, 0xfc, 0x3b, 0x8e, 0x41, 0x98, 0xfe, 0x29, 0x72, 0x39,
5239
+	0x95, 0x79, 0x0a, 0xf1, 0x86, 0x8a, 0x1e, 0x43, 0x99, 0xd3, 0x50, 0x10, 0x2e, 0xbe, 0xef, 0x92,
5240
+	0x71, 0x0c, 0x99, 0x30, 0xcf, 0xb5, 0x57, 0x38, 0x61, 0xa0, 0xc7, 0x50, 0x0d, 0x3c, 0x62, 0x2b,
5241
+	0xaf, 0xd6, 0x9e, 0xa2, 0xff, 0x20, 0x8f, 0x3e, 0x49, 0x40, 0x78, 0x83, 0x47, 0x9f, 0x03, 0x78,
5242
+	0x6c, 0x36, 0x75, 0xb8, 0xbb, 0xa4, 0x5c, 0x8b, 0xa4, 0x91, 0xc7, 0x1e, 0x2a, 0x04, 0xae, 0x7a,
5243
+	0x6c, 0x16, 0x2f, 0xd1, 0xf1, 0xff, 0xa5, 0x90, 0x8c, 0x3a, 0x4e, 0x00, 0x48, 0xfa, 0x54, 0xeb,
5244
+	0xe3, 0x93, 0x37, 0x72, 0xa5, 0x6f, 0x24, 0x43, 0x47, 0xf7, 0xa1, 0x76, 0xc5, 0xb8, 0x4d, 0xa7,
5245
+	0x5a, 0xf7, 0x55, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0xab, 0x50, 0xe6, 0x91, 0x2f, 0xdc,
5246
+	0x05, 0xed, 0x9c, 0xc0, 0xbb, 0xb9, 0x4e, 0x51, 0x17, 0x6a, 0xe9, 0x35, 0x4f, 0x5d, 0x47, 0xe9,
5247
+	0xa3, 0xda, 0xbf, 0xbb, 0xbe, 0x69, 0x99, 0xa9, 0x1e, 0xc6, 0x43, 0x6c, 0xa6, 0xa0, 0xb1, 0xd3,
5248
+	0xf9, 0x7b, 0x19, 0xf6, 0xb7, 0xc4, 0x82, 0xde, 0x81, 0x3d, 0x77, 0x41, 0x66, 0x34, 0xa6, 0xe3,
5249
+	0x78, 0x83, 0x46, 0x50, 0xf2, 0xc8, 0x25, 0xf5, 0xa4, 0x64, 0x64, 0xda, 0x7e, 0xf2, 0x5a, 0xd5,
5250
+	0x1d, 0xfd, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x2b, 0xac, 0xc9, 0xc8, 0x82, 0xb2, 0xcd, 0x16, 0x0b,
5251
+	0xe2, 0xcb, 0xf2, 0xb2, 0x7b, 0x58, 0xc5, 0xc9, 0x16, 0x21, 0x28, 0x12, 0x3e, 0x0b, 0xad, 0xa2,
5252
+	0x32, 0xab, 0x35, 0xaa, 0xc3, 0x2e, 0xf5, 0x97, 0xd6, 0x9e, 0x32, 0xc9, 0xa5, 0xb4, 0x38, 0x6e,
5253
+	0x7c, 0xe7, 0x55, 0x2c, 0x97, 0x92, 0x17, 0x85, 0x94, 0x5b, 0x65, 0x65, 0x52, 0x6b, 0xf4, 0x4b,
5254
+	0x28, 0x2d, 0x58, 0xe4, 0x8b, 0xd0, 0xaa, 0xa8, 0x60, 0xef, 0xe5, 0x05, 0x7b, 0x2a, 0x11, 0xba,
5255
+	0xfc, 0x69, 0x38, 0x7a, 0x02, 0x07, 0xa1, 0x60, 0xc1, 0x74, 0xc6, 0x89, 0x4d, 0xa7, 0x01, 0xe5,
5256
+	0x2e, 0x73, 0xd4, 0x6d, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0xbb, 0x92, 0x76, 0x2c, 0x59,
5257
+	0x13, 0x45, 0x42, 0x13, 0xa8, 0x05, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e, 0xde,
5258
+	0x20, 0x6b, 0x93, 0xc8, 0xf3, 0xbe, 0x8c, 0x49, 0xd8, 0x0c, 0x36, 0x1b, 0xf4, 0x1e, 0x94, 0x66,
5259
+	0x9c, 0x45, 0x41, 0x68, 0x99, 0x2a, 0x1f, 0x7a, 0x87, 0xbe, 0x80, 0x72, 0x48, 0x6d, 0x4e, 0x45,
5260
+	0x68, 0xd5, 0xd4, 0x69, 0x3f, 0xca, 0x7b, 0xc9, 0xb9, 0x82, 0x60, 0x7a, 0x45, 0x39, 0xf5, 0x6d,
5261
+	0x8a, 0x13, 0x0e, 0xba, 0x07, 0xbb, 0x42, 0xac, 0xac, 0xfd, 0xb6, 0x71, 0x58, 0xe9, 0x97, 0xd7,
5262
+	0x37, 0xad, 0xdd, 0x8b, 0x8b, 0x67, 0x58, 0xda, 0x64, 0x99, 0x9a, 0xb3, 0x50, 0xf8, 0x64, 0x41,
5263
+	0xad, 0x3b, 0x2a, 0xbd, 0xe9, 0x1e, 0x3d, 0x03, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2, 0xba,
5264
+	0xab, 0x4e, 0xf7, 0xe9, 0xeb, 0x4f, 0x37, 0x3c, 0x3b, 0xd7, 0xc5, 0x76, 0x7f, 0x7d, 0xd3, 0xaa,
5265
+	0xa6, 0x5b, 0x5c, 0x75, 0xfc, 0x30, 0x5e, 0xa2, 0x3e, 0x98, 0x73, 0x4a, 0x3c, 0x31, 0xb7, 0xe7,
5266
+	0xd4, 0xbe, 0xb6, 0xea, 0xb7, 0xd7, 0xde, 0x27, 0x0a, 0xa6, 0x3d, 0x64, 0x49, 0x52, 0xc4, 0x32,
5267
+	0xd4, 0xd0, 0x3a, 0x50, 0xb9, 0x8a, 0x37, 0x8d, 0xcf, 0xc1, 0xcc, 0x88, 0x52, 0x8a, 0xe9, 0x9a,
5268
+	0xae, 0xb4, 0xce, 0xe5, 0x52, 0xd2, 0x96, 0xc4, 0x8b, 0xe2, 0x69, 0xaa, 0x8a, 0xe3, 0xcd, 0xaf,
5269
+	0x76, 0x1e, 0x19, 0x8d, 0x2e, 0x98, 0x99, 0x9b, 0x41, 0x1f, 0xc1, 0x3e, 0xa7, 0x33, 0x37, 0x14,
5270
+	0x7c, 0x35, 0x25, 0x91, 0x98, 0x5b, 0xbf, 0x55, 0x84, 0x5a, 0x62, 0xec, 0x45, 0x62, 0xde, 0x98,
5271
+	0xc2, 0xe6, 0x80, 0xa8, 0x0d, 0xa6, 0x4c, 0x5c, 0x48, 0xf9, 0x92, 0x72, 0x59, 0xf6, 0x65, 0x5c,
5272
+	0x59, 0x93, 0xbc, 0xe0, 0x90, 0x12, 0x6e, 0xcf, 0xd5, 0x27, 0x56, 0xc5, 0x7a, 0x27, 0xbf, 0x99,
5273
+	0x44, 0x45, 0xfa, 0x9b, 0xd1, 0xdb, 0xce, 0x7f, 0x0c, 0xa8, 0x65, 0xfb, 0x0f, 0x1a, 0xc4, 0x5d,
5274
+	0x47, 0x1d, 0xe9, 0x4e, 0xf7, 0xe1, 0xeb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x9d, 0xca,
5275
+	0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x2f, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb7, 0x2e, 0x33,
5276
+	0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x9d, 0x6d, 0x6f, 0xe8, 0x01, 0xec, 0x3e, 0x1d, 0x4f,
5277
+	0xea, 0x85, 0xc6, 0x07, 0xcf, 0x5f, 0xb4, 0xdf, 0xdf, 0x7e, 0xf8, 0xd4, 0xe5, 0x22, 0x22, 0xde,
5278
+	0x78, 0x82, 0x7e, 0x0c, 0x7b, 0xc3, 0xb3, 0x73, 0x8c, 0xeb, 0x46, 0xa3, 0xf5, 0xfc, 0x45, 0xfb,
5279
+	0x83, 0x6d, 0x9c, 0x7c, 0xc4, 0x22, 0xdf, 0xc1, 0xec, 0x32, 0x1d, 0x9b, 0xfe, 0xb6, 0x03, 0xa6,
5280
+	0x2e, 0x80, 0x6f, 0x77, 0x6c, 0xfa, 0x0d, 0xec, 0xc7, 0x3d, 0x25, 0x91, 0xf5, 0xce, 0x6b, 0x5b,
5281
+	0x4b, 0x2d, 0x26, 0xe8, 0x3b, 0xbe, 0x0f, 0x35, 0x37, 0x58, 0x7e, 0x36, 0xa5, 0x3e, 0xb9, 0xf4,
5282
+	0xf4, 0x04, 0x55, 0xc1, 0xa6, 0xb4, 0x8d, 0x62, 0x93, 0xfc, 0xa6, 0x5c, 0x5f, 0x50, 0xee, 0xeb,
5283
+	0xd9, 0xa8, 0x82, 0xd3, 0x3d, 0xfa, 0x02, 0x8a, 0x6e, 0x40, 0x16, 0xba, 0x1f, 0xe6, 0x9e, 0x60,
5284
+	0x3c, 0xe9, 0x9d, 0x6a, 0x0d, 0xf6, 0x2b, 0xeb, 0x9b, 0x56, 0x51, 0x1a, 0xb0, 0xa2, 0xa1, 0x66,
5285
+	0xd2, 0x92, 0xe4, 0x9b, 0x54, 0x89, 0xac, 0xe0, 0x8c, 0xa5, 0xf3, 0xdf, 0x22, 0x98, 0x03, 0x2f,
5286
+	0x0a, 0x85, 0x2e, 0xf4, 0x6f, 0x2d, 0x6f, 0xcf, 0xe0, 0x80, 0xa8, 0x21, 0x9b, 0xf8, 0xb2, 0x6a,
5287
+	0xaa, 0x56, 0xaf, 0x73, 0xf7, 0x20, 0xd7, 0x5d, 0x0a, 0x8e, 0xc7, 0x82, 0x7e, 0x49, 0xfa, 0xb4,
5288
+	0x0c, 0x5c, 0x27, 0xdf, 0x7a, 0x82, 0xce, 0x61, 0x9f, 0x71, 0x7b, 0x4e, 0x43, 0x11, 0x17, 0x5a,
5289
+	0x3d, 0x94, 0xe6, 0xfe, 0xae, 0x7c, 0x99, 0x05, 0xea, 0x2a, 0x13, 0x47, 0xbb, 0xed, 0x03, 0x3d,
5290
+	0x82, 0x22, 0x27, 0x57, 0xc9, 0xd8, 0x92, 0xab, 0x6f, 0x4c, 0xae, 0xc4, 0x96, 0x0b, 0xc5, 0x40,
5291
+	0xbf, 0x07, 0x70, 0xdc, 0x30, 0x20, 0xc2, 0x9e, 0x53, 0xae, 0xef, 0x29, 0xf7, 0x88, 0xc3, 0x14,
5292
+	0xb5, 0xe5, 0x25, 0xc3, 0x46, 0x27, 0x50, 0xb5, 0x49, 0xa2, 0xb4, 0xd2, 0xed, 0x3d, 0x66, 0xd0,
5293
+	0xd3, 0x2e, 0xea, 0xd2, 0xc5, 0xfa, 0xa6, 0x55, 0x49, 0x2c, 0xb8, 0x62, 0x13, 0xad, 0xbc, 0x13,
5294
+	0xd8, 0x97, 0x13, 0xfc, 0xd4, 0xa1, 0x57, 0x24, 0xf2, 0x44, 0xa8, 0xda, 0xe1, 0x2d, 0x55, 0x53,
5295
+	0x0e, 0x93, 0x43, 0x8d, 0xd3, 0x71, 0xd5, 0x44, 0xc6, 0x86, 0xfe, 0x08, 0x07, 0xd4, 0xb7, 0xf9,
5296
+	0x4a, 0xe9, 0x2c, 0x89, 0xb0, 0x72, 0xfb, 0x61, 0x47, 0x29, 0x78, 0xeb, 0xb0, 0x75, 0xfa, 0x2d,
5297
+	0x7b, 0xc7, 0x05, 0x88, 0xfb, 0xd0, 0xdb, 0xd5, 0x1f, 0x82, 0xa2, 0x43, 0x04, 0x51, 0x92, 0xab,
5298
+	0x61, 0xb5, 0xee, 0x7f, 0xf8, 0xf2, 0x55, 0xb3, 0xf0, 0xcf, 0x57, 0xcd, 0xc2, 0xbf, 0x5f, 0x35,
5299
+	0x8d, 0x3f, 0xaf, 0x9b, 0xc6, 0xcb, 0x75, 0xd3, 0xf8, 0xc7, 0xba, 0x69, 0xfc, 0x6b, 0xdd, 0x34,
5300
+	0x2e, 0x4b, 0xea, 0x8f, 0xfc, 0xe7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xce, 0x13, 0x97, 0xa2,
5301
+	0xf0, 0x0f, 0x00, 0x00,
5300 5302
 }
... ...
@@ -323,6 +323,9 @@ message ClusterSpec {
323 323
 
324 324
 	// TaskDefaults specifies the default values to use for task creation.
325 325
 	TaskDefaults task_defaults = 7 [(gogoproto.nullable) = false];
326
+
327
+	// EncryptionConfig defines the cluster's encryption settings.
328
+	EncryptionConfig encryption_config = 8 [(gogoproto.nullable) = false];
326 329
 }
327 330
 
328 331
 // SecretSpec specifies a user-provided secret.
... ...
@@ -52,6 +52,7 @@
52 52
 		TaskDefaults
53 53
 		DispatcherConfig
54 54
 		RaftConfig
55
+		EncryptionConfig
55 56
 		Placement
56 57
 		JoinTokens
57 58
 		RootCA
... ...
@@ -118,7 +119,7 @@
118 118
 		GetClusterResponse
119 119
 		ListClustersRequest
120 120
 		ListClustersResponse
121
-		JoinTokenRotation
121
+		KeyRotation
122 122
 		UpdateClusterRequest
123 123
 		UpdateClusterResponse
124 124
 		GetSecretRequest
... ...
@@ -149,6 +150,8 @@
149 149
 		IssueNodeCertificateResponse
150 150
 		GetRootCACertificateRequest
151 151
 		GetRootCACertificateResponse
152
+		GetUnlockKeyRequest
153
+		GetUnlockKeyResponse
152 154
 		StoreSnapshot
153 155
 		ClusterSnapshot
154 156
 		Snapshot
... ...
@@ -662,7 +665,7 @@ func (x EncryptionKey_Algorithm) String() string {
662 662
 	return proto.EnumName(EncryptionKey_Algorithm_name, int32(x))
663 663
 }
664 664
 func (EncryptionKey_Algorithm) EnumDescriptor() ([]byte, []int) {
665
-	return fileDescriptorTypes, []int{37, 0}
665
+	return fileDescriptorTypes, []int{38, 0}
666 666
 }
667 667
 
668 668
 type MaybeEncryptedRecord_Algorithm int32
... ...
@@ -685,7 +688,7 @@ func (x MaybeEncryptedRecord_Algorithm) String() string {
685 685
 	return proto.EnumName(MaybeEncryptedRecord_Algorithm_name, int32(x))
686 686
 }
687 687
 func (MaybeEncryptedRecord_Algorithm) EnumDescriptor() ([]byte, []int) {
688
-	return fileDescriptorTypes, []int{42, 0}
688
+	return fileDescriptorTypes, []int{43, 0}
689 689
 }
690 690
 
691 691
 // Version tracks the last time an object in the store was updated.
... ...
@@ -1359,6 +1362,17 @@ func (m *RaftConfig) Reset()                    { *m = RaftConfig{} }
1359 1359
 func (*RaftConfig) ProtoMessage()               {}
1360 1360
 func (*RaftConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{32} }
1361 1361
 
1362
+type EncryptionConfig struct {
1363
+	// AutoLockManagers specifies whether or not managers TLS keys and raft data
1364
+	// should be encrypted at rest in such a way that they must be unlocked
1365
+	// before the manager node starts up again.
1366
+	AutoLockManagers bool `protobuf:"varint,1,opt,name=auto_lock_managers,json=autoLockManagers,proto3" json:"auto_lock_managers,omitempty"`
1367
+}
1368
+
1369
+func (m *EncryptionConfig) Reset()                    { *m = EncryptionConfig{} }
1370
+func (*EncryptionConfig) ProtoMessage()               {}
1371
+func (*EncryptionConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} }
1372
+
1362 1373
 // Placement specifies task distribution constraints.
1363 1374
 type Placement struct {
1364 1375
 	// constraints specifies a set of requirements a node should meet for a task.
... ...
@@ -1367,7 +1381,7 @@ type Placement struct {
1367 1367
 
1368 1368
 func (m *Placement) Reset()                    { *m = Placement{} }
1369 1369
 func (*Placement) ProtoMessage()               {}
1370
-func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} }
1370
+func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} }
1371 1371
 
1372 1372
 // JoinToken contains the join tokens for workers and managers.
1373 1373
 type JoinTokens struct {
... ...
@@ -1379,7 +1393,7 @@ type JoinTokens struct {
1379 1379
 
1380 1380
 func (m *JoinTokens) Reset()                    { *m = JoinTokens{} }
1381 1381
 func (*JoinTokens) ProtoMessage()               {}
1382
-func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} }
1382
+func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} }
1383 1383
 
1384 1384
 type RootCA struct {
1385 1385
 	// CAKey is the root CA private key.
... ...
@@ -1394,7 +1408,7 @@ type RootCA struct {
1394 1394
 
1395 1395
 func (m *RootCA) Reset()                    { *m = RootCA{} }
1396 1396
 func (*RootCA) ProtoMessage()               {}
1397
-func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} }
1397
+func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} }
1398 1398
 
1399 1399
 type Certificate struct {
1400 1400
 	Role        NodeRole       `protobuf:"varint,1,opt,name=role,proto3,enum=docker.swarmkit.v1.NodeRole" json:"role,omitempty"`
... ...
@@ -1407,7 +1421,7 @@ type Certificate struct {
1407 1407
 
1408 1408
 func (m *Certificate) Reset()                    { *m = Certificate{} }
1409 1409
 func (*Certificate) ProtoMessage()               {}
1410
-func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} }
1410
+func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} }
1411 1411
 
1412 1412
 // Symmetric keys to encrypt inter-agent communication.
1413 1413
 type EncryptionKey struct {
... ...
@@ -1423,7 +1437,7 @@ type EncryptionKey struct {
1423 1423
 
1424 1424
 func (m *EncryptionKey) Reset()                    { *m = EncryptionKey{} }
1425 1425
 func (*EncryptionKey) ProtoMessage()               {}
1426
-func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} }
1426
+func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} }
1427 1427
 
1428 1428
 // ManagerStatus provides informations about the state of a manager in the cluster.
1429 1429
 type ManagerStatus struct {
... ...
@@ -1440,7 +1454,7 @@ type ManagerStatus struct {
1440 1440
 
1441 1441
 func (m *ManagerStatus) Reset()                    { *m = ManagerStatus{} }
1442 1442
 func (*ManagerStatus) ProtoMessage()               {}
1443
-func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} }
1443
+func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} }
1444 1444
 
1445 1445
 // SecretReference is the linkage between a service and a secret that it uses.
1446 1446
 type SecretReference struct {
... ...
@@ -1460,7 +1474,7 @@ type SecretReference struct {
1460 1460
 
1461 1461
 func (m *SecretReference) Reset()                    { *m = SecretReference{} }
1462 1462
 func (*SecretReference) ProtoMessage()               {}
1463
-func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} }
1463
+func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} }
1464 1464
 
1465 1465
 type isSecretReference_Target interface {
1466 1466
 	isSecretReference_Target()
... ...
@@ -1558,7 +1572,7 @@ type SecretReference_FileTarget struct {
1558 1558
 func (m *SecretReference_FileTarget) Reset()      { *m = SecretReference_FileTarget{} }
1559 1559
 func (*SecretReference_FileTarget) ProtoMessage() {}
1560 1560
 func (*SecretReference_FileTarget) Descriptor() ([]byte, []int) {
1561
-	return fileDescriptorTypes, []int{39, 0}
1561
+	return fileDescriptorTypes, []int{40, 0}
1562 1562
 }
1563 1563
 
1564 1564
 // BlacklistedCertificate is a record for a blacklisted certificate. It does not
... ...
@@ -1571,7 +1585,7 @@ type BlacklistedCertificate struct {
1571 1571
 
1572 1572
 func (m *BlacklistedCertificate) Reset()                    { *m = BlacklistedCertificate{} }
1573 1573
 func (*BlacklistedCertificate) ProtoMessage()               {}
1574
-func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} }
1574
+func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} }
1575 1575
 
1576 1576
 // HealthConfig holds configuration settings for the HEALTHCHECK feature.
1577 1577
 type HealthConfig struct {
... ...
@@ -1595,7 +1609,7 @@ type HealthConfig struct {
1595 1595
 
1596 1596
 func (m *HealthConfig) Reset()                    { *m = HealthConfig{} }
1597 1597
 func (*HealthConfig) ProtoMessage()               {}
1598
-func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} }
1598
+func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} }
1599 1599
 
1600 1600
 type MaybeEncryptedRecord struct {
1601 1601
 	Algorithm MaybeEncryptedRecord_Algorithm `protobuf:"varint,1,opt,name=algorithm,proto3,enum=docker.swarmkit.v1.MaybeEncryptedRecord_Algorithm" json:"algorithm,omitempty"`
... ...
@@ -1605,7 +1619,7 @@ type MaybeEncryptedRecord struct {
1605 1605
 
1606 1606
 func (m *MaybeEncryptedRecord) Reset()                    { *m = MaybeEncryptedRecord{} }
1607 1607
 func (*MaybeEncryptedRecord) ProtoMessage()               {}
1608
-func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} }
1608
+func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{43} }
1609 1609
 
1610 1610
 func init() {
1611 1611
 	proto.RegisterType((*Version)(nil), "docker.swarmkit.v1.Version")
... ...
@@ -1646,6 +1660,7 @@ func init() {
1646 1646
 	proto.RegisterType((*TaskDefaults)(nil), "docker.swarmkit.v1.TaskDefaults")
1647 1647
 	proto.RegisterType((*DispatcherConfig)(nil), "docker.swarmkit.v1.DispatcherConfig")
1648 1648
 	proto.RegisterType((*RaftConfig)(nil), "docker.swarmkit.v1.RaftConfig")
1649
+	proto.RegisterType((*EncryptionConfig)(nil), "docker.swarmkit.v1.EncryptionConfig")
1649 1650
 	proto.RegisterType((*Placement)(nil), "docker.swarmkit.v1.Placement")
1650 1651
 	proto.RegisterType((*JoinTokens)(nil), "docker.swarmkit.v1.JoinTokens")
1651 1652
 	proto.RegisterType((*RootCA)(nil), "docker.swarmkit.v1.RootCA")
... ...
@@ -2276,6 +2291,18 @@ func (m *RaftConfig) Copy() *RaftConfig {
2276 2276
 	return o
2277 2277
 }
2278 2278
 
2279
+func (m *EncryptionConfig) Copy() *EncryptionConfig {
2280
+	if m == nil {
2281
+		return nil
2282
+	}
2283
+
2284
+	o := &EncryptionConfig{
2285
+		AutoLockManagers: m.AutoLockManagers,
2286
+	}
2287
+
2288
+	return o
2289
+}
2290
+
2279 2291
 func (m *Placement) Copy() *Placement {
2280 2292
 	if m == nil {
2281 2293
 		return nil
... ...
@@ -3028,6 +3055,16 @@ func (this *RaftConfig) GoString() string {
3028 3028
 	s = append(s, "}")
3029 3029
 	return strings.Join(s, "")
3030 3030
 }
3031
+func (this *EncryptionConfig) GoString() string {
3032
+	if this == nil {
3033
+		return "nil"
3034
+	}
3035
+	s := make([]string, 0, 5)
3036
+	s = append(s, "&api.EncryptionConfig{")
3037
+	s = append(s, "AutoLockManagers: "+fmt.Sprintf("%#v", this.AutoLockManagers)+",\n")
3038
+	s = append(s, "}")
3039
+	return strings.Join(s, "")
3040
+}
3031 3041
 func (this *Placement) GoString() string {
3032 3042
 	if this == nil {
3033 3043
 		return "nil"
... ...
@@ -4708,6 +4745,34 @@ func (m *RaftConfig) MarshalTo(data []byte) (int, error) {
4708 4708
 	return i, nil
4709 4709
 }
4710 4710
 
4711
+func (m *EncryptionConfig) Marshal() (data []byte, err error) {
4712
+	size := m.Size()
4713
+	data = make([]byte, size)
4714
+	n, err := m.MarshalTo(data)
4715
+	if err != nil {
4716
+		return nil, err
4717
+	}
4718
+	return data[:n], nil
4719
+}
4720
+
4721
+func (m *EncryptionConfig) MarshalTo(data []byte) (int, error) {
4722
+	var i int
4723
+	_ = i
4724
+	var l int
4725
+	_ = l
4726
+	if m.AutoLockManagers {
4727
+		data[i] = 0x8
4728
+		i++
4729
+		if m.AutoLockManagers {
4730
+			data[i] = 1
4731
+		} else {
4732
+			data[i] = 0
4733
+		}
4734
+		i++
4735
+	}
4736
+	return i, nil
4737
+}
4738
+
4711 4739
 func (m *Placement) Marshal() (data []byte, err error) {
4712 4740
 	size := m.Size()
4713 4741
 	data = make([]byte, size)
... ...
@@ -5836,6 +5901,15 @@ func (m *RaftConfig) Size() (n int) {
5836 5836
 	return n
5837 5837
 }
5838 5838
 
5839
+func (m *EncryptionConfig) Size() (n int) {
5840
+	var l int
5841
+	_ = l
5842
+	if m.AutoLockManagers {
5843
+		n += 2
5844
+	}
5845
+	return n
5846
+}
5847
+
5839 5848
 func (m *Placement) Size() (n int) {
5840 5849
 	var l int
5841 5850
 	_ = l
... ...
@@ -6569,6 +6643,16 @@ func (this *RaftConfig) String() string {
6569 6569
 	}, "")
6570 6570
 	return s
6571 6571
 }
6572
+func (this *EncryptionConfig) String() string {
6573
+	if this == nil {
6574
+		return "nil"
6575
+	}
6576
+	s := strings.Join([]string{`&EncryptionConfig{`,
6577
+		`AutoLockManagers:` + fmt.Sprintf("%v", this.AutoLockManagers) + `,`,
6578
+		`}`,
6579
+	}, "")
6580
+	return s
6581
+}
6572 6582
 func (this *Placement) String() string {
6573 6583
 	if this == nil {
6574 6584
 		return "nil"
... ...
@@ -11863,6 +11947,76 @@ func (m *RaftConfig) Unmarshal(data []byte) error {
11863 11863
 	}
11864 11864
 	return nil
11865 11865
 }
11866
+func (m *EncryptionConfig) Unmarshal(data []byte) error {
11867
+	l := len(data)
11868
+	iNdEx := 0
11869
+	for iNdEx < l {
11870
+		preIndex := iNdEx
11871
+		var wire uint64
11872
+		for shift := uint(0); ; shift += 7 {
11873
+			if shift >= 64 {
11874
+				return ErrIntOverflowTypes
11875
+			}
11876
+			if iNdEx >= l {
11877
+				return io.ErrUnexpectedEOF
11878
+			}
11879
+			b := data[iNdEx]
11880
+			iNdEx++
11881
+			wire |= (uint64(b) & 0x7F) << shift
11882
+			if b < 0x80 {
11883
+				break
11884
+			}
11885
+		}
11886
+		fieldNum := int32(wire >> 3)
11887
+		wireType := int(wire & 0x7)
11888
+		if wireType == 4 {
11889
+			return fmt.Errorf("proto: EncryptionConfig: wiretype end group for non-group")
11890
+		}
11891
+		if fieldNum <= 0 {
11892
+			return fmt.Errorf("proto: EncryptionConfig: illegal tag %d (wire type %d)", fieldNum, wire)
11893
+		}
11894
+		switch fieldNum {
11895
+		case 1:
11896
+			if wireType != 0 {
11897
+				return fmt.Errorf("proto: wrong wireType = %d for field AutoLockManagers", wireType)
11898
+			}
11899
+			var v int
11900
+			for shift := uint(0); ; shift += 7 {
11901
+				if shift >= 64 {
11902
+					return ErrIntOverflowTypes
11903
+				}
11904
+				if iNdEx >= l {
11905
+					return io.ErrUnexpectedEOF
11906
+				}
11907
+				b := data[iNdEx]
11908
+				iNdEx++
11909
+				v |= (int(b) & 0x7F) << shift
11910
+				if b < 0x80 {
11911
+					break
11912
+				}
11913
+			}
11914
+			m.AutoLockManagers = bool(v != 0)
11915
+		default:
11916
+			iNdEx = preIndex
11917
+			skippy, err := skipTypes(data[iNdEx:])
11918
+			if err != nil {
11919
+				return err
11920
+			}
11921
+			if skippy < 0 {
11922
+				return ErrInvalidLengthTypes
11923
+			}
11924
+			if (iNdEx + skippy) > l {
11925
+				return io.ErrUnexpectedEOF
11926
+			}
11927
+			iNdEx += skippy
11928
+		}
11929
+	}
11930
+
11931
+	if iNdEx > l {
11932
+		return io.ErrUnexpectedEOF
11933
+	}
11934
+	return nil
11935
+}
11866 11936
 func (m *Placement) Unmarshal(data []byte) error {
11867 11937
 	l := len(data)
11868 11938
 	iNdEx := 0
... ...
@@ -13478,252 +13632,254 @@ var (
13478 13478
 func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) }
13479 13479
 
13480 13480
 var fileDescriptorTypes = []byte{
13481
-	// 3946 bytes of a gzipped FileDescriptorProto
13481
+	// 3975 bytes of a gzipped FileDescriptorProto
13482 13482
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x79, 0x4d, 0x6c, 0x1b, 0x49,
13483
-	0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0xcf,
13484
-	0x78, 0x67, 0xc6, 0x3b, 0x7f, 0x8e, 0xad, 0xf9, 0x80, 0x77, 0xfc, 0xcf, 0x7a, 0x9a, 0x1f, 0xb2,
13485
-	0xb8, 0x96, 0x48, 0xa2, 0x48, 0xd9, 0x19, 0x04, 0x08, 0x51, 0xea, 0x2e, 0x91, 0x3d, 0x6a, 0x76,
13486
-	0x33, 0xdd, 0x45, 0xc9, 0x4c, 0x10, 0xc4, 0xc8, 0x21, 0x09, 0x74, 0xca, 0x3d, 0x10, 0x82, 0x20,
13487
-	0x41, 0x0e, 0x39, 0xec, 0x25, 0x87, 0x00, 0x39, 0x0d, 0x72, 0x9a, 0xe3, 0x26, 0x01, 0x82, 0x45,
13488
-	0x82, 0x18, 0x19, 0xe5, 0x1c, 0x60, 0x2f, 0x8b, 0x1c, 0x92, 0x00, 0x41, 0x7d, 0x74, 0xb3, 0x29,
13489
-	0xd3, 0xb2, 0x27, 0xbb, 0x17, 0xb2, 0xeb, 0xd5, 0xef, 0xbd, 0xfa, 0x7a, 0x55, 0xf5, 0x7b, 0xaf,
13490
-	0xa0, 0xc0, 0xa6, 0x63, 0x1a, 0x54, 0xc6, 0xbe, 0xc7, 0x3c, 0x84, 0x2c, 0xcf, 0x3c, 0xa2, 0x7e,
13491
-	0x25, 0x38, 0x21, 0xfe, 0xe8, 0xc8, 0x66, 0x95, 0xe3, 0xbb, 0xa5, 0x1b, 0xcc, 0x1e, 0xd1, 0x80,
13492
-	0x91, 0xd1, 0xf8, 0xa3, 0xe8, 0x4b, 0xc2, 0x4b, 0x6f, 0x58, 0x13, 0x9f, 0x30, 0xdb, 0x73, 0x3f,
13493
-	0x0a, 0x3f, 0x54, 0xc5, 0xb5, 0x81, 0x37, 0xf0, 0xc4, 0xe7, 0x47, 0xfc, 0x4b, 0x4a, 0xf5, 0x4d,
13494
-	0x58, 0x7e, 0x4c, 0xfd, 0xc0, 0xf6, 0x5c, 0x74, 0x0d, 0x32, 0xb6, 0x6b, 0xd1, 0xa7, 0xeb, 0x89,
13495
-	0x72, 0xe2, 0xfd, 0x34, 0x96, 0x05, 0xfd, 0xcf, 0x12, 0x50, 0x30, 0x5c, 0xd7, 0x63, 0xc2, 0x56,
13496
-	0x80, 0x10, 0xa4, 0x5d, 0x32, 0xa2, 0x02, 0x94, 0xc7, 0xe2, 0x1b, 0xd5, 0x20, 0xeb, 0x90, 0x03,
13497
-	0xea, 0x04, 0xeb, 0xc9, 0x72, 0xea, 0xfd, 0xc2, 0xd6, 0x0f, 0x2a, 0x2f, 0xf6, 0xb9, 0x12, 0x33,
13498
-	0x52, 0xd9, 0x15, 0xe8, 0x86, 0xcb, 0xfc, 0x29, 0x56, 0xaa, 0xa5, 0x1f, 0x42, 0x21, 0x26, 0x46,
13499
-	0x1a, 0xa4, 0x8e, 0xe8, 0x54, 0x35, 0xc3, 0x3f, 0x79, 0xff, 0x8e, 0x89, 0x33, 0xa1, 0xeb, 0x49,
13500
-	0x21, 0x93, 0x85, 0xcf, 0x93, 0xf7, 0x12, 0xfa, 0x97, 0x90, 0xc7, 0x34, 0xf0, 0x26, 0xbe, 0x49,
13501
-	0x03, 0xf4, 0x01, 0xe4, 0x5d, 0xe2, 0x7a, 0x7d, 0x73, 0x3c, 0x09, 0x84, 0x7a, 0xaa, 0x5a, 0x3c,
13502
-	0x7f, 0xbe, 0x99, 0x6b, 0x11, 0xd7, 0xab, 0x75, 0xf6, 0x03, 0x9c, 0xe3, 0xd5, 0xb5, 0xf1, 0x24,
13503
-	0x40, 0x6f, 0x43, 0x71, 0x44, 0x47, 0x9e, 0x3f, 0xed, 0x1f, 0x4c, 0x19, 0x0d, 0x84, 0xe1, 0x14,
13504
-	0x2e, 0x48, 0x59, 0x95, 0x8b, 0xf4, 0x3f, 0x4e, 0xc0, 0xb5, 0xd0, 0x36, 0xa6, 0xbf, 0x35, 0xb1,
13505
-	0x7d, 0x3a, 0xa2, 0x2e, 0x0b, 0xd0, 0xa7, 0x90, 0x75, 0xec, 0x91, 0xcd, 0x64, 0x1b, 0x85, 0xad,
13506
-	0xb7, 0x16, 0x8d, 0x39, 0xea, 0x15, 0x56, 0x60, 0x64, 0x40, 0xd1, 0xa7, 0x01, 0xf5, 0x8f, 0xe5,
13507
-	0x4c, 0x88, 0x26, 0x5f, 0xa9, 0x3c, 0xa7, 0xa2, 0x6f, 0x43, 0xae, 0xe3, 0x10, 0x76, 0xe8, 0xf9,
13508
-	0x23, 0xa4, 0x43, 0x91, 0xf8, 0xe6, 0xd0, 0x66, 0xd4, 0x64, 0x13, 0x3f, 0x5c, 0x95, 0x39, 0x19,
13509
-	0xba, 0x0e, 0x49, 0x4f, 0x36, 0x94, 0xaf, 0x66, 0xcf, 0x9f, 0x6f, 0x26, 0xdb, 0x5d, 0x9c, 0xf4,
13510
-	0x02, 0xfd, 0x3e, 0x5c, 0xe9, 0x38, 0x93, 0x81, 0xed, 0xd6, 0x69, 0x60, 0xfa, 0xf6, 0x98, 0x5b,
13511
-	0xe7, 0xcb, 0xcb, 0x9d, 0x2f, 0x5c, 0x5e, 0xfe, 0x1d, 0x2d, 0x79, 0x72, 0xb6, 0xe4, 0xfa, 0x1f,
13512
-	0x26, 0xe1, 0x4a, 0xc3, 0x1d, 0xd8, 0x2e, 0x8d, 0x6b, 0xdf, 0x82, 0x55, 0x2a, 0x84, 0xfd, 0x63,
13513
-	0xe9, 0x54, 0xca, 0xce, 0x8a, 0x94, 0x86, 0x9e, 0xd6, 0xbc, 0xe0, 0x2f, 0x77, 0x17, 0x0d, 0xff,
13514
-	0x05, 0xeb, 0x8b, 0xbc, 0x06, 0x35, 0x60, 0x79, 0x2c, 0x06, 0x11, 0xac, 0xa7, 0x84, 0xad, 0x5b,
13515
-	0x8b, 0x6c, 0xbd, 0x30, 0xce, 0x6a, 0xfa, 0x9b, 0xe7, 0x9b, 0x4b, 0x38, 0xd4, 0xfd, 0x65, 0x9c,
13516
-	0xef, 0xdf, 0x13, 0xb0, 0xd6, 0xf2, 0xac, 0xb9, 0x79, 0x28, 0x41, 0x6e, 0xe8, 0x05, 0x2c, 0xb6,
13517
-	0x51, 0xa2, 0x32, 0xba, 0x07, 0xb9, 0xb1, 0x5a, 0x3e, 0xb5, 0xfa, 0x37, 0x17, 0x77, 0x59, 0x62,
13518
-	0x70, 0x84, 0x46, 0xf7, 0x21, 0xef, 0x87, 0x3e, 0xb1, 0x9e, 0x7a, 0x1d, 0xc7, 0x99, 0xe1, 0xd1,
13519
-	0xaf, 0x41, 0x56, 0x2e, 0xc2, 0x7a, 0x5a, 0x68, 0xde, 0x7a, 0xad, 0x39, 0xc7, 0x4a, 0x49, 0xff,
13520
-	0x59, 0x02, 0x34, 0x4c, 0x0e, 0xd9, 0x1e, 0x1d, 0x1d, 0x50, 0xbf, 0xcb, 0x08, 0x9b, 0x04, 0xe8,
13521
-	0x3a, 0x64, 0x1d, 0x4a, 0x2c, 0xea, 0x8b, 0x41, 0xe6, 0xb0, 0x2a, 0xa1, 0x7d, 0xee, 0xe4, 0xc4,
13522
-	0x1c, 0x92, 0x03, 0xdb, 0xb1, 0xd9, 0x54, 0x0c, 0x73, 0x75, 0xf1, 0x2a, 0x5f, 0xb4, 0x59, 0xc1,
13523
-	0x31, 0x45, 0x3c, 0x67, 0x06, 0xad, 0xc3, 0xf2, 0x88, 0x06, 0x01, 0x19, 0x50, 0x31, 0xfa, 0x3c,
13524
-	0x0e, 0x8b, 0xfa, 0x7d, 0x28, 0xc6, 0xf5, 0x50, 0x01, 0x96, 0xf7, 0x5b, 0x8f, 0x5a, 0xed, 0x27,
13525
-	0x2d, 0x6d, 0x09, 0xad, 0x41, 0x61, 0xbf, 0x85, 0x1b, 0x46, 0x6d, 0xc7, 0xa8, 0xee, 0x36, 0xb4,
13526
-	0x04, 0x5a, 0x81, 0xfc, 0xac, 0x98, 0xd4, 0xff, 0x3a, 0x01, 0xc0, 0x17, 0x50, 0x0d, 0xea, 0x73,
13527
-	0xc8, 0x04, 0x8c, 0x30, 0xb9, 0x70, 0xab, 0x5b, 0xef, 0x2e, 0xea, 0xf5, 0x0c, 0x5e, 0xe1, 0x7f,
13528
-	0x14, 0x4b, 0x95, 0x78, 0x0f, 0x93, 0x73, 0x3d, 0xe4, 0x7b, 0x88, 0x58, 0x96, 0xaf, 0x3a, 0x2e,
13529
-	0xbe, 0xf5, 0xfb, 0x90, 0x11, 0xda, 0xf3, 0xdd, 0xcd, 0x41, 0xba, 0xce, 0xbf, 0x12, 0x28, 0x0f,
13530
-	0x19, 0xdc, 0x30, 0xea, 0x5f, 0x6a, 0x49, 0xa4, 0x41, 0xb1, 0xde, 0xec, 0xd6, 0xda, 0xad, 0x56,
13531
-	0xa3, 0xd6, 0x6b, 0xd4, 0xb5, 0x94, 0x7e, 0x0b, 0x32, 0xcd, 0x11, 0xb7, 0x7c, 0x93, 0x7b, 0xc5,
13532
-	0x21, 0xf5, 0xa9, 0x6b, 0x86, 0xce, 0x36, 0x13, 0xe8, 0x3f, 0xcd, 0x43, 0x66, 0xcf, 0x9b, 0xb8,
13533
-	0x0c, 0x6d, 0xc5, 0x76, 0xf6, 0xea, 0xd6, 0xc6, 0xa2, 0x61, 0x09, 0x60, 0xa5, 0x37, 0x1d, 0x53,
13534
-	0xb5, 0xf3, 0xaf, 0x43, 0x56, 0xfa, 0x8f, 0x1a, 0x8e, 0x2a, 0x71, 0x39, 0x23, 0xfe, 0x80, 0x32,
13535
-	0x35, 0x1e, 0x55, 0x42, 0xef, 0x43, 0xce, 0xa7, 0xc4, 0xf2, 0x5c, 0x67, 0x2a, 0xdc, 0x2c, 0x27,
13536
-	0x8f, 0x5e, 0x4c, 0x89, 0xd5, 0x76, 0x9d, 0x29, 0x8e, 0x6a, 0xd1, 0x0e, 0x14, 0x0f, 0x6c, 0xd7,
13537
-	0xea, 0x7b, 0x63, 0x79, 0x0e, 0x66, 0x5e, 0xee, 0x94, 0xb2, 0x57, 0x55, 0xdb, 0xb5, 0xda, 0x12,
13538
-	0x8c, 0x0b, 0x07, 0xb3, 0x02, 0x6a, 0xc1, 0xea, 0xb1, 0xe7, 0x4c, 0x46, 0x34, 0xb2, 0x95, 0x15,
13539
-	0xb6, 0xde, 0x7b, 0xb9, 0xad, 0xc7, 0x02, 0x1f, 0x5a, 0x5b, 0x39, 0x8e, 0x17, 0xd1, 0x23, 0x58,
13540
-	0x61, 0xa3, 0xf1, 0x61, 0x10, 0x99, 0x5b, 0x16, 0xe6, 0xbe, 0x7f, 0xc9, 0x84, 0x71, 0x78, 0x68,
13541
-	0xad, 0xc8, 0x62, 0xa5, 0xd2, 0xef, 0xa7, 0xa0, 0x10, 0xeb, 0x39, 0xea, 0x42, 0x61, 0xec, 0x7b,
13542
-	0x63, 0x32, 0x10, 0x67, 0xb9, 0x5a, 0x8b, 0xbb, 0xaf, 0x35, 0xea, 0x4a, 0x67, 0xa6, 0x88, 0xe3,
13543
-	0x56, 0xf4, 0xb3, 0x24, 0x14, 0x62, 0x95, 0xe8, 0x36, 0xe4, 0x70, 0x07, 0x37, 0x1f, 0x1b, 0xbd,
13544
-	0x86, 0xb6, 0x54, 0xba, 0x79, 0x7a, 0x56, 0x5e, 0x17, 0xd6, 0xe2, 0x06, 0x3a, 0xbe, 0x7d, 0xcc,
13545
-	0x5d, 0xef, 0x7d, 0x58, 0x0e, 0xa1, 0x89, 0xd2, 0x9b, 0xa7, 0x67, 0xe5, 0x37, 0x2e, 0x42, 0x63,
13546
-	0x48, 0xdc, 0xdd, 0x31, 0x70, 0xa3, 0xae, 0x25, 0x17, 0x23, 0x71, 0x77, 0x48, 0x7c, 0x6a, 0xa1,
13547
-	0xef, 0x43, 0x56, 0x01, 0x53, 0xa5, 0xd2, 0xe9, 0x59, 0xf9, 0xfa, 0x45, 0xe0, 0x0c, 0x87, 0xbb,
13548
-	0xbb, 0xc6, 0xe3, 0x86, 0x96, 0x5e, 0x8c, 0xc3, 0x5d, 0x87, 0x1c, 0x53, 0xf4, 0x2e, 0x64, 0x24,
13549
-	0x2c, 0x53, 0xba, 0x71, 0x7a, 0x56, 0xfe, 0xde, 0x0b, 0xe6, 0x38, 0xaa, 0xb4, 0xfe, 0x47, 0x7f,
13550
-	0xbe, 0xb1, 0xf4, 0xb7, 0x7f, 0xb1, 0xa1, 0x5d, 0xac, 0x2e, 0xfd, 0x77, 0x02, 0x56, 0xe6, 0x96,
13551
-	0x1c, 0xe9, 0x90, 0x75, 0x3d, 0xd3, 0x1b, 0xcb, 0x23, 0x3e, 0x57, 0x85, 0xf3, 0xe7, 0x9b, 0xd9,
13552
-	0x96, 0x57, 0xf3, 0xc6, 0x53, 0xac, 0x6a, 0xd0, 0xa3, 0x0b, 0x97, 0xd4, 0xc7, 0xaf, 0xe9, 0x4f,
13553
-	0x0b, 0xaf, 0xa9, 0x07, 0xb0, 0x62, 0xf9, 0xf6, 0x31, 0xf5, 0xfb, 0xa6, 0xe7, 0x1e, 0xda, 0x03,
13554
-	0x75, 0x7c, 0x97, 0x16, 0xd9, 0xac, 0x0b, 0x20, 0x2e, 0x4a, 0x85, 0x9a, 0xc0, 0xff, 0x12, 0x17,
13555
-	0x54, 0xe9, 0x31, 0x14, 0xe3, 0x1e, 0x8a, 0xde, 0x02, 0x08, 0xec, 0xdf, 0xa6, 0x8a, 0xf3, 0x08,
13556
-	0x86, 0x84, 0xf3, 0x5c, 0x22, 0x18, 0x0f, 0x7a, 0x0f, 0xd2, 0x23, 0xcf, 0x92, 0x76, 0x56, 0xaa,
13557
-	0x57, 0xf9, 0x3d, 0xf9, 0xcf, 0xcf, 0x37, 0x0b, 0x5e, 0x50, 0xd9, 0xb6, 0x1d, 0xba, 0xe7, 0x59,
13558
-	0x14, 0x0b, 0x80, 0x7e, 0x0c, 0x69, 0x7e, 0x54, 0xa0, 0x37, 0x21, 0x5d, 0x6d, 0xb6, 0xea, 0xda,
13559
-	0x52, 0xe9, 0xca, 0xe9, 0x59, 0x79, 0x45, 0x4c, 0x09, 0xaf, 0xe0, 0xbe, 0x8b, 0x36, 0x21, 0xfb,
13560
-	0xb8, 0xbd, 0xbb, 0xbf, 0xc7, 0xdd, 0xeb, 0xea, 0xe9, 0x59, 0x79, 0x2d, 0xaa, 0x96, 0x93, 0x86,
13561
-	0xde, 0x82, 0x4c, 0x6f, 0xaf, 0xb3, 0xdd, 0xd5, 0x92, 0x25, 0x74, 0x7a, 0x56, 0x5e, 0x8d, 0xea,
13562
-	0x45, 0x9f, 0x4b, 0x57, 0xd4, 0xaa, 0xe6, 0x23, 0xb9, 0xfe, 0x5f, 0x49, 0x58, 0xc1, 0x9c, 0xf3,
13563
-	0xfa, 0xac, 0xe3, 0x39, 0xb6, 0x39, 0x45, 0x1d, 0xc8, 0x9b, 0x9e, 0x6b, 0xd9, 0xb1, 0x3d, 0xb5,
13564
-	0xf5, 0x92, 0x8b, 0x71, 0xa6, 0x15, 0x96, 0x6a, 0xa1, 0x26, 0x9e, 0x19, 0x41, 0x5b, 0x90, 0xb1,
13565
-	0xa8, 0x43, 0xa6, 0x97, 0xdd, 0xd0, 0x75, 0xc5, 0xaf, 0xb1, 0x84, 0x0a, 0x36, 0x49, 0x9e, 0xf6,
13566
-	0x09, 0x63, 0x74, 0x34, 0x66, 0xf2, 0x86, 0x4e, 0xe3, 0xc2, 0x88, 0x3c, 0x35, 0x94, 0x08, 0x7d,
13567
-	0x02, 0xd9, 0x13, 0xdb, 0xb5, 0xbc, 0x13, 0x75, 0x09, 0x5f, 0x6e, 0x57, 0x61, 0xf5, 0x53, 0x7e,
13568
-	0xf7, 0x5e, 0xe8, 0x2c, 0x9f, 0xf5, 0x56, 0xbb, 0xd5, 0x08, 0x67, 0x5d, 0xd5, 0xb7, 0xdd, 0x96,
13569
-	0xe7, 0xf2, 0x1d, 0x03, 0xed, 0x56, 0x7f, 0xdb, 0x68, 0xee, 0xee, 0x63, 0x3e, 0xf3, 0xd7, 0x4e,
13570
-	0xcf, 0xca, 0x5a, 0x04, 0xd9, 0x26, 0xb6, 0xc3, 0x89, 0xe1, 0x0d, 0x48, 0x19, 0xad, 0x2f, 0xb5,
13571
-	0x64, 0x49, 0x3b, 0x3d, 0x2b, 0x17, 0xa3, 0x6a, 0xc3, 0x9d, 0xce, 0x36, 0xd3, 0xc5, 0x76, 0xf5,
13572
-	0x7f, 0x4d, 0x42, 0x71, 0x7f, 0x6c, 0x11, 0x46, 0xa5, 0x67, 0xa2, 0x32, 0x14, 0xc6, 0xc4, 0x27,
13573
-	0x8e, 0x43, 0x1d, 0x3b, 0x18, 0xa9, 0xe0, 0x21, 0x2e, 0x42, 0xf7, 0xbe, 0xc3, 0x64, 0x2a, 0x62,
13574
-	0xa6, 0xa6, 0x74, 0x1f, 0x56, 0x0f, 0x65, 0x67, 0xfb, 0xc4, 0x14, 0xab, 0x9b, 0x12, 0xab, 0x5b,
13575
-	0x59, 0x64, 0x22, 0xde, 0xab, 0x8a, 0x1a, 0xa3, 0x21, 0xb4, 0xf0, 0xca, 0x61, 0xbc, 0x88, 0x3e,
13576
-	0x83, 0xe5, 0x91, 0xe7, 0xda, 0xcc, 0xf3, 0x5f, 0x6b, 0x1d, 0x42, 0x30, 0xba, 0x0d, 0x57, 0xf8,
13577
-	0x0a, 0x87, 0x5d, 0x12, 0xd5, 0xe2, 0xe6, 0x4a, 0xe2, 0xb5, 0x11, 0x79, 0xaa, 0xda, 0xc4, 0x5c,
13578
-	0xac, 0x7f, 0x06, 0x2b, 0x73, 0x7d, 0xe0, 0xb7, 0x79, 0xc7, 0xd8, 0xef, 0x36, 0xb4, 0x25, 0x54,
13579
-	0x84, 0x5c, 0xad, 0xdd, 0xea, 0x35, 0x5b, 0xfb, 0x9c, 0x8e, 0x14, 0x21, 0x87, 0xdb, 0xbb, 0xbb,
13580
-	0x55, 0xa3, 0xf6, 0x48, 0x4b, 0xea, 0xbf, 0x88, 0xe6, 0x57, 0xf1, 0x91, 0xea, 0x3c, 0x1f, 0xf9,
13581
-	0xf0, 0xe5, 0x43, 0x57, 0x8c, 0x64, 0x56, 0x88, 0x78, 0xc9, 0xff, 0x07, 0x10, 0xcb, 0x48, 0xad,
13582
-	0x3e, 0x61, 0x97, 0xc5, 0x1c, 0xbd, 0x30, 0x9a, 0xc4, 0x79, 0xa5, 0x60, 0x30, 0xf4, 0x05, 0x14,
13583
-	0x4d, 0x6f, 0x34, 0x76, 0xa8, 0xd2, 0x4f, 0xbd, 0x8e, 0x7e, 0x21, 0x52, 0x31, 0x58, 0x9c, 0x17,
13584
-	0xa5, 0xe7, 0x99, 0xdb, 0x1f, 0x24, 0xa0, 0x10, 0xeb, 0xf0, 0x3c, 0x15, 0x2a, 0x42, 0x6e, 0xbf,
13585
-	0x53, 0x37, 0x7a, 0xcd, 0xd6, 0x43, 0x2d, 0x81, 0x00, 0xb2, 0x62, 0x02, 0xeb, 0x5a, 0x92, 0x53,
13586
-	0xb8, 0x5a, 0x7b, 0xaf, 0xb3, 0xdb, 0x10, 0x64, 0x08, 0x5d, 0x03, 0x2d, 0x9c, 0xc2, 0x7e, 0xb7,
13587
-	0x67, 0x60, 0x2e, 0x4d, 0xa3, 0xab, 0xb0, 0x16, 0x49, 0x95, 0x66, 0x06, 0x5d, 0x07, 0x14, 0x09,
13588
-	0x67, 0x26, 0xb2, 0xfa, 0xef, 0xc2, 0x5a, 0xcd, 0x73, 0x19, 0xb1, 0xdd, 0x88, 0xde, 0x6e, 0xf1,
13589
-	0x71, 0x2b, 0x51, 0xdf, 0xb6, 0xe4, 0x69, 0x5b, 0x5d, 0x3b, 0x7f, 0xbe, 0x59, 0x88, 0xa0, 0xcd,
13590
-	0x3a, 0x1f, 0x69, 0x58, 0xb0, 0xf8, 0x9e, 0x1a, 0xdb, 0x96, 0x98, 0xe2, 0x4c, 0x75, 0xf9, 0xfc,
13591
-	0xf9, 0x66, 0xaa, 0xd3, 0xac, 0x63, 0x2e, 0x43, 0x6f, 0x42, 0x9e, 0x3e, 0xb5, 0x59, 0xdf, 0xe4,
13592
-	0xa7, 0x2b, 0x9f, 0xc3, 0x0c, 0xce, 0x71, 0x41, 0x8d, 0x1f, 0xa6, 0x55, 0x80, 0x8e, 0xe7, 0x33,
13593
-	0xd5, 0xf2, 0x27, 0x90, 0x19, 0x7b, 0xbe, 0x88, 0x2d, 0xf9, 0xd5, 0xb3, 0x90, 0xac, 0x71, 0xb8,
13594
-	0x74, 0x76, 0x2c, 0xc1, 0xfa, 0xdf, 0x25, 0x01, 0x7a, 0x24, 0x38, 0x52, 0x46, 0xee, 0x43, 0x3e,
13595
-	0x4a, 0x0e, 0x5c, 0x16, 0xa4, 0xc6, 0xd6, 0x3c, 0xc2, 0xa3, 0x8f, 0x43, 0xaf, 0x93, 0xdc, 0x7d,
13596
-	0xb1, 0xa2, 0x6a, 0x6b, 0x11, 0xfd, 0x9d, 0x27, 0xe8, 0xfc, 0xbe, 0xa2, 0xbe, 0xaf, 0x16, 0x9f,
13597
-	0x7f, 0xa2, 0x9a, 0x38, 0xb3, 0xe5, 0xbc, 0x29, 0xf6, 0xf7, 0xce, 0xa2, 0x46, 0x2e, 0x2c, 0xca,
13598
-	0xce, 0x12, 0x9e, 0xe9, 0xa1, 0x07, 0x50, 0xe0, 0x43, 0xef, 0x07, 0xa2, 0x4e, 0x11, 0xbf, 0x97,
13599
-	0xce, 0x96, 0xb4, 0x80, 0x61, 0x1c, 0x7d, 0x57, 0x35, 0x58, 0xf5, 0x27, 0x2e, 0x1f, 0xb6, 0xb2,
13600
-	0xa1, 0xdb, 0xf0, 0x46, 0x8b, 0xb2, 0x13, 0xcf, 0x3f, 0x32, 0x18, 0x23, 0xe6, 0x90, 0x47, 0xfb,
13601
-	0xea, 0xa4, 0x9b, 0xb1, 0xde, 0xc4, 0x1c, 0xeb, 0x5d, 0x87, 0x65, 0xe2, 0xd8, 0x24, 0xa0, 0x92,
13602
-	0x2a, 0xe4, 0x71, 0x58, 0xe4, 0xdc, 0x9c, 0x33, 0x7d, 0x1a, 0x04, 0x54, 0xc6, 0xa7, 0x79, 0x3c,
13603
-	0x13, 0xe8, 0xff, 0x98, 0x04, 0x68, 0x76, 0x8c, 0x3d, 0x65, 0xbe, 0x0e, 0xd9, 0x43, 0x32, 0xb2,
13604
-	0x9d, 0xe9, 0x65, 0x3b, 0x7d, 0x86, 0xaf, 0x18, 0xd2, 0xd0, 0xb6, 0xd0, 0xc1, 0x4a, 0x57, 0x50,
13605
-	0xf6, 0xc9, 0x81, 0x4b, 0x59, 0x44, 0xd9, 0x45, 0x89, 0xf3, 0x03, 0x9f, 0xb8, 0xd1, 0xca, 0xc8,
13606
-	0x02, 0xef, 0xfa, 0x80, 0x30, 0x7a, 0x42, 0xa6, 0xe1, 0xc6, 0x54, 0x45, 0xb4, 0xc3, 0xa9, 0x7c,
13607
-	0x40, 0xfd, 0x63, 0x6a, 0xad, 0x67, 0x84, 0x17, 0xbe, 0xaa, 0x3f, 0x58, 0xc1, 0x25, 0xf3, 0x89,
13608
-	0xb4, 0x4b, 0xf7, 0xc5, 0x75, 0x3d, 0xab, 0xfa, 0x4e, 0xd1, 0xf5, 0x1d, 0x58, 0x99, 0x1b, 0xe7,
13609
-	0x0b, 0xb1, 0x52, 0xb3, 0xf3, 0xf8, 0x13, 0x2d, 0xad, 0xbe, 0x3e, 0xd3, 0xb2, 0xfa, 0x5f, 0xa5,
13610
-	0xe4, 0x56, 0x52, 0xb3, 0xba, 0x38, 0x5f, 0x95, 0x13, 0xd9, 0x2f, 0xd3, 0x73, 0x94, 0x7f, 0xbf,
13611
-	0x77, 0xf9, 0x0e, 0xe3, 0xdc, 0x5b, 0xc0, 0x71, 0xa4, 0x88, 0x36, 0xa1, 0x20, 0xd7, 0xbf, 0xcf,
13612
-	0xfd, 0x49, 0x4c, 0xeb, 0x0a, 0x06, 0x29, 0xe2, 0x9a, 0xe8, 0x16, 0xac, 0x8e, 0x27, 0x07, 0x8e,
13613
-	0x1d, 0x0c, 0xa9, 0x25, 0x31, 0x69, 0x81, 0x59, 0x89, 0xa4, 0x02, 0xb6, 0x07, 0x45, 0x25, 0xe8,
13614
-	0x0b, 0xde, 0x95, 0x11, 0x1d, 0xba, 0xfd, 0xaa, 0x0e, 0x49, 0x15, 0x41, 0xc7, 0x0a, 0xe3, 0x59,
13615
-	0x41, 0xaf, 0x43, 0x2e, 0xec, 0x2c, 0x5a, 0x87, 0x54, 0xaf, 0xd6, 0xd1, 0x96, 0x4a, 0x6b, 0xa7,
13616
-	0x67, 0xe5, 0x42, 0x28, 0xee, 0xd5, 0x3a, 0xbc, 0x66, 0xbf, 0xde, 0xd1, 0x12, 0xf3, 0x35, 0xfb,
13617
-	0xf5, 0x4e, 0x29, 0xcd, 0x6f, 0x7e, 0xfd, 0x10, 0x0a, 0xb1, 0x16, 0xd0, 0x3b, 0xb0, 0xdc, 0x6c,
13618
-	0x3d, 0xc4, 0x8d, 0x6e, 0x57, 0x5b, 0x2a, 0x5d, 0x3f, 0x3d, 0x2b, 0xa3, 0x58, 0x6d, 0xd3, 0x1d,
13619
-	0xf0, 0xf5, 0x41, 0x6f, 0x41, 0x7a, 0xa7, 0xdd, 0xed, 0x85, 0x44, 0x2f, 0x86, 0xd8, 0xf1, 0x02,
13620
-	0x56, 0xba, 0xaa, 0x28, 0x45, 0xdc, 0xb0, 0xfe, 0x27, 0x09, 0xc8, 0x4a, 0xbe, 0xbb, 0x70, 0xa1,
13621
-	0x0c, 0x58, 0x0e, 0xa3, 0x30, 0x49, 0xc2, 0xdf, 0x7b, 0x39, 0x61, 0xae, 0x28, 0x7e, 0x2b, 0xdd,
13622
-	0x2f, 0xd4, 0x2b, 0x7d, 0x0e, 0xc5, 0x78, 0xc5, 0x77, 0x72, 0xbe, 0xdf, 0x81, 0x02, 0xf7, 0xef,
13623
-	0x90, 0x38, 0x6f, 0x41, 0x56, 0x72, 0x72, 0x75, 0x9a, 0x5e, 0xc6, 0xde, 0x15, 0x12, 0xdd, 0x83,
13624
-	0x65, 0xc9, 0xf8, 0xc3, 0xfc, 0xd4, 0xc6, 0xe5, 0xbb, 0x08, 0x87, 0x70, 0xfd, 0x01, 0xa4, 0x3b,
13625
-	0x94, 0xfa, 0x7c, 0xee, 0x5d, 0xcf, 0xa2, 0xb3, 0x0b, 0x48, 0x05, 0x2b, 0x16, 0x6d, 0xd6, 0x79,
13626
-	0xb0, 0x62, 0xd1, 0xa6, 0x15, 0xa5, 0x17, 0x92, 0xb1, 0xf4, 0x42, 0x0f, 0x8a, 0x4f, 0xa8, 0x3d,
13627
-	0x18, 0x32, 0x6a, 0x09, 0x43, 0x1f, 0x42, 0x7a, 0x4c, 0xa3, 0xce, 0xaf, 0x2f, 0x74, 0x30, 0x4a,
13628
-	0x7d, 0x2c, 0x50, 0xfc, 0x1c, 0x39, 0x11, 0xda, 0x2a, 0x2b, 0xaa, 0x4a, 0xfa, 0x3f, 0x24, 0x61,
13629
-	0xb5, 0x19, 0x04, 0x13, 0xe2, 0x9a, 0x21, 0x43, 0xf9, 0xd1, 0x3c, 0x43, 0x79, 0x7f, 0xe1, 0x08,
13630
-	0xe7, 0x54, 0xe6, 0xb3, 0x26, 0xea, 0x72, 0x48, 0x46, 0x97, 0x83, 0xfe, 0x1f, 0x89, 0x30, 0x35,
13631
-	0x72, 0x2b, 0xb6, 0xdd, 0x4b, 0xeb, 0xa7, 0x67, 0xe5, 0x6b, 0x71, 0x4b, 0x74, 0xdf, 0x3d, 0x72,
13632
-	0xbd, 0x13, 0x17, 0xbd, 0x0d, 0x19, 0xdc, 0x68, 0x35, 0x9e, 0x68, 0x09, 0xe9, 0x9e, 0x73, 0x20,
13633
-	0x4c, 0x5d, 0x7a, 0xc2, 0x2d, 0x75, 0x1a, 0xad, 0x3a, 0xe7, 0x12, 0xc9, 0x05, 0x96, 0x3a, 0xd4,
13634
-	0xb5, 0x6c, 0x77, 0x80, 0xde, 0x81, 0x6c, 0xb3, 0xdb, 0xdd, 0x17, 0xc1, 0xeb, 0x1b, 0xa7, 0x67,
13635
-	0xe5, 0xab, 0x73, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x02, 0x10, 0xe7,
13636
-	0x7d, 0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0x16, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb,
13637
-	0x97, 0x24, 0x68, 0x86, 0x69, 0xd2, 0x31, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xcc, 0xbf,
13638
-	0x6c, 0x1a, 0xf2, 0x80, 0x7b, 0x0b, 0xf3, 0xea, 0x17, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x8d,
13639
-	0xec, 0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0xba, 0x00, 0x81,
13640
-	0xee, 0x40, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0x6f, 0xbe, 0x2c, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2,
13641
-	0x0d, 0x00, 0x32, 0x61, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x81, 0x6c,
13642
-	0x40, 0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xc1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9,
13643
-	0x52, 0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde,
13644
-	0x44, 0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x65, 0xd4, 0x77,
13645
-	0x89, 0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0xb0, 0x30, 0x17, 0x1a, 0x69, 0x54,
13646
-	0x6a, 0xc6, 0x82, 0xf3, 0xff, 0x06, 0xa4, 0x26, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe3,
13647
-	0x5d, 0xcc, 0x65, 0xa8, 0x31, 0x3b, 0xb6, 0x52, 0x2f, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f,
13648
-	0xba, 0x3e, 0x04, 0x98, 0xf5, 0x1a, 0x6d, 0x40, 0xa6, 0xb6, 0xdd, 0xed, 0xee, 0x6a, 0x4b, 0xf2,
13649
-	0x6c, 0x9e, 0x55, 0x09, 0xb1, 0xfe, 0x97, 0x09, 0xc8, 0xd5, 0x0c, 0x75, 0x63, 0x6e, 0x83, 0x26,
13650
-	0x0e, 0x1c, 0x93, 0xfa, 0xac, 0x4f, 0x9f, 0x8e, 0x6d, 0x7f, 0xaa, 0xce, 0x8c, 0xcb, 0xc3, 0xa4,
13651
-	0x55, 0xae, 0x55, 0xa3, 0x3e, 0x6b, 0x08, 0x1d, 0x84, 0xa1, 0x48, 0xd5, 0x10, 0xfb, 0x26, 0x09,
13652
-	0x4f, 0xf0, 0x8d, 0xcb, 0xa7, 0x42, 0xd2, 0xeb, 0x59, 0x39, 0xc0, 0x85, 0xd0, 0x48, 0x8d, 0x04,
13653
-	0xfa, 0x63, 0xb8, 0xda, 0xf6, 0xcd, 0x21, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x01, 0xdc, 0x64,
13654
-	0x24, 0x38, 0xea, 0x0f, 0xed, 0x80, 0x79, 0xfe, 0xb4, 0xef, 0x53, 0x46, 0x5d, 0x5e, 0xdf, 0x17,
13655
-	0xcf, 0x2e, 0x2a, 0xc9, 0x71, 0x83, 0x63, 0x76, 0x24, 0x04, 0x87, 0x88, 0x5d, 0x0e, 0xd0, 0x9b,
13656
-	0x50, 0xe4, 0x6c, 0xb6, 0x4e, 0x0f, 0xc9, 0xc4, 0x61, 0x01, 0xfa, 0x21, 0x80, 0xe3, 0x0d, 0xfa,
13657
-	0xaf, 0x7d, 0xdc, 0xe7, 0x1d, 0x6f, 0x20, 0x3f, 0xf5, 0xdf, 0x00, 0xad, 0x6e, 0x07, 0x63, 0xc2,
13658
-	0xcc, 0x61, 0x98, 0xbd, 0x41, 0x0f, 0x41, 0x1b, 0x52, 0xe2, 0xb3, 0x03, 0x4a, 0x58, 0x7f, 0x4c,
13659
-	0x7d, 0xdb, 0xb3, 0x5e, 0x6b, 0x4a, 0xd7, 0x22, 0xad, 0x8e, 0x50, 0xd2, 0xff, 0x33, 0x01, 0x80,
13660
-	0xc9, 0x61, 0x48, 0x6e, 0x7e, 0x00, 0x57, 0x02, 0x97, 0x8c, 0x83, 0xa1, 0xc7, 0xfa, 0xb6, 0xcb,
13661
-	0xa8, 0x7f, 0x4c, 0x1c, 0x15, 0x81, 0x6b, 0x61, 0x45, 0x53, 0xc9, 0xd1, 0x87, 0x80, 0x8e, 0x28,
13662
-	0x1d, 0xf7, 0x3d, 0xc7, 0xea, 0x87, 0x95, 0xf2, 0x5d, 0x28, 0x8d, 0x35, 0x5e, 0xd3, 0x76, 0xac,
13663
-	0x6e, 0x28, 0x47, 0x55, 0xd8, 0xe0, 0x33, 0x40, 0x5d, 0xe6, 0xdb, 0x34, 0xe8, 0x1f, 0x7a, 0x7e,
13664
-	0x3f, 0x70, 0xbc, 0x93, 0xfe, 0xa1, 0xe7, 0x38, 0xde, 0x09, 0xf5, 0xc3, 0xfc, 0x46, 0xc9, 0xf1,
13665
-	0x06, 0x0d, 0x09, 0xda, 0xf6, 0xfc, 0xae, 0xe3, 0x9d, 0x6c, 0x87, 0x08, 0xce, 0x80, 0x66, 0xc3,
13666
-	0x66, 0xb6, 0x79, 0x14, 0x32, 0xa0, 0x48, 0xda, 0xb3, 0xcd, 0x23, 0xf4, 0x0e, 0xac, 0x50, 0x87,
13667
-	0x8a, 0x28, 0x59, 0xa2, 0x32, 0x02, 0x55, 0x0c, 0x85, 0x1c, 0xa4, 0xff, 0x3f, 0xc8, 0x77, 0x1c,
13668
-	0x62, 0x8a, 0xd7, 0x37, 0x54, 0x06, 0x1e, 0x74, 0x71, 0x27, 0xb0, 0x5d, 0x15, 0x25, 0xe5, 0x71,
13669
-	0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x23, 0xea, 0x8a, 0x87, 0x0a, 0xce,
13670
-	0xe8, 0xd5, 0x52, 0xe6, 0xb1, 0x2a, 0x89, 0x80, 0x85, 0xb8, 0x64, 0x40, 0xfd, 0x28, 0x5f, 0x2f,
13671
-	0x8b, 0xfa, 0x37, 0x09, 0xc8, 0x62, 0xcf, 0x63, 0x35, 0x03, 0x95, 0x21, 0x6b, 0x92, 0x7e, 0xb8,
13672
-	0xf3, 0x8a, 0xd5, 0xfc, 0xf9, 0xf3, 0xcd, 0x4c, 0xcd, 0x78, 0x44, 0xa7, 0x38, 0x63, 0x92, 0x47,
13673
-	0x74, 0xca, 0xaf, 0x68, 0x93, 0x88, 0xfd, 0x22, 0xcc, 0x14, 0xe5, 0x15, 0x5d, 0x33, 0xf8, 0x66,
13674
-	0xc0, 0x59, 0x93, 0xf0, 0x7f, 0x74, 0x07, 0x8a, 0x0a, 0xd4, 0x1f, 0x92, 0x60, 0x28, 0x79, 0x78,
13675
-	0x75, 0xf5, 0xfc, 0xf9, 0x26, 0x48, 0xe4, 0x0e, 0x09, 0x86, 0x18, 0x24, 0x9a, 0x7f, 0xa3, 0x06,
13676
-	0x14, 0xbe, 0xf2, 0x6c, 0xb7, 0xcf, 0xc4, 0x20, 0x54, 0xaa, 0x62, 0xe1, 0xfe, 0x99, 0x0d, 0x55,
13677
-	0xe5, 0x4f, 0xe0, 0xab, 0x48, 0xa2, 0xff, 0x53, 0x02, 0x0a, 0xdc, 0xa6, 0x7d, 0x68, 0x9b, 0xfc,
13678
-	0x4a, 0xfd, 0xee, 0x27, 0xfd, 0x0d, 0x48, 0x99, 0x81, 0xaf, 0xc6, 0x26, 0x8e, 0xba, 0x5a, 0x17,
13679
-	0x63, 0x2e, 0x43, 0x5f, 0x40, 0x56, 0x05, 0x5f, 0xf2, 0x90, 0xd7, 0x5f, 0x7d, 0xf9, 0xab, 0x2e,
13680
-	0x2a, 0x3d, 0xb1, 0x96, 0xb3, 0xde, 0x89, 0x51, 0x16, 0x71, 0x5c, 0x84, 0xae, 0x43, 0xd2, 0x74,
13681
-	0x85, 0x53, 0xa8, 0x07, 0xcc, 0x5a, 0x0b, 0x27, 0x4d, 0x57, 0xff, 0xfb, 0x04, 0xac, 0x34, 0x5c,
13682
-	0xd3, 0x9f, 0x8a, 0x43, 0x92, 0x2f, 0xc4, 0x4d, 0xc8, 0x07, 0x93, 0x83, 0x60, 0x1a, 0x30, 0x3a,
13683
-	0x0a, 0xdf, 0x42, 0x22, 0x01, 0x6a, 0x42, 0x9e, 0x38, 0x03, 0xcf, 0xb7, 0xd9, 0x70, 0xa4, 0x78,
13684
-	0xff, 0xe2, 0x83, 0x39, 0x6e, 0xb3, 0x62, 0x84, 0x2a, 0x78, 0xa6, 0x1d, 0x1e, 0xc5, 0x29, 0xd1,
13685
-	0x59, 0x71, 0x14, 0xbf, 0x0d, 0x45, 0x87, 0x8c, 0x44, 0x34, 0xca, 0xc3, 0x49, 0x31, 0x8e, 0x34,
13686
-	0x2e, 0x28, 0x19, 0x8f, 0xb1, 0x75, 0x1d, 0xf2, 0x91, 0x31, 0xb4, 0x06, 0x05, 0xa3, 0xd1, 0xed,
13687
-	0xdf, 0xdd, 0xba, 0xd7, 0x7f, 0x58, 0xdb, 0xd3, 0x96, 0x14, 0x13, 0xf8, 0x9b, 0x04, 0xac, 0xec,
13688
-	0x49, 0x1f, 0x54, 0xec, 0xea, 0x1d, 0x58, 0xf6, 0xc9, 0x21, 0x0b, 0xf9, 0x5f, 0x5a, 0x3a, 0x17,
13689
-	0x3f, 0x04, 0x38, 0xff, 0xe3, 0x55, 0x8b, 0xf9, 0x5f, 0xec, 0x75, 0x2e, 0x75, 0xe9, 0xeb, 0x5c,
13690
-	0xfa, 0x57, 0xf2, 0x3a, 0xa7, 0xff, 0x24, 0x09, 0x6b, 0xea, 0xa2, 0x0e, 0x5f, 0x9f, 0xd0, 0x07,
13691
-	0x90, 0x97, 0x77, 0xf6, 0x8c, 0xbd, 0x8a, 0x07, 0x21, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37,
13692
-	0x2d, 0x1e, 0x4e, 0x29, 0x68, 0xec, 0xad, 0x19, 0xa4, 0xa8, 0xc5, 0x63, 0x81, 0x3a, 0xa4, 0x0f,
13693
-	0x6d, 0x87, 0x2a, 0x3f, 0x5b, 0x98, 0x01, 0xbc, 0xd0, 0xbc, 0x48, 0x58, 0xf7, 0x44, 0x40, 0xb6,
13694
-	0xb3, 0x84, 0x85, 0x76, 0xe9, 0xf7, 0x00, 0x66, 0xd2, 0x85, 0x31, 0x07, 0xbf, 0xd7, 0x55, 0x06,
13695
-	0x27, 0xbc, 0xd7, 0x9b, 0x75, 0xcc, 0x65, 0xbc, 0x6a, 0x60, 0x5b, 0x6a, 0xe7, 0x8a, 0xaa, 0x87,
13696
-	0xbc, 0x6a, 0x60, 0x5b, 0x51, 0xd6, 0x3c, 0xfd, 0x8a, 0xac, 0x79, 0x35, 0x17, 0x26, 0x11, 0xf4,
13697
-	0x36, 0x5c, 0xaf, 0x3a, 0xc4, 0x3c, 0x72, 0xec, 0x80, 0x51, 0x2b, 0xbe, 0x43, 0x3f, 0x85, 0xec,
13698
-	0xdc, 0xbd, 0xfb, 0x8a, 0xb4, 0x8d, 0x02, 0xeb, 0x3f, 0x49, 0x40, 0x71, 0x87, 0x12, 0x87, 0x0d,
13699
-	0x67, 0xb1, 0x2f, 0xa3, 0x01, 0x53, 0xe7, 0xa3, 0xf8, 0x46, 0xf7, 0x20, 0x17, 0xdd, 0x14, 0xaf,
13700
-	0x93, 0xdc, 0x8e, 0xd0, 0xe8, 0x33, 0x58, 0xe6, 0x9e, 0xed, 0x4d, 0x42, 0x42, 0xf7, 0x8a, 0xac,
13701
-	0xa9, 0x02, 0xf3, 0x43, 0xd6, 0xa7, 0xe2, 0x82, 0x10, 0xb3, 0x93, 0xc1, 0x61, 0x51, 0xff, 0x9f,
13702
-	0x04, 0x5c, 0xdb, 0x23, 0xd3, 0x03, 0xaa, 0x76, 0x1c, 0xb5, 0x30, 0x35, 0x3d, 0xdf, 0x42, 0x9d,
13703
-	0xf8, 0x4e, 0xbd, 0x24, 0xa1, 0xbf, 0x48, 0x79, 0xf1, 0x86, 0x0d, 0x99, 0x62, 0x32, 0xc6, 0x14,
13704
-	0xaf, 0x41, 0xc6, 0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28,
13705
-	0xcb, 0x2e, 0x72, 0xe4, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x2f, 0xa0, 0xd4, 0x6d, 0xd4, 0x70, 0xa3,
13706
-	0x57, 0x6d, 0xff, 0x7a, 0xbf, 0x6b, 0xec, 0x76, 0x8d, 0xad, 0x3b, 0xfd, 0x4e, 0x7b, 0xf7, 0xcb,
13707
-	0xbb, 0x1f, 0xdf, 0xf9, 0x54, 0x4b, 0x94, 0xca, 0xa7, 0x67, 0xe5, 0x9b, 0x2d, 0xa3, 0xb6, 0x2b,
13708
-	0xdd, 0xf2, 0xc0, 0x7b, 0xda, 0x25, 0x4e, 0x40, 0xb6, 0xee, 0x74, 0x3c, 0x67, 0xca, 0x31, 0xb7,
13709
-	0x7f, 0x91, 0x82, 0x7c, 0x94, 0x44, 0xe3, 0xde, 0xc5, 0x23, 0x18, 0xd5, 0x54, 0x24, 0x6f, 0xd1,
13710
-	0x13, 0xf4, 0xf6, 0x2c, 0x76, 0xf9, 0x42, 0x26, 0xf3, 0xa3, 0xea, 0x30, 0x6e, 0x79, 0x17, 0x72,
13711
-	0x46, 0xb7, 0xdb, 0x7c, 0xd8, 0x6a, 0xd4, 0xb5, 0xaf, 0x13, 0xa5, 0xef, 0x9d, 0x9e, 0x95, 0xaf,
13712
-	0x44, 0x20, 0x23, 0x08, 0xec, 0x81, 0x4b, 0x2d, 0x81, 0xaa, 0xd5, 0x1a, 0x9d, 0x5e, 0xa3, 0xae,
13713
-	0x3d, 0x4b, 0x5e, 0x44, 0x09, 0x2e, 0x2e, 0x1e, 0xe6, 0xf2, 0x1d, 0xdc, 0xe8, 0x18, 0x98, 0x37,
13714
-	0xf8, 0x75, 0x52, 0x86, 0x54, 0xb3, 0x16, 0x7d, 0x3a, 0x26, 0x3e, 0x6f, 0x73, 0x23, 0x7c, 0xa0,
13715
-	0x7e, 0x96, 0x92, 0x8f, 0x37, 0xb3, 0x8c, 0x20, 0x25, 0xd6, 0x94, 0xb7, 0x26, 0xb2, 0xb1, 0xc2,
13716
-	0x4c, 0xea, 0x42, 0x6b, 0x5d, 0x46, 0x7c, 0xc6, 0xad, 0xe8, 0xb0, 0x8c, 0xf7, 0x5b, 0x2d, 0x0e,
13717
-	0x7a, 0x96, 0xbe, 0x30, 0x3a, 0x3c, 0x71, 0x5d, 0x8e, 0xb9, 0x05, 0xb9, 0x30, 0x59, 0xab, 0x7d,
13718
-	0x9d, 0xbe, 0xd0, 0xa1, 0x5a, 0x98, 0x69, 0x16, 0x0d, 0xee, 0xec, 0xf7, 0xc4, 0xfb, 0xf9, 0xb3,
13719
-	0xcc, 0xc5, 0x06, 0x87, 0x13, 0x66, 0xf1, 0x60, 0xb1, 0x1c, 0x45, 0x6f, 0x5f, 0x67, 0x24, 0x1f,
13720
-	0x8e, 0x30, 0x2a, 0x74, 0x7b, 0x17, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0xd4, 0xfe, 0x2c, 0x7b, 0xc1,
13721
-	0x0e, 0xa6, 0x5f, 0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x1d, 0x43, 0x4c, 0xf9, 0x45, 0x54, 0xdb,
13722
-	0x1f, 0x0f, 0x89, 0x4b, 0xad, 0xd9, 0x0b, 0x56, 0x54, 0x75, 0xfb, 0x37, 0x21, 0x17, 0xde, 0xb0,
13723
-	0x68, 0x03, 0xb2, 0x4f, 0xda, 0xf8, 0x51, 0x03, 0x6b, 0x4b, 0x72, 0x0e, 0xc3, 0x9a, 0x27, 0x92,
13724
-	0xa2, 0x94, 0x61, 0x79, 0xcf, 0x68, 0x19, 0x0f, 0x1b, 0x38, 0x4c, 0xac, 0x84, 0x00, 0x75, 0x4d,
13725
-	0x94, 0x34, 0xd5, 0x40, 0x64, 0xb3, 0x7a, 0xf3, 0x9b, 0x6f, 0x37, 0x96, 0x7e, 0xf6, 0xed, 0xc6,
13726
-	0xd2, 0xcf, 0xbf, 0xdd, 0x48, 0x3c, 0x3b, 0xdf, 0x48, 0x7c, 0x73, 0xbe, 0x91, 0xf8, 0xe9, 0xf9,
13727
-	0x46, 0xe2, 0xdf, 0xce, 0x37, 0x12, 0x07, 0x59, 0x11, 0xc2, 0x7c, 0xfc, 0xbf, 0x01, 0x00, 0x00,
13728
-	0xff, 0xff, 0x6b, 0x1c, 0x13, 0xe7, 0x66, 0x26, 0x00, 0x00,
13483
+	0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0x7b,
13484
+	0xbc, 0xe3, 0xf1, 0xfa, 0xcf, 0xb1, 0x35, 0x1f, 0xf0, 0x8e, 0xff, 0x59, 0xbb, 0xf9, 0x21, 0x8b,
13485
+	0x6b, 0x89, 0x24, 0x8a, 0x94, 0x9d, 0x41, 0x80, 0x10, 0xa5, 0xee, 0x12, 0xd5, 0xa3, 0x66, 0x37,
13486
+	0xd3, 0x5d, 0x94, 0xcc, 0x04, 0x41, 0x8c, 0x1c, 0x92, 0x40, 0xa7, 0xdc, 0x03, 0x21, 0x08, 0x12,
13487
+	0xe4, 0x90, 0xc3, 0x5e, 0x72, 0x08, 0x90, 0xd3, 0x20, 0xa7, 0x39, 0x6e, 0x12, 0x20, 0x58, 0x24,
13488
+	0x88, 0x91, 0x51, 0xce, 0x01, 0xf6, 0xb2, 0xc8, 0x21, 0x09, 0x10, 0xd4, 0x47, 0x37, 0x9b, 0x32,
13489
+	0x2d, 0x7b, 0xb2, 0x7b, 0x21, 0xbb, 0x5e, 0xfd, 0xde, 0xab, 0xaf, 0x57, 0x55, 0xbf, 0xf7, 0x0a,
13490
+	0x0a, 0x6c, 0x32, 0xa2, 0x41, 0x65, 0xe4, 0x7b, 0xcc, 0x43, 0xc8, 0xf2, 0xcc, 0x43, 0xea, 0x57,
13491
+	0x82, 0x63, 0xe2, 0x0f, 0x0f, 0x6d, 0x56, 0x39, 0xba, 0x57, 0xba, 0xc6, 0xec, 0x21, 0x0d, 0x18,
13492
+	0x19, 0x8e, 0x3e, 0x8e, 0xbe, 0x24, 0xbc, 0xf4, 0x8e, 0x35, 0xf6, 0x09, 0xb3, 0x3d, 0xf7, 0xe3,
13493
+	0xf0, 0x43, 0x55, 0x5c, 0x19, 0x78, 0x03, 0x4f, 0x7c, 0x7e, 0xcc, 0xbf, 0xa4, 0x54, 0x5f, 0x87,
13494
+	0xc5, 0xa7, 0xd4, 0x0f, 0x6c, 0xcf, 0x45, 0x57, 0x20, 0x63, 0xbb, 0x16, 0x7d, 0xbe, 0x9a, 0x28,
13495
+	0x27, 0x6e, 0xa5, 0xb1, 0x2c, 0xe8, 0x7f, 0x96, 0x80, 0x82, 0xe1, 0xba, 0x1e, 0x13, 0xb6, 0x02,
13496
+	0x84, 0x20, 0xed, 0x92, 0x21, 0x15, 0xa0, 0x3c, 0x16, 0xdf, 0xa8, 0x06, 0x59, 0x87, 0xec, 0x51,
13497
+	0x27, 0x58, 0x4d, 0x96, 0x53, 0xb7, 0x0a, 0x1b, 0x3f, 0xa8, 0xbc, 0xda, 0xe7, 0x4a, 0xcc, 0x48,
13498
+	0x65, 0x5b, 0xa0, 0x1b, 0x2e, 0xf3, 0x27, 0x58, 0xa9, 0x96, 0x7e, 0x08, 0x85, 0x98, 0x18, 0x69,
13499
+	0x90, 0x3a, 0xa4, 0x13, 0xd5, 0x0c, 0xff, 0xe4, 0xfd, 0x3b, 0x22, 0xce, 0x98, 0xae, 0x26, 0x85,
13500
+	0x4c, 0x16, 0xbe, 0x48, 0xde, 0x4f, 0xe8, 0x5f, 0x42, 0x1e, 0xd3, 0xc0, 0x1b, 0xfb, 0x26, 0x0d,
13501
+	0xd0, 0x47, 0x90, 0x77, 0x89, 0xeb, 0xf5, 0xcd, 0xd1, 0x38, 0x10, 0xea, 0xa9, 0x6a, 0xf1, 0xec,
13502
+	0xe5, 0x7a, 0xae, 0x45, 0x5c, 0xaf, 0xd6, 0xd9, 0x0d, 0x70, 0x8e, 0x57, 0xd7, 0x46, 0xe3, 0x00,
13503
+	0xbd, 0x0f, 0xc5, 0x21, 0x1d, 0x7a, 0xfe, 0xa4, 0xbf, 0x37, 0x61, 0x34, 0x10, 0x86, 0x53, 0xb8,
13504
+	0x20, 0x65, 0x55, 0x2e, 0xd2, 0xff, 0x38, 0x01, 0x57, 0x42, 0xdb, 0x98, 0xfe, 0xd6, 0xd8, 0xf6,
13505
+	0xe9, 0x90, 0xba, 0x2c, 0x40, 0x9f, 0x41, 0xd6, 0xb1, 0x87, 0x36, 0x93, 0x6d, 0x14, 0x36, 0xde,
13506
+	0x9b, 0x37, 0xe6, 0xa8, 0x57, 0x58, 0x81, 0x91, 0x01, 0x45, 0x9f, 0x06, 0xd4, 0x3f, 0x92, 0x33,
13507
+	0x21, 0x9a, 0x7c, 0xa3, 0xf2, 0x8c, 0x8a, 0xbe, 0x09, 0xb9, 0x8e, 0x43, 0xd8, 0xbe, 0xe7, 0x0f,
13508
+	0x91, 0x0e, 0x45, 0xe2, 0x9b, 0x07, 0x36, 0xa3, 0x26, 0x1b, 0xfb, 0xe1, 0xaa, 0xcc, 0xc8, 0xd0,
13509
+	0x55, 0x48, 0x7a, 0xb2, 0xa1, 0x7c, 0x35, 0x7b, 0xf6, 0x72, 0x3d, 0xd9, 0xee, 0xe2, 0xa4, 0x17,
13510
+	0xe8, 0x0f, 0xe0, 0x52, 0xc7, 0x19, 0x0f, 0x6c, 0xb7, 0x4e, 0x03, 0xd3, 0xb7, 0x47, 0xdc, 0x3a,
13511
+	0x5f, 0x5e, 0xee, 0x7c, 0xe1, 0xf2, 0xf2, 0xef, 0x68, 0xc9, 0x93, 0xd3, 0x25, 0xd7, 0xff, 0x30,
13512
+	0x09, 0x97, 0x1a, 0xee, 0xc0, 0x76, 0x69, 0x5c, 0xfb, 0x26, 0x2c, 0x53, 0x21, 0xec, 0x1f, 0x49,
13513
+	0xa7, 0x52, 0x76, 0x96, 0xa4, 0x34, 0xf4, 0xb4, 0xe6, 0x39, 0x7f, 0xb9, 0x37, 0x6f, 0xf8, 0xaf,
13514
+	0x58, 0x9f, 0xe7, 0x35, 0xa8, 0x01, 0x8b, 0x23, 0x31, 0x88, 0x60, 0x35, 0x25, 0x6c, 0xdd, 0x9c,
13515
+	0x67, 0xeb, 0x95, 0x71, 0x56, 0xd3, 0xdf, 0xbc, 0x5c, 0x5f, 0xc0, 0xa1, 0xee, 0x2f, 0xe3, 0x7c,
13516
+	0xff, 0x9e, 0x80, 0x95, 0x96, 0x67, 0xcd, 0xcc, 0x43, 0x09, 0x72, 0x07, 0x5e, 0xc0, 0x62, 0x1b,
13517
+	0x25, 0x2a, 0xa3, 0xfb, 0x90, 0x1b, 0xa9, 0xe5, 0x53, 0xab, 0x7f, 0x7d, 0x7e, 0x97, 0x25, 0x06,
13518
+	0x47, 0x68, 0xf4, 0x00, 0xf2, 0x7e, 0xe8, 0x13, 0xab, 0xa9, 0xb7, 0x71, 0x9c, 0x29, 0x1e, 0xfd,
13519
+	0x1a, 0x64, 0xe5, 0x22, 0xac, 0xa6, 0x85, 0xe6, 0xcd, 0xb7, 0x9a, 0x73, 0xac, 0x94, 0xf4, 0x9f,
13520
+	0x25, 0x40, 0xc3, 0x64, 0x9f, 0xed, 0xd0, 0xe1, 0x1e, 0xf5, 0xbb, 0x8c, 0xb0, 0x71, 0x80, 0xae,
13521
+	0x42, 0xd6, 0xa1, 0xc4, 0xa2, 0xbe, 0x18, 0x64, 0x0e, 0xab, 0x12, 0xda, 0xe5, 0x4e, 0x4e, 0xcc,
13522
+	0x03, 0xb2, 0x67, 0x3b, 0x36, 0x9b, 0x88, 0x61, 0x2e, 0xcf, 0x5f, 0xe5, 0xf3, 0x36, 0x2b, 0x38,
13523
+	0xa6, 0x88, 0x67, 0xcc, 0xa0, 0x55, 0x58, 0x1c, 0xd2, 0x20, 0x20, 0x03, 0x2a, 0x46, 0x9f, 0xc7,
13524
+	0x61, 0x51, 0x7f, 0x00, 0xc5, 0xb8, 0x1e, 0x2a, 0xc0, 0xe2, 0x6e, 0xeb, 0x49, 0xab, 0xfd, 0xac,
13525
+	0xa5, 0x2d, 0xa0, 0x15, 0x28, 0xec, 0xb6, 0x70, 0xc3, 0xa8, 0x6d, 0x19, 0xd5, 0xed, 0x86, 0x96,
13526
+	0x40, 0x4b, 0x90, 0x9f, 0x16, 0x93, 0xfa, 0x5f, 0x27, 0x00, 0xf8, 0x02, 0xaa, 0x41, 0x7d, 0x01,
13527
+	0x99, 0x80, 0x11, 0x26, 0x17, 0x6e, 0x79, 0xe3, 0x83, 0x79, 0xbd, 0x9e, 0xc2, 0x2b, 0xfc, 0x8f,
13528
+	0x62, 0xa9, 0x12, 0xef, 0x61, 0x72, 0xa6, 0x87, 0x7c, 0x0f, 0x11, 0xcb, 0xf2, 0x55, 0xc7, 0xc5,
13529
+	0xb7, 0xfe, 0x00, 0x32, 0x42, 0x7b, 0xb6, 0xbb, 0x39, 0x48, 0xd7, 0xf9, 0x57, 0x02, 0xe5, 0x21,
13530
+	0x83, 0x1b, 0x46, 0xfd, 0x4b, 0x2d, 0x89, 0x34, 0x28, 0xd6, 0x9b, 0xdd, 0x5a, 0xbb, 0xd5, 0x6a,
13531
+	0xd4, 0x7a, 0x8d, 0xba, 0x96, 0xd2, 0x6f, 0x42, 0xa6, 0x39, 0xe4, 0x96, 0xaf, 0x73, 0xaf, 0xd8,
13532
+	0xa7, 0x3e, 0x75, 0xcd, 0xd0, 0xd9, 0xa6, 0x02, 0xfd, 0xa7, 0x79, 0xc8, 0xec, 0x78, 0x63, 0x97,
13533
+	0xa1, 0x8d, 0xd8, 0xce, 0x5e, 0xde, 0x58, 0x9b, 0x37, 0x2c, 0x01, 0xac, 0xf4, 0x26, 0x23, 0xaa,
13534
+	0x76, 0xfe, 0x55, 0xc8, 0x4a, 0xff, 0x51, 0xc3, 0x51, 0x25, 0x2e, 0x67, 0xc4, 0x1f, 0x50, 0xa6,
13535
+	0xc6, 0xa3, 0x4a, 0xe8, 0x16, 0xe4, 0x7c, 0x4a, 0x2c, 0xcf, 0x75, 0x26, 0xc2, 0xcd, 0x72, 0xf2,
13536
+	0xe8, 0xc5, 0x94, 0x58, 0x6d, 0xd7, 0x99, 0xe0, 0xa8, 0x16, 0x6d, 0x41, 0x71, 0xcf, 0x76, 0xad,
13537
+	0xbe, 0x37, 0x92, 0xe7, 0x60, 0xe6, 0xf5, 0x4e, 0x29, 0x7b, 0x55, 0xb5, 0x5d, 0xab, 0x2d, 0xc1,
13538
+	0xb8, 0xb0, 0x37, 0x2d, 0xa0, 0x16, 0x2c, 0x1f, 0x79, 0xce, 0x78, 0x48, 0x23, 0x5b, 0x59, 0x61,
13539
+	0xeb, 0xc3, 0xd7, 0xdb, 0x7a, 0x2a, 0xf0, 0xa1, 0xb5, 0xa5, 0xa3, 0x78, 0x11, 0x3d, 0x81, 0x25,
13540
+	0x36, 0x1c, 0xed, 0x07, 0x91, 0xb9, 0x45, 0x61, 0xee, 0xfb, 0x17, 0x4c, 0x18, 0x87, 0x87, 0xd6,
13541
+	0x8a, 0x2c, 0x56, 0x2a, 0xfd, 0x7e, 0x0a, 0x0a, 0xb1, 0x9e, 0xa3, 0x2e, 0x14, 0x46, 0xbe, 0x37,
13542
+	0x22, 0x03, 0x71, 0x96, 0xab, 0xb5, 0xb8, 0xf7, 0x56, 0xa3, 0xae, 0x74, 0xa6, 0x8a, 0x38, 0x6e,
13543
+	0x45, 0x3f, 0x4d, 0x42, 0x21, 0x56, 0x89, 0x6e, 0x43, 0x0e, 0x77, 0x70, 0xf3, 0xa9, 0xd1, 0x6b,
13544
+	0x68, 0x0b, 0xa5, 0xeb, 0x27, 0xa7, 0xe5, 0x55, 0x61, 0x2d, 0x6e, 0xa0, 0xe3, 0xdb, 0x47, 0xdc,
13545
+	0xf5, 0x6e, 0xc1, 0x62, 0x08, 0x4d, 0x94, 0xde, 0x3d, 0x39, 0x2d, 0xbf, 0x73, 0x1e, 0x1a, 0x43,
13546
+	0xe2, 0xee, 0x96, 0x81, 0x1b, 0x75, 0x2d, 0x39, 0x1f, 0x89, 0xbb, 0x07, 0xc4, 0xa7, 0x16, 0xfa,
13547
+	0x3e, 0x64, 0x15, 0x30, 0x55, 0x2a, 0x9d, 0x9c, 0x96, 0xaf, 0x9e, 0x07, 0x4e, 0x71, 0xb8, 0xbb,
13548
+	0x6d, 0x3c, 0x6d, 0x68, 0xe9, 0xf9, 0x38, 0xdc, 0x75, 0xc8, 0x11, 0x45, 0x1f, 0x40, 0x46, 0xc2,
13549
+	0x32, 0xa5, 0x6b, 0x27, 0xa7, 0xe5, 0xef, 0xbd, 0x62, 0x8e, 0xa3, 0x4a, 0xab, 0x7f, 0xf4, 0xe7,
13550
+	0x6b, 0x0b, 0x7f, 0xfb, 0x17, 0x6b, 0xda, 0xf9, 0xea, 0xd2, 0x7f, 0x27, 0x60, 0x69, 0x66, 0xc9,
13551
+	0x91, 0x0e, 0x59, 0xd7, 0x33, 0xbd, 0x91, 0x3c, 0xe2, 0x73, 0x55, 0x38, 0x7b, 0xb9, 0x9e, 0x6d,
13552
+	0x79, 0x35, 0x6f, 0x34, 0xc1, 0xaa, 0x06, 0x3d, 0x39, 0x77, 0x49, 0x7d, 0xf2, 0x96, 0xfe, 0x34,
13553
+	0xf7, 0x9a, 0x7a, 0x08, 0x4b, 0x96, 0x6f, 0x1f, 0x51, 0xbf, 0x6f, 0x7a, 0xee, 0xbe, 0x3d, 0x50,
13554
+	0xc7, 0x77, 0x69, 0x9e, 0xcd, 0xba, 0x00, 0xe2, 0xa2, 0x54, 0xa8, 0x09, 0xfc, 0x2f, 0x71, 0x41,
13555
+	0x95, 0x9e, 0x42, 0x31, 0xee, 0xa1, 0xe8, 0x3d, 0x80, 0xc0, 0xfe, 0x6d, 0xaa, 0x38, 0x8f, 0x60,
13556
+	0x48, 0x38, 0xcf, 0x25, 0x82, 0xf1, 0xa0, 0x0f, 0x21, 0x3d, 0xf4, 0x2c, 0x69, 0x67, 0xa9, 0x7a,
13557
+	0x99, 0xdf, 0x93, 0xff, 0xfc, 0x72, 0xbd, 0xe0, 0x05, 0x95, 0x4d, 0xdb, 0xa1, 0x3b, 0x9e, 0x45,
13558
+	0xb1, 0x00, 0xe8, 0x47, 0x90, 0xe6, 0x47, 0x05, 0x7a, 0x17, 0xd2, 0xd5, 0x66, 0xab, 0xae, 0x2d,
13559
+	0x94, 0x2e, 0x9d, 0x9c, 0x96, 0x97, 0xc4, 0x94, 0xf0, 0x0a, 0xee, 0xbb, 0x68, 0x1d, 0xb2, 0x4f,
13560
+	0xdb, 0xdb, 0xbb, 0x3b, 0xdc, 0xbd, 0x2e, 0x9f, 0x9c, 0x96, 0x57, 0xa2, 0x6a, 0x39, 0x69, 0xe8,
13561
+	0x3d, 0xc8, 0xf4, 0x76, 0x3a, 0x9b, 0x5d, 0x2d, 0x59, 0x42, 0x27, 0xa7, 0xe5, 0xe5, 0xa8, 0x5e,
13562
+	0xf4, 0xb9, 0x74, 0x49, 0xad, 0x6a, 0x3e, 0x92, 0xeb, 0xff, 0x95, 0x84, 0x25, 0xcc, 0x39, 0xaf,
13563
+	0xcf, 0x3a, 0x9e, 0x63, 0x9b, 0x13, 0xd4, 0x81, 0xbc, 0xe9, 0xb9, 0x96, 0x1d, 0xdb, 0x53, 0x1b,
13564
+	0xaf, 0xb9, 0x18, 0xa7, 0x5a, 0x61, 0xa9, 0x16, 0x6a, 0xe2, 0xa9, 0x11, 0xb4, 0x01, 0x19, 0x8b,
13565
+	0x3a, 0x64, 0x72, 0xd1, 0x0d, 0x5d, 0x57, 0xfc, 0x1a, 0x4b, 0xa8, 0x60, 0x93, 0xe4, 0x79, 0x9f,
13566
+	0x30, 0x46, 0x87, 0x23, 0x26, 0x6f, 0xe8, 0x34, 0x2e, 0x0c, 0xc9, 0x73, 0x43, 0x89, 0xd0, 0xa7,
13567
+	0x90, 0x3d, 0xb6, 0x5d, 0xcb, 0x3b, 0x56, 0x97, 0xf0, 0xc5, 0x76, 0x15, 0x56, 0x3f, 0xe1, 0x77,
13568
+	0xef, 0xb9, 0xce, 0xf2, 0x59, 0x6f, 0xb5, 0x5b, 0x8d, 0x70, 0xd6, 0x55, 0x7d, 0xdb, 0x6d, 0x79,
13569
+	0x2e, 0xdf, 0x31, 0xd0, 0x6e, 0xf5, 0x37, 0x8d, 0xe6, 0xf6, 0x2e, 0xe6, 0x33, 0x7f, 0xe5, 0xe4,
13570
+	0xb4, 0xac, 0x45, 0x90, 0x4d, 0x62, 0x3b, 0x9c, 0x18, 0x5e, 0x83, 0x94, 0xd1, 0xfa, 0x52, 0x4b,
13571
+	0x96, 0xb4, 0x93, 0xd3, 0x72, 0x31, 0xaa, 0x36, 0xdc, 0xc9, 0x74, 0x33, 0x9d, 0x6f, 0x57, 0xff,
13572
+	0xd7, 0x24, 0x14, 0x77, 0x47, 0x16, 0x61, 0x54, 0x7a, 0x26, 0x2a, 0x43, 0x61, 0x44, 0x7c, 0xe2,
13573
+	0x38, 0xd4, 0xb1, 0x83, 0xa1, 0x0a, 0x1e, 0xe2, 0x22, 0x74, 0xff, 0x3b, 0x4c, 0xa6, 0x22, 0x66,
13574
+	0x6a, 0x4a, 0x77, 0x61, 0x79, 0x5f, 0x76, 0xb6, 0x4f, 0x4c, 0xb1, 0xba, 0x29, 0xb1, 0xba, 0x95,
13575
+	0x79, 0x26, 0xe2, 0xbd, 0xaa, 0xa8, 0x31, 0x1a, 0x42, 0x0b, 0x2f, 0xed, 0xc7, 0x8b, 0xe8, 0x73,
13576
+	0x58, 0x1c, 0x7a, 0xae, 0xcd, 0x3c, 0xff, 0xad, 0xd6, 0x21, 0x04, 0xa3, 0xdb, 0x70, 0x89, 0xaf,
13577
+	0x70, 0xd8, 0x25, 0x51, 0x2d, 0x6e, 0xae, 0x24, 0x5e, 0x19, 0x92, 0xe7, 0xaa, 0x4d, 0xcc, 0xc5,
13578
+	0xfa, 0xe7, 0xb0, 0x34, 0xd3, 0x07, 0x7e, 0x9b, 0x77, 0x8c, 0xdd, 0x6e, 0x43, 0x5b, 0x40, 0x45,
13579
+	0xc8, 0xd5, 0xda, 0xad, 0x5e, 0xb3, 0xb5, 0xcb, 0xe9, 0x48, 0x11, 0x72, 0xb8, 0xbd, 0xbd, 0x5d,
13580
+	0x35, 0x6a, 0x4f, 0xb4, 0xa4, 0xfe, 0x8b, 0x68, 0x7e, 0x15, 0x1f, 0xa9, 0xce, 0xf2, 0x91, 0x3b,
13581
+	0xaf, 0x1f, 0xba, 0x62, 0x24, 0xd3, 0x42, 0xc4, 0x4b, 0xfe, 0x3f, 0x80, 0x58, 0x46, 0x6a, 0xf5,
13582
+	0x09, 0xbb, 0x28, 0xe6, 0xe8, 0x85, 0xd1, 0x24, 0xce, 0x2b, 0x05, 0x83, 0xa1, 0x47, 0x50, 0x34,
13583
+	0xbd, 0xe1, 0xc8, 0xa1, 0x4a, 0x3f, 0xf5, 0x36, 0xfa, 0x85, 0x48, 0xc5, 0x60, 0x71, 0x5e, 0x94,
13584
+	0x9e, 0x65, 0x6e, 0x7f, 0x90, 0x80, 0x42, 0xac, 0xc3, 0xb3, 0x54, 0xa8, 0x08, 0xb9, 0xdd, 0x4e,
13585
+	0xdd, 0xe8, 0x35, 0x5b, 0x8f, 0xb5, 0x04, 0x02, 0xc8, 0x8a, 0x09, 0xac, 0x6b, 0x49, 0x4e, 0xe1,
13586
+	0x6a, 0xed, 0x9d, 0xce, 0x76, 0x43, 0x90, 0x21, 0x74, 0x05, 0xb4, 0x70, 0x0a, 0xfb, 0xdd, 0x9e,
13587
+	0x81, 0xb9, 0x34, 0x8d, 0x2e, 0xc3, 0x4a, 0x24, 0x55, 0x9a, 0x19, 0x74, 0x15, 0x50, 0x24, 0x9c,
13588
+	0x9a, 0xc8, 0xea, 0xbf, 0x0b, 0x2b, 0x35, 0xcf, 0x65, 0xc4, 0x76, 0x23, 0x7a, 0xbb, 0xc1, 0xc7,
13589
+	0xad, 0x44, 0x7d, 0xdb, 0x92, 0xa7, 0x6d, 0x75, 0xe5, 0xec, 0xe5, 0x7a, 0x21, 0x82, 0x36, 0xeb,
13590
+	0x7c, 0xa4, 0x61, 0xc1, 0xe2, 0x7b, 0x6a, 0x64, 0x5b, 0x62, 0x8a, 0x33, 0xd5, 0xc5, 0xb3, 0x97,
13591
+	0xeb, 0xa9, 0x4e, 0xb3, 0x8e, 0xb9, 0x0c, 0xbd, 0x0b, 0x79, 0xfa, 0xdc, 0x66, 0x7d, 0x93, 0x9f,
13592
+	0xae, 0x7c, 0x0e, 0x33, 0x38, 0xc7, 0x05, 0x35, 0x7e, 0x98, 0x56, 0x01, 0x3a, 0x9e, 0xcf, 0x54,
13593
+	0xcb, 0x9f, 0x42, 0x66, 0xe4, 0xf9, 0x22, 0xb6, 0xe4, 0x57, 0xcf, 0x5c, 0xb2, 0xc6, 0xe1, 0xd2,
13594
+	0xd9, 0xb1, 0x04, 0xeb, 0x7f, 0x97, 0x04, 0xe8, 0x91, 0xe0, 0x50, 0x19, 0x79, 0x00, 0xf9, 0x28,
13595
+	0x39, 0x70, 0x51, 0x90, 0x1a, 0x5b, 0xf3, 0x08, 0x8f, 0x3e, 0x09, 0xbd, 0x4e, 0x72, 0xf7, 0xf9,
13596
+	0x8a, 0xaa, 0xad, 0x79, 0xf4, 0x77, 0x96, 0xa0, 0xf3, 0xfb, 0x8a, 0xfa, 0xbe, 0x5a, 0x7c, 0xfe,
13597
+	0x89, 0x6a, 0xe2, 0xcc, 0x96, 0xf3, 0xa6, 0xd8, 0xdf, 0x8d, 0x79, 0x8d, 0x9c, 0x5b, 0x94, 0xad,
13598
+	0x05, 0x3c, 0xd5, 0x43, 0x0f, 0xa1, 0xc0, 0x87, 0xde, 0x0f, 0x44, 0x9d, 0x22, 0x7e, 0xaf, 0x9d,
13599
+	0x2d, 0x69, 0x01, 0xc3, 0x28, 0xfa, 0xae, 0x6a, 0xb0, 0xec, 0x8f, 0x5d, 0x3e, 0x6c, 0x65, 0x43,
13600
+	0xb7, 0xe1, 0x9d, 0x16, 0x65, 0xc7, 0x9e, 0x7f, 0x68, 0x30, 0x46, 0xcc, 0x03, 0x1e, 0xed, 0xab,
13601
+	0x93, 0x6e, 0xca, 0x7a, 0x13, 0x33, 0xac, 0x77, 0x15, 0x16, 0x89, 0x63, 0x93, 0x80, 0x4a, 0xaa,
13602
+	0x90, 0xc7, 0x61, 0x91, 0x73, 0x73, 0xce, 0xf4, 0x69, 0x10, 0x50, 0x19, 0x9f, 0xe6, 0xf1, 0x54,
13603
+	0xa0, 0xff, 0x63, 0x12, 0xa0, 0xd9, 0x31, 0x76, 0x94, 0xf9, 0x3a, 0x64, 0xf7, 0xc9, 0xd0, 0x76,
13604
+	0x26, 0x17, 0xed, 0xf4, 0x29, 0xbe, 0x62, 0x48, 0x43, 0x9b, 0x42, 0x07, 0x2b, 0x5d, 0x41, 0xd9,
13605
+	0xc7, 0x7b, 0x2e, 0x65, 0x11, 0x65, 0x17, 0x25, 0xce, 0x0f, 0x7c, 0xe2, 0x46, 0x2b, 0x23, 0x0b,
13606
+	0xbc, 0xeb, 0x03, 0xc2, 0xe8, 0x31, 0x99, 0x84, 0x1b, 0x53, 0x15, 0xd1, 0x16, 0xa7, 0xf2, 0x01,
13607
+	0xf5, 0x8f, 0xa8, 0xb5, 0x9a, 0x11, 0x5e, 0xf8, 0xa6, 0xfe, 0x60, 0x05, 0x97, 0xcc, 0x27, 0xd2,
13608
+	0x2e, 0x3d, 0x10, 0xd7, 0xf5, 0xb4, 0xea, 0x3b, 0x45, 0xd7, 0x77, 0x61, 0x69, 0x66, 0x9c, 0xaf,
13609
+	0xc4, 0x4a, 0xcd, 0xce, 0xd3, 0x4f, 0xb5, 0xb4, 0xfa, 0xfa, 0x5c, 0xcb, 0xea, 0x7f, 0x95, 0x92,
13610
+	0x5b, 0x49, 0xcd, 0xea, 0xfc, 0x7c, 0x55, 0x4e, 0x64, 0xbf, 0x4c, 0xcf, 0x51, 0xfe, 0xfd, 0xe1,
13611
+	0xc5, 0x3b, 0x8c, 0x73, 0x6f, 0x01, 0xc7, 0x91, 0x22, 0x5a, 0x87, 0x82, 0x5c, 0xff, 0x3e, 0xf7,
13612
+	0x27, 0x31, 0xad, 0x4b, 0x18, 0xa4, 0x88, 0x6b, 0xa2, 0x9b, 0xb0, 0x3c, 0x1a, 0xef, 0x39, 0x76,
13613
+	0x70, 0x40, 0x2d, 0x89, 0x49, 0x0b, 0xcc, 0x52, 0x24, 0x15, 0xb0, 0x1d, 0x28, 0x2a, 0x41, 0x5f,
13614
+	0xf0, 0xae, 0x8c, 0xe8, 0xd0, 0xed, 0x37, 0x75, 0x48, 0xaa, 0x08, 0x3a, 0x56, 0x18, 0x4d, 0x0b,
13615
+	0x7a, 0x1d, 0x72, 0x61, 0x67, 0xd1, 0x2a, 0xa4, 0x7a, 0xb5, 0x8e, 0xb6, 0x50, 0x5a, 0x39, 0x39,
13616
+	0x2d, 0x17, 0x42, 0x71, 0xaf, 0xd6, 0xe1, 0x35, 0xbb, 0xf5, 0x8e, 0x96, 0x98, 0xad, 0xd9, 0xad,
13617
+	0x77, 0x4a, 0x69, 0x7e, 0xf3, 0xeb, 0xfb, 0x50, 0x88, 0xb5, 0x80, 0x6e, 0xc0, 0x62, 0xb3, 0xf5,
13618
+	0x18, 0x37, 0xba, 0x5d, 0x6d, 0xa1, 0x74, 0xf5, 0xe4, 0xb4, 0x8c, 0x62, 0xb5, 0x4d, 0x77, 0xc0,
13619
+	0xd7, 0x07, 0xbd, 0x07, 0xe9, 0xad, 0x76, 0xb7, 0x17, 0x12, 0xbd, 0x18, 0x62, 0xcb, 0x0b, 0x58,
13620
+	0xe9, 0xb2, 0xa2, 0x14, 0x71, 0xc3, 0xfa, 0x9f, 0x24, 0x20, 0x2b, 0xf9, 0xee, 0xdc, 0x85, 0x32,
13621
+	0x60, 0x31, 0x8c, 0xc2, 0x24, 0x09, 0xff, 0xf0, 0xf5, 0x84, 0xb9, 0xa2, 0xf8, 0xad, 0x74, 0xbf,
13622
+	0x50, 0xaf, 0xf4, 0x05, 0x14, 0xe3, 0x15, 0xdf, 0xc9, 0xf9, 0x7e, 0x07, 0x0a, 0xdc, 0xbf, 0x43,
13623
+	0xe2, 0xbc, 0x01, 0x59, 0xc9, 0xc9, 0xd5, 0x69, 0x7a, 0x11, 0x7b, 0x57, 0x48, 0x74, 0x1f, 0x16,
13624
+	0x25, 0xe3, 0x0f, 0xf3, 0x53, 0x6b, 0x17, 0xef, 0x22, 0x1c, 0xc2, 0xf5, 0x87, 0x90, 0xee, 0x50,
13625
+	0xea, 0xf3, 0xb9, 0x77, 0x3d, 0x8b, 0x4e, 0x2f, 0x20, 0x15, 0xac, 0x58, 0xb4, 0x59, 0xe7, 0xc1,
13626
+	0x8a, 0x45, 0x9b, 0x56, 0x94, 0x5e, 0x48, 0xc6, 0xd2, 0x0b, 0x3d, 0x28, 0x3e, 0xa3, 0xf6, 0xe0,
13627
+	0x80, 0x51, 0x4b, 0x18, 0xba, 0x03, 0xe9, 0x11, 0x8d, 0x3a, 0xbf, 0x3a, 0xd7, 0xc1, 0x28, 0xf5,
13628
+	0xb1, 0x40, 0xf1, 0x73, 0xe4, 0x58, 0x68, 0xab, 0xac, 0xa8, 0x2a, 0xe9, 0xff, 0x90, 0x84, 0xe5,
13629
+	0x66, 0x10, 0x8c, 0x89, 0x6b, 0x86, 0x0c, 0xe5, 0x47, 0xb3, 0x0c, 0xe5, 0xd6, 0xdc, 0x11, 0xce,
13630
+	0xa8, 0xcc, 0x66, 0x4d, 0xd4, 0xe5, 0x90, 0x8c, 0x2e, 0x07, 0xfd, 0x3f, 0x12, 0x61, 0x6a, 0xe4,
13631
+	0x66, 0x6c, 0xbb, 0x97, 0x56, 0x4f, 0x4e, 0xcb, 0x57, 0xe2, 0x96, 0xe8, 0xae, 0x7b, 0xe8, 0x7a,
13632
+	0xc7, 0x2e, 0x7a, 0x1f, 0x32, 0xb8, 0xd1, 0x6a, 0x3c, 0xd3, 0x12, 0xd2, 0x3d, 0x67, 0x40, 0x98,
13633
+	0xba, 0xf4, 0x98, 0x5b, 0xea, 0x34, 0x5a, 0x75, 0xce, 0x25, 0x92, 0x73, 0x2c, 0x75, 0xa8, 0x6b,
13634
+	0xd9, 0xee, 0x00, 0xdd, 0x80, 0x6c, 0xb3, 0xdb, 0xdd, 0x15, 0xc1, 0xeb, 0x3b, 0x27, 0xa7, 0xe5,
13635
+	0xcb, 0x33, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x1c, 0x10, 0xe7, 0x7d,
13636
+	0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0xe6, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb, 0x97,
13637
+	0x24, 0x68, 0x86, 0x69, 0xd2, 0x11, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xc4, 0xbf, 0x6c,
13638
+	0x1a, 0xf2, 0x80, 0xfb, 0x73, 0xf3, 0xea, 0xe7, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x0d, 0xed,
13639
+	0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0x3c, 0x07, 0x81, 0xee,
13640
+	0x42, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0xaf, 0xbf, 0x2e, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2, 0x35,
13641
+	0x00, 0x32, 0x66, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x83, 0x6c, 0x40,
13642
+	0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xe1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9, 0x52,
13643
+	0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde, 0x44,
13644
+	0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x67, 0xd4, 0x77, 0x89,
13645
+	0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0x34, 0x37, 0x17, 0x1a, 0x69, 0x54, 0x6a,
13646
+	0xc6, 0x9c, 0xf3, 0xff, 0x1a, 0xa4, 0xc6, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe2, 0x6d,
13647
+	0xcc, 0x65, 0xa8, 0x31, 0x3d, 0xb6, 0x52, 0xaf, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f, 0xba,
13648
+	0xee, 0x00, 0x4c, 0x7b, 0x8d, 0xd6, 0x20, 0x53, 0xdb, 0xec, 0x76, 0xb7, 0xb5, 0x05, 0x79, 0x36,
13649
+	0x4f, 0xab, 0x84, 0x58, 0xff, 0xcb, 0x04, 0xe4, 0x6a, 0x86, 0xba, 0x31, 0x37, 0x41, 0x13, 0x07,
13650
+	0x8e, 0x49, 0x7d, 0xd6, 0xa7, 0xcf, 0x47, 0xb6, 0x3f, 0x51, 0x67, 0xc6, 0xc5, 0x61, 0xd2, 0x32,
13651
+	0xd7, 0xaa, 0x51, 0x9f, 0x35, 0x84, 0x0e, 0xc2, 0x50, 0xa4, 0x6a, 0x88, 0x7d, 0x93, 0x84, 0x27,
13652
+	0xf8, 0xda, 0xc5, 0x53, 0x21, 0xe9, 0xf5, 0xb4, 0x1c, 0xe0, 0x42, 0x68, 0xa4, 0x46, 0x02, 0xfd,
13653
+	0x29, 0x5c, 0x6e, 0xfb, 0xe6, 0x01, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x21, 0x5c, 0x67, 0x24,
13654
+	0x38, 0xec, 0x1f, 0xd8, 0x01, 0xf3, 0xfc, 0x49, 0xdf, 0xa7, 0x8c, 0xba, 0xbc, 0xbe, 0x2f, 0x9e,
13655
+	0x5d, 0x54, 0x92, 0xe3, 0x1a, 0xc7, 0x6c, 0x49, 0x08, 0x0e, 0x11, 0xdb, 0x1c, 0xa0, 0x37, 0xa1,
13656
+	0xc8, 0xd9, 0x6c, 0x9d, 0xee, 0x93, 0xb1, 0xc3, 0x02, 0xf4, 0x43, 0x00, 0xc7, 0x1b, 0xf4, 0xdf,
13657
+	0xfa, 0xb8, 0xcf, 0x3b, 0xde, 0x40, 0x7e, 0xea, 0xbf, 0x01, 0x5a, 0xdd, 0x0e, 0x46, 0x84, 0x99,
13658
+	0x07, 0x61, 0xf6, 0x06, 0x3d, 0x06, 0xed, 0x80, 0x12, 0x9f, 0xed, 0x51, 0xc2, 0xfa, 0x23, 0xea,
13659
+	0xdb, 0x9e, 0xf5, 0x56, 0x53, 0xba, 0x12, 0x69, 0x75, 0x84, 0x92, 0xfe, 0x9f, 0x09, 0x00, 0x4c,
13660
+	0xf6, 0x43, 0x72, 0xf3, 0x03, 0xb8, 0x14, 0xb8, 0x64, 0x14, 0x1c, 0x78, 0xac, 0x6f, 0xbb, 0x8c,
13661
+	0xfa, 0x47, 0xc4, 0x51, 0x11, 0xb8, 0x16, 0x56, 0x34, 0x95, 0x1c, 0xdd, 0x01, 0x74, 0x48, 0xe9,
13662
+	0xa8, 0xef, 0x39, 0x56, 0x3f, 0xac, 0x94, 0xef, 0x42, 0x69, 0xac, 0xf1, 0x9a, 0xb6, 0x63, 0x75,
13663
+	0x43, 0x39, 0xaa, 0xc2, 0x1a, 0x9f, 0x01, 0xea, 0x32, 0xdf, 0xa6, 0x41, 0x7f, 0xdf, 0xf3, 0xfb,
13664
+	0x81, 0xe3, 0x1d, 0xf7, 0xf7, 0x3d, 0xc7, 0xf1, 0x8e, 0xa9, 0x1f, 0xe6, 0x37, 0x4a, 0x8e, 0x37,
13665
+	0x68, 0x48, 0xd0, 0xa6, 0xe7, 0x77, 0x1d, 0xef, 0x78, 0x33, 0x44, 0x70, 0x06, 0x34, 0x1d, 0x36,
13666
+	0xb3, 0xcd, 0xc3, 0x90, 0x01, 0x45, 0xd2, 0x9e, 0x6d, 0x1e, 0xa2, 0x1b, 0xb0, 0x44, 0x1d, 0x2a,
13667
+	0xa2, 0x64, 0x89, 0xca, 0x08, 0x54, 0x31, 0x14, 0x72, 0x90, 0xfe, 0x08, 0xb4, 0x86, 0x6b, 0xfa,
13668
+	0x93, 0x51, 0x6c, 0xd9, 0xef, 0x00, 0xe2, 0xe7, 0x4d, 0xdf, 0xf1, 0xcc, 0xc3, 0xfe, 0x90, 0xb8,
13669
+	0x64, 0xc0, 0xfb, 0x25, 0xdf, 0x22, 0x34, 0x5e, 0xb3, 0xed, 0x99, 0x87, 0x3b, 0x4a, 0xae, 0xff,
13670
+	0x3f, 0xc8, 0x77, 0x1c, 0x62, 0x8a, 0xf7, 0x3b, 0x54, 0x06, 0x1e, 0xb6, 0x71, 0x37, 0xb2, 0x5d,
13671
+	0x15, 0x67, 0xe5, 0x71, 0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x43, 0xea,
13672
+	0x8a, 0xa7, 0x0e, 0x1e, 0x13, 0x28, 0x67, 0xc8, 0x63, 0x55, 0x12, 0x21, 0x8f, 0x6c, 0x20, 0xca,
13673
+	0xf8, 0xcb, 0xa2, 0xfe, 0x4d, 0x02, 0xb2, 0xd8, 0xf3, 0x58, 0xcd, 0x40, 0x65, 0xc8, 0x9a, 0xa4,
13674
+	0x1f, 0xee, 0xdd, 0x62, 0x35, 0x7f, 0xf6, 0x72, 0x3d, 0x53, 0x33, 0x9e, 0xd0, 0x09, 0xce, 0x98,
13675
+	0xe4, 0x09, 0x9d, 0xf0, 0x4b, 0xde, 0x24, 0x62, 0xc7, 0x09, 0x33, 0x45, 0x79, 0xc9, 0xd7, 0x0c,
13676
+	0xbe, 0x9d, 0x70, 0xd6, 0x24, 0xfc, 0x1f, 0xdd, 0x85, 0xa2, 0x02, 0xf5, 0x0f, 0x48, 0x70, 0x20,
13677
+	0x99, 0x7c, 0x75, 0xf9, 0xec, 0xe5, 0x3a, 0x48, 0xe4, 0x16, 0x09, 0x0e, 0x30, 0x48, 0x34, 0xff,
13678
+	0x46, 0x0d, 0x28, 0x7c, 0xe5, 0xd9, 0x6e, 0x9f, 0x89, 0x41, 0xa8, 0x64, 0xc7, 0xdc, 0x1d, 0x38,
13679
+	0x1d, 0xaa, 0xca, 0xc0, 0xc0, 0x57, 0x91, 0x44, 0xff, 0xa7, 0x04, 0x14, 0xb8, 0x4d, 0x7b, 0xdf,
13680
+	0x36, 0xf9, 0xa5, 0xfc, 0xdd, 0xef, 0x8a, 0x6b, 0x90, 0x32, 0x03, 0x5f, 0x8d, 0x4d, 0x1c, 0x96,
13681
+	0xb5, 0x2e, 0xc6, 0x5c, 0x86, 0x1e, 0x41, 0x56, 0x85, 0x6f, 0xf2, 0x9a, 0xd0, 0xdf, 0x4c, 0x1f,
13682
+	0x54, 0x17, 0x95, 0x9e, 0x58, 0xcb, 0x69, 0xef, 0xc4, 0x28, 0x8b, 0x38, 0x2e, 0x42, 0x57, 0x21,
13683
+	0x69, 0xba, 0xc2, 0xad, 0xd4, 0x13, 0x68, 0xad, 0x85, 0x93, 0xa6, 0xab, 0xff, 0x7d, 0x02, 0x96,
13684
+	0xa6, 0x5e, 0xc5, 0x17, 0xe2, 0x3a, 0xe4, 0x83, 0xf1, 0x5e, 0x30, 0x09, 0x18, 0x1d, 0x86, 0xaf,
13685
+	0x29, 0x91, 0x00, 0x35, 0x21, 0x4f, 0x9c, 0x81, 0xe7, 0xdb, 0xec, 0x60, 0xa8, 0x22, 0x87, 0xf9,
13686
+	0x47, 0x7b, 0xdc, 0x66, 0xc5, 0x08, 0x55, 0xf0, 0x54, 0x3b, 0x3c, 0xcc, 0x53, 0xa2, 0xb3, 0xe2,
13687
+	0x30, 0x7f, 0x1f, 0x8a, 0x0e, 0x19, 0x8a, 0x78, 0x96, 0x07, 0xa4, 0x62, 0x1c, 0x69, 0x5c, 0x50,
13688
+	0x32, 0x1e, 0xa5, 0xeb, 0x3a, 0xe4, 0x23, 0x63, 0x68, 0x05, 0x0a, 0x46, 0xa3, 0xdb, 0xbf, 0xb7,
13689
+	0x71, 0xbf, 0xff, 0xb8, 0xb6, 0xa3, 0x2d, 0x28, 0x2e, 0xf1, 0x37, 0x09, 0x58, 0x52, 0x3e, 0xaf,
13690
+	0xf8, 0xd9, 0x0d, 0x58, 0xf4, 0xc9, 0x3e, 0x0b, 0x19, 0x64, 0x5a, 0x3a, 0x17, 0x3f, 0x46, 0x38,
13691
+	0x83, 0xe4, 0x55, 0xf3, 0x19, 0x64, 0xec, 0x7d, 0x2f, 0x75, 0xe1, 0xfb, 0x5e, 0xfa, 0x57, 0xf2,
13692
+	0xbe, 0xa7, 0xff, 0x24, 0x09, 0x2b, 0xea, 0xaa, 0x0f, 0xdf, 0xaf, 0xd0, 0x47, 0x90, 0x97, 0xb7,
13693
+	0xfe, 0x94, 0xff, 0x8a, 0x27, 0x25, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37, 0x2d, 0x1e, 0x90,
13694
+	0x29, 0x68, 0xec, 0xb5, 0x1a, 0xa4, 0xa8, 0xc5, 0xa3, 0x89, 0x3a, 0xa4, 0xf7, 0x6d, 0x87, 0x2a,
13695
+	0x3f, 0x9b, 0x9b, 0x43, 0x3c, 0xd7, 0xbc, 0x48, 0x79, 0xf7, 0x44, 0x48, 0xb7, 0xb5, 0x80, 0x85,
13696
+	0x76, 0xe9, 0xf7, 0x00, 0xa6, 0xd2, 0xb9, 0x51, 0x0b, 0x67, 0x06, 0x2a, 0x07, 0x14, 0x32, 0x83,
13697
+	0x66, 0x1d, 0x73, 0x19, 0xaf, 0x1a, 0xd8, 0x96, 0xda, 0xb9, 0xa2, 0xea, 0x31, 0xaf, 0x1a, 0xd8,
13698
+	0x56, 0x94, 0x77, 0x4f, 0xbf, 0x21, 0xef, 0x5e, 0xcd, 0x85, 0x69, 0x08, 0xbd, 0x0d, 0x57, 0xab,
13699
+	0x0e, 0x31, 0x0f, 0x1d, 0x3b, 0x60, 0xd4, 0x8a, 0xef, 0xd0, 0xcf, 0x20, 0x3b, 0x73, 0x73, 0xbf,
13700
+	0x21, 0xf1, 0xa3, 0xc0, 0xfa, 0x4f, 0x12, 0x50, 0xdc, 0xa2, 0xc4, 0x61, 0x07, 0xd3, 0xe8, 0x99,
13701
+	0xd1, 0x80, 0xa9, 0xf3, 0x51, 0x7c, 0xa3, 0xfb, 0x90, 0x8b, 0xee, 0x9a, 0xb7, 0x49, 0x8f, 0x47,
13702
+	0x68, 0xf4, 0x39, 0x2c, 0x72, 0xcf, 0xf6, 0xc6, 0x21, 0x25, 0x7c, 0x43, 0xde, 0x55, 0x81, 0xf9,
13703
+	0x21, 0xeb, 0x53, 0x71, 0xc5, 0x88, 0xd9, 0xc9, 0xe0, 0xb0, 0xa8, 0xff, 0x4f, 0x02, 0xae, 0xec,
13704
+	0x90, 0xc9, 0x1e, 0x55, 0x3b, 0x8e, 0x5a, 0x98, 0x9a, 0x9e, 0x6f, 0xa1, 0x4e, 0x7c, 0xa7, 0x5e,
13705
+	0xf0, 0x24, 0x30, 0x4f, 0x79, 0xfe, 0x86, 0x0d, 0xb9, 0x66, 0x32, 0xc6, 0x35, 0xaf, 0x40, 0xc6,
13706
+	0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28, 0x4f, 0x2f, 0xb2,
13707
+	0xec, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x47, 0x50, 0xea, 0x36, 0x6a, 0xb8, 0xd1, 0xab, 0xb6, 0x7f,
13708
+	0xbd, 0xdf, 0x35, 0xb6, 0xbb, 0xc6, 0xc6, 0xdd, 0x7e, 0xa7, 0xbd, 0xfd, 0xe5, 0xbd, 0x4f, 0xee,
13709
+	0x7e, 0xa6, 0x25, 0x4a, 0xe5, 0x93, 0xd3, 0xf2, 0xf5, 0x96, 0x51, 0xdb, 0x96, 0x6e, 0xb9, 0xe7,
13710
+	0x3d, 0xef, 0x12, 0x27, 0x20, 0x1b, 0x77, 0x3b, 0x9e, 0x33, 0xe1, 0x98, 0xdb, 0xbf, 0x48, 0x41,
13711
+	0x3e, 0x4a, 0xc3, 0x71, 0xef, 0xe2, 0x31, 0x90, 0x6a, 0x2a, 0x92, 0xb7, 0xe8, 0x31, 0x7a, 0x7f,
13712
+	0x1a, 0xfd, 0x3c, 0x92, 0xcf, 0x01, 0x51, 0x75, 0x18, 0xf9, 0x7c, 0x00, 0x39, 0xa3, 0xdb, 0x6d,
13713
+	0x3e, 0x6e, 0x35, 0xea, 0xda, 0xd7, 0x89, 0xd2, 0xf7, 0x4e, 0x4e, 0xcb, 0x97, 0x22, 0x90, 0x11,
13714
+	0x04, 0xf6, 0xc0, 0xa5, 0x96, 0x40, 0xd5, 0x6a, 0x8d, 0x4e, 0xaf, 0x51, 0xd7, 0x5e, 0x24, 0xcf,
13715
+	0xa3, 0x04, 0x9b, 0x17, 0x4f, 0x7b, 0xf9, 0x0e, 0x6e, 0x74, 0x0c, 0xcc, 0x1b, 0xfc, 0x3a, 0x29,
13716
+	0x83, 0xb2, 0x69, 0x8b, 0x3e, 0x1d, 0x11, 0x9f, 0xb7, 0xb9, 0x16, 0x3e, 0x71, 0xbf, 0x48, 0xc9,
13717
+	0xe7, 0x9f, 0x69, 0x4e, 0x91, 0x12, 0x6b, 0xc2, 0x5b, 0x13, 0xf9, 0x5c, 0x61, 0x26, 0x75, 0xae,
13718
+	0xb5, 0x2e, 0x23, 0x3e, 0xe3, 0x56, 0x74, 0x58, 0xc4, 0xbb, 0xad, 0x16, 0x07, 0xbd, 0x48, 0x9f,
13719
+	0x1b, 0x1d, 0x1e, 0xbb, 0x2e, 0xc7, 0xdc, 0x84, 0x5c, 0x98, 0xee, 0xd5, 0xbe, 0x4e, 0x9f, 0xeb,
13720
+	0x50, 0x2d, 0xcc, 0x55, 0x8b, 0x06, 0xb7, 0x76, 0x7b, 0xe2, 0x05, 0xfe, 0x45, 0xe6, 0x7c, 0x83,
13721
+	0x07, 0x63, 0x66, 0xf1, 0x70, 0xb3, 0x1c, 0xc5, 0x7f, 0x5f, 0x67, 0x24, 0xa3, 0x8e, 0x30, 0x2a,
13722
+	0xf8, 0xfb, 0x00, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0x58, 0xff, 0x22, 0x7b, 0xce, 0x0e, 0xa6, 0x5f,
13723
+	0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x2d, 0x43, 0x4c, 0xf9, 0x79, 0x54, 0xdb, 0x1f, 0x1d, 0x10,
13724
+	0x97, 0x5a, 0xd3, 0x37, 0xb0, 0xa8, 0xea, 0xf6, 0x6f, 0x42, 0x2e, 0xbc, 0x61, 0xd1, 0x1a, 0x64,
13725
+	0x9f, 0xb5, 0xf1, 0x93, 0x06, 0xd6, 0x16, 0xe4, 0x1c, 0x86, 0x35, 0xcf, 0x24, 0x45, 0x29, 0xc3,
13726
+	0xe2, 0x8e, 0xd1, 0x32, 0x1e, 0x37, 0x70, 0x98, 0x9a, 0x09, 0x01, 0xea, 0x9a, 0x28, 0x69, 0xaa,
13727
+	0x81, 0xc8, 0x66, 0xf5, 0xfa, 0x37, 0xdf, 0xae, 0x2d, 0xfc, 0xec, 0xdb, 0xb5, 0x85, 0x9f, 0x7f,
13728
+	0xbb, 0x96, 0x78, 0x71, 0xb6, 0x96, 0xf8, 0xe6, 0x6c, 0x2d, 0xf1, 0xd3, 0xb3, 0xb5, 0xc4, 0xbf,
13729
+	0x9d, 0xad, 0x25, 0xf6, 0xb2, 0x22, 0x08, 0xfa, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x7d,
13730
+	0xfe, 0xa7, 0xa7, 0xa8, 0x26, 0x00, 0x00,
13729 13731
 }
... ...
@@ -694,6 +694,13 @@ message RaftConfig {
694 694
 	uint32 election_tick = 5;
695 695
 }
696 696
 
697
+message EncryptionConfig {
698
+	// AutoLockManagers specifies whether or not managers TLS keys and raft data
699
+	// should be encrypted at rest in such a way that they must be unlocked
700
+	// before the manager node starts up again.
701
+	bool auto_lock_managers = 1;
702
+}
703
+
697 704
 // Placement specifies task distribution constraints.
698 705
 message Placement {
699 706
 	// constraints specifies a set of requirements a node should meet for a task.
... ...
@@ -23,12 +23,12 @@ import (
23 23
 	"github.com/docker/distribution/digest"
24 24
 	"github.com/docker/go-events"
25 25
 	"github.com/docker/swarmkit/api"
26
-	"github.com/docker/swarmkit/identity"
27 26
 	"github.com/docker/swarmkit/ioutils"
28 27
 	"github.com/docker/swarmkit/remotes"
29 28
 	"github.com/pkg/errors"
30 29
 	"golang.org/x/net/context"
31 30
 	"google.golang.org/grpc"
31
+	"google.golang.org/grpc/codes"
32 32
 	"google.golang.org/grpc/credentials"
33 33
 )
34 34
 
... ...
@@ -122,8 +122,8 @@ func (rca *RootCA) CanSign() bool {
122 122
 
123 123
 // IssueAndSaveNewCertificates generates a new key-pair, signs it with the local root-ca, and returns a
124 124
 // tls certificate
125
-func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org string) (*tls.Certificate, error) {
126
-	csr, key, err := GenerateAndWriteNewKey(paths)
125
+func (rca *RootCA) IssueAndSaveNewCertificates(kw KeyWriter, cn, ou, org string) (*tls.Certificate, error) {
126
+	csr, key, err := GenerateNewCSR()
127 127
 	if err != nil {
128 128
 		return nil, errors.Wrap(err, "error when generating new node certs")
129 129
 	}
... ...
@@ -138,20 +138,13 @@ func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org stri
138 138
 		return nil, errors.Wrap(err, "failed to sign node certificate")
139 139
 	}
140 140
 
141
-	// Ensure directory exists
142
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
141
+	// Create a valid TLSKeyPair out of the PEM encoded private key and certificate
142
+	tlsKeyPair, err := tls.X509KeyPair(certChain, key)
143 143
 	if err != nil {
144 144
 		return nil, err
145 145
 	}
146 146
 
147
-	// Write the chain to disk
148
-	if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil {
149
-		return nil, err
150
-	}
151
-
152
-	// Create a valid TLSKeyPair out of the PEM encoded private key and certificate
153
-	tlsKeyPair, err := tls.X509KeyPair(certChain, key)
154
-	if err != nil {
147
+	if err := kw.Write(certChain, key, nil); err != nil {
155 148
 		return nil, err
156 149
 	}
157 150
 
... ...
@@ -160,11 +153,9 @@ func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org stri
160 160
 
161 161
 // RequestAndSaveNewCertificates gets new certificates issued, either by signing them locally if a signer is
162 162
 // available, or by requesting them from the remote server at remoteAddr.
163
-func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths CertPaths, token string, remotes remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) {
164
-	// Create a new key/pair and CSR for the new manager
165
-	// Write the new CSR and the new key to a temporary location so we can survive crashes on rotation
166
-	tempPaths := genTempPaths(paths)
167
-	csr, key, err := GenerateAndWriteNewKey(tempPaths)
163
+func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, kw KeyWriter, token string, r remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) {
164
+	// Create a new key/pair and CSR
165
+	csr, key, err := GenerateNewCSR()
168 166
 	if err != nil {
169 167
 		return nil, errors.Wrap(err, "error when generating new node certs")
170 168
 	}
... ...
@@ -174,7 +165,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert
174 174
 	// responding properly (for example, it may have just been demoted).
175 175
 	var signedCert []byte
176 176
 	for i := 0; i != 5; i++ {
177
-		signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, remotes, transport, nodeInfo)
177
+		signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, r, transport, nodeInfo)
178 178
 		if err == nil {
179 179
 			break
180 180
 		}
... ...
@@ -184,7 +175,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert
184 184
 	}
185 185
 
186 186
 	// Доверяй, но проверяй.
187
-	// Before we overwrite our local certificate, let's make sure the server gave us one that is valid
187
+	// Before we overwrite our local key + certificate, let's make sure the server gave us one that is valid
188 188
 	// Create an X509Cert so we can .Verify()
189 189
 	certBlock, _ := pem.Decode(signedCert)
190 190
 	if certBlock == nil {
... ...
@@ -209,23 +200,60 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert
209 209
 		return nil, err
210 210
 	}
211 211
 
212
-	// Ensure directory exists
213
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
212
+	var kekUpdate *KEKData
213
+	for i := 0; i < 5; i++ {
214
+		kekUpdate, err = rca.getKEKUpdate(ctx, X509Cert, tlsKeyPair, r)
215
+		if err == nil {
216
+			break
217
+		}
218
+	}
214 219
 	if err != nil {
215 220
 		return nil, err
216 221
 	}
217 222
 
218
-	// Write the chain to disk
219
-	if err := ioutils.AtomicWriteFile(paths.Cert, signedCert, 0644); err != nil {
223
+	if err := kw.Write(signedCert, key, kekUpdate); err != nil {
220 224
 		return nil, err
221 225
 	}
222 226
 
223
-	// Move the new key to the final location
224
-	if err := os.Rename(tempPaths.Key, paths.Key); err != nil {
225
-		return nil, err
227
+	return &tlsKeyPair, nil
228
+}
229
+
230
+func (rca *RootCA) getKEKUpdate(ctx context.Context, cert *x509.Certificate, keypair tls.Certificate, r remotes.Remotes) (*KEKData, error) {
231
+	var managerRole bool
232
+	for _, ou := range cert.Subject.OrganizationalUnit {
233
+		if ou == ManagerRole {
234
+			managerRole = true
235
+			break
236
+		}
226 237
 	}
227 238
 
228
-	return &tlsKeyPair, nil
239
+	if managerRole {
240
+		mtlsCreds := credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rca.Pool, Certificates: []tls.Certificate{keypair}})
241
+		conn, peer, err := getGRPCConnection(mtlsCreds, r)
242
+		if err != nil {
243
+			return nil, err
244
+		}
245
+		defer conn.Close()
246
+
247
+		client := api.NewCAClient(conn)
248
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
249
+		defer cancel()
250
+		response, err := client.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{})
251
+		if err != nil {
252
+			if grpc.Code(err) == codes.Unimplemented { // if the server does not support keks, return as if no encryption key was specified
253
+				return &KEKData{}, nil
254
+			}
255
+
256
+			r.Observe(peer, -remotes.DefaultObservationWeight)
257
+			return nil, err
258
+		}
259
+		r.Observe(peer, remotes.DefaultObservationWeight)
260
+		return &KEKData{KEK: response.UnlockKey, Version: response.Version.Index}, nil
261
+	}
262
+
263
+	// If this is a worker, set to never encrypt. We always want to set to the lock key to nil,
264
+	// in case this was a manager that was demoted to a worker.
265
+	return &KEKData{}, nil
229 266
 }
230 267
 
231 268
 // PrepareCSR creates a CFSSL Sign Request based on the given raw CSR and
... ...
@@ -388,11 +416,9 @@ func ensureCertKeyMatch(cert *x509.Certificate, key crypto.PublicKey) error {
388 388
 
389 389
 // GetLocalRootCA validates if the contents of the file are a valid self-signed
390 390
 // CA certificate, and returns the PEM-encoded Certificate if so
391
-func GetLocalRootCA(baseDir string) (RootCA, error) {
392
-	paths := NewConfigPaths(baseDir)
393
-
391
+func GetLocalRootCA(paths CertPaths) (RootCA, error) {
394 392
 	// Check if we have a Certificate file
395
-	cert, err := ioutil.ReadFile(paths.RootCA.Cert)
393
+	cert, err := ioutil.ReadFile(paths.Cert)
396 394
 	if err != nil {
397 395
 		if os.IsNotExist(err) {
398 396
 			err = ErrNoLocalRootCA
... ...
@@ -401,7 +427,7 @@ func GetLocalRootCA(baseDir string) (RootCA, error) {
401 401
 		return RootCA{}, err
402 402
 	}
403 403
 
404
-	key, err := ioutil.ReadFile(paths.RootCA.Key)
404
+	key, err := ioutil.ReadFile(paths.Key)
405 405
 	if err != nil {
406 406
 		if !os.IsNotExist(err) {
407 407
 			return RootCA{}, err
... ...
@@ -414,24 +440,31 @@ func GetLocalRootCA(baseDir string) (RootCA, error) {
414 414
 	return NewRootCA(cert, key, DefaultNodeCertExpiration)
415 415
 }
416 416
 
417
-// GetRemoteCA returns the remote endpoint's CA certificate
418
-func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) {
419
-	// This TLS Config is intentionally using InsecureSkipVerify. Either we're
420
-	// doing TOFU, in which case we don't validate the remote CA, or we're using
421
-	// a user supplied hash to check the integrity of the CA certificate.
422
-	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
417
+func getGRPCConnection(creds credentials.TransportCredentials, r remotes.Remotes) (*grpc.ClientConn, api.Peer, error) {
418
+	peer, err := r.Select()
419
+	if err != nil {
420
+		return nil, api.Peer{}, err
421
+	}
422
+
423 423
 	opts := []grpc.DialOption{
424
-		grpc.WithTransportCredentials(insecureCreds),
424
+		grpc.WithTransportCredentials(creds),
425 425
 		grpc.WithTimeout(5 * time.Second),
426 426
 		grpc.WithBackoffMaxDelay(5 * time.Second),
427 427
 	}
428 428
 
429
-	peer, err := r.Select()
429
+	conn, err := grpc.Dial(peer.Addr, opts...)
430 430
 	if err != nil {
431
-		return RootCA{}, err
431
+		return nil, api.Peer{}, err
432 432
 	}
433
+	return conn, peer, nil
434
+}
433 435
 
434
-	conn, err := grpc.Dial(peer.Addr, opts...)
436
+// GetRemoteCA returns the remote endpoint's CA certificate
437
+func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) {
438
+	// This TLS Config is intentionally using InsecureSkipVerify. We use the
439
+	// digest instead to check the integrity of the CA certificate.
440
+	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
441
+	conn, peer, err := getGRPCConnection(insecureCreds, r)
435 442
 	if err != nil {
436 443
 		return RootCA{}, err
437 444
 	}
... ...
@@ -481,9 +514,9 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootC
481 481
 	return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil
482 482
 }
483 483
 
484
-// CreateAndWriteRootCA creates a Certificate authority for a new Swarm Cluster, potentially
484
+// CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially
485 485
 // overwriting any existing CAs.
486
-func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) {
486
+func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) {
487 487
 	// Create a simple CSR for the CA using the default CA validator and policy
488 488
 	req := cfcsr.CertificateRequest{
489 489
 		CN:         rootCN,
... ...
@@ -497,99 +530,17 @@ func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) {
497 497
 		return RootCA{}, err
498 498
 	}
499 499
 
500
-	// Ensure directory exists
501
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
500
+	rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration)
502 501
 	if err != nil {
503 502
 		return RootCA{}, err
504 503
 	}
505 504
 
506
-	// Write the Private Key and Certificate to disk, using decent permissions
507
-	if err := ioutils.AtomicWriteFile(paths.Cert, cert, 0644); err != nil {
508
-		return RootCA{}, err
509
-	}
510
-	if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
505
+	// save the cert to disk
506
+	if err := saveRootCA(rootCA, paths); err != nil {
511 507
 		return RootCA{}, err
512 508
 	}
513 509
 
514
-	return NewRootCA(cert, key, DefaultNodeCertExpiration)
515
-}
516
-
517
-// BootstrapCluster receives a directory and creates both new Root CA key material
518
-// and a ManagerRole key/certificate pair to be used by the initial cluster manager
519
-func BootstrapCluster(baseCertDir string) error {
520
-	paths := NewConfigPaths(baseCertDir)
521
-
522
-	rootCA, err := CreateAndWriteRootCA(rootCN, paths.RootCA)
523
-	if err != nil {
524
-		return err
525
-	}
526
-
527
-	nodeID := identity.NewID()
528
-	newOrg := identity.NewID()
529
-	_, err = GenerateAndSignNewTLSCert(rootCA, nodeID, ManagerRole, newOrg, paths.Node)
530
-
531
-	return err
532
-}
533
-
534
-// GenerateAndSignNewTLSCert creates a new keypair, signs the certificate using signer,
535
-// and saves the certificate and key to disk. This method is used to bootstrap the first
536
-// manager TLS certificates.
537
-func GenerateAndSignNewTLSCert(rootCA RootCA, cn, ou, org string, paths CertPaths) (*tls.Certificate, error) {
538
-	// Generate and new keypair and CSR
539
-	csr, key, err := generateNewCSR()
540
-	if err != nil {
541
-		return nil, err
542
-	}
543
-
544
-	// Obtain a signed Certificate
545
-	certChain, err := rootCA.ParseValidateAndSignCSR(csr, cn, ou, org)
546
-	if err != nil {
547
-		return nil, errors.Wrap(err, "failed to sign node certificate")
548
-	}
549
-
550
-	// Ensure directory exists
551
-	err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
552
-	if err != nil {
553
-		return nil, err
554
-	}
555
-
556
-	// Write both the chain and key to disk
557
-	if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil {
558
-		return nil, err
559
-	}
560
-	if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
561
-		return nil, err
562
-	}
563
-
564
-	// Load a valid tls.Certificate from the chain and the key
565
-	serverCert, err := tls.X509KeyPair(certChain, key)
566
-	if err != nil {
567
-		return nil, err
568
-	}
569
-
570
-	return &serverCert, nil
571
-}
572
-
573
-// GenerateAndWriteNewKey generates a new pub/priv key pair, writes it to disk
574
-// and returns the CSR and the private key material
575
-func GenerateAndWriteNewKey(paths CertPaths) (csr, key []byte, err error) {
576
-	// Generate a new key pair
577
-	csr, key, err = generateNewCSR()
578
-	if err != nil {
579
-		return
580
-	}
581
-
582
-	// Ensure directory exists
583
-	err = os.MkdirAll(filepath.Dir(paths.Key), 0755)
584
-	if err != nil {
585
-		return
586
-	}
587
-
588
-	if err = ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
589
-		return
590
-	}
591
-
592
-	return
510
+	return rootCA, nil
593 511
 }
594 512
 
595 513
 // GetRemoteSignedCertificate submits a CSR to a remote CA server address,
... ...
@@ -605,18 +556,7 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r
605 605
 		creds = credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rootCAPool})
606 606
 	}
607 607
 
608
-	peer, err := r.Select()
609
-	if err != nil {
610
-		return nil, err
611
-	}
612
-
613
-	opts := []grpc.DialOption{
614
-		grpc.WithTransportCredentials(creds),
615
-		grpc.WithTimeout(5 * time.Second),
616
-		grpc.WithBackoffMaxDelay(5 * time.Second),
617
-	}
618
-
619
-	conn, err := grpc.Dial(peer.Addr, opts...)
608
+	conn, peer, err := getGRPCConnection(creds, r)
620 609
 	if err != nil {
621 610
 		return nil, err
622 611
 	}
... ...
@@ -681,10 +621,10 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r
681 681
 }
682 682
 
683 683
 // readCertValidity returns the certificate issue and expiration time
684
-func readCertValidity(paths CertPaths) (time.Time, time.Time, error) {
684
+func readCertValidity(kr KeyReader) (time.Time, time.Time, error) {
685 685
 	var zeroTime time.Time
686 686
 	// Read the Cert
687
-	cert, err := ioutil.ReadFile(paths.Cert)
687
+	cert, _, err := kr.Read()
688 688
 	if err != nil {
689 689
 		return zeroTime, zeroTime, err
690 690
 	}
... ...
@@ -714,7 +654,8 @@ func saveRootCA(rootCA RootCA, paths CertPaths) error {
714 714
 	return ioutils.AtomicWriteFile(paths.Cert, rootCA.Cert, 0644)
715 715
 }
716 716
 
717
-func generateNewCSR() (csr, key []byte, err error) {
717
+// GenerateNewCSR returns a newly generated key and CSR signed with said key
718
+func GenerateNewCSR() (csr, key []byte, err error) {
718 719
 	req := &cfcsr.CertificateRequest{
719 720
 		KeyRequest: cfcsr.NewBasicKeyRequest(),
720 721
 	}
... ...
@@ -6,7 +6,6 @@ import (
6 6
 	"crypto/x509"
7 7
 	"encoding/pem"
8 8
 	"fmt"
9
-	"io/ioutil"
10 9
 	"math/big"
11 10
 	"math/rand"
12 11
 	"path/filepath"
... ...
@@ -33,7 +32,8 @@ const (
33 33
 	nodeTLSKeyFilename  = "swarm-node.key"
34 34
 	nodeCSRFilename     = "swarm-node.csr"
35 35
 
36
-	rootCN = "swarm-ca"
36
+	// DefaultRootCN represents the root CN that we should create roots CAs with by default
37
+	DefaultRootCN = "swarm-ca"
37 38
 	// ManagerRole represents the Manager node type, and is used for authorization to endpoints
38 39
 	ManagerRole = "swarm-manager"
39 40
 	// WorkerRole represents the Worker node type, and is used for authorization to endpoints
... ...
@@ -54,8 +54,9 @@ const (
54 54
 type SecurityConfig struct {
55 55
 	mu sync.Mutex
56 56
 
57
-	rootCA     *RootCA
58
-	externalCA *ExternalCA
57
+	rootCA        *RootCA
58
+	externalCA    *ExternalCA
59
+	keyReadWriter *KeyReadWriter
59 60
 
60 61
 	ServerTLSCreds *MutableTLSCreds
61 62
 	ClientTLSCreds *MutableTLSCreds
... ...
@@ -69,7 +70,7 @@ type CertificateUpdate struct {
69 69
 }
70 70
 
71 71
 // NewSecurityConfig initializes and returns a new SecurityConfig.
72
-func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig {
72
+func NewSecurityConfig(rootCA *RootCA, krw *KeyReadWriter, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig {
73 73
 	// Make a new TLS config for the external CA client without a
74 74
 	// ServerName value set.
75 75
 	clientTLSConfig := clientTLSCreds.Config()
... ...
@@ -82,6 +83,7 @@ func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTL
82 82
 
83 83
 	return &SecurityConfig{
84 84
 		rootCA:         rootCA,
85
+		keyReadWriter:  krw,
85 86
 		externalCA:     NewExternalCA(rootCA, externalCATLSConfig),
86 87
 		ClientTLSCreds: clientTLSCreds,
87 88
 		ServerTLSCreds: serverTLSCreds,
... ...
@@ -96,6 +98,16 @@ func (s *SecurityConfig) RootCA() *RootCA {
96 96
 	return s.rootCA
97 97
 }
98 98
 
99
+// KeyWriter returns the object that can write keys to disk
100
+func (s *SecurityConfig) KeyWriter() KeyWriter {
101
+	return s.keyReadWriter
102
+}
103
+
104
+// KeyReader returns the object that can read keys from disk
105
+func (s *SecurityConfig) KeyReader() KeyReader {
106
+	return s.keyReadWriter
107
+}
108
+
99 109
 // UpdateRootCA replaces the root CA with a new root CA based on the specified
100 110
 // certificate, key, and the number of hours the certificates issue should last.
101 111
 func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration) error {
... ...
@@ -181,70 +193,63 @@ func getCAHashFromToken(token string) (digest.Digest, error) {
181 181
 	return digest.ParseDigest(fmt.Sprintf("sha256:%0[1]*s", 64, digestInt.Text(16)))
182 182
 }
183 183
 
184
-// LoadOrCreateSecurityConfig encapsulates the security logic behind joining a cluster.
185
-// Every node requires at least a set of TLS certificates with which to join the cluster with.
186
-// In the case of a manager, these certificates will be used both for client and server credentials.
187
-func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse) (*SecurityConfig, error) {
188
-	ctx = log.WithModule(ctx, "tls")
189
-	paths := NewConfigPaths(baseCertDir)
190
-
184
+// DownloadRootCA tries to retrieve a remote root CA and matches the digest against the provided token.
185
+func DownloadRootCA(ctx context.Context, paths CertPaths, token string, r remotes.Remotes) (RootCA, error) {
186
+	var rootCA RootCA
187
+	// Get a digest for the optional CA hash string that we've been provided
188
+	// If we were provided a non-empty string, and it is an invalid hash, return
189
+	// otherwise, allow the invalid digest through.
191 190
 	var (
192
-		rootCA                         RootCA
193
-		serverTLSCreds, clientTLSCreds *MutableTLSCreds
194
-		err                            error
191
+		d   digest.Digest
192
+		err error
195 193
 	)
196
-
197
-	// Check if we already have a CA certificate on disk. We need a CA to have a valid SecurityConfig
198
-	rootCA, err = GetLocalRootCA(baseCertDir)
199
-	switch err {
200
-	case nil:
201
-		log.G(ctx).Debug("loaded CA certificate")
202
-	case ErrNoLocalRootCA:
203
-		log.G(ctx).WithError(err).Debugf("failed to load local CA certificate")
204
-
205
-		// Get a digest for the optional CA hash string that we've been provided
206
-		// If we were provided a non-empty string, and it is an invalid hash, return
207
-		// otherwise, allow the invalid digest through.
208
-		var d digest.Digest
209
-		if token != "" {
210
-			d, err = getCAHashFromToken(token)
211
-			if err != nil {
212
-				return nil, err
213
-			}
214
-		}
215
-
216
-		// Get the remote CA certificate, verify integrity with the
217
-		// hash provided. Retry up to 5 times, in case the manager we
218
-		// first try to contact is not responding properly (it may have
219
-		// just been demoted, for example).
220
-
221
-		for i := 0; i != 5; i++ {
222
-			rootCA, err = GetRemoteCA(ctx, d, remotes)
223
-			if err == nil {
224
-				break
225
-			}
226
-			log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate")
227
-		}
194
+	if token != "" {
195
+		d, err = getCAHashFromToken(token)
228 196
 		if err != nil {
229
-			return nil, err
197
+			return RootCA{}, err
230 198
 		}
231
-
232
-		// Save root CA certificate to disk
233
-		if err = saveRootCA(rootCA, paths.RootCA); err != nil {
234
-			return nil, err
199
+	}
200
+	// Get the remote CA certificate, verify integrity with the
201
+	// hash provided. Retry up to 5 times, in case the manager we
202
+	// first try to contact is not responding properly (it may have
203
+	// just been demoted, for example).
204
+
205
+	for i := 0; i != 5; i++ {
206
+		rootCA, err = GetRemoteCA(ctx, d, r)
207
+		if err == nil {
208
+			break
235 209
 		}
210
+		log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate")
211
+	}
212
+	if err != nil {
213
+		return RootCA{}, err
214
+	}
236 215
 
237
-		log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.RootCA.Cert)
238
-	default:
239
-		return nil, err
216
+	// Save root CA certificate to disk
217
+	if err = saveRootCA(rootCA, paths); err != nil {
218
+		return RootCA{}, err
240 219
 	}
241 220
 
221
+	log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.Cert)
222
+	return rootCA, nil
223
+}
224
+
225
+// LoadOrCreateSecurityConfig encapsulates the security logic behind joining a cluster.
226
+// Every node requires at least a set of TLS certificates with which to join the cluster with.
227
+// In the case of a manager, these certificates will be used both for client and server credentials.
228
+func LoadOrCreateSecurityConfig(ctx context.Context, rootCA RootCA, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse, krw *KeyReadWriter) (*SecurityConfig, error) {
229
+	ctx = log.WithModule(ctx, "tls")
230
+
242 231
 	// At this point we've successfully loaded the CA details from disk, or
243 232
 	// successfully downloaded them remotely. The next step is to try to
244 233
 	// load our certificates.
245
-	clientTLSCreds, serverTLSCreds, err = LoadTLSCreds(rootCA, paths.Node)
234
+	clientTLSCreds, serverTLSCreds, err := LoadTLSCreds(rootCA, krw)
246 235
 	if err != nil {
247
-		log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", paths.Node.Cert)
236
+		if _, ok := errors.Cause(err).(ErrInvalidKEK); ok {
237
+			return nil, err
238
+		}
239
+
240
+		log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target())
248 241
 
249 242
 		var (
250 243
 			tlsKeyPair *tls.Certificate
... ...
@@ -262,7 +267,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
262 262
 					NodeMembership: api.NodeMembershipAccepted,
263 263
 				}
264 264
 			}
265
-			tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(paths.Node, cn, proposedRole, org)
265
+			tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(krw, cn, proposedRole, org)
266 266
 			if err != nil {
267 267
 				log.G(ctx).WithFields(logrus.Fields{
268 268
 					"node.id":   cn,
... ...
@@ -278,7 +283,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
278 278
 		} else {
279 279
 			// There was an error loading our Credentials, let's get a new certificate issued
280 280
 			// Last argument is nil because at this point we don't have any valid TLS creds
281
-			tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, paths.Node, token, remotes, nil, nodeInfo)
281
+			tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, krw, token, remotes, nil, nodeInfo)
282 282
 			if err != nil {
283 283
 				log.G(ctx).WithError(err).Error("failed to request save new certificate")
284 284
 				return nil, err
... ...
@@ -299,7 +304,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
299 299
 		log.G(ctx).WithFields(logrus.Fields{
300 300
 			"node.id":   clientTLSCreds.NodeID(),
301 301
 			"node.role": clientTLSCreds.Role(),
302
-		}).Debugf("new node credentials generated: %s", paths.Node.Cert)
302
+		}).Debugf("new node credentials generated: %s", krw.Target())
303 303
 	} else {
304 304
 		if nodeInfo != nil {
305 305
 			nodeInfo <- api.IssueNodeCertificateResponse{
... ...
@@ -313,13 +318,66 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose
313 313
 		}).Debug("loaded node credentials")
314 314
 	}
315 315
 
316
-	return NewSecurityConfig(&rootCA, clientTLSCreds, serverTLSCreds), nil
316
+	return NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds), nil
317
+}
318
+
319
+// RenewTLSConfigNow gets a new TLS cert and key, and updates the security config if provided.  This is similar to
320
+// RenewTLSConfig, except while that monitors for expiry, and periodically renews, this renews once and is blocking
321
+func RenewTLSConfigNow(ctx context.Context, s *SecurityConfig, r remotes.Remotes) error {
322
+	ctx = log.WithModule(ctx, "tls")
323
+	log := log.G(ctx).WithFields(logrus.Fields{
324
+		"node.id":   s.ClientTLSCreds.NodeID(),
325
+		"node.role": s.ClientTLSCreds.Role(),
326
+	})
327
+
328
+	// Let's request new certs. Renewals don't require a token.
329
+	rootCA := s.RootCA()
330
+	tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx,
331
+		s.KeyWriter(),
332
+		"",
333
+		r,
334
+		s.ClientTLSCreds,
335
+		nil)
336
+	if err != nil {
337
+		log.WithError(err).Errorf("failed to renew the certificate")
338
+		return err
339
+	}
340
+
341
+	clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole)
342
+	if err != nil {
343
+		log.WithError(err).Errorf("failed to create a new client config")
344
+		return err
345
+	}
346
+	serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool)
347
+	if err != nil {
348
+		log.WithError(err).Errorf("failed to create a new server config")
349
+		return err
350
+	}
351
+
352
+	if err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig); err != nil {
353
+		log.WithError(err).Errorf("failed to update the client credentials")
354
+		return err
355
+	}
356
+
357
+	// Update the external CA to use the new client TLS
358
+	// config using a copy without a serverName specified.
359
+	s.externalCA.UpdateTLSConfig(&tls.Config{
360
+		Certificates: clientTLSConfig.Certificates,
361
+		RootCAs:      clientTLSConfig.RootCAs,
362
+		MinVersion:   tls.VersionTLS12,
363
+	})
364
+
365
+	if err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig); err != nil {
366
+		log.WithError(err).Errorf("failed to update the server TLS credentials")
367
+		return err
368
+	}
369
+
370
+	return nil
317 371
 }
318 372
 
319 373
 // RenewTLSConfig will continuously monitor for the necessity of renewing the local certificates, either by
320 374
 // issuing them locally if key-material is available, or requesting them from a remote CA.
321
-func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate {
322
-	paths := NewConfigPaths(baseCertDir)
375
+func RenewTLSConfig(ctx context.Context, s *SecurityConfig, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate {
323 376
 	updates := make(chan CertificateUpdate)
324 377
 
325 378
 	go func() {
... ...
@@ -337,10 +395,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string,
337 337
 			// Since the expiration of the certificate is managed remotely we should update our
338 338
 			// retry timer on every iteration of this loop.
339 339
 			// Retrieve the current certificate expiration information.
340
-			validFrom, validUntil, err := readCertValidity(paths.Node)
340
+			validFrom, validUntil, err := readCertValidity(s.KeyReader())
341 341
 			if err != nil {
342 342
 				// We failed to read the expiration, let's stick with the starting default
343
-				log.Errorf("failed to read the expiration of the TLS certificate in: %s", paths.Node.Cert)
343
+				log.Errorf("failed to read the expiration of the TLS certificate in: %s", s.KeyReader().Target())
344 344
 				updates <- CertificateUpdate{Err: errors.New("failed to read certificate expiration")}
345 345
 			} else {
346 346
 				// If we have an expired certificate, we let's stick with the starting default in
... ...
@@ -368,52 +426,12 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string,
368 368
 				return
369 369
 			}
370 370
 
371
-			// Let's request new certs. Renewals don't require a token.
372
-			rootCA := s.RootCA()
373
-			tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx,
374
-				paths.Node,
375
-				"",
376
-				remotes,
377
-				s.ClientTLSCreds,
378
-				nil)
379
-			if err != nil {
380
-				log.WithError(err).Errorf("failed to renew the certificate")
381
-				updates <- CertificateUpdate{Err: err}
382
-				continue
383
-			}
384
-
385
-			clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole)
386
-			if err != nil {
387
-				log.WithError(err).Errorf("failed to create a new client config")
388
-				updates <- CertificateUpdate{Err: err}
389
-			}
390
-			serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool)
391
-			if err != nil {
392
-				log.WithError(err).Errorf("failed to create a new server config")
393
-				updates <- CertificateUpdate{Err: err}
394
-			}
395
-
396
-			err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig)
397
-			if err != nil {
398
-				log.WithError(err).Errorf("failed to update the client credentials")
399
-				updates <- CertificateUpdate{Err: err}
400
-			}
401
-
402
-			// Update the external CA to use the new client TLS
403
-			// config using a copy without a serverName specified.
404
-			s.externalCA.UpdateTLSConfig(&tls.Config{
405
-				Certificates: clientTLSConfig.Certificates,
406
-				RootCAs:      clientTLSConfig.RootCAs,
407
-				MinVersion:   tls.VersionTLS12,
408
-			})
409
-
410
-			err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig)
411
-			if err != nil {
412
-				log.WithError(err).Errorf("failed to update the server TLS credentials")
371
+			// ignore errors - it will just try again laster
372
+			if err := RenewTLSConfigNow(ctx, s, remotes); err != nil {
413 373
 				updates <- CertificateUpdate{Err: err}
374
+			} else {
375
+				updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()}
414 376
 			}
415
-
416
-			updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()}
417 377
 		}
418 378
 	}()
419 379
 
... ...
@@ -447,13 +465,9 @@ func calculateRandomExpiry(validFrom, validUntil time.Time) time.Duration {
447 447
 
448 448
 // LoadTLSCreds loads tls credentials from the specified path and verifies that
449 449
 // thay are valid for the RootCA.
450
-func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLSCreds, error) {
450
+func LoadTLSCreds(rootCA RootCA, kr KeyReader) (*MutableTLSCreds, *MutableTLSCreds, error) {
451 451
 	// Read both the Cert and Key from disk
452
-	cert, err := ioutil.ReadFile(paths.Cert)
453
-	if err != nil {
454
-		return nil, nil, err
455
-	}
456
-	key, err := ioutil.ReadFile(paths.Key)
452
+	cert, key, err := kr.Read()
457 453
 	if err != nil {
458 454
 		return nil, nil, err
459 455
 	}
... ...
@@ -482,24 +496,9 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS
482 482
 
483 483
 	// Now that we know this certificate is valid, create a TLS Certificate for our
484 484
 	// credentials
485
-	var (
486
-		keyPair tls.Certificate
487
-		newErr  error
488
-	)
489
-	keyPair, err = tls.X509KeyPair(cert, key)
485
+	keyPair, err := tls.X509KeyPair(cert, key)
490 486
 	if err != nil {
491
-		// This current keypair isn't valid. It's possible we crashed before we
492
-		// overwrote the current key. Let's try loading it from disk.
493
-		tempPaths := genTempPaths(paths)
494
-		key, newErr = ioutil.ReadFile(tempPaths.Key)
495
-		if newErr != nil {
496
-			return nil, nil, err
497
-		}
498
-
499
-		keyPair, newErr = tls.X509KeyPair(cert, key)
500
-		if newErr != nil {
501
-			return nil, nil, err
502
-		}
487
+		return nil, nil, err
503 488
 	}
504 489
 
505 490
 	// Load the Certificates as server credentials
... ...
@@ -519,13 +518,6 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS
519 519
 	return clientTLSCreds, serverTLSCreds, nil
520 520
 }
521 521
 
522
-func genTempPaths(path CertPaths) CertPaths {
523
-	return CertPaths{
524
-		Key:  filepath.Join(filepath.Dir(path.Key), "."+filepath.Base(path.Key)),
525
-		Cert: filepath.Join(filepath.Dir(path.Cert), "."+filepath.Base(path.Cert)),
526
-	}
527
-}
528
-
529 522
 // NewServerTLSConfig returns a tls.Config configured for a TLS Server, given a tls.Certificate
530 523
 // and the PEM-encoded root CA Certificate
531 524
 func NewServerTLSConfig(cert *tls.Certificate, rootCAPool *x509.CertPool) (*tls.Config, error) {
532 525
new file mode 100644
... ...
@@ -0,0 +1,388 @@
0
+package ca
1
+
2
+import (
3
+	"crypto/rand"
4
+	"crypto/x509"
5
+	"encoding/pem"
6
+	"io/ioutil"
7
+	"os"
8
+	"path/filepath"
9
+	"strconv"
10
+	"strings"
11
+	"sync"
12
+
13
+	"crypto/tls"
14
+
15
+	"github.com/docker/swarmkit/ioutils"
16
+	"github.com/pkg/errors"
17
+)
18
+
19
+const (
20
+	// keyPerms are the permissions used to write the TLS keys
21
+	keyPerms = 0600
22
+	// certPerms are the permissions used to write TLS certificates
23
+	certPerms = 0644
24
+	// versionHeader is the TLS PEM key header that contains the KEK version
25
+	versionHeader = "kek-version"
26
+)
27
+
28
+// PEMKeyHeaders is something that needs to know about PEM headers when reading
29
+// or writing TLS keys.
30
+type PEMKeyHeaders interface {
31
+	// UnmarshalHeaders loads the headers map given the current KEK
32
+	UnmarshalHeaders(map[string]string, KEKData) (PEMKeyHeaders, error)
33
+	// MarshalHeaders returns a header map given the current KEK
34
+	MarshalHeaders(KEKData) (map[string]string, error)
35
+	// UpdateKEK may get a new PEMKeyHeaders if the KEK changes
36
+	UpdateKEK(KEKData, KEKData) PEMKeyHeaders
37
+}
38
+
39
+// KeyReader reads a TLS cert and key from disk
40
+type KeyReader interface {
41
+	Read() ([]byte, []byte, error)
42
+	Target() string
43
+}
44
+
45
+// KeyWriter writes a TLS key and cert to disk
46
+type KeyWriter interface {
47
+	Write([]byte, []byte, *KEKData) error
48
+	ViewAndUpdateHeaders(func(PEMKeyHeaders) (PEMKeyHeaders, error)) error
49
+	ViewAndRotateKEK(func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error
50
+	GetCurrentState() (PEMKeyHeaders, KEKData)
51
+	Target() string
52
+}
53
+
54
+// KEKData provides an optional update to the kek when writing.  The structure
55
+// is needed so that we can tell the difference between "do not encrypt anymore"
56
+// and there is "no update".
57
+type KEKData struct {
58
+	KEK     []byte
59
+	Version uint64
60
+}
61
+
62
+// ErrInvalidKEK means that we cannot decrypt the TLS key for some reason
63
+type ErrInvalidKEK struct {
64
+	Wrapped error
65
+}
66
+
67
+func (e ErrInvalidKEK) Error() string {
68
+	return e.Wrapped.Error()
69
+}
70
+
71
+// KeyReadWriter is an object that knows how to read and write TLS keys and certs to disk,
72
+// optionally encrypted and optionally updating PEM headers.
73
+type KeyReadWriter struct {
74
+	mu         sync.Mutex
75
+	kekData    KEKData
76
+	paths      CertPaths
77
+	headersObj PEMKeyHeaders
78
+}
79
+
80
+// NewKeyReadWriter creates a new KeyReadWriter
81
+func NewKeyReadWriter(paths CertPaths, kek []byte, headersObj PEMKeyHeaders) *KeyReadWriter {
82
+	return &KeyReadWriter{
83
+		kekData:    KEKData{KEK: kek},
84
+		paths:      paths,
85
+		headersObj: headersObj,
86
+	}
87
+}
88
+
89
+// Migrate checks to see if a temporary key file exists.  Older versions of
90
+// swarmkit wrote temporary keys instead of temporary certificates, so
91
+// migrate that temporary key if it exists.  We want to write temporary certificates,
92
+// instead of temporary keys, because we may need to periodically re-encrypt the
93
+// keys and modify the headers, and it's easier to have a single canonical key
94
+// location than two possible key locations.
95
+func (k *KeyReadWriter) Migrate() error {
96
+	tmpPaths := k.genTempPaths()
97
+	keyBytes, err := ioutil.ReadFile(tmpPaths.Key)
98
+	if err != nil {
99
+		return nil // no key?  no migration
100
+	}
101
+
102
+	// it does exist - no need to decrypt, because previous versions of swarmkit
103
+	// which supported this temporary key did not support encrypting TLS keys
104
+	cert, err := ioutil.ReadFile(k.paths.Cert)
105
+	if err != nil {
106
+		return os.RemoveAll(tmpPaths.Key) // no cert?  no migration
107
+	}
108
+
109
+	// nope, this does not match the cert
110
+	if _, err = tls.X509KeyPair(cert, keyBytes); err != nil {
111
+		return os.RemoveAll(tmpPaths.Key)
112
+	}
113
+
114
+	return os.Rename(tmpPaths.Key, k.paths.Key)
115
+}
116
+
117
+// Read will read a TLS cert and key from the given paths
118
+func (k *KeyReadWriter) Read() ([]byte, []byte, error) {
119
+	k.mu.Lock()
120
+	defer k.mu.Unlock()
121
+	keyBlock, err := k.readKey()
122
+	if err != nil {
123
+		return nil, nil, err
124
+	}
125
+
126
+	if version, ok := keyBlock.Headers[versionHeader]; ok {
127
+		if versionInt, err := strconv.ParseUint(version, 10, 64); err == nil {
128
+			k.kekData.Version = versionInt
129
+		}
130
+	}
131
+	delete(keyBlock.Headers, versionHeader)
132
+
133
+	if k.headersObj != nil {
134
+		newHeaders, err := k.headersObj.UnmarshalHeaders(keyBlock.Headers, k.kekData)
135
+		if err != nil {
136
+			return nil, nil, errors.Wrap(err, "unable to read TLS key headers")
137
+		}
138
+		k.headersObj = newHeaders
139
+	}
140
+
141
+	keyBytes := pem.EncodeToMemory(keyBlock)
142
+	cert, err := ioutil.ReadFile(k.paths.Cert)
143
+	// The cert is written to a temporary file first, then the key, and then
144
+	// the cert gets renamed - so, if interrupted, it's possible to end up with
145
+	// a cert that only exists in the temporary location.
146
+	switch {
147
+	case err == nil:
148
+		_, err = tls.X509KeyPair(cert, keyBytes)
149
+	case os.IsNotExist(err): //continue to try temp location
150
+		break
151
+	default:
152
+		return nil, nil, err
153
+	}
154
+
155
+	// either the cert doesn't exist, or it doesn't match the key - try the temp file, if it exists
156
+	if err != nil {
157
+		var tempErr error
158
+		tmpPaths := k.genTempPaths()
159
+		cert, tempErr = ioutil.ReadFile(tmpPaths.Cert)
160
+		if tempErr != nil {
161
+			return nil, nil, err // return the original error
162
+		}
163
+		if _, tempErr := tls.X509KeyPair(cert, keyBytes); tempErr != nil {
164
+			os.RemoveAll(tmpPaths.Cert) // nope, it doesn't match either - remove and return the original error
165
+			return nil, nil, err
166
+		}
167
+		os.Rename(tmpPaths.Cert, k.paths.Cert) // try to move the temp cert back to the regular location
168
+
169
+	}
170
+
171
+	return cert, keyBytes, nil
172
+}
173
+
174
+// ViewAndRotateKEK re-encrypts the key with a new KEK
175
+func (k *KeyReadWriter) ViewAndRotateKEK(cb func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error {
176
+	k.mu.Lock()
177
+	defer k.mu.Unlock()
178
+
179
+	updatedKEK, updatedHeaderObj, err := cb(k.kekData, k.headersObj)
180
+	if err != nil {
181
+		return err
182
+	}
183
+
184
+	keyBlock, err := k.readKey()
185
+	if err != nil {
186
+		return err
187
+	}
188
+
189
+	if err := k.writeKey(keyBlock, updatedKEK, updatedHeaderObj); err != nil {
190
+		return err
191
+	}
192
+	return nil
193
+}
194
+
195
+// ViewAndUpdateHeaders updates the header manager, and updates any headers on the existing key
196
+func (k *KeyReadWriter) ViewAndUpdateHeaders(cb func(PEMKeyHeaders) (PEMKeyHeaders, error)) error {
197
+	k.mu.Lock()
198
+	defer k.mu.Unlock()
199
+
200
+	pkh, err := cb(k.headersObj)
201
+	if err != nil {
202
+		return err
203
+	}
204
+
205
+	keyBlock, err := k.readKeyblock()
206
+	if err != nil {
207
+		return err
208
+	}
209
+
210
+	headers := make(map[string]string)
211
+	if pkh != nil {
212
+		var err error
213
+		headers, err = pkh.MarshalHeaders(k.kekData)
214
+		if err != nil {
215
+			return err
216
+		}
217
+	}
218
+	// we WANT any original encryption headers
219
+	for key, value := range keyBlock.Headers {
220
+		normalizedKey := strings.TrimSpace(strings.ToLower(key))
221
+		if normalizedKey == "proc-type" || normalizedKey == "dek-info" {
222
+			headers[key] = value
223
+		}
224
+	}
225
+	headers[versionHeader] = strconv.FormatUint(k.kekData.Version, 10)
226
+	keyBlock.Headers = headers
227
+
228
+	if err = ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil {
229
+		return err
230
+	}
231
+	k.headersObj = pkh
232
+	return nil
233
+}
234
+
235
+// GetCurrentState returns the current KEK data, including version
236
+func (k *KeyReadWriter) GetCurrentState() (PEMKeyHeaders, KEKData) {
237
+	k.mu.Lock()
238
+	defer k.mu.Unlock()
239
+	return k.headersObj, k.kekData
240
+}
241
+
242
+// Write attempts write a cert and key to text.  This can also optionally update
243
+// the KEK while writing, if an updated KEK is provided.  If the pointer to the
244
+// update KEK is nil, then we don't update. If the updated KEK itself is nil,
245
+// then we update the KEK to be nil (data should be unencrypted).
246
+func (k *KeyReadWriter) Write(certBytes, plaintextKeyBytes []byte, kekData *KEKData) error {
247
+	k.mu.Lock()
248
+	defer k.mu.Unlock()
249
+
250
+	// current assumption is that the cert and key will be in the same directory
251
+	if err := os.MkdirAll(filepath.Dir(k.paths.Key), 0755); err != nil {
252
+		return err
253
+	}
254
+
255
+	// Ensure that we will have a keypair on disk at all times by writing the cert to a
256
+	// temp path first.  This is because we want to have only a single copy of the key
257
+	// for rotation and header modification.
258
+	tmpPaths := k.genTempPaths()
259
+	if err := ioutils.AtomicWriteFile(tmpPaths.Cert, certBytes, certPerms); err != nil {
260
+		return err
261
+	}
262
+
263
+	keyBlock, _ := pem.Decode(plaintextKeyBytes)
264
+	if keyBlock == nil {
265
+		return errors.New("invalid PEM-encoded private key")
266
+	}
267
+
268
+	if kekData == nil {
269
+		kekData = &k.kekData
270
+	}
271
+	pkh := k.headersObj
272
+	if k.headersObj != nil {
273
+		pkh = k.headersObj.UpdateKEK(k.kekData, *kekData)
274
+	}
275
+
276
+	if err := k.writeKey(keyBlock, *kekData, pkh); err != nil {
277
+		return err
278
+	}
279
+	return os.Rename(tmpPaths.Cert, k.paths.Cert)
280
+}
281
+
282
+func (k *KeyReadWriter) genTempPaths() CertPaths {
283
+	return CertPaths{
284
+		Key:  filepath.Join(filepath.Dir(k.paths.Key), "."+filepath.Base(k.paths.Key)),
285
+		Cert: filepath.Join(filepath.Dir(k.paths.Cert), "."+filepath.Base(k.paths.Cert)),
286
+	}
287
+}
288
+
289
+// Target returns a string representation of this KeyReadWriter, namely where
290
+// it is writing to
291
+func (k *KeyReadWriter) Target() string {
292
+	return k.paths.Cert
293
+}
294
+
295
+func (k *KeyReadWriter) readKeyblock() (*pem.Block, error) {
296
+	key, err := ioutil.ReadFile(k.paths.Key)
297
+	if err != nil {
298
+		return nil, err
299
+	}
300
+
301
+	// Decode the PEM private key
302
+	keyBlock, _ := pem.Decode(key)
303
+	if keyBlock == nil {
304
+		return nil, errors.New("invalid PEM-encoded private key")
305
+	}
306
+
307
+	return keyBlock, nil
308
+}
309
+
310
+// readKey returns the decrypted key pem bytes, and enforces the KEK if applicable
311
+// (writes it back with the correct encryption if it is not correctly encrypted)
312
+func (k *KeyReadWriter) readKey() (*pem.Block, error) {
313
+	keyBlock, err := k.readKeyblock()
314
+	if err != nil {
315
+		return nil, err
316
+	}
317
+
318
+	if !x509.IsEncryptedPEMBlock(keyBlock) {
319
+		return keyBlock, nil
320
+	}
321
+
322
+	// If it's encrypted, we can't read without a passphrase (we're assuming
323
+	// empty passphrases iare invalid)
324
+	if k.kekData.KEK == nil {
325
+		return nil, ErrInvalidKEK{Wrapped: x509.IncorrectPasswordError}
326
+	}
327
+
328
+	derBytes, err := x509.DecryptPEMBlock(keyBlock, k.kekData.KEK)
329
+	if err != nil {
330
+		return nil, ErrInvalidKEK{Wrapped: err}
331
+	}
332
+	// remove encryption PEM headers
333
+	headers := make(map[string]string)
334
+	mergePEMHeaders(headers, keyBlock.Headers)
335
+
336
+	return &pem.Block{
337
+		Type:    keyBlock.Type, // the key type doesn't change
338
+		Bytes:   derBytes,
339
+		Headers: headers,
340
+	}, nil
341
+}
342
+
343
+// writeKey takes an unencrypted keyblock and, if the kek is not nil, encrypts it before
344
+// writing it to disk.  If the kek is nil, writes it to disk unencrypted.
345
+func (k *KeyReadWriter) writeKey(keyBlock *pem.Block, kekData KEKData, pkh PEMKeyHeaders) error {
346
+	if kekData.KEK != nil {
347
+		encryptedPEMBlock, err := x509.EncryptPEMBlock(rand.Reader,
348
+			keyBlock.Type,
349
+			keyBlock.Bytes,
350
+			kekData.KEK,
351
+			x509.PEMCipherAES256)
352
+		if err != nil {
353
+			return err
354
+		}
355
+		if encryptedPEMBlock.Headers == nil {
356
+			return errors.New("unable to encrypt key - invalid PEM file produced")
357
+		}
358
+		keyBlock = encryptedPEMBlock
359
+	}
360
+
361
+	if pkh != nil {
362
+		headers, err := pkh.MarshalHeaders(kekData)
363
+		if err != nil {
364
+			return err
365
+		}
366
+		mergePEMHeaders(keyBlock.Headers, headers)
367
+	}
368
+	keyBlock.Headers[versionHeader] = strconv.FormatUint(kekData.Version, 10)
369
+
370
+	if err := ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil {
371
+		return err
372
+	}
373
+	k.kekData = kekData
374
+	k.headersObj = pkh
375
+	return nil
376
+}
377
+
378
+// merges one set of PEM headers onto another, excepting for key encryption value
379
+// "proc-type" and "dek-info"
380
+func mergePEMHeaders(original, newSet map[string]string) {
381
+	for key, value := range newSet {
382
+		normalizedKey := strings.TrimSpace(strings.ToLower(key))
383
+		if normalizedKey != "proc-type" && normalizedKey != "dek-info" {
384
+			original[key] = value
385
+		}
386
+	}
387
+}
... ...
@@ -69,6 +69,33 @@ func (s *Server) SetReconciliationRetryInterval(reconciliationRetryInterval time
69 69
 	s.reconciliationRetryInterval = reconciliationRetryInterval
70 70
 }
71 71
 
72
+// GetUnlockKey is responsible for returning the current unlock key used for encrypting TLS private keys and
73
+// other at rest data.  Access to this RPC call should only be allowed via mutual TLS from managers.
74
+func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequest) (*api.GetUnlockKeyResponse, error) {
75
+	// This directly queries the store, rather than storing the unlock key and version on
76
+	// the `Server` object and updating it `updateCluster` is called, because we need this
77
+	// API to return the latest version of the key.  Otherwise, there might be a slight delay
78
+	// between when the cluster gets updated, and when this function returns the latest key.
79
+	// This delay is currently unacceptable because this RPC call is the only way, after a
80
+	// cluster update, to get the actual value of the unlock key, and we don't want to return
81
+	// a cached value.
82
+	resp := api.GetUnlockKeyResponse{}
83
+	s.store.View(func(tx store.ReadTx) {
84
+		cluster := store.GetCluster(tx, s.securityConfig.ClientTLSCreds.Organization())
85
+		resp.Version = cluster.Meta.Version
86
+		if cluster.Spec.EncryptionConfig.AutoLockManagers {
87
+			for _, encryptionKey := range cluster.UnlockKeys {
88
+				if encryptionKey.Subsystem == ManagerRole {
89
+					resp.UnlockKey = encryptionKey.Key
90
+					return
91
+				}
92
+			}
93
+		}
94
+	})
95
+
96
+	return &resp, nil
97
+}
98
+
72 99
 // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID
73 100
 func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) {
74 101
 	if request.NodeID == "" {
... ...
@@ -6,6 +6,7 @@ import (
6 6
 
7 7
 	"github.com/docker/swarmkit/api"
8 8
 	"github.com/docker/swarmkit/ca"
9
+	"github.com/docker/swarmkit/manager/encryption"
9 10
 	"github.com/docker/swarmkit/manager/state/store"
10 11
 	"github.com/docker/swarmkit/protobuf/ptypes"
11 12
 	"golang.org/x/net/context"
... ...
@@ -107,12 +108,38 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe
107 107
 
108 108
 		expireBlacklistedCerts(cluster)
109 109
 
110
-		if request.Rotation.RotateWorkerToken {
110
+		if request.Rotation.WorkerJoinToken {
111 111
 			cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA)
112 112
 		}
113
-		if request.Rotation.RotateManagerToken {
113
+		if request.Rotation.ManagerJoinToken {
114 114
 			cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA)
115 115
 		}
116
+
117
+		var unlockKeys []*api.EncryptionKey
118
+		var managerKey *api.EncryptionKey
119
+		for _, eKey := range cluster.UnlockKeys {
120
+			if eKey.Subsystem == ca.ManagerRole {
121
+				if !cluster.Spec.EncryptionConfig.AutoLockManagers {
122
+					continue
123
+				}
124
+				managerKey = eKey
125
+			}
126
+			unlockKeys = append(unlockKeys, eKey)
127
+		}
128
+
129
+		switch {
130
+		case !cluster.Spec.EncryptionConfig.AutoLockManagers:
131
+			break
132
+		case managerKey == nil:
133
+			unlockKeys = append(unlockKeys, &api.EncryptionKey{
134
+				Subsystem: ca.ManagerRole,
135
+				Key:       encryption.GenerateSecretKey(),
136
+			})
137
+		case request.Rotation.ManagerUnlockKey:
138
+			managerKey.Key = encryption.GenerateSecretKey()
139
+		}
140
+		cluster.UnlockKeys = unlockKeys
141
+
116 142
 		return store.UpdateCluster(tx, cluster)
117 143
 	})
118 144
 	if err != nil {
119 145
new file mode 100644
... ...
@@ -0,0 +1,269 @@
0
+package manager
1
+
2
+import (
3
+	"crypto/subtle"
4
+	"encoding/base64"
5
+	"fmt"
6
+
7
+	"github.com/docker/swarmkit/ca"
8
+	"github.com/docker/swarmkit/manager/encryption"
9
+	"github.com/docker/swarmkit/manager/state/raft"
10
+)
11
+
12
+const (
13
+	// the raft DEK (data encryption key) is stored in the TLS key as a header
14
+	// these are the header values
15
+	pemHeaderRaftDEK              = "raft-dek"
16
+	pemHeaderRaftPendingDEK       = "raft-dek-pending"
17
+	pemHeaderRaftDEKNeedsRotation = "raft-dek-needs-rotation"
18
+)
19
+
20
+// RaftDEKData contains all the data stored in TLS pem headers
21
+type RaftDEKData struct {
22
+	raft.EncryptionKeys
23
+	NeedsRotation bool
24
+}
25
+
26
+// UnmarshalHeaders loads the state of the DEK manager given the current TLS headers
27
+func (r RaftDEKData) UnmarshalHeaders(headers map[string]string, kekData ca.KEKData) (ca.PEMKeyHeaders, error) {
28
+	var (
29
+		currentDEK, pendingDEK []byte
30
+		err                    error
31
+	)
32
+
33
+	if currentDEKStr, ok := headers[pemHeaderRaftDEK]; ok {
34
+		currentDEK, err = decodePEMHeaderValue(currentDEKStr, kekData.KEK)
35
+		if err != nil {
36
+			return nil, err
37
+		}
38
+	}
39
+	if pendingDEKStr, ok := headers[pemHeaderRaftPendingDEK]; ok {
40
+		pendingDEK, err = decodePEMHeaderValue(pendingDEKStr, kekData.KEK)
41
+		if err != nil {
42
+			return nil, err
43
+		}
44
+	}
45
+
46
+	if pendingDEK != nil && currentDEK == nil {
47
+		return nil, fmt.Errorf("there is a pending DEK, but no current DEK")
48
+	}
49
+
50
+	_, ok := headers[pemHeaderRaftDEKNeedsRotation]
51
+	return RaftDEKData{
52
+		NeedsRotation: ok,
53
+		EncryptionKeys: raft.EncryptionKeys{
54
+			CurrentDEK: currentDEK,
55
+			PendingDEK: pendingDEK,
56
+		},
57
+	}, nil
58
+}
59
+
60
+// MarshalHeaders returns new headers given the current KEK
61
+func (r RaftDEKData) MarshalHeaders(kekData ca.KEKData) (map[string]string, error) {
62
+	headers := make(map[string]string)
63
+	for headerKey, contents := range map[string][]byte{
64
+		pemHeaderRaftDEK:        r.CurrentDEK,
65
+		pemHeaderRaftPendingDEK: r.PendingDEK,
66
+	} {
67
+		if contents != nil {
68
+			dekStr, err := encodePEMHeaderValue(contents, kekData.KEK)
69
+			if err != nil {
70
+				return nil, err
71
+			}
72
+			headers[headerKey] = dekStr
73
+		}
74
+	}
75
+
76
+	if r.NeedsRotation {
77
+		headers[pemHeaderRaftDEKNeedsRotation] = "true"
78
+	}
79
+
80
+	// return a function that updates the dek data on write success
81
+	return headers, nil
82
+}
83
+
84
+// UpdateKEK optionally sets NeedRotation to true if we go from unlocked to locked
85
+func (r RaftDEKData) UpdateKEK(oldKEK, candidateKEK ca.KEKData) ca.PEMKeyHeaders {
86
+	if _, unlockedToLocked, err := compareKEKs(oldKEK, candidateKEK); err == nil && unlockedToLocked {
87
+		return RaftDEKData{
88
+			EncryptionKeys: r.EncryptionKeys,
89
+			NeedsRotation:  true,
90
+		}
91
+	}
92
+	return r
93
+}
94
+
95
+// Returns whether the old KEK should be replaced with the new KEK, whether we went from
96
+// unlocked to locked, and whether there was an error (the versions are the same, but the
97
+// keks are different)
98
+func compareKEKs(oldKEK, candidateKEK ca.KEKData) (bool, bool, error) {
99
+	keksEqual := subtle.ConstantTimeCompare(oldKEK.KEK, candidateKEK.KEK) == 1
100
+	switch {
101
+	case oldKEK.Version == candidateKEK.Version && !keksEqual:
102
+		return false, false, fmt.Errorf("candidate KEK has the same version as the current KEK, but a different KEK value")
103
+	case oldKEK.Version >= candidateKEK.Version || keksEqual:
104
+		return false, false, nil
105
+	default:
106
+		return true, oldKEK.KEK == nil, nil
107
+	}
108
+}
109
+
110
+// RaftDEKManager manages the raft DEK keys using TLS headers
111
+type RaftDEKManager struct {
112
+	kw         ca.KeyWriter
113
+	rotationCh chan struct{}
114
+}
115
+
116
+var errNoUpdateNeeded = fmt.Errorf("don't need to rotate or update")
117
+
118
+// this error is returned if the KeyReadWriter's PEMKeyHeaders object is no longer a RaftDEKData object -
119
+// this can happen if the node is no longer a manager, for example
120
+var errNotUsingRaftDEKData = fmt.Errorf("RaftDEKManager can no longer store and manage TLS key headers")
121
+
122
+// NewRaftDEKManager returns a RaftDEKManager that uses the current key writer
123
+// and header manager
124
+func NewRaftDEKManager(kw ca.KeyWriter) (*RaftDEKManager, error) {
125
+	// If there is no current DEK, generate one and write it to disk
126
+	err := kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
127
+		dekData, ok := h.(RaftDEKData)
128
+		// it wasn't a raft DEK manager before - just replace it
129
+		if !ok || dekData.CurrentDEK == nil {
130
+			return RaftDEKData{
131
+				EncryptionKeys: raft.EncryptionKeys{
132
+					CurrentDEK: encryption.GenerateSecretKey(),
133
+				},
134
+			}, nil
135
+		}
136
+		return nil, errNoUpdateNeeded
137
+	})
138
+	if err != nil && err != errNoUpdateNeeded {
139
+		return nil, err
140
+	}
141
+	return &RaftDEKManager{
142
+		kw:         kw,
143
+		rotationCh: make(chan struct{}, 1),
144
+	}, nil
145
+}
146
+
147
+// NeedsRotation returns a boolean about whether we should do a rotation
148
+func (r *RaftDEKManager) NeedsRotation() bool {
149
+	h, _ := r.kw.GetCurrentState()
150
+	data, ok := h.(RaftDEKData)
151
+	if !ok {
152
+		return false
153
+	}
154
+	return data.NeedsRotation || data.EncryptionKeys.PendingDEK != nil
155
+}
156
+
157
+// GetKeys returns the current set of DEKs.  If NeedsRotation is true, and there
158
+// is no existing PendingDEK, it will try to create one.  If there are any errors
159
+// doing so, just return the original.
160
+func (r *RaftDEKManager) GetKeys() raft.EncryptionKeys {
161
+	var newKeys, originalKeys raft.EncryptionKeys
162
+	err := r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
163
+		data, ok := h.(RaftDEKData)
164
+		if !ok {
165
+			return nil, errNotUsingRaftDEKData
166
+		}
167
+		originalKeys = data.EncryptionKeys
168
+		if !data.NeedsRotation || data.PendingDEK != nil {
169
+			return nil, errNoUpdateNeeded
170
+		}
171
+		newKeys = raft.EncryptionKeys{
172
+			CurrentDEK: data.CurrentDEK,
173
+			PendingDEK: encryption.GenerateSecretKey(),
174
+		}
175
+		return RaftDEKData{EncryptionKeys: newKeys}, nil
176
+	})
177
+	if err != nil {
178
+		return originalKeys
179
+	}
180
+	return newKeys
181
+}
182
+
183
+// RotationNotify the channel used to notify subscribers as to whether there
184
+// should be a rotation done
185
+func (r *RaftDEKManager) RotationNotify() chan struct{} {
186
+	return r.rotationCh
187
+}
188
+
189
+// UpdateKeys will set the updated encryption keys in the headers.  This finishes
190
+// a rotation, and is expected to set the CurrentDEK to the previous PendingDEK.
191
+func (r *RaftDEKManager) UpdateKeys(newKeys raft.EncryptionKeys) error {
192
+	return r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
193
+		data, ok := h.(RaftDEKData)
194
+		if !ok {
195
+			return nil, errNotUsingRaftDEKData
196
+		}
197
+		// If there is no current DEK, we are basically wiping out all DEKs (no header object)
198
+		if newKeys.CurrentDEK == nil {
199
+			return nil, nil
200
+		}
201
+		return RaftDEKData{
202
+			EncryptionKeys: newKeys,
203
+			NeedsRotation:  data.NeedsRotation,
204
+		}, nil
205
+	})
206
+}
207
+
208
+// MaybeUpdateKEK does a KEK rotation if one is required.  Returns whether
209
+// the kek was updated, whether it went from unlocked to locked, and any errors.
210
+func (r *RaftDEKManager) MaybeUpdateKEK(candidateKEK ca.KEKData) (bool, bool, error) {
211
+	var updated, unlockedToLocked bool
212
+	err := r.kw.ViewAndRotateKEK(func(currentKEK ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
213
+		var err error
214
+		updated, unlockedToLocked, err = compareKEKs(currentKEK, candidateKEK)
215
+		if err == nil && !updated { // if we don't need to rotate the KEK, don't bother updating
216
+			err = errNoUpdateNeeded
217
+		}
218
+		if err != nil {
219
+			return ca.KEKData{}, nil, err
220
+		}
221
+
222
+		data, ok := h.(RaftDEKData)
223
+		if !ok {
224
+			return ca.KEKData{}, nil, errNotUsingRaftDEKData
225
+		}
226
+
227
+		if unlockedToLocked {
228
+			data.NeedsRotation = true
229
+		}
230
+		return candidateKEK, data, nil
231
+	})
232
+	if err == errNoUpdateNeeded {
233
+		err = nil
234
+	}
235
+
236
+	if err == nil && unlockedToLocked {
237
+		r.rotationCh <- struct{}{}
238
+	}
239
+	return updated, unlockedToLocked, err
240
+}
241
+
242
+func decodePEMHeaderValue(headerValue string, kek []byte) ([]byte, error) {
243
+	var decrypter encryption.Decrypter = encryption.NoopCrypter
244
+	if kek != nil {
245
+		_, decrypter = encryption.Defaults(kek)
246
+	}
247
+	valueBytes, err := base64.StdEncoding.DecodeString(headerValue)
248
+	if err != nil {
249
+		return nil, err
250
+	}
251
+	result, err := encryption.Decrypt(valueBytes, decrypter)
252
+	if err != nil {
253
+		return nil, ca.ErrInvalidKEK{Wrapped: err}
254
+	}
255
+	return result, nil
256
+}
257
+
258
+func encodePEMHeaderValue(headerValue []byte, kek []byte) (string, error) {
259
+	var encrypter encryption.Encrypter = encryption.NoopCrypter
260
+	if kek != nil {
261
+		encrypter, _ = encryption.Defaults(kek)
262
+	}
263
+	encrypted, err := encryption.Encrypt(headerValue, encrypter)
264
+	if err != nil {
265
+		return "", err
266
+	}
267
+	return base64.StdEncoding.EncodeToString(encrypted), nil
268
+}
0 269
new file mode 100644
... ...
@@ -0,0 +1,132 @@
0
+package encryption
1
+
2
+import (
3
+	"crypto/rand"
4
+	"encoding/base64"
5
+	"fmt"
6
+	"io"
7
+	"strings"
8
+
9
+	"github.com/docker/swarmkit/api"
10
+	"github.com/gogo/protobuf/proto"
11
+	"github.com/pkg/errors"
12
+)
13
+
14
+// This package defines the interfaces and encryption package
15
+
16
+const humanReadablePrefix = "SWMKEY-1-"
17
+
18
+// ErrCannotDecrypt is the type of error returned when some data cannot be decryptd as plaintext
19
+type ErrCannotDecrypt struct {
20
+	msg string
21
+}
22
+
23
+func (e ErrCannotDecrypt) Error() string {
24
+	return e.msg
25
+}
26
+
27
+// A Decrypter can decrypt an encrypted record
28
+type Decrypter interface {
29
+	Decrypt(api.MaybeEncryptedRecord) ([]byte, error)
30
+}
31
+
32
+// A Encrypter can encrypt some bytes into an encrypted record
33
+type Encrypter interface {
34
+	Encrypt(data []byte) (*api.MaybeEncryptedRecord, error)
35
+}
36
+
37
+type noopCrypter struct{}
38
+
39
+func (n noopCrypter) Decrypt(e api.MaybeEncryptedRecord) ([]byte, error) {
40
+	if e.Algorithm != n.Algorithm() {
41
+		return nil, fmt.Errorf("record is encrypted")
42
+	}
43
+	return e.Data, nil
44
+}
45
+
46
+func (n noopCrypter) Encrypt(data []byte) (*api.MaybeEncryptedRecord, error) {
47
+	return &api.MaybeEncryptedRecord{
48
+		Algorithm: n.Algorithm(),
49
+		Data:      data,
50
+	}, nil
51
+}
52
+
53
+func (n noopCrypter) Algorithm() api.MaybeEncryptedRecord_Algorithm {
54
+	return api.MaybeEncryptedRecord_NotEncrypted
55
+}
56
+
57
+// NoopCrypter is just a pass-through crypter - it does not actually encrypt or
58
+// decrypt any data
59
+var NoopCrypter = noopCrypter{}
60
+
61
+// Decrypt turns a slice of bytes serialized as an MaybeEncryptedRecord into a slice of plaintext bytes
62
+func Decrypt(encryptd []byte, decrypter Decrypter) ([]byte, error) {
63
+	if decrypter == nil {
64
+		return nil, ErrCannotDecrypt{msg: "no decrypter specified"}
65
+	}
66
+	r := api.MaybeEncryptedRecord{}
67
+	if err := proto.Unmarshal(encryptd, &r); err != nil {
68
+		// nope, this wasn't marshalled as a MaybeEncryptedRecord
69
+		return nil, ErrCannotDecrypt{msg: "unable to unmarshal as MaybeEncryptedRecord"}
70
+	}
71
+	plaintext, err := decrypter.Decrypt(r)
72
+	if err != nil {
73
+		return nil, ErrCannotDecrypt{msg: err.Error()}
74
+	}
75
+	return plaintext, nil
76
+}
77
+
78
+// Encrypt turns a slice of bytes into a serialized MaybeEncryptedRecord slice of bytes
79
+func Encrypt(plaintext []byte, encrypter Encrypter) ([]byte, error) {
80
+	if encrypter == nil {
81
+		return nil, fmt.Errorf("no encrypter specified")
82
+	}
83
+
84
+	encryptedRecord, err := encrypter.Encrypt(plaintext)
85
+	if err != nil {
86
+		return nil, errors.Wrap(err, "unable to encrypt data")
87
+	}
88
+
89
+	data, err := proto.Marshal(encryptedRecord)
90
+	if err != nil {
91
+		return nil, errors.Wrap(err, "unable to marshal as MaybeEncryptedRecord")
92
+	}
93
+
94
+	return data, nil
95
+}
96
+
97
+// Defaults returns a default encrypter and decrypter
98
+func Defaults(key []byte) (Encrypter, Decrypter) {
99
+	n := NewNACLSecretbox(key)
100
+	return n, n
101
+}
102
+
103
+// GenerateSecretKey generates a secret key that can be used for encrypting data
104
+// using this package
105
+func GenerateSecretKey() []byte {
106
+	secretData := make([]byte, naclSecretboxKeySize)
107
+	if _, err := io.ReadFull(rand.Reader, secretData); err != nil {
108
+		// panic if we can't read random data
109
+		panic(errors.Wrap(err, "failed to read random bytes"))
110
+	}
111
+	return secretData
112
+}
113
+
114
+// HumanReadableKey displays a secret key in a human readable way
115
+func HumanReadableKey(key []byte) string {
116
+	// base64-encode the key
117
+	return humanReadablePrefix + base64.RawStdEncoding.EncodeToString(key)
118
+}
119
+
120
+// ParseHumanReadableKey returns a key as bytes from recognized serializations of
121
+// said keys
122
+func ParseHumanReadableKey(key string) ([]byte, error) {
123
+	if !strings.HasPrefix(key, humanReadablePrefix) {
124
+		return nil, fmt.Errorf("invalid key string")
125
+	}
126
+	keyBytes, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(key, humanReadablePrefix))
127
+	if err != nil {
128
+		return nil, fmt.Errorf("invalid key string")
129
+	}
130
+	return keyBytes, nil
131
+}
0 132
new file mode 100644
... ...
@@ -0,0 +1,73 @@
0
+package encryption
1
+
2
+import (
3
+	"crypto/rand"
4
+	"fmt"
5
+	"io"
6
+
7
+	"github.com/docker/swarmkit/api"
8
+
9
+	"golang.org/x/crypto/nacl/secretbox"
10
+)
11
+
12
+const naclSecretboxKeySize = 32
13
+const naclSecretboxNonceSize = 24
14
+
15
+// This provides the default implementation of an encrypter and decrypter, as well
16
+// as the default KDF function.
17
+
18
+// NACLSecretbox is an implementation of an encrypter/decrypter.  Encrypting
19
+// generates random Nonces.
20
+type NACLSecretbox struct {
21
+	key [naclSecretboxKeySize]byte
22
+}
23
+
24
+// NewNACLSecretbox returns a new NACL secretbox encrypter/decrypter with the given key
25
+func NewNACLSecretbox(key []byte) NACLSecretbox {
26
+	secretbox := NACLSecretbox{}
27
+	copy(secretbox.key[:], key)
28
+	return secretbox
29
+}
30
+
31
+// Algorithm returns the type of algorhtm this is (NACL Secretbox using XSalsa20 and Poly1305)
32
+func (n NACLSecretbox) Algorithm() api.MaybeEncryptedRecord_Algorithm {
33
+	return api.MaybeEncryptedRecord_NACLSecretboxSalsa20Poly1305
34
+}
35
+
36
+// Encrypt encrypts some bytes and returns an encrypted record
37
+func (n NACLSecretbox) Encrypt(data []byte) (*api.MaybeEncryptedRecord, error) {
38
+	var nonce [24]byte
39
+	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
40
+		return nil, err
41
+	}
42
+
43
+	// Seal's first argument is an "out", the data that the new encrypted message should be
44
+	// appended to.  Since we don't want to append anything, we pass nil.
45
+	encrypted := secretbox.Seal(nil, data, &nonce, &n.key)
46
+	return &api.MaybeEncryptedRecord{
47
+		Algorithm: n.Algorithm(),
48
+		Data:      encrypted,
49
+		Nonce:     nonce[:],
50
+	}, nil
51
+}
52
+
53
+// Decrypt decrypts a MaybeEncryptedRecord and returns some bytes
54
+func (n NACLSecretbox) Decrypt(record api.MaybeEncryptedRecord) ([]byte, error) {
55
+	if record.Algorithm != n.Algorithm() {
56
+		return nil, fmt.Errorf("not a NACL secretbox record")
57
+	}
58
+	if len(record.Nonce) != naclSecretboxNonceSize {
59
+		return nil, fmt.Errorf("invalid nonce size for NACL secretbox: require 24, got %d", len(record.Nonce))
60
+	}
61
+
62
+	var decryptNonce [naclSecretboxNonceSize]byte
63
+	copy(decryptNonce[:], record.Nonce[:naclSecretboxNonceSize])
64
+
65
+	// Open's first argument is an "out", the data that the decrypted message should be
66
+	// appended to.  Since we don't want to append anything, we pass nil.
67
+	decrypted, ok := secretbox.Open(nil, record.Data, &decryptNonce, &n.key)
68
+	if !ok {
69
+		return nil, fmt.Errorf("decryption error using NACL secretbox")
70
+	}
71
+	return decrypted, nil
72
+}
... ...
@@ -29,9 +29,11 @@ import (
29 29
 	"github.com/docker/swarmkit/manager/orchestrator/taskreaper"
30 30
 	"github.com/docker/swarmkit/manager/resourceapi"
31 31
 	"github.com/docker/swarmkit/manager/scheduler"
32
+	"github.com/docker/swarmkit/manager/state"
32 33
 	"github.com/docker/swarmkit/manager/state/raft"
33 34
 	"github.com/docker/swarmkit/manager/state/store"
34 35
 	"github.com/docker/swarmkit/protobuf/ptypes"
36
+	"github.com/docker/swarmkit/remotes"
35 37
 	"github.com/docker/swarmkit/xnet"
36 38
 	"github.com/pkg/errors"
37 39
 	"golang.org/x/net/context"
... ...
@@ -86,6 +88,17 @@ type Config struct {
86 86
 	// HeartbeatTick defines the amount of ticks between each
87 87
 	// heartbeat sent to other members for health-check purposes
88 88
 	HeartbeatTick uint32
89
+
90
+	// AutoLockManagers determines whether or not managers require an unlock key
91
+	// when starting from a stopped state.  This configuration parameter is only
92
+	// applicable when bootstrapping a new cluster for the first time.
93
+	AutoLockManagers bool
94
+
95
+	// UnlockKey is the key to unlock a node - used for decrypting manager TLS keys
96
+	// as well as the raft data encryption key (DEK).  It is applicable when
97
+	// bootstrapping a cluster for the first time (it's a cluster-wide setting),
98
+	// and also when loading up any raft data on disk (as a KEK for the raft DEK).
99
+	UnlockKey []byte
89 100
 }
90 101
 
91 102
 // Manager is the cluster manager for Swarm.
... ...
@@ -108,6 +121,7 @@ type Manager struct {
108 108
 	server                 *grpc.Server
109 109
 	localserver            *grpc.Server
110 110
 	raftNode               *raft.Node
111
+	dekRotator             *RaftDEKManager
111 112
 
112 113
 	cancelFunc context.CancelFunc
113 114
 
... ...
@@ -217,6 +231,11 @@ func New(config *Config) (*Manager, error) {
217 217
 		raftCfg.HeartbeatTick = int(config.HeartbeatTick)
218 218
 	}
219 219
 
220
+	dekRotator, err := NewRaftDEKManager(config.SecurityConfig.KeyWriter())
221
+	if err != nil {
222
+		return nil, err
223
+	}
224
+
220 225
 	newNodeOpts := raft.NodeOptions{
221 226
 		ID:              config.SecurityConfig.ClientTLSCreds.NodeID(),
222 227
 		Addr:            advertiseAddr,
... ...
@@ -225,6 +244,7 @@ func New(config *Config) (*Manager, error) {
225 225
 		StateDir:        raftStateDir,
226 226
 		ForceNewCluster: config.ForceNewCluster,
227 227
 		TLSCredentials:  config.SecurityConfig.ClientTLSCreds,
228
+		KeyRotator:      dekRotator,
228 229
 	}
229 230
 	raftNode := raft.NewNode(newNodeOpts)
230 231
 
... ...
@@ -241,6 +261,7 @@ func New(config *Config) (*Manager, error) {
241 241
 		localserver: grpc.NewServer(opts...),
242 242
 		raftNode:    raftNode,
243 243
 		started:     make(chan struct{}),
244
+		dekRotator:  dekRotator,
244 245
 	}
245 246
 
246 247
 	return m, nil
... ...
@@ -320,6 +341,7 @@ func (m *Manager) Run(parent context.Context) error {
320 320
 	forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil }
321 321
 	localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.raftNode, forwardAsOwnRequest)
322 322
 	localProxyLogsAPI := api.NewRaftProxyLogsServer(m.logbroker, m.raftNode, forwardAsOwnRequest)
323
+	localCAAPI := api.NewRaftProxyCAServer(m.caserver, m.raftNode, forwardAsOwnRequest)
323 324
 
324 325
 	// Everything registered on m.server should be an authenticated
325 326
 	// wrapper, or a proxy wrapping an authenticated wrapper!
... ...
@@ -337,6 +359,7 @@ func (m *Manager) Run(parent context.Context) error {
337 337
 	api.RegisterControlServer(m.localserver, localProxyControlAPI)
338 338
 	api.RegisterLogsServer(m.localserver, localProxyLogsAPI)
339 339
 	api.RegisterHealthServer(m.localserver, localHealthServer)
340
+	api.RegisterCAServer(m.localserver, localCAAPI)
340 341
 
341 342
 	healthServer.SetServingStatus("Raft", api.HealthCheckResponse_NOT_SERVING)
342 343
 	localHealthServer.SetServingStatus("ControlAPI", api.HealthCheckResponse_NOT_SERVING)
... ...
@@ -362,8 +385,12 @@ func (m *Manager) Run(parent context.Context) error {
362 362
 
363 363
 	close(m.started)
364 364
 
365
+	watchDone := make(chan struct{})
366
+	watchCtx, watchCtxCancel := context.WithCancel(parent)
365 367
 	go func() {
366 368
 		err := m.raftNode.Run(ctx)
369
+		watchCtxCancel()
370
+		<-watchDone
367 371
 		if err != nil {
368 372
 			log.G(ctx).Error(err)
369 373
 			m.Stop(ctx)
... ...
@@ -380,6 +407,10 @@ func (m *Manager) Run(parent context.Context) error {
380 380
 	}
381 381
 	raftConfig := c.Spec.Raft
382 382
 
383
+	if err := m.watchForKEKChanges(watchCtx, watchDone); err != nil {
384
+		return err
385
+	}
386
+
383 387
 	if int(raftConfig.ElectionTick) != m.raftNode.Config.ElectionTick {
384 388
 		log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.ElectionTick, raftConfig.ElectionTick)
385 389
 	}
... ...
@@ -475,6 +506,78 @@ func (m *Manager) Stop(ctx context.Context) {
475 475
 	// mutex is released and Run can return now
476 476
 }
477 477
 
478
+func (m *Manager) updateKEK(ctx context.Context, cluster *api.Cluster) error {
479
+	securityConfig := m.config.SecurityConfig
480
+	nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID()
481
+	logger := log.G(ctx).WithFields(logrus.Fields{
482
+		"node.id":   nodeID,
483
+		"node.role": ca.ManagerRole,
484
+	})
485
+
486
+	// we are our own peer from which we get certs - try to connect over the local socket
487
+	r := remotes.NewRemotes(api.Peer{Addr: m.Addr(), NodeID: nodeID})
488
+
489
+	kekData := ca.KEKData{Version: cluster.Meta.Version.Index}
490
+	for _, encryptionKey := range cluster.UnlockKeys {
491
+		if encryptionKey.Subsystem == ca.ManagerRole {
492
+			kekData.KEK = encryptionKey.Key
493
+			break
494
+		}
495
+	}
496
+	updated, unlockedToLocked, err := m.dekRotator.MaybeUpdateKEK(kekData)
497
+	if err != nil {
498
+		logger.WithError(err).Errorf("failed to re-encrypt TLS key with a new KEK")
499
+		return err
500
+	}
501
+	if updated {
502
+		logger.Debug("successfully rotated KEK")
503
+	}
504
+	if unlockedToLocked {
505
+		// a best effort attempt to update the TLS certificate - if it fails, it'll be updated the next time it renews;
506
+		// don't wait because it might take a bit
507
+		go func() {
508
+			if err := ca.RenewTLSConfigNow(ctx, securityConfig, r); err != nil {
509
+				logger.WithError(err).Errorf("failed to download new TLS certificate after locking the cluster")
510
+			}
511
+		}()
512
+	}
513
+	return nil
514
+}
515
+
516
+func (m *Manager) watchForKEKChanges(ctx context.Context, watchDone chan struct{}) error {
517
+	defer close(watchDone)
518
+	clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization()
519
+	clusterWatch, clusterWatchCancel, err := store.ViewAndWatch(m.raftNode.MemoryStore(),
520
+		func(tx store.ReadTx) error {
521
+			cluster := store.GetCluster(tx, clusterID)
522
+			if cluster == nil {
523
+				return fmt.Errorf("unable to get current cluster")
524
+			}
525
+			return m.updateKEK(ctx, cluster)
526
+		},
527
+		state.EventUpdateCluster{
528
+			Cluster: &api.Cluster{ID: clusterID},
529
+			Checks:  []state.ClusterCheckFunc{state.ClusterCheckID},
530
+		},
531
+	)
532
+	if err != nil {
533
+		return err
534
+	}
535
+	go func() {
536
+		for {
537
+			select {
538
+			case event := <-clusterWatch:
539
+				clusterEvent := event.(state.EventUpdateCluster)
540
+				m.updateKEK(ctx, clusterEvent.Cluster)
541
+			case <-ctx.Done():
542
+				clusterWatchCancel()
543
+				return
544
+			}
545
+		}
546
+	}()
547
+	return nil
548
+}
549
+
478 550
 // rotateRootCAKEK will attempt to rotate the key-encryption-key for root CA key-material in raft.
479 551
 // If there is no passphrase set in ENV, it returns.
480 552
 // If there is plain-text root key-material, and a passphrase set, it encrypts it.
... ...
@@ -625,12 +728,26 @@ func (m *Manager) becomeLeader(ctx context.Context) {
625 625
 	initialCAConfig := ca.DefaultCAConfig()
626 626
 	initialCAConfig.ExternalCAs = m.config.ExternalCAs
627 627
 
628
+	var unlockKeys []*api.EncryptionKey
629
+	if m.config.AutoLockManagers {
630
+		unlockKeys = []*api.EncryptionKey{{
631
+			Subsystem: ca.ManagerRole,
632
+			Key:       m.config.UnlockKey,
633
+		}}
634
+	}
635
+
628 636
 	s.Update(func(tx store.Tx) error {
629 637
 		// Add a default cluster object to the
630 638
 		// store. Don't check the error because
631 639
 		// we expect this to fail unless this
632 640
 		// is a brand new cluster.
633
-		store.CreateCluster(tx, defaultClusterObject(clusterID, initialCAConfig, raftCfg, rootCA))
641
+		store.CreateCluster(tx, defaultClusterObject(
642
+			clusterID,
643
+			initialCAConfig,
644
+			raftCfg,
645
+			api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers},
646
+			unlockKeys,
647
+			rootCA))
634 648
 		// Add Node entry for ourself, if one
635 649
 		// doesn't exist already.
636 650
 		store.CreateNode(tx, managerNode(nodeID))
... ...
@@ -759,7 +876,14 @@ func (m *Manager) becomeFollower() {
759 759
 }
760 760
 
761 761
 // defaultClusterObject creates a default cluster.
762
-func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCfg api.RaftConfig, rootCA *ca.RootCA) *api.Cluster {
762
+func defaultClusterObject(
763
+	clusterID string,
764
+	initialCAConfig api.CAConfig,
765
+	raftCfg api.RaftConfig,
766
+	encryptionConfig api.EncryptionConfig,
767
+	initialUnlockKeys []*api.EncryptionKey,
768
+	rootCA *ca.RootCA) *api.Cluster {
769
+
763 770
 	return &api.Cluster{
764 771
 		ID: clusterID,
765 772
 		Spec: api.ClusterSpec{
... ...
@@ -772,8 +896,9 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf
772 772
 			Dispatcher: api.DispatcherConfig{
773 773
 				HeartbeatPeriod: ptypes.DurationProto(dispatcher.DefaultHeartBeatPeriod),
774 774
 			},
775
-			Raft:     raftCfg,
776
-			CAConfig: initialCAConfig,
775
+			Raft:             raftCfg,
776
+			CAConfig:         initialCAConfig,
777
+			EncryptionConfig: encryptionConfig,
777 778
 		},
778 779
 		RootCA: api.RootCA{
779 780
 			CAKey:      rootCA.Key,
... ...
@@ -784,6 +909,7 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf
784 784
 				Manager: ca.GenerateJoinToken(rootCA),
785 785
 			},
786 786
 		},
787
+		UnlockKeys: initialUnlockKeys,
787 788
 	}
788 789
 }
789 790
 
... ...
@@ -20,14 +20,13 @@ import (
20 20
 	"github.com/coreos/etcd/pkg/idutil"
21 21
 	"github.com/coreos/etcd/raft"
22 22
 	"github.com/coreos/etcd/raft/raftpb"
23
-	"github.com/coreos/etcd/snap"
24
-	"github.com/coreos/etcd/wal"
25 23
 	"github.com/docker/go-events"
26 24
 	"github.com/docker/swarmkit/api"
27 25
 	"github.com/docker/swarmkit/ca"
28 26
 	"github.com/docker/swarmkit/log"
29 27
 	"github.com/docker/swarmkit/manager/raftselector"
30 28
 	"github.com/docker/swarmkit/manager/state/raft/membership"
29
+	"github.com/docker/swarmkit/manager/state/raft/storage"
31 30
 	"github.com/docker/swarmkit/manager/state/store"
32 31
 	"github.com/docker/swarmkit/watch"
33 32
 	"github.com/gogo/protobuf/proto"
... ...
@@ -75,6 +74,21 @@ const (
75 75
 	IsFollower
76 76
 )
77 77
 
78
+// EncryptionKeys are the current and, if necessary, pending DEKs with which to
79
+// encrypt raft data
80
+type EncryptionKeys struct {
81
+	CurrentDEK []byte
82
+	PendingDEK []byte
83
+}
84
+
85
+// EncryptionKeyRotator is an interface to find out if any keys need rotating.
86
+type EncryptionKeyRotator interface {
87
+	GetKeys() EncryptionKeys
88
+	UpdateKeys(EncryptionKeys) error
89
+	NeedsRotation() bool
90
+	RotationNotify() chan struct{}
91
+}
92
+
78 93
 // Node represents the Raft Node useful
79 94
 // configuration.
80 95
 type Node struct {
... ...
@@ -87,8 +101,6 @@ type Node struct {
87 87
 	opts                NodeOptions
88 88
 	reqIDGen            *idutil.Generator
89 89
 	wait                *wait
90
-	wal                 *wal.WAL
91
-	snapshotter         *snap.Snapshotter
92 90
 	campaignWhenAble    bool
93 91
 	signalledLeadership uint32
94 92
 	isMember            uint32
... ...
@@ -122,6 +134,9 @@ type Node struct {
122 122
 	stopped chan struct{}
123 123
 
124 124
 	lastSendToMember map[uint64]chan struct{}
125
+	raftLogger       *storage.EncryptedRaftLogger
126
+	keyRotator       EncryptionKeyRotator
127
+	rotationQueued   bool
125 128
 }
126 129
 
127 130
 // NodeOptions provides node-level options.
... ...
@@ -150,6 +165,8 @@ type NodeOptions struct {
150 150
 	// nodes. Leave this as 0 to get the default value.
151 151
 	SendTimeout    time.Duration
152 152
 	TLSCredentials credentials.TransportCredentials
153
+
154
+	KeyRotator EncryptionKeyRotator
153 155
 }
154 156
 
155 157
 func init() {
... ...
@@ -188,6 +205,7 @@ func NewNode(opts NodeOptions) *Node {
188 188
 		stopped:             make(chan struct{}),
189 189
 		leadershipBroadcast: watch.NewQueue(),
190 190
 		lastSendToMember:    make(map[uint64]chan struct{}),
191
+		keyRotator:          opts.KeyRotator,
191 192
 	}
192 193
 	n.memoryStore = store.NewMemoryStore(n)
193 194
 
... ...
@@ -238,7 +256,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
238 238
 	}()
239 239
 
240 240
 	loadAndStartErr := n.loadAndStart(ctx, n.opts.ForceNewCluster)
241
-	if loadAndStartErr != nil && loadAndStartErr != errNoWAL {
241
+	if loadAndStartErr != nil && loadAndStartErr != storage.ErrNoWAL {
242 242
 		return loadAndStartErr
243 243
 	}
244 244
 
... ...
@@ -252,7 +270,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
252 252
 	n.appliedIndex = snapshot.Metadata.Index
253 253
 	n.snapshotIndex = snapshot.Metadata.Index
254 254
 
255
-	if loadAndStartErr == errNoWAL {
255
+	if loadAndStartErr == storage.ErrNoWAL {
256 256
 		if n.opts.JoinAddr != "" {
257 257
 			c, err := n.ConnectToMember(n.opts.JoinAddr, 10*time.Second)
258 258
 			if err != nil {
... ...
@@ -274,22 +292,20 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
274 274
 
275 275
 			n.Config.ID = resp.RaftID
276 276
 
277
-			if _, err := n.createWAL(n.opts.ID); err != nil {
277
+			if _, err := n.newRaftLogs(n.opts.ID); err != nil {
278 278
 				return err
279 279
 			}
280 280
 
281 281
 			n.raftNode = raft.StartNode(n.Config, []raft.Peer{})
282 282
 
283 283
 			if err := n.registerNodes(resp.Members); err != nil {
284
-				if walErr := n.wal.Close(); err != nil {
285
-					log.G(ctx).WithError(walErr).Error("raft: error closing WAL")
286
-				}
284
+				n.raftLogger.Close(ctx)
287 285
 				return err
288 286
 			}
289 287
 		} else {
290 288
 			// First member in the cluster, self-assign ID
291 289
 			n.Config.ID = uint64(rand.Int63()) + 1
292
-			peer, err := n.createWAL(n.opts.ID)
290
+			peer, err := n.newRaftLogs(n.opts.ID)
293 291
 			if err != nil {
294 292
 				return err
295 293
 			}
... ...
@@ -367,9 +383,13 @@ func (n *Node) Run(ctx context.Context) error {
367 367
 		if nodeRemoved {
368 368
 			// Move WAL and snapshot out of the way, since
369 369
 			// they are no longer usable.
370
-			if err := n.moveWALAndSnap(); err != nil {
370
+			if err := n.raftLogger.Clear(ctx); err != nil {
371 371
 				log.G(ctx).WithError(err).Error("failed to move wal after node removal")
372 372
 			}
373
+			// clear out the DEKs
374
+			if err := n.keyRotator.UpdateKeys(EncryptionKeys{}); err != nil {
375
+				log.G(ctx).WithError(err).Error("could not remove DEKs")
376
+			}
373 377
 		}
374 378
 		n.done()
375 379
 	}()
... ...
@@ -382,16 +402,10 @@ func (n *Node) Run(ctx context.Context) error {
382 382
 			n.raftNode.Tick()
383 383
 			n.cluster.Tick()
384 384
 		case rd := <-n.raftNode.Ready():
385
-			raftConfig := DefaultRaftConfig()
386
-			n.memoryStore.View(func(readTx store.ReadTx) {
387
-				clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
388
-				if err == nil && len(clusters) == 1 {
389
-					raftConfig = clusters[0].Spec.Raft
390
-				}
391
-			})
385
+			raftConfig := n.getCurrentRaftConfig()
392 386
 
393 387
 			// Save entries to storage
394
-			if err := n.saveToStorage(&raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil {
388
+			if err := n.saveToStorage(ctx, &raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil {
395 389
 				log.G(ctx).WithError(err).Error("failed to save entries to storage")
396 390
 			}
397 391
 
... ...
@@ -459,8 +473,8 @@ func (n *Node) Run(ctx context.Context) error {
459 459
 
460 460
 			// Trigger a snapshot every once in awhile
461 461
 			if n.snapshotInProgress == nil &&
462
-				raftConfig.SnapshotInterval > 0 &&
463
-				n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval {
462
+				(n.keyRotator.NeedsRotation() || raftConfig.SnapshotInterval > 0 &&
463
+					n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval) {
464 464
 				n.doSnapshot(ctx, raftConfig)
465 465
 			}
466 466
 
... ...
@@ -496,6 +510,24 @@ func (n *Node) Run(ctx context.Context) error {
496 496
 				n.snapshotIndex = snapshotIndex
497 497
 			}
498 498
 			n.snapshotInProgress = nil
499
+			if n.rotationQueued {
500
+				// there was a key rotation that took place before while the snapshot
501
+				// was in progress - we have to take another snapshot and encrypt with the new key
502
+				n.doSnapshot(ctx, n.getCurrentRaftConfig())
503
+			}
504
+		case <-n.keyRotator.RotationNotify():
505
+			// There are 2 separate checks:  rotationQueued, and keyRotator.NeedsRotation().
506
+			// We set rotationQueued so that when we are notified of a rotation, we try to
507
+			// do a snapshot as soon as possible.  However, if there is an error while doing
508
+			// the snapshot, we don't want to hammer the node attempting to do snapshots over
509
+			// and over.  So if doing a snapshot fails, wait until the next entry comes in to
510
+			// try again.
511
+			switch {
512
+			case n.snapshotInProgress != nil:
513
+				n.rotationQueued = true
514
+			case n.keyRotator.NeedsRotation():
515
+				n.doSnapshot(ctx, n.getCurrentRaftConfig())
516
+			}
499 517
 		case <-n.removeRaftCh:
500 518
 			nodeRemoved = true
501 519
 			// If the node was removed from other members,
... ...
@@ -508,6 +540,17 @@ func (n *Node) Run(ctx context.Context) error {
508 508
 	}
509 509
 }
510 510
 
511
+func (n *Node) getCurrentRaftConfig() api.RaftConfig {
512
+	raftConfig := DefaultRaftConfig()
513
+	n.memoryStore.View(func(readTx store.ReadTx) {
514
+		clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName))
515
+		if err == nil && len(clusters) == 1 {
516
+			raftConfig = clusters[0].Spec.Raft
517
+		}
518
+	})
519
+	return raftConfig
520
+}
521
+
511 522
 // Done returns channel which is closed when raft node is fully stopped.
512 523
 func (n *Node) Done() <-chan struct{} {
513 524
 	return n.doneCh
... ...
@@ -524,9 +567,7 @@ func (n *Node) stop(ctx context.Context) {
524 524
 
525 525
 	n.raftNode.Stop()
526 526
 	n.ticker.Stop()
527
-	if err := n.wal.Close(); err != nil {
528
-		log.G(ctx).WithError(err).Error("raft: failed to close WAL")
529
-	}
527
+	n.raftLogger.Close(ctx)
530 528
 	atomic.StoreUint32(&n.isMember, 0)
531 529
 	// TODO(stevvooe): Handle ctx.Done()
532 530
 }
... ...
@@ -1123,17 +1164,27 @@ func (n *Node) canSubmitProposal() bool {
1123 1123
 }
1124 1124
 
1125 1125
 // Saves a log entry to our Store
1126
-func (n *Node) saveToStorage(raftConfig *api.RaftConfig, hardState raftpb.HardState, entries []raftpb.Entry, snapshot raftpb.Snapshot) (err error) {
1126
+func (n *Node) saveToStorage(
1127
+	ctx context.Context,
1128
+	raftConfig *api.RaftConfig,
1129
+	hardState raftpb.HardState,
1130
+	entries []raftpb.Entry,
1131
+	snapshot raftpb.Snapshot,
1132
+) (err error) {
1133
+
1127 1134
 	if !raft.IsEmptySnap(snapshot) {
1128
-		if err := n.saveSnapshot(snapshot, raftConfig.KeepOldSnapshots); err != nil {
1135
+		if err := n.raftLogger.SaveSnapshot(snapshot); err != nil {
1129 1136
 			return ErrApplySnapshot
1130 1137
 		}
1138
+		if err := n.raftLogger.GC(snapshot.Metadata.Index, snapshot.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil {
1139
+			log.G(ctx).WithError(err).Error("unable to clean old snapshots and WALs")
1140
+		}
1131 1141
 		if err = n.raftStore.ApplySnapshot(snapshot); err != nil {
1132 1142
 			return ErrApplySnapshot
1133 1143
 		}
1134 1144
 	}
1135 1145
 
1136
-	if err := n.wal.Save(hardState, entries); err != nil {
1146
+	if err := n.raftLogger.SaveEntries(hardState, entries); err != nil {
1137 1147
 		// TODO(aaronl): These error types should really wrap more
1138 1148
 		// detailed errors.
1139 1149
 		return ErrApplySnapshot
... ...
@@ -2,282 +2,72 @@ package raft
2 2
 
3 3
 import (
4 4
 	"fmt"
5
-	"io"
6
-	"io/ioutil"
7
-	"os"
8
-	"path/filepath"
9
-	"sort"
10
-	"strings"
11
-
12
-	"github.com/coreos/etcd/pkg/fileutil"
5
+
13 6
 	"github.com/coreos/etcd/raft"
14 7
 	"github.com/coreos/etcd/raft/raftpb"
15
-	"github.com/coreos/etcd/snap"
16
-	"github.com/coreos/etcd/wal"
17
-	"github.com/coreos/etcd/wal/walpb"
18 8
 	"github.com/docker/swarmkit/api"
19 9
 	"github.com/docker/swarmkit/log"
10
+	"github.com/docker/swarmkit/manager/encryption"
20 11
 	"github.com/docker/swarmkit/manager/state/raft/membership"
12
+	"github.com/docker/swarmkit/manager/state/raft/storage"
21 13
 	"github.com/docker/swarmkit/manager/state/store"
22 14
 	"github.com/pkg/errors"
23 15
 	"golang.org/x/net/context"
24 16
 )
25 17
 
26
-var errNoWAL = errors.New("no WAL present")
27
-
28
-func (n *Node) legacyWALDir() string {
29
-	return filepath.Join(n.opts.StateDir, "wal")
30
-}
31
-
32
-func (n *Node) walDir() string {
33
-	return filepath.Join(n.opts.StateDir, "wal-v3")
34
-}
35
-
36
-func (n *Node) legacySnapDir() string {
37
-	return filepath.Join(n.opts.StateDir, "snap")
38
-}
39
-
40
-func (n *Node) snapDir() string {
41
-	return filepath.Join(n.opts.StateDir, "snap-v3")
42
-}
43
-
44
-func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
45
-	walDir := n.walDir()
46
-	snapDir := n.snapDir()
47
-
48
-	if !fileutil.Exist(snapDir) {
49
-		// If snapshots created by the etcd-v2 code exist, hard link
50
-		// them at the new path. This prevents etc-v2 creating
51
-		// snapshots that are visible to us, but out of sync with our
52
-		// WALs, after a downgrade.
53
-		legacySnapDir := n.legacySnapDir()
54
-		if fileutil.Exist(legacySnapDir) {
55
-			if err := migrateSnapshots(legacySnapDir, snapDir); err != nil {
56
-				return err
57
-			}
58
-		} else if err := os.MkdirAll(snapDir, 0700); err != nil {
59
-			return errors.Wrap(err, "failed to create snapshot directory")
60
-		}
61
-	}
62
-
63
-	// Create a snapshotter
64
-	n.snapshotter = snap.New(snapDir)
65
-
66
-	if !wal.Exist(walDir) {
67
-		// If wals created by the etcd-v2 wal code exist, copy them to
68
-		// the new path to avoid adding backwards-incompatible entries
69
-		// to those files.
70
-		legacyWALDir := n.legacyWALDir()
71
-		if !wal.Exist(legacyWALDir) {
72
-			return errNoWAL
73
-		}
74
-
75
-		if err := migrateWALs(legacyWALDir, walDir); err != nil {
76
-			return err
77
-		}
78
-	}
79
-
80
-	// Load snapshot data
81
-	snapshot, err := n.snapshotter.Load()
82
-	if err != nil && err != snap.ErrNoSnapshot {
83
-		return err
84
-	}
85
-
86
-	if snapshot != nil {
87
-		// Load the snapshot data into the store
88
-		if err := n.restoreFromSnapshot(snapshot.Data, forceNewCluster); err != nil {
89
-			return err
90
-		}
91
-	}
92
-
93
-	// Read logs to fully catch up store
94
-	if err := n.readWAL(ctx, snapshot, forceNewCluster); err != nil {
95
-		return err
96
-	}
97
-
98
-	return nil
99
-}
18
+func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) {
19
+	keys := n.keyRotator.GetKeys()
100 20
 
101
-func migrateWALs(legacyWALDir, walDir string) error {
102
-	// keep temporary wal directory so WAL initialization appears atomic
103
-	tmpdirpath := filepath.Clean(walDir) + ".tmp"
104
-	if fileutil.Exist(tmpdirpath) {
105
-		if err := os.RemoveAll(tmpdirpath); err != nil {
106
-			return errors.Wrap(err, "could not remove temporary WAL directory")
107
-		}
108
-	}
109
-	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
110
-		return errors.Wrap(err, "could not create temporary WAL directory")
21
+	n.raftLogger = &storage.EncryptedRaftLogger{
22
+		StateDir:      n.opts.StateDir,
23
+		EncryptionKey: keys.CurrentDEK,
111 24
 	}
112
-
113
-	walNames, err := fileutil.ReadDir(legacyWALDir)
114
-	if err != nil {
115
-		return errors.Wrapf(err, "could not list WAL directory %s", legacyWALDir)
25
+	if keys.PendingDEK != nil {
26
+		n.raftLogger.EncryptionKey = keys.PendingDEK
116 27
 	}
117 28
 
118
-	for _, fname := range walNames {
119
-		_, err := copyFile(filepath.Join(legacyWALDir, fname), filepath.Join(tmpdirpath, fname), 0600)
120
-		if err != nil {
121
-			return errors.Wrap(err, "error copying WAL file")
122
-		}
123
-	}
29
+	snap, walData, err := n.raftLogger.BootstrapFromDisk(ctx)
124 30
 
125
-	if err := os.Rename(tmpdirpath, walDir); err != nil {
126
-		return err
127
-	}
128
-
129
-	return nil
130
-}
131
-
132
-func migrateSnapshots(legacySnapDir, snapDir string) error {
133
-	// use temporary snaphot directory so initialization appears atomic
134
-	tmpdirpath := filepath.Clean(snapDir) + ".tmp"
135
-	if fileutil.Exist(tmpdirpath) {
136
-		if err := os.RemoveAll(tmpdirpath); err != nil {
137
-			return errors.Wrap(err, "could not remove temporary snapshot directory")
138
-		}
139
-	}
140
-	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
141
-		return errors.Wrap(err, "could not create temporary snapshot directory")
142
-	}
143
-
144
-	snapshotNames, err := fileutil.ReadDir(legacySnapDir)
145
-	if err != nil {
146
-		return errors.Wrapf(err, "could not list snapshot directory %s", legacySnapDir)
147
-	}
148
-
149
-	for _, fname := range snapshotNames {
150
-		err := os.Link(filepath.Join(legacySnapDir, fname), filepath.Join(tmpdirpath, fname))
151
-		if err != nil {
152
-			return errors.Wrap(err, "error linking snapshot file")
31
+	if keys.PendingDEK != nil {
32
+		switch errors.Cause(err).(type) {
33
+		case nil:
34
+			if err = n.keyRotator.UpdateKeys(EncryptionKeys{CurrentDEK: keys.PendingDEK}); err != nil {
35
+				err = errors.Wrap(err, "previous key rotation was successful, but unable mark rotation as complete")
36
+			}
37
+		case encryption.ErrCannotDecrypt:
38
+			snap, walData, err = n.raftLogger.BootstrapFromDisk(ctx, keys.CurrentDEK)
153 39
 		}
154 40
 	}
155 41
 
156
-	if err := os.Rename(tmpdirpath, snapDir); err != nil {
157
-		return err
158
-	}
159
-
160
-	return nil
161
-}
162
-
163
-// copyFile copies from src to dst until either EOF is reached
164
-// on src or an error occurs. It verifies src exists and removes
165
-// the dst if it exists.
166
-func copyFile(src, dst string, perm os.FileMode) (int64, error) {
167
-	cleanSrc := filepath.Clean(src)
168
-	cleanDst := filepath.Clean(dst)
169
-	if cleanSrc == cleanDst {
170
-		return 0, nil
171
-	}
172
-	sf, err := os.Open(cleanSrc)
173
-	if err != nil {
174
-		return 0, err
175
-	}
176
-	defer sf.Close()
177
-	if err := os.Remove(cleanDst); err != nil && !os.IsNotExist(err) {
178
-		return 0, err
179
-	}
180
-	df, err := os.OpenFile(cleanDst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
181 42
 	if err != nil {
182
-		return 0, err
43
+		return nil, storage.WALData{}, err
183 44
 	}
184
-	defer df.Close()
185
-	return io.Copy(df, sf)
45
+	return snap, walData, nil
186 46
 }
187 47
 
188
-func (n *Node) createWAL(nodeID string) (raft.Peer, error) {
189
-	raftNode := &api.RaftMember{
190
-		RaftID: n.Config.ID,
191
-		NodeID: nodeID,
192
-		Addr:   n.opts.Addr,
193
-	}
194
-	metadata, err := raftNode.Marshal()
195
-	if err != nil {
196
-		return raft.Peer{}, errors.Wrap(err, "error marshalling raft node")
197
-	}
198
-	n.wal, err = wal.Create(n.walDir(), metadata)
199
-	if err != nil {
200
-		return raft.Peer{}, errors.Wrap(err, "failed to create WAL")
201
-	}
202
-
203
-	n.cluster.AddMember(&membership.Member{RaftMember: raftNode})
204
-	return raft.Peer{ID: n.Config.ID, Context: metadata}, nil
205
-}
206
-
207
-// moveWALAndSnap moves away the WAL and snapshot because we were removed
208
-// from the cluster and will need to recreate them if we are readded.
209
-func (n *Node) moveWALAndSnap() error {
210
-	newWALDir, err := ioutil.TempDir(n.opts.StateDir, "wal.")
211
-	if err != nil {
212
-		return err
213
-	}
214
-	err = os.Rename(n.walDir(), newWALDir)
215
-	if err != nil {
216
-		return err
217
-	}
218
-
219
-	newSnapDir, err := ioutil.TempDir(n.opts.StateDir, "snap.")
220
-	if err != nil {
221
-		return err
222
-	}
223
-	err = os.Rename(n.snapDir(), newSnapDir)
48
+// bootstraps a node's raft store from the raft logs and snapshots on disk
49
+func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
50
+	snapshot, waldata, err := n.readFromDisk(ctx)
224 51
 	if err != nil {
225 52
 		return err
226 53
 	}
227 54
 
228
-	return nil
229
-}
230
-
231
-func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewCluster bool) (err error) {
232
-	var (
233
-		walsnap  walpb.Snapshot
234
-		metadata []byte
235
-		st       raftpb.HardState
236
-		ents     []raftpb.Entry
237
-	)
238
-
239 55
 	if snapshot != nil {
240
-		walsnap.Index = snapshot.Metadata.Index
241
-		walsnap.Term = snapshot.Metadata.Term
242
-	}
243
-
244
-	repaired := false
245
-	for {
246
-		if n.wal, err = wal.Open(n.walDir(), walsnap); err != nil {
247
-			return errors.Wrap(err, "failed to open WAL")
248
-		}
249
-		if metadata, st, ents, err = n.wal.ReadAll(); err != nil {
250
-			if err := n.wal.Close(); err != nil {
251
-				return err
252
-			}
253
-			// we can only repair ErrUnexpectedEOF and we never repair twice.
254
-			if repaired || err != io.ErrUnexpectedEOF {
255
-				return errors.Wrap(err, "irreparable WAL error")
256
-			}
257
-			if !wal.Repair(n.walDir()) {
258
-				return errors.Wrap(err, "WAL error cannot be repaired")
259
-			}
260
-			log.G(ctx).WithError(err).Info("repaired WAL error")
261
-			repaired = true
262
-			continue
56
+		// Load the snapshot data into the store
57
+		if err := n.restoreFromSnapshot(snapshot.Data, forceNewCluster); err != nil {
58
+			return err
263 59
 		}
264
-		break
265 60
 	}
266 61
 
267
-	defer func() {
268
-		if err != nil {
269
-			if walErr := n.wal.Close(); walErr != nil {
270
-				log.G(ctx).WithError(walErr).Error("error closing raft WAL")
271
-			}
272
-		}
273
-	}()
274
-
62
+	// Read logs to fully catch up store
275 63
 	var raftNode api.RaftMember
276
-	if err := raftNode.Unmarshal(metadata); err != nil {
64
+	if err := raftNode.Unmarshal(waldata.Metadata); err != nil {
277 65
 		return errors.Wrap(err, "failed to unmarshal WAL metadata")
278 66
 	}
279 67
 	n.Config.ID = raftNode.RaftID
280 68
 
69
+	ents, st := waldata.Entries, waldata.HardState
70
+
281 71
 	// All members that are no longer part of the cluster must be added to
282 72
 	// the removed list right away, so that we don't try to connect to them
283 73
 	// before processing the configuration change entries, which could make
... ...
@@ -326,7 +116,7 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC
326 326
 		ents = append(ents, toAppEnts...)
327 327
 
328 328
 		// force commit newly appended entries
329
-		err := n.wal.Save(st, toAppEnts)
329
+		err := n.raftLogger.SaveEntries(st, toAppEnts)
330 330
 		if err != nil {
331 331
 			log.G(ctx).WithError(err).Fatalf("failed to save WAL while forcing new cluster")
332 332
 		}
... ...
@@ -343,146 +133,24 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC
343 343
 	if err := n.raftStore.SetHardState(st); err != nil {
344 344
 		return err
345 345
 	}
346
-	if err := n.raftStore.Append(ents); err != nil {
347
-		return err
348
-	}
349
-
350
-	return nil
346
+	return n.raftStore.Append(ents)
351 347
 }
352 348
 
353
-func (n *Node) saveSnapshot(snapshot raftpb.Snapshot, keepOldSnapshots uint64) error {
354
-	err := n.wal.SaveSnapshot(walpb.Snapshot{
355
-		Index: snapshot.Metadata.Index,
356
-		Term:  snapshot.Metadata.Term,
357
-	})
358
-	if err != nil {
359
-		return err
360
-	}
361
-	err = n.snapshotter.SaveSnap(snapshot)
362
-	if err != nil {
363
-		return err
364
-	}
365
-	err = n.wal.ReleaseLockTo(snapshot.Metadata.Index)
366
-	if err != nil {
367
-		return err
368
-	}
369
-
370
-	// Delete any older snapshots
371
-	curSnapshot := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, ".snap")
372
-
373
-	dirents, err := ioutil.ReadDir(n.snapDir())
374
-	if err != nil {
375
-		return err
376
-	}
377
-
378
-	var snapshots []string
379
-	for _, dirent := range dirents {
380
-		if strings.HasSuffix(dirent.Name(), ".snap") {
381
-			snapshots = append(snapshots, dirent.Name())
382
-		}
383
-	}
384
-
385
-	// Sort snapshot filenames in reverse lexical order
386
-	sort.Sort(sort.Reverse(sort.StringSlice(snapshots)))
387
-
388
-	// Ignore any snapshots that are older than the current snapshot.
389
-	// Delete the others. Rather than doing lexical comparisons, we look
390
-	// at what exists before/after the current snapshot in the slice.
391
-	// This means that if the current snapshot doesn't appear in the
392
-	// directory for some strange reason, we won't delete anything, which
393
-	// is the safe behavior.
394
-	curSnapshotIdx := -1
395
-	var (
396
-		removeErr      error
397
-		oldestSnapshot string
398
-	)
399
-
400
-	for i, snapFile := range snapshots {
401
-		if curSnapshotIdx >= 0 && i > curSnapshotIdx {
402
-			if uint64(i-curSnapshotIdx) > keepOldSnapshots {
403
-				err := os.Remove(filepath.Join(n.snapDir(), snapFile))
404
-				if err != nil && removeErr == nil {
405
-					removeErr = err
406
-				}
407
-				continue
408
-			}
409
-		} else if snapFile == curSnapshot {
410
-			curSnapshotIdx = i
411
-		}
412
-		oldestSnapshot = snapFile
413
-	}
414
-
415
-	if removeErr != nil {
416
-		return removeErr
417
-	}
418
-
419
-	// Remove any WAL files that only contain data from before the oldest
420
-	// remaining snapshot.
421
-
422
-	if oldestSnapshot == "" {
423
-		return nil
424
-	}
425
-
426
-	// Parse index out of oldest snapshot's filename
427
-	var snapTerm, snapIndex uint64
428
-	_, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex)
429
-	if err != nil {
430
-		return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot)
349
+func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) {
350
+	raftNode := &api.RaftMember{
351
+		RaftID: n.Config.ID,
352
+		NodeID: nodeID,
353
+		Addr:   n.opts.Addr,
431 354
 	}
432
-
433
-	// List the WALs
434
-	dirents, err = ioutil.ReadDir(n.walDir())
355
+	metadata, err := raftNode.Marshal()
435 356
 	if err != nil {
436
-		return err
437
-	}
438
-
439
-	var wals []string
440
-	for _, dirent := range dirents {
441
-		if strings.HasSuffix(dirent.Name(), ".wal") {
442
-			wals = append(wals, dirent.Name())
443
-		}
444
-	}
445
-
446
-	// Sort WAL filenames in lexical order
447
-	sort.Sort(sort.StringSlice(wals))
448
-
449
-	found := false
450
-	deleteUntil := -1
451
-
452
-	for i, walName := range wals {
453
-		var walSeq, walIndex uint64
454
-		_, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex)
455
-		if err != nil {
456
-			return errors.Wrapf(err, "could not parse WAL name %s", walName)
457
-		}
458
-
459
-		if walIndex >= snapIndex {
460
-			deleteUntil = i - 1
461
-			found = true
462
-			break
463
-		}
464
-	}
465
-
466
-	// If all WAL files started with indices below the oldest snapshot's
467
-	// index, we can delete all but the newest WAL file.
468
-	if !found && len(wals) != 0 {
469
-		deleteUntil = len(wals) - 1
357
+		return raft.Peer{}, errors.Wrap(err, "error marshalling raft node")
470 358
 	}
471
-
472
-	for i := 0; i < deleteUntil; i++ {
473
-		walPath := filepath.Join(n.walDir(), wals[i])
474
-		l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode)
475
-		if err != nil {
476
-			return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i])
477
-		}
478
-		err = os.Remove(walPath)
479
-		l.Close()
480
-		if err != nil {
481
-			return errors.Wrapf(err, "error removing old WAL file %s", wals[i])
482
-		}
359
+	if err := n.raftLogger.BootstrapNew(metadata); err != nil {
360
+		return raft.Peer{}, err
483 361
 	}
484
-
485
-	return nil
362
+	n.cluster.AddMember(&membership.Member{RaftMember: raftNode})
363
+	return raft.Peer{ID: n.Config.ID, Context: metadata}, nil
486 364
 }
487 365
 
488 366
 func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
... ...
@@ -497,6 +165,17 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
497 497
 	}
498 498
 	snapshot.Membership.Removed = n.cluster.Removed()
499 499
 
500
+	// maybe start rotation
501
+	n.rotationQueued = false
502
+	var newEncryptionKeys *EncryptionKeys
503
+	if n.keyRotator.NeedsRotation() {
504
+		keys := n.keyRotator.GetKeys()
505
+		if keys.PendingDEK != nil {
506
+			n.raftLogger.RotateEncryptionKey(keys.PendingDEK)
507
+			newEncryptionKeys = &EncryptionKeys{CurrentDEK: keys.PendingDEK}
508
+		}
509
+	}
510
+
500 511
 	viewStarted := make(chan struct{})
501 512
 	n.asyncTasks.Add(1)
502 513
 	n.snapshotInProgress = make(chan uint64, 1) // buffered in case Shutdown is called during the snapshot
... ...
@@ -505,7 +184,6 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
505 505
 			n.asyncTasks.Done()
506 506
 			n.snapshotInProgress <- snapshotIndex
507 507
 		}()
508
-
509 508
 		var err error
510 509
 		n.memoryStore.View(func(tx store.ReadTx) {
511 510
 			close(viewStarted)
... ...
@@ -526,11 +204,18 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
526 526
 		}
527 527
 		snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d)
528 528
 		if err == nil {
529
-			if err := n.saveSnapshot(snap, raftConfig.KeepOldSnapshots); err != nil {
529
+			if err := n.raftLogger.SaveSnapshot(snap); err != nil {
530 530
 				log.G(ctx).WithError(err).Error("failed to save snapshot")
531 531
 				return
532 532
 			}
533 533
 			snapshotIndex = appliedIndex
534
+			if newEncryptionKeys != nil {
535
+				// this means we tried to rotate - so finish the rotation
536
+				if err := n.keyRotator.UpdateKeys(*newEncryptionKeys); err != nil {
537
+					log.G(ctx).WithError(err).Error(
538
+						"failed to update encryption keys after a rotation - will wait for the next snapshot")
539
+				}
540
+			}
534 541
 
535 542
 			if appliedIndex > raftConfig.LogEntriesForSlowFollowers {
536 543
 				err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers)
... ...
@@ -538,6 +223,10 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) {
538 538
 					log.G(ctx).WithError(err).Error("failed to compact snapshot")
539 539
 				}
540 540
 			}
541
+
542
+			if err := n.raftLogger.GC(snap.Metadata.Index, snap.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil {
543
+				log.G(ctx).WithError(err).Error("failed to clean up old snapshots and WALs")
544
+			}
541 545
 		} else if err != raft.ErrSnapOutOfDate {
542 546
 			log.G(ctx).WithError(err).Error("failed to create snapshot")
543 547
 		}
544 548
new file mode 100644
... ...
@@ -0,0 +1,158 @@
0
+package storage
1
+
2
+import (
3
+	"io/ioutil"
4
+	"os"
5
+	"path/filepath"
6
+	"sort"
7
+	"strings"
8
+
9
+	"github.com/coreos/etcd/pkg/fileutil"
10
+	"github.com/coreos/etcd/raft/raftpb"
11
+	"github.com/coreos/etcd/snap"
12
+	"github.com/docker/swarmkit/manager/encryption"
13
+	"github.com/pkg/errors"
14
+)
15
+
16
+// This package wraps the github.com/coreos/etcd/snap package, and encrypts
17
+// the bytes of whatever snapshot is passed to it, and decrypts the bytes of
18
+// whatever snapshot it reads.
19
+
20
+// Snapshotter is the interface presented by github.com/coreos/etcd/snap.Snapshotter that we depend upon
21
+type Snapshotter interface {
22
+	SaveSnap(snapshot raftpb.Snapshot) error
23
+	Load() (*raftpb.Snapshot, error)
24
+}
25
+
26
+// SnapFactory provides an interface for the different ways to get a Snapshotter object.
27
+// For instance, the etcd/snap package itself provides this
28
+type SnapFactory interface {
29
+	New(dirpath string) Snapshotter
30
+}
31
+
32
+var _ Snapshotter = &wrappedSnap{}
33
+var _ Snapshotter = &snap.Snapshotter{}
34
+var _ SnapFactory = snapCryptor{}
35
+
36
+// wrappedSnap wraps a github.com/coreos/etcd/snap.Snapshotter, and handles
37
+// encrypting/decrypting.
38
+type wrappedSnap struct {
39
+	*snap.Snapshotter
40
+	encrypter encryption.Encrypter
41
+	decrypter encryption.Decrypter
42
+}
43
+
44
+// SaveSnap encrypts the snapshot data (if an encrypter is exists) before passing it onto the
45
+// wrapped snap.Snapshotter's SaveSnap function.
46
+func (s *wrappedSnap) SaveSnap(snapshot raftpb.Snapshot) error {
47
+	toWrite := snapshot
48
+	var err error
49
+	toWrite.Data, err = encryption.Encrypt(snapshot.Data, s.encrypter)
50
+	if err != nil {
51
+		return err
52
+	}
53
+	return s.Snapshotter.SaveSnap(toWrite)
54
+}
55
+
56
+// Load decrypts the snapshot data (if a decrypter is exists) after reading it using the
57
+// wrapped snap.Snapshotter's Load function.
58
+func (s *wrappedSnap) Load() (*raftpb.Snapshot, error) {
59
+	snapshot, err := s.Snapshotter.Load()
60
+	if err != nil {
61
+		return nil, err
62
+	}
63
+	snapshot.Data, err = encryption.Decrypt(snapshot.Data, s.decrypter)
64
+	if err != nil {
65
+		return nil, err
66
+	}
67
+
68
+	return snapshot, nil
69
+}
70
+
71
+// snapCryptor is an object that provides the same functions as `etcd/wal`
72
+// and `etcd/snap` that we need to open a WAL object or Snapshotter object
73
+type snapCryptor struct {
74
+	encrypter encryption.Encrypter
75
+	decrypter encryption.Decrypter
76
+}
77
+
78
+// NewSnapFactory returns a new object that can read from and write to encrypted
79
+// snapshots on disk
80
+func NewSnapFactory(encrypter encryption.Encrypter, decrypter encryption.Decrypter) SnapFactory {
81
+	return snapCryptor{
82
+		encrypter: encrypter,
83
+		decrypter: decrypter,
84
+	}
85
+}
86
+
87
+// NewSnapshotter returns a new Snapshotter with the given encrypters and decrypters
88
+func (sc snapCryptor) New(dirpath string) Snapshotter {
89
+	return &wrappedSnap{
90
+		Snapshotter: snap.New(dirpath),
91
+		encrypter:   sc.encrypter,
92
+		decrypter:   sc.decrypter,
93
+	}
94
+}
95
+
96
+type originalSnap struct{}
97
+
98
+func (o originalSnap) New(dirpath string) Snapshotter {
99
+	return snap.New(dirpath)
100
+}
101
+
102
+// OriginalSnap is the original `snap` package as an implemntation of the SnapFactory interface
103
+var OriginalSnap SnapFactory = originalSnap{}
104
+
105
+// MigrateSnapshot reads the latest existing snapshot from one directory, encoded one way, and writes
106
+// it to a new directory, encoded a different way
107
+func MigrateSnapshot(oldDir, newDir string, oldFactory, newFactory SnapFactory) error {
108
+	// use temporary snapshot directory so initialization appears atomic
109
+	oldSnapshotter := oldFactory.New(oldDir)
110
+	snapshot, err := oldSnapshotter.Load()
111
+	switch err {
112
+	case snap.ErrNoSnapshot: // if there's no snapshot, the migration succeeded
113
+		return nil
114
+	case nil:
115
+		break
116
+	default:
117
+		return err
118
+	}
119
+
120
+	tmpdirpath := filepath.Clean(newDir) + ".tmp"
121
+	if fileutil.Exist(tmpdirpath) {
122
+		if err := os.RemoveAll(tmpdirpath); err != nil {
123
+			return errors.Wrap(err, "could not remove temporary snapshot directory")
124
+		}
125
+	}
126
+	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
127
+		return errors.Wrap(err, "could not create temporary snapshot directory")
128
+	}
129
+	tmpSnapshotter := newFactory.New(tmpdirpath)
130
+
131
+	// write the new snapshot to the temporary location
132
+	if err = tmpSnapshotter.SaveSnap(*snapshot); err != nil {
133
+		return err
134
+	}
135
+
136
+	return os.Rename(tmpdirpath, newDir)
137
+}
138
+
139
+// ListSnapshots lists all the snapshot files in a particular directory and returns
140
+// the snapshot files in reverse lexical order (newest first)
141
+func ListSnapshots(dirpath string) ([]string, error) {
142
+	dirents, err := ioutil.ReadDir(dirpath)
143
+	if err != nil {
144
+		return nil, err
145
+	}
146
+
147
+	var snapshots []string
148
+	for _, dirent := range dirents {
149
+		if strings.HasSuffix(dirent.Name(), ".snap") {
150
+			snapshots = append(snapshots, dirent.Name())
151
+		}
152
+	}
153
+
154
+	// Sort snapshot filenames in reverse lexical order
155
+	sort.Sort(sort.Reverse(sort.StringSlice(snapshots)))
156
+	return snapshots, nil
157
+}
0 158
new file mode 100644
... ...
@@ -0,0 +1,391 @@
0
+package storage
1
+
2
+import (
3
+	"fmt"
4
+	"io/ioutil"
5
+	"os"
6
+	"path/filepath"
7
+	"sync"
8
+
9
+	"golang.org/x/net/context"
10
+
11
+	"github.com/coreos/etcd/pkg/fileutil"
12
+	"github.com/coreos/etcd/raft/raftpb"
13
+	"github.com/coreos/etcd/snap"
14
+	"github.com/coreos/etcd/wal"
15
+	"github.com/coreos/etcd/wal/walpb"
16
+	"github.com/docker/swarmkit/api"
17
+	"github.com/docker/swarmkit/log"
18
+	"github.com/docker/swarmkit/manager/encryption"
19
+	"github.com/pkg/errors"
20
+)
21
+
22
+// ErrNoWAL is returned if there are no WALs on disk
23
+var ErrNoWAL = errors.New("no WAL present")
24
+
25
+type walSnapDirs struct {
26
+	wal  string
27
+	snap string
28
+}
29
+
30
+// the wal/snap directories in decreasing order of preference/version
31
+var versionedWALSnapDirs = []walSnapDirs{
32
+	{wal: "wal-v3-encrypted", snap: "snap-v3-encrypted"},
33
+	{wal: "wal-v3", snap: "snap-v3"},
34
+	{wal: "wal", snap: "snap"},
35
+}
36
+
37
+// MultiDecrypter attempts to decrypt with a list of decrypters
38
+type MultiDecrypter []encryption.Decrypter
39
+
40
+// Decrypt tries to decrypt using all the decrypters
41
+func (m MultiDecrypter) Decrypt(r api.MaybeEncryptedRecord) (result []byte, err error) {
42
+	for _, d := range m {
43
+		result, err = d.Decrypt(r)
44
+		if err == nil {
45
+			return
46
+		}
47
+	}
48
+	return
49
+}
50
+
51
+// EncryptedRaftLogger saves raft data to disk
52
+type EncryptedRaftLogger struct {
53
+	StateDir      string
54
+	EncryptionKey []byte
55
+
56
+	// mutex is locked for writing only when we need to replace the wal object and snapshotter
57
+	// object, not when we're writing snapshots or wals (in which case it's locked for reading)
58
+	encoderMu   sync.RWMutex
59
+	wal         WAL
60
+	snapshotter Snapshotter
61
+}
62
+
63
+// BootstrapFromDisk creates a new snapshotter and wal, and also reads the latest snapshot and WALs from disk
64
+func (e *EncryptedRaftLogger) BootstrapFromDisk(ctx context.Context, oldEncryptionKeys ...[]byte) (*raftpb.Snapshot, WALData, error) {
65
+	e.encoderMu.Lock()
66
+	defer e.encoderMu.Unlock()
67
+
68
+	walDir := e.walDir()
69
+	snapDir := e.snapDir()
70
+
71
+	encrypter, decrypter := encryption.Defaults(e.EncryptionKey)
72
+	if oldEncryptionKeys != nil {
73
+		decrypters := []encryption.Decrypter{decrypter}
74
+		for _, key := range oldEncryptionKeys {
75
+			_, d := encryption.Defaults(key)
76
+			decrypters = append(decrypters, d)
77
+		}
78
+		decrypter = MultiDecrypter(decrypters)
79
+	}
80
+
81
+	snapFactory := NewSnapFactory(encrypter, decrypter)
82
+
83
+	if !fileutil.Exist(snapDir) {
84
+		// If snapshots created by the etcd-v2 code exist, or by swarmkit development version,
85
+		// read the latest snapshot and write it encoded to the new path.  The new path
86
+		// prevents etc-v2 creating snapshots that are visible to us, but not encoded and
87
+		// out of sync with our WALs, after a downgrade.
88
+		for _, dirs := range versionedWALSnapDirs[1:] {
89
+			legacySnapDir := filepath.Join(e.StateDir, dirs.snap)
90
+			if fileutil.Exist(legacySnapDir) {
91
+				if err := MigrateSnapshot(legacySnapDir, snapDir, OriginalSnap, snapFactory); err != nil {
92
+					return nil, WALData{}, err
93
+				}
94
+				break
95
+			}
96
+		}
97
+	}
98
+	// ensure the new directory exists
99
+	if err := os.MkdirAll(snapDir, 0700); err != nil {
100
+		return nil, WALData{}, errors.Wrap(err, "failed to create snapshot directory")
101
+	}
102
+
103
+	var (
104
+		snapshotter Snapshotter
105
+		walObj      WAL
106
+		err         error
107
+	)
108
+
109
+	// Create a snapshotter and load snapshot data
110
+	snapshotter = snapFactory.New(snapDir)
111
+	snapshot, err := snapshotter.Load()
112
+	if err != nil && err != snap.ErrNoSnapshot {
113
+		return nil, WALData{}, err
114
+	}
115
+
116
+	walFactory := NewWALFactory(encrypter, decrypter)
117
+	var walsnap walpb.Snapshot
118
+	if snapshot != nil {
119
+		walsnap.Index = snapshot.Metadata.Index
120
+		walsnap.Term = snapshot.Metadata.Term
121
+	}
122
+
123
+	if !wal.Exist(walDir) {
124
+		var walExists bool
125
+		// If wals created by the etcd-v2 wal code exist, read the latest ones based
126
+		// on this snapshot and encode them to wals in the new path to avoid adding
127
+		// backwards-incompatible entries to those files.
128
+		for _, dirs := range versionedWALSnapDirs[1:] {
129
+			legacyWALDir := filepath.Join(e.StateDir, dirs.wal)
130
+			if !wal.Exist(legacyWALDir) {
131
+				continue
132
+			}
133
+			if err = MigrateWALs(ctx, legacyWALDir, walDir, OriginalWAL, walFactory, walsnap); err != nil {
134
+				return nil, WALData{}, err
135
+			}
136
+			walExists = true
137
+			break
138
+		}
139
+		if !walExists {
140
+			return nil, WALData{}, ErrNoWAL
141
+		}
142
+	}
143
+
144
+	walObj, waldata, err := ReadRepairWAL(ctx, walDir, walsnap, walFactory)
145
+	if err != nil {
146
+		return nil, WALData{}, err
147
+	}
148
+
149
+	e.snapshotter = snapshotter
150
+	e.wal = walObj
151
+
152
+	return snapshot, waldata, nil
153
+}
154
+
155
+// BootstrapNew creates a new snapshotter and WAL writer, expecting that there is nothing on disk
156
+func (e *EncryptedRaftLogger) BootstrapNew(metadata []byte) error {
157
+	e.encoderMu.Lock()
158
+	defer e.encoderMu.Unlock()
159
+	encrypter, decrypter := encryption.Defaults(e.EncryptionKey)
160
+	walFactory := NewWALFactory(encrypter, decrypter)
161
+
162
+	for _, dirpath := range []string{e.walDir(), e.snapDir()} {
163
+		if err := os.MkdirAll(dirpath, 0700); err != nil {
164
+			return errors.Wrapf(err, "failed to create %s", dirpath)
165
+		}
166
+	}
167
+	var err error
168
+	e.wal, err = walFactory.Create(e.walDir(), metadata)
169
+	if err != nil {
170
+		return errors.Wrap(err, "failed to create WAL")
171
+	}
172
+
173
+	e.snapshotter = NewSnapFactory(encrypter, decrypter).New(e.snapDir())
174
+	return nil
175
+}
176
+
177
+func (e *EncryptedRaftLogger) walDir() string {
178
+	return filepath.Join(e.StateDir, versionedWALSnapDirs[0].wal)
179
+}
180
+
181
+func (e *EncryptedRaftLogger) snapDir() string {
182
+	return filepath.Join(e.StateDir, versionedWALSnapDirs[0].snap)
183
+}
184
+
185
+// RotateEncryptionKey swaps out the encoders and decoders used by the wal and snapshotter
186
+func (e *EncryptedRaftLogger) RotateEncryptionKey(newKey []byte) {
187
+	e.encoderMu.Lock()
188
+	defer e.encoderMu.Unlock()
189
+
190
+	if e.wal != nil { // if the wal exists, the snapshotter exists
191
+		// We don't want to have to close the WAL, because we can't open a new one.
192
+		// We need to know the previous snapshot, because when you open a WAL you
193
+		// have to read out all the entries from a particular snapshot, or you can't
194
+		// write.  So just rotate the encoders out from under it.  We already
195
+		// have a lock on writing to snapshots and WALs.
196
+		wrapped, ok := e.wal.(*wrappedWAL)
197
+		if !ok {
198
+			panic(fmt.Errorf("EncryptedRaftLogger's WAL is not a wrappedWAL"))
199
+		}
200
+
201
+		wrapped.encrypter, wrapped.decrypter = encryption.Defaults(newKey)
202
+
203
+		e.snapshotter = NewSnapFactory(wrapped.encrypter, wrapped.decrypter).New(e.snapDir())
204
+	}
205
+	e.EncryptionKey = newKey
206
+}
207
+
208
+// SaveSnapshot actually saves a given snapshot to both the WAL and the snapshot.
209
+func (e *EncryptedRaftLogger) SaveSnapshot(snapshot raftpb.Snapshot) error {
210
+
211
+	walsnap := walpb.Snapshot{
212
+		Index: snapshot.Metadata.Index,
213
+		Term:  snapshot.Metadata.Term,
214
+	}
215
+
216
+	e.encoderMu.RLock()
217
+	if err := e.wal.SaveSnapshot(walsnap); err != nil {
218
+		e.encoderMu.RUnlock()
219
+		return err
220
+	}
221
+
222
+	snapshotter := e.snapshotter
223
+	e.encoderMu.RUnlock()
224
+
225
+	if err := snapshotter.SaveSnap(snapshot); err != nil {
226
+		return err
227
+	}
228
+	if err := e.wal.ReleaseLockTo(snapshot.Metadata.Index); err != nil {
229
+		return err
230
+	}
231
+	return nil
232
+}
233
+
234
+// GC garbage collects snapshots and wals older than the provided index and term
235
+func (e *EncryptedRaftLogger) GC(index uint64, term uint64, keepOldSnapshots uint64) error {
236
+	// Delete any older snapshots
237
+	curSnapshot := fmt.Sprintf("%016x-%016x%s", term, index, ".snap")
238
+
239
+	snapshots, err := ListSnapshots(e.snapDir())
240
+	if err != nil {
241
+		return err
242
+	}
243
+
244
+	// Ignore any snapshots that are older than the current snapshot.
245
+	// Delete the others. Rather than doing lexical comparisons, we look
246
+	// at what exists before/after the current snapshot in the slice.
247
+	// This means that if the current snapshot doesn't appear in the
248
+	// directory for some strange reason, we won't delete anything, which
249
+	// is the safe behavior.
250
+	curSnapshotIdx := -1
251
+	var (
252
+		removeErr      error
253
+		oldestSnapshot string
254
+	)
255
+
256
+	for i, snapFile := range snapshots {
257
+		if curSnapshotIdx >= 0 && i > curSnapshotIdx {
258
+			if uint64(i-curSnapshotIdx) > keepOldSnapshots {
259
+				err := os.Remove(filepath.Join(e.snapDir(), snapFile))
260
+				if err != nil && removeErr == nil {
261
+					removeErr = err
262
+				}
263
+				continue
264
+			}
265
+		} else if snapFile == curSnapshot {
266
+			curSnapshotIdx = i
267
+		}
268
+		oldestSnapshot = snapFile
269
+	}
270
+
271
+	if removeErr != nil {
272
+		return removeErr
273
+	}
274
+
275
+	// Remove any WAL files that only contain data from before the oldest
276
+	// remaining snapshot.
277
+
278
+	if oldestSnapshot == "" {
279
+		return nil
280
+	}
281
+
282
+	// Parse index out of oldest snapshot's filename
283
+	var snapTerm, snapIndex uint64
284
+	_, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex)
285
+	if err != nil {
286
+		return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot)
287
+	}
288
+
289
+	wals, err := ListWALs(e.walDir())
290
+	if err != nil {
291
+		return err
292
+	}
293
+
294
+	found := false
295
+	deleteUntil := -1
296
+
297
+	for i, walName := range wals {
298
+		var walSeq, walIndex uint64
299
+		_, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex)
300
+		if err != nil {
301
+			return errors.Wrapf(err, "could not parse WAL name %s", walName)
302
+		}
303
+
304
+		if walIndex >= snapIndex {
305
+			deleteUntil = i - 1
306
+			found = true
307
+			break
308
+		}
309
+	}
310
+
311
+	// If all WAL files started with indices below the oldest snapshot's
312
+	// index, we can delete all but the newest WAL file.
313
+	if !found && len(wals) != 0 {
314
+		deleteUntil = len(wals) - 1
315
+	}
316
+
317
+	for i := 0; i < deleteUntil; i++ {
318
+		walPath := filepath.Join(e.walDir(), wals[i])
319
+		l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode)
320
+		if err != nil {
321
+			return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i])
322
+		}
323
+		err = os.Remove(walPath)
324
+		l.Close()
325
+		if err != nil {
326
+			return errors.Wrapf(err, "error removing old WAL file %s", wals[i])
327
+		}
328
+	}
329
+
330
+	return nil
331
+}
332
+
333
+// SaveEntries saves only entries to disk
334
+func (e *EncryptedRaftLogger) SaveEntries(st raftpb.HardState, entries []raftpb.Entry) error {
335
+	e.encoderMu.RLock()
336
+	defer e.encoderMu.RUnlock()
337
+
338
+	if e.wal == nil {
339
+		return fmt.Errorf("raft WAL has either been closed or has never been created")
340
+	}
341
+	return e.wal.Save(st, entries)
342
+}
343
+
344
+// Close closes the logger - it will have to be bootstrapped again to start writing
345
+func (e *EncryptedRaftLogger) Close(ctx context.Context) {
346
+	e.encoderMu.Lock()
347
+	defer e.encoderMu.Unlock()
348
+
349
+	if e.wal != nil {
350
+		if err := e.wal.Close(); err != nil {
351
+			log.G(ctx).WithError(err).Error("error closing raft WAL")
352
+		}
353
+	}
354
+
355
+	e.wal = nil
356
+	e.snapshotter = nil
357
+}
358
+
359
+// Clear closes the existing WAL and moves away the WAL and snapshot.
360
+func (e *EncryptedRaftLogger) Clear(ctx context.Context) error {
361
+	e.encoderMu.Lock()
362
+	defer e.encoderMu.Unlock()
363
+
364
+	if e.wal != nil {
365
+		if err := e.wal.Close(); err != nil {
366
+			log.G(ctx).WithError(err).Error("error closing raft WAL")
367
+		}
368
+	}
369
+	e.snapshotter = nil
370
+
371
+	newWALDir, err := ioutil.TempDir(e.StateDir, "wal.")
372
+	if err != nil {
373
+		return err
374
+	}
375
+	err = os.Rename(e.walDir(), newWALDir)
376
+	if err != nil {
377
+		return err
378
+	}
379
+
380
+	newSnapDir, err := ioutil.TempDir(e.StateDir, "snap.")
381
+	if err != nil {
382
+		return err
383
+	}
384
+	err = os.Rename(e.snapDir(), newSnapDir)
385
+	if err != nil {
386
+		return err
387
+	}
388
+
389
+	return nil
390
+}
0 391
new file mode 100644
... ...
@@ -0,0 +1,253 @@
0
+package storage
1
+
2
+import (
3
+	"context"
4
+	"io"
5
+	"io/ioutil"
6
+	"os"
7
+	"path/filepath"
8
+	"sort"
9
+	"strings"
10
+
11
+	"github.com/coreos/etcd/pkg/fileutil"
12
+	"github.com/coreos/etcd/raft/raftpb"
13
+	"github.com/coreos/etcd/wal"
14
+	"github.com/coreos/etcd/wal/walpb"
15
+	"github.com/docker/swarmkit/log"
16
+	"github.com/docker/swarmkit/manager/encryption"
17
+	"github.com/pkg/errors"
18
+)
19
+
20
+// This package wraps the github.com/coreos/etcd/wal package, and encrypts
21
+// the bytes of whatever entry is passed to it, and decrypts the bytes of
22
+// whatever entry it reads.
23
+
24
+// WAL is the interface presented by github.com/coreos/etcd/wal.WAL that we depend upon
25
+type WAL interface {
26
+	ReadAll() ([]byte, raftpb.HardState, []raftpb.Entry, error)
27
+	ReleaseLockTo(index uint64) error
28
+	Close() error
29
+	Save(st raftpb.HardState, ents []raftpb.Entry) error
30
+	SaveSnapshot(e walpb.Snapshot) error
31
+}
32
+
33
+// WALFactory provides an interface for the different ways to get a WAL object.
34
+// For instance, the etcd/wal package itself provides this
35
+type WALFactory interface {
36
+	Create(dirpath string, metadata []byte) (WAL, error)
37
+	Open(dirpath string, walsnap walpb.Snapshot) (WAL, error)
38
+}
39
+
40
+var _ WAL = &wrappedWAL{}
41
+var _ WAL = &wal.WAL{}
42
+var _ WALFactory = walCryptor{}
43
+
44
+// wrappedWAL wraps a github.com/coreos/etcd/wal.WAL, and handles encrypting/decrypting
45
+type wrappedWAL struct {
46
+	*wal.WAL
47
+	encrypter encryption.Encrypter
48
+	decrypter encryption.Decrypter
49
+}
50
+
51
+// ReadAll wraps the wal.WAL.ReadAll() function, but it first checks to see if the
52
+// metadata indicates that the entries are encryptd, and if so, decrypts them.
53
+func (w *wrappedWAL) ReadAll() ([]byte, raftpb.HardState, []raftpb.Entry, error) {
54
+	metadata, state, ents, err := w.WAL.ReadAll()
55
+	if err != nil {
56
+		return metadata, state, ents, err
57
+	}
58
+	for i, ent := range ents {
59
+		ents[i].Data, err = encryption.Decrypt(ent.Data, w.decrypter)
60
+		if err != nil {
61
+			return nil, raftpb.HardState{}, nil, err
62
+		}
63
+	}
64
+
65
+	return metadata, state, ents, nil
66
+}
67
+
68
+// Save encrypts the entry data (if an encrypter is exists) before passing it onto the
69
+// wrapped wal.WAL's Save function.
70
+func (w *wrappedWAL) Save(st raftpb.HardState, ents []raftpb.Entry) error {
71
+	var writeEnts []raftpb.Entry
72
+	for _, ent := range ents {
73
+		data, err := encryption.Encrypt(ent.Data, w.encrypter)
74
+		if err != nil {
75
+			return err
76
+		}
77
+		writeEnts = append(writeEnts, raftpb.Entry{
78
+			Index: ent.Index,
79
+			Term:  ent.Term,
80
+			Type:  ent.Type,
81
+			Data:  data,
82
+		})
83
+	}
84
+
85
+	return w.WAL.Save(st, writeEnts)
86
+}
87
+
88
+// walCryptor is an object that provides the same functions as `etcd/wal`
89
+// and `etcd/snap` that we need to open a WAL object or Snapshotter object
90
+type walCryptor struct {
91
+	encrypter encryption.Encrypter
92
+	decrypter encryption.Decrypter
93
+}
94
+
95
+// NewWALFactory returns an object that can be used to produce objects that
96
+// will read from and write to encrypted WALs on disk.
97
+func NewWALFactory(encrypter encryption.Encrypter, decrypter encryption.Decrypter) WALFactory {
98
+	return walCryptor{
99
+		encrypter: encrypter,
100
+		decrypter: decrypter,
101
+	}
102
+}
103
+
104
+// Create returns a new WAL object with the given encrypters and decrypters.
105
+func (wc walCryptor) Create(dirpath string, metadata []byte) (WAL, error) {
106
+	w, err := wal.Create(dirpath, metadata)
107
+	if err != nil {
108
+		return nil, err
109
+	}
110
+	return &wrappedWAL{
111
+		WAL:       w,
112
+		encrypter: wc.encrypter,
113
+		decrypter: wc.decrypter,
114
+	}, nil
115
+}
116
+
117
+// Open returns a new WAL object with the given encrypters and decrypters.
118
+func (wc walCryptor) Open(dirpath string, snap walpb.Snapshot) (WAL, error) {
119
+	w, err := wal.Open(dirpath, snap)
120
+	if err != nil {
121
+		return nil, err
122
+	}
123
+	return &wrappedWAL{
124
+		WAL:       w,
125
+		encrypter: wc.encrypter,
126
+		decrypter: wc.decrypter,
127
+	}, nil
128
+}
129
+
130
+type originalWAL struct{}
131
+
132
+func (o originalWAL) Create(dirpath string, metadata []byte) (WAL, error) {
133
+	return wal.Create(dirpath, metadata)
134
+}
135
+func (o originalWAL) Open(dirpath string, walsnap walpb.Snapshot) (WAL, error) {
136
+	return wal.Open(dirpath, walsnap)
137
+}
138
+
139
+// OriginalWAL is the original `wal` package as an implemntation of the WALFactory interface
140
+var OriginalWAL WALFactory = originalWAL{}
141
+
142
+// WALData contains all the data returned by a WAL's ReadAll() function
143
+// (metadata, hardwate, and entries)
144
+type WALData struct {
145
+	Metadata  []byte
146
+	HardState raftpb.HardState
147
+	Entries   []raftpb.Entry
148
+}
149
+
150
+// ReadRepairWAL opens a WAL for reading, and attempts to read it.  If we can't read it, attempts to repair
151
+// and read again.
152
+func ReadRepairWAL(
153
+	ctx context.Context,
154
+	walDir string,
155
+	walsnap walpb.Snapshot,
156
+	factory WALFactory,
157
+) (WAL, WALData, error) {
158
+	var (
159
+		reader   WAL
160
+		metadata []byte
161
+		st       raftpb.HardState
162
+		ents     []raftpb.Entry
163
+		err      error
164
+	)
165
+	repaired := false
166
+	for {
167
+		if reader, err = factory.Open(walDir, walsnap); err != nil {
168
+			return nil, WALData{}, errors.Wrap(err, "failed to open WAL")
169
+		}
170
+		if metadata, st, ents, err = reader.ReadAll(); err != nil {
171
+			if closeErr := reader.Close(); closeErr != nil {
172
+				return nil, WALData{}, closeErr
173
+			}
174
+			if _, ok := err.(encryption.ErrCannotDecrypt); ok {
175
+				return nil, WALData{}, errors.Wrap(err, "failed to decrypt WAL")
176
+			}
177
+			// we can only repair ErrUnexpectedEOF and we never repair twice.
178
+			if repaired || err != io.ErrUnexpectedEOF {
179
+				return nil, WALData{}, errors.Wrap(err, "irreparable WAL error")
180
+			}
181
+			if !wal.Repair(walDir) {
182
+				return nil, WALData{}, errors.Wrap(err, "WAL error cannot be repaired")
183
+			}
184
+			log.G(ctx).WithError(err).Info("repaired WAL error")
185
+			repaired = true
186
+			continue
187
+		}
188
+		break
189
+	}
190
+	return reader, WALData{
191
+		Metadata:  metadata,
192
+		HardState: st,
193
+		Entries:   ents,
194
+	}, nil
195
+}
196
+
197
+// MigrateWALs reads existing WALs (from a particular snapshot and beyond) from one directory, encoded one way,
198
+// and writes them to a new directory, encoded a different way
199
+func MigrateWALs(ctx context.Context, oldDir, newDir string, oldFactory, newFactory WALFactory, snapshot walpb.Snapshot) error {
200
+	oldReader, waldata, err := ReadRepairWAL(ctx, oldDir, snapshot, oldFactory)
201
+	if err != nil {
202
+		return err
203
+	}
204
+	oldReader.Close()
205
+
206
+	// keep temporary wal directory so WAL initialization appears atomic
207
+	tmpdirpath := filepath.Clean(newDir) + ".tmp"
208
+	if fileutil.Exist(tmpdirpath) {
209
+		if err := os.RemoveAll(tmpdirpath); err != nil {
210
+			return errors.Wrap(err, "could not remove temporary WAL directory")
211
+		}
212
+	}
213
+	if err := fileutil.CreateDirAll(tmpdirpath); err != nil {
214
+		return errors.Wrap(err, "could not create temporary WAL directory")
215
+	}
216
+
217
+	tmpWAL, err := newFactory.Create(tmpdirpath, waldata.Metadata)
218
+	if err != nil {
219
+		return errors.Wrap(err, "could not create new WAL in temporary WAL directory")
220
+	}
221
+	defer tmpWAL.Close()
222
+
223
+	if err := tmpWAL.SaveSnapshot(snapshot); err != nil {
224
+		return errors.Wrap(err, "could not write WAL snapshot in temporary directory")
225
+	}
226
+
227
+	if err := tmpWAL.Save(waldata.HardState, waldata.Entries); err != nil {
228
+		return errors.Wrap(err, "could not migrate WALs to temporary directory")
229
+	}
230
+
231
+	return os.Rename(tmpdirpath, newDir)
232
+}
233
+
234
+// ListWALs lists all the wals in a directory and returns the list in lexical
235
+// order (oldest first)
236
+func ListWALs(dirpath string) ([]string, error) {
237
+	dirents, err := ioutil.ReadDir(dirpath)
238
+	if err != nil {
239
+		return nil, err
240
+	}
241
+
242
+	var wals []string
243
+	for _, dirent := range dirents {
244
+		if strings.HasSuffix(dirent.Name(), ".wal") {
245
+			wals = append(wals, dirent.Name())
246
+		}
247
+	}
248
+
249
+	// Sort WAL filenames in lexical order
250
+	sort.Sort(sort.StringSlice(wals))
251
+	return wals, nil
252
+}
... ...
@@ -21,6 +21,7 @@ import (
21 21
 	"github.com/docker/swarmkit/ioutils"
22 22
 	"github.com/docker/swarmkit/log"
23 23
 	"github.com/docker/swarmkit/manager"
24
+	"github.com/docker/swarmkit/manager/encryption"
24 25
 	"github.com/docker/swarmkit/remotes"
25 26
 	"github.com/docker/swarmkit/xnet"
26 27
 	"github.com/pkg/errors"
... ...
@@ -34,6 +35,10 @@ const stateFilename = "state.json"
34 34
 var (
35 35
 	errNodeStarted    = errors.New("node: already started")
36 36
 	errNodeNotStarted = errors.New("node: not started")
37
+	certDirectory     = "certificates"
38
+
39
+	// ErrInvalidUnlockKey is returned when we can't decrypt the TLS certificate
40
+	ErrInvalidUnlockKey = errors.New("node is locked, and needs a valid unlock key")
37 41
 )
38 42
 
39 43
 // Config provides values for a Node.
... ...
@@ -81,6 +86,14 @@ type Config struct {
81 81
 	// HeartbeatTick defines the amount of ticks between each
82 82
 	// heartbeat sent to other members for health-check purposes
83 83
 	HeartbeatTick uint32
84
+
85
+	// AutoLockManagers determines whether or not an unlock key will be generated
86
+	// when bootstrapping a new cluster for the first time
87
+	AutoLockManagers bool
88
+
89
+	// UnlockKey is the key to unlock a node - used for decrypting at rest.  This
90
+	// only applies to nodes that have already joined a cluster.
91
+	UnlockKey []byte
84 92
 }
85 93
 
86 94
 // Node implements the primary node functionality for a member of a swarm
... ...
@@ -106,6 +119,7 @@ type Node struct {
106 106
 	agent                *agent.Agent
107 107
 	manager              *manager.Manager
108 108
 	notifyNodeChange     chan *api.Node // used to send role updates from the dispatcher api on promotion/demotion
109
+	unlockKey            []byte
109 110
 }
110 111
 
111 112
 // RemoteAPIAddr returns address on which remote manager api listens.
... ...
@@ -150,12 +164,18 @@ func New(c *Config) (*Node, error) {
150 150
 		ready:                make(chan struct{}),
151 151
 		certificateRequested: make(chan struct{}),
152 152
 		notifyNodeChange:     make(chan *api.Node, 1),
153
+		unlockKey:            c.UnlockKey,
153 154
 	}
155
+
156
+	if n.config.JoinAddr != "" || n.config.ForceNewCluster {
157
+		n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
158
+		if n.config.JoinAddr != "" {
159
+			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight)
160
+		}
161
+	}
162
+
154 163
 	n.roleCond = sync.NewCond(n.RLocker())
155 164
 	n.connCond = sync.NewCond(n.RLocker())
156
-	if err := n.loadCertificates(); err != nil {
157
-		return nil, err
158
-	}
159 165
 	return n, nil
160 166
 }
161 167
 
... ...
@@ -189,46 +209,7 @@ func (n *Node) run(ctx context.Context) (err error) {
189 189
 		}
190 190
 	}()
191 191
 
192
-	// NOTE: When this node is created by NewNode(), our nodeID is set if
193
-	// n.loadCertificates() succeeded in loading TLS credentials.
194
-	if n.config.JoinAddr == "" && n.nodeID == "" {
195
-		if err := n.bootstrapCA(); err != nil {
196
-			return err
197
-		}
198
-	}
199
-
200
-	if n.config.JoinAddr != "" || n.config.ForceNewCluster {
201
-		n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
202
-		if n.config.JoinAddr != "" {
203
-			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight)
204
-		}
205
-	}
206
-
207
-	// Obtain new certs and setup TLS certificates renewal for this node:
208
-	// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
209
-	// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
210
-	// us to display the ID before the certificate gets issued (for potential approval).
211
-	// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
212
-	// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
213
-	// up to date.
214
-	issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
215
-	go func() {
216
-		select {
217
-		case <-ctx.Done():
218
-		case resp := <-issueResponseChan:
219
-			log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
220
-				"node.id": resp.NodeID,
221
-			}).Debugf("requesting certificate")
222
-			n.Lock()
223
-			n.nodeID = resp.NodeID
224
-			n.nodeMembership = resp.NodeMembership
225
-			n.Unlock()
226
-			close(n.certificateRequested)
227
-		}
228
-	}()
229
-
230
-	certDir := filepath.Join(n.config.StateDir, "certificates")
231
-	securityConfig, err := ca.LoadOrCreateSecurityConfig(ctx, certDir, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan)
192
+	securityConfig, err := n.loadSecurityConfig(ctx)
232 193
 	if err != nil {
233 194
 		return err
234 195
 	}
... ...
@@ -244,10 +225,6 @@ func (n *Node) run(ctx context.Context) (err error) {
244 244
 	}
245 245
 	defer db.Close()
246 246
 
247
-	if err := n.loadCertificates(); err != nil {
248
-		return err
249
-	}
250
-
251 247
 	forceCertRenewal := make(chan struct{})
252 248
 	renewCert := func() {
253 249
 		select {
... ...
@@ -289,7 +266,7 @@ func (n *Node) run(ctx context.Context) (err error) {
289 289
 		}
290 290
 	}()
291 291
 
292
-	updates := ca.RenewTLSConfig(ctx, securityConfig, certDir, n.remotes, forceCertRenewal)
292
+	updates := ca.RenewTLSConfig(ctx, securityConfig, n.remotes, forceCertRenewal)
293 293
 	go func() {
294 294
 		for {
295 295
 			select {
... ...
@@ -515,40 +492,100 @@ func (n *Node) Remotes() []api.Peer {
515 515
 	return remotes
516 516
 }
517 517
 
518
-func (n *Node) loadCertificates() error {
519
-	certDir := filepath.Join(n.config.StateDir, "certificates")
520
-	rootCA, err := ca.GetLocalRootCA(certDir)
521
-	if err != nil {
522
-		if err == ca.ErrNoLocalRootCA {
523
-			return nil
518
+func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
519
+	paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
520
+	var securityConfig *ca.SecurityConfig
521
+
522
+	krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
523
+	if err := krw.Migrate(); err != nil {
524
+		return nil, err
525
+	}
526
+
527
+	// Check if we already have a valid certificates on disk.
528
+	rootCA, err := ca.GetLocalRootCA(paths.RootCA)
529
+	if err != nil && err != ca.ErrNoLocalRootCA {
530
+		return nil, err
531
+	}
532
+	if err == nil {
533
+		clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw)
534
+		_, ok := errors.Cause(err).(ca.ErrInvalidKEK)
535
+		switch {
536
+		case err == nil:
537
+			securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds)
538
+			log.G(ctx).Debug("loaded CA and TLS certificates")
539
+		case ok:
540
+			return nil, ErrInvalidUnlockKey
541
+		case os.IsNotExist(err):
542
+			break
543
+		default:
544
+			return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
524 545
 		}
525
-		return err
526 546
 	}
527
-	configPaths := ca.NewConfigPaths(certDir)
528
-	clientTLSCreds, _, err := ca.LoadTLSCreds(rootCA, configPaths.Node)
529
-	if err != nil {
530
-		if os.IsNotExist(err) {
531
-			return nil
547
+
548
+	if securityConfig == nil {
549
+		if n.config.JoinAddr == "" {
550
+			// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
551
+			n.unlockKey = nil
552
+			if n.config.AutoLockManagers {
553
+				n.unlockKey = encryption.GenerateSecretKey()
554
+			}
555
+			krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
556
+			rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
557
+			if err != nil {
558
+				return nil, err
559
+			}
560
+			log.G(ctx).Debug("generated CA key and certificate")
561
+		} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
562
+			rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
563
+			if err != nil {
564
+				return nil, err
565
+			}
566
+			log.G(ctx).Debug("downloaded CA certificate")
532 567
 		}
533 568
 
534
-		return errors.Wrapf(err, "error while loading TLS Certificate in %s", configPaths.Node.Cert)
569
+		// Obtain new certs and setup TLS certificates renewal for this node:
570
+		// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
571
+		// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
572
+		// us to display the ID before the certificate gets issued (for potential approval).
573
+		// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
574
+		// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
575
+		// up to date.
576
+		issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
577
+		go func() {
578
+			select {
579
+			case <-ctx.Done():
580
+			case resp := <-issueResponseChan:
581
+				log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
582
+					"node.id": resp.NodeID,
583
+				}).Debugf("loaded TLS certificate")
584
+				n.Lock()
585
+				n.nodeID = resp.NodeID
586
+				n.nodeMembership = resp.NodeMembership
587
+				n.Unlock()
588
+				close(n.certificateRequested)
589
+			}
590
+		}()
591
+
592
+		// LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS
593
+		// certificates and write them to disk
594
+		securityConfig, err = ca.LoadOrCreateSecurityConfig(
595
+			ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw)
596
+		if err != nil {
597
+			if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
598
+				return nil, ErrInvalidUnlockKey
599
+			}
600
+			return nil, err
601
+		}
535 602
 	}
536
-	// todo: try csr if no cert or store nodeID/role in some other way
603
+
537 604
 	n.Lock()
538
-	n.role = clientTLSCreds.Role()
539
-	n.nodeID = clientTLSCreds.NodeID()
605
+	n.role = securityConfig.ClientTLSCreds.Role()
606
+	n.nodeID = securityConfig.ClientTLSCreds.NodeID()
540 607
 	n.nodeMembership = api.NodeMembershipAccepted
541 608
 	n.roleCond.Broadcast()
542 609
 	n.Unlock()
543 610
 
544
-	return nil
545
-}
546
-
547
-func (n *Node) bootstrapCA() error {
548
-	if err := ca.BootstrapCluster(filepath.Join(n.config.StateDir, "certificates")); err != nil {
549
-		return err
550
-	}
551
-	return n.loadCertificates()
611
+	return securityConfig, nil
552 612
 }
553 613
 
554 614
 func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
... ...
@@ -626,13 +663,15 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig
626 626
 				ListenAddr:    n.config.ListenRemoteAPI,
627 627
 				AdvertiseAddr: n.config.AdvertiseRemoteAPI,
628 628
 			},
629
-			ControlAPI:     n.config.ListenControlAPI,
630
-			SecurityConfig: securityConfig,
631
-			ExternalCAs:    n.config.ExternalCAs,
632
-			JoinRaft:       remoteAddr.Addr,
633
-			StateDir:       n.config.StateDir,
634
-			HeartbeatTick:  n.config.HeartbeatTick,
635
-			ElectionTick:   n.config.ElectionTick,
629
+			ControlAPI:       n.config.ListenControlAPI,
630
+			SecurityConfig:   securityConfig,
631
+			ExternalCAs:      n.config.ExternalCAs,
632
+			JoinRaft:         remoteAddr.Addr,
633
+			StateDir:         n.config.StateDir,
634
+			HeartbeatTick:    n.config.HeartbeatTick,
635
+			ElectionTick:     n.config.ElectionTick,
636
+			AutoLockManagers: n.config.AutoLockManagers,
637
+			UnlockKey:        n.unlockKey,
636 638
 		})
637 639
 		if err != nil {
638 640
 			return err