Signed-off-by: Alessandro Boch <aboch@docker.com>
| ... | ... |
@@ -105,7 +105,7 @@ github.com/docker/containerd 422e31ce907fd9c3833a38d7b8fdd023e5a76e73 |
| 105 | 105 |
github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4 |
| 106 | 106 |
|
| 107 | 107 |
# cluster |
| 108 |
-github.com/docker/swarmkit 0e2d9ebcea9d5bbd4a06b3b964fb96356801f880 |
|
| 108 |
+github.com/docker/swarmkit 9fdea50c14492b6e1f472813849794d36bfef217 |
|
| 109 | 109 |
github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 |
| 110 | 110 |
github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2 |
| 111 | 111 |
github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a |
| ... | ... |
@@ -591,6 +591,11 @@ type NetworkSpec struct {
|
| 591 | 591 |
// enabled(default case) no manual attachment to this network |
| 592 | 592 |
// can happen. |
| 593 | 593 |
Attachable bool `protobuf:"varint,6,opt,name=attachable,proto3" json:"attachable,omitempty"` |
| 594 |
+ // Ingress indicates this network will provide the routing-mesh. |
|
| 595 |
+ // In older versions, the network providing the routing mesh was |
|
| 596 |
+ // swarm internally created only and it was identified by the name |
|
| 597 |
+ // "ingress" and the label "com.docker.swarm.internal": "true". |
|
| 598 |
+ Ingress bool `protobuf:"varint,7,opt,name=ingress,proto3" json:"ingress,omitempty"` |
|
| 594 | 599 |
} |
| 595 | 600 |
|
| 596 | 601 |
func (m *NetworkSpec) Reset() { *m = NetworkSpec{} }
|
| ... | ... |
@@ -1795,6 +1800,16 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
|
| 1795 | 1795 |
} |
| 1796 | 1796 |
i++ |
| 1797 | 1797 |
} |
| 1798 |
+ if m.Ingress {
|
|
| 1799 |
+ dAtA[i] = 0x38 |
|
| 1800 |
+ i++ |
|
| 1801 |
+ if m.Ingress {
|
|
| 1802 |
+ dAtA[i] = 1 |
|
| 1803 |
+ } else {
|
|
| 1804 |
+ dAtA[i] = 0 |
|
| 1805 |
+ } |
|
| 1806 |
+ i++ |
|
| 1807 |
+ } |
|
| 1798 | 1808 |
return i, nil |
| 1799 | 1809 |
} |
| 1800 | 1810 |
|
| ... | ... |
@@ -2255,6 +2270,9 @@ func (m *NetworkSpec) Size() (n int) {
|
| 2255 | 2255 |
if m.Attachable {
|
| 2256 | 2256 |
n += 2 |
| 2257 | 2257 |
} |
| 2258 |
+ if m.Ingress {
|
|
| 2259 |
+ n += 2 |
|
| 2260 |
+ } |
|
| 2258 | 2261 |
return n |
| 2259 | 2262 |
} |
| 2260 | 2263 |
|
| ... | ... |
@@ -2502,6 +2520,7 @@ func (this *NetworkSpec) String() string {
|
| 2502 | 2502 |
`Internal:` + fmt.Sprintf("%v", this.Internal) + `,`,
|
| 2503 | 2503 |
`IPAM:` + strings.Replace(fmt.Sprintf("%v", this.IPAM), "IPAMOptions", "IPAMOptions", 1) + `,`,
|
| 2504 | 2504 |
`Attachable:` + fmt.Sprintf("%v", this.Attachable) + `,`,
|
| 2505 |
+ `Ingress:` + fmt.Sprintf("%v", this.Ingress) + `,`,
|
|
| 2505 | 2506 |
`}`, |
| 2506 | 2507 |
}, "") |
| 2507 | 2508 |
return s |
| ... | ... |
@@ -4688,6 +4707,26 @@ func (m *NetworkSpec) Unmarshal(dAtA []byte) error {
|
| 4688 | 4688 |
} |
| 4689 | 4689 |
} |
| 4690 | 4690 |
m.Attachable = bool(v != 0) |
| 4691 |
+ case 7: |
|
| 4692 |
+ if wireType != 0 {
|
|
| 4693 |
+ return fmt.Errorf("proto: wrong wireType = %d for field Ingress", wireType)
|
|
| 4694 |
+ } |
|
| 4695 |
+ var v int |
|
| 4696 |
+ for shift := uint(0); ; shift += 7 {
|
|
| 4697 |
+ if shift >= 64 {
|
|
| 4698 |
+ return ErrIntOverflowSpecs |
|
| 4699 |
+ } |
|
| 4700 |
+ if iNdEx >= l {
|
|
| 4701 |
+ return io.ErrUnexpectedEOF |
|
| 4702 |
+ } |
|
| 4703 |
+ b := dAtA[iNdEx] |
|
| 4704 |
+ iNdEx++ |
|
| 4705 |
+ v |= (int(b) & 0x7F) << shift |
|
| 4706 |
+ if b < 0x80 {
|
|
| 4707 |
+ break |
|
| 4708 |
+ } |
|
| 4709 |
+ } |
|
| 4710 |
+ m.Ingress = bool(v != 0) |
|
| 4691 | 4711 |
default: |
| 4692 | 4712 |
iNdEx = preIndex |
| 4693 | 4713 |
skippy, err := skipSpecs(dAtA[iNdEx:]) |
| ... | ... |
@@ -5218,112 +5257,113 @@ var ( |
| 5218 | 5218 |
func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
|
| 5219 | 5219 |
|
| 5220 | 5220 |
var fileDescriptorSpecs = []byte{
|
| 5221 |
- // 1707 bytes of a gzipped FileDescriptorProto |
|
| 5221 |
+ // 1717 bytes of a gzipped FileDescriptorProto |
|
| 5222 | 5222 |
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x41, 0x73, 0x1b, 0xb7, |
| 5223 |
- 0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3, |
|
| 5223 |
+ 0x15, 0x16, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x50, 0x34, 0xe3, |
|
| 5224 | 5224 |
0xa6, 0x4a, 0x33, 0xa5, 0xa6, 0x6a, 0x27, 0x75, 0xea, 0x66, 0x5a, 0x52, 0x64, 0x65, 0x55, 0x95, |
| 5225 | 5225 |
0xcc, 0x01, 0x15, 0x77, 0x7c, 0xe2, 0x80, 0xbb, 0x10, 0xb9, 0xa3, 0xe5, 0x62, 0x0b, 0x60, 0x99, |
| 5226 |
- 0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0xbf, 0xa1, 0xff, 0xc1, 0xc7, 0x1e, |
|
| 5227 |
- 0x7b, 0xd2, 0x34, 0xfc, 0x0b, 0xfd, 0x01, 0xed, 0x00, 0x0b, 0x92, 0xcb, 0x64, 0x15, 0x7b, 0x26, |
|
| 5228 |
- 0xbe, 0xe1, 0xbd, 0xfd, 0xbe, 0x07, 0xe0, 0xe1, 0xc3, 0xc3, 0x5b, 0xb0, 0x45, 0x44, 0x5d, 0xd1, |
|
| 5229 |
- 0x8a, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x57, 0x94, 0xb7, 0xc4, 0x97, 0x84, 0x4f, 0xaf, 0x7c, |
|
| 5230 |
- 0xd9, 0x9a, 0xfd, 0xbc, 0x6a, 0xcb, 0x79, 0x44, 0x0d, 0xa0, 0x7a, 0x7f, 0xcc, 0xc6, 0x4c, 0x0f, |
|
| 5231 |
- 0x0f, 0xd4, 0xc8, 0x78, 0x6b, 0x63, 0xc6, 0xc6, 0x01, 0x3d, 0xd0, 0xd6, 0x28, 0xbe, 0x3c, 0xf0, |
|
| 5232 |
- 0x62, 0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xba, 0x00, 0xd6, 0x39, 0xf3, 0xe8, 0x20, 0xa2, |
|
| 5233 |
- 0x2e, 0x3a, 0x06, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0xbf, 0x6f, 0x1f, 0xd6, |
|
| 5234 |
- 0x5b, 0xdf, 0x9e, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0x37, 0xf5, 0x1c, 0x4e, 0x33, 0xd1, |
|
| 5235 |
- 0x6f, 0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0xef, 0xdf, |
|
| 5236 |
- 0x39, 0xfc, 0x51, 0x56, 0x24, 0x35, 0x39, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0x28, 0x03, 0x1d, 0x03, |
|
| 5237 |
- 0x4c, 0xe9, 0x74, 0x44, 0xb9, 0x98, 0xf8, 0x91, 0xb3, 0xad, 0xe9, 0x3f, 0xb9, 0x8d, 0xae, 0xd6, |
|
| 5238 |
- 0xde, 0x3a, 0x5b, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa0, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f, |
|
| 5239 |
- 0x7c, 0x39, 0x77, 0x0a, 0x3a, 0xd4, 0xc7, 0xdf, 0x19, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3, |
|
| 5240 |
- 0x03, 0x58, 0x4f, 0x84, 0x3e, 0x82, 0xdd, 0x7e, 0xef, 0xbc, 0x7b, 0x72, 0x7e, 0x5c, 0xc9, 0x55, |
|
| 5241 |
- 0x1f, 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb, |
|
| 5242 |
- 0x60, 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0x5d, 0x37, 0xde, |
|
| 5243 |
- 0xdf, 0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0x57, 0xff, 0xa8, 0xe5, 0x9a, 0x5f, |
|
| 5244 |
- 0xe5, 0xa1, 0x9c, 0x5e, 0x04, 0xfa, 0x08, 0x8a, 0xed, 0xa3, 0x8b, 0x93, 0x17, 0xbd, 0x4a, 0x6e, |
|
| 5245 |
- 0x4d, 0x4f, 0x23, 0xda, 0xae, 0xf4, 0x67, 0x14, 0x3d, 0x86, 0x9d, 0x7e, 0xfb, 0x8b, 0x41, 0xaf, |
|
| 5246 |
- 0x92, 0x5f, 0x2f, 0x27, 0x0d, 0xeb, 0x93, 0x58, 0x68, 0x54, 0x17, 0xb7, 0x4f, 0xce, 0x2b, 0x5b, |
|
| 5247 |
- 0xd9, 0xa8, 0x2e, 0x27, 0x7e, 0x68, 0x96, 0xf2, 0xf7, 0x02, 0xd8, 0x03, 0xca, 0x67, 0xbe, 0xfb, |
|
| 5248 |
- 0x8e, 0x25, 0xf2, 0x29, 0x14, 0x24, 0x11, 0x57, 0x5a, 0x1a, 0x76, 0xb6, 0x34, 0x2e, 0x88, 0xb8, |
|
| 5249 |
- 0x52, 0x93, 0x1a, 0xba, 0xc6, 0x2b, 0x65, 0x70, 0x1a, 0x05, 0xbe, 0x4b, 0x24, 0xf5, 0xb4, 0x32, |
|
| 5250 |
- 0xec, 0xc3, 0x1f, 0x67, 0xb1, 0xf1, 0x0a, 0x65, 0xd6, 0xff, 0x2c, 0x87, 0x53, 0x54, 0xf4, 0x14, |
|
| 5251 |
- 0x8a, 0xe3, 0x80, 0x8d, 0x48, 0xa0, 0x35, 0x61, 0x1f, 0x3e, 0xca, 0x0a, 0x72, 0xac, 0x11, 0xeb, |
|
| 5252 |
- 0x00, 0x86, 0x82, 0x9e, 0x40, 0x31, 0x8e, 0x3c, 0x22, 0xa9, 0x53, 0xd4, 0xe4, 0x46, 0x16, 0xf9, |
|
| 5253 |
- 0x0b, 0x8d, 0x38, 0x62, 0xe1, 0xa5, 0x3f, 0xc6, 0x06, 0x8f, 0x4e, 0xc1, 0x0a, 0xa9, 0xfc, 0x92, |
|
| 5254 |
- 0xf1, 0x2b, 0xe1, 0xec, 0x36, 0xb6, 0xf7, 0xed, 0xc3, 0x4f, 0x32, 0xc5, 0x98, 0x60, 0xda, 0x52, |
|
| 5255 |
- 0x12, 0x77, 0x32, 0xa5, 0xa1, 0x4c, 0xc2, 0x74, 0xb6, 0x9c, 0x3c, 0x5e, 0x05, 0x40, 0xbf, 0x01, |
|
| 5256 |
- 0x8b, 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x3a, 0xd6, 0xed, 0x0b, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x5e, |
|
| 5257 |
- 0x31, 0x14, 0x9b, 0xb3, 0x20, 0x18, 0x11, 0xf7, 0xca, 0x29, 0xbd, 0xe5, 0x36, 0x56, 0x8c, 0x4e, |
|
| 5258 |
- 0x11, 0x0a, 0x53, 0xe6, 0xd1, 0xe6, 0x01, 0xdc, 0xfb, 0x56, 0xaa, 0x51, 0x15, 0x2c, 0x93, 0xea, |
|
| 5259 |
- 0x44, 0x23, 0x05, 0xbc, 0xb2, 0x9b, 0x77, 0x61, 0x6f, 0x23, 0xad, 0xcd, 0xbf, 0x16, 0xc0, 0x5a, |
|
| 5260 |
- 0x9e, 0x35, 0x6a, 0x43, 0xc9, 0x65, 0xa1, 0x24, 0x7e, 0x48, 0xb9, 0x91, 0x57, 0xe6, 0xc9, 0x1c, |
|
| 5261 |
- 0x2d, 0x41, 0x8a, 0xf5, 0x2c, 0x87, 0xd7, 0x2c, 0xf4, 0x7b, 0x28, 0x71, 0x2a, 0x58, 0xcc, 0x5d, |
|
| 5262 |
- 0x2a, 0x8c, 0xbe, 0xf6, 0xb3, 0x15, 0x92, 0x80, 0x30, 0xfd, 0x73, 0xec, 0x73, 0xaa, 0xb2, 0x2c, |
|
| 5263 |
- 0xf0, 0x9a, 0x8a, 0x9e, 0xc2, 0x2e, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0x4b, 0x22, 0x38, 0x81, 0xf4, |
|
| 5264 |
- 0x59, 0xe0, 0xbb, 0x73, 0xbc, 0x64, 0xa0, 0xa7, 0x50, 0x8a, 0x02, 0xe2, 0xea, 0xa8, 0xce, 0x8e, |
|
| 5265 |
- 0xa6, 0x7f, 0x90, 0x45, 0xef, 0x2f, 0x41, 0x78, 0x8d, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4, |
|
| 5266 |
- 0xb8, 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x9a, 0xc5, 0xee, 0x6a, 0x04, 0x2e, 0x05, 0x6c, 0x9c, 0x0c, |
|
| 5267 |
- 0xd1, 0xf1, 0xf7, 0xd2, 0x57, 0x4a, 0x5b, 0xa7, 0x00, 0x64, 0xf5, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7, |
|
| 5268 |
- 0x0a, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x8f, 0xa0, 0x7c, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94, |
|
| 5269 |
- 0xb4, 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc1, 0x2e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3, |
|
| 5270 |
- 0x14, 0xde, 0xcb, 0x0c, 0x8a, 0x0e, 0xa1, 0xbc, 0x3a, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3a, |
|
| 5271 |
- 0x77, 0x17, 0x37, 0x75, 0x7b, 0xa5, 0x87, 0x93, 0x2e, 0xb6, 0x57, 0xa0, 0x13, 0xaf, 0xf9, 0x37, |
|
| 5272 |
- 0x0b, 0xf6, 0x36, 0xc4, 0x82, 0xee, 0xc3, 0x8e, 0x3f, 0x25, 0x63, 0x9a, 0xd0, 0x71, 0x62, 0xa0, |
|
| 5273 |
- 0x1e, 0x14, 0x03, 0x32, 0xa2, 0x81, 0x92, 0x8c, 0x4a, 0xdb, 0xcf, 0xde, 0xa8, 0xba, 0xd6, 0x1f, |
|
| 5274 |
- 0x35, 0xbe, 0x17, 0x4a, 0x3e, 0xc7, 0x86, 0x8c, 0x1c, 0xd8, 0x75, 0xd9, 0x74, 0x4a, 0x42, 0x55, |
|
| 5275 |
- 0x9c, 0xb6, 0xf7, 0x4b, 0x78, 0x69, 0x22, 0x04, 0x05, 0xc2, 0xc7, 0xc2, 0x29, 0x68, 0xb7, 0x1e, |
|
| 5276 |
- 0xa3, 0x0a, 0x6c, 0xd3, 0x70, 0xe6, 0xec, 0x68, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e, |
|
| 5277 |
- 0xc2, 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd5, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xe2, 0x94, |
|
| 5278 |
- 0xc5, 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6b, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81, |
|
| 5279 |
- 0xa3, 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33, |
|
| 5280 |
- 0x97, 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0xcb, 0x5e, 0xa1, 0xd5, 0x35, 0xbd, 0x02, 0xbe, 0xab, |
|
| 5281 |
- 0x38, 0xc7, 0x8a, 0xd2, 0xd7, 0x0c, 0xd4, 0x87, 0x72, 0x14, 0x07, 0xc1, 0x90, 0x45, 0xc9, 0x3b, |
|
| 5282 |
- 0x00, 0x3a, 0xc2, 0x5b, 0xa4, 0xac, 0x1f, 0x07, 0xc1, 0xf3, 0x84, 0x84, 0xed, 0x68, 0x6d, 0xa0, |
|
| 5283 |
- 0xf7, 0xa1, 0x38, 0xe6, 0x2c, 0x8e, 0x84, 0x63, 0xeb, 0x64, 0x18, 0x0b, 0x7d, 0x0e, 0xbb, 0x82, |
|
| 5284 |
- 0xba, 0x9c, 0x4a, 0xe1, 0x94, 0xf5, 0x56, 0x3f, 0xcc, 0x9a, 0x64, 0xa0, 0x21, 0x98, 0x5e, 0x52, |
|
| 5285 |
- 0x4e, 0x43, 0x97, 0xe2, 0x25, 0x07, 0x3d, 0x80, 0x6d, 0x29, 0xe7, 0xce, 0x5e, 0x23, 0xbf, 0x6f, |
|
| 5286 |
- 0x75, 0x76, 0x17, 0x37, 0xf5, 0xed, 0x8b, 0x8b, 0x97, 0x58, 0xf9, 0x54, 0x8d, 0x9a, 0x30, 0x21, |
|
| 5287 |
- 0x43, 0x32, 0xa5, 0xce, 0x1d, 0x9d, 0xdb, 0x95, 0x8d, 0x5e, 0x02, 0x78, 0xa1, 0x18, 0xba, 0xfa, |
|
| 5288 |
- 0x52, 0x38, 0x77, 0xf5, 0xee, 0x3e, 0x79, 0xf3, 0xee, 0xba, 0xe7, 0x03, 0x53, 0xa7, 0xf7, 0x16, |
|
| 5289 |
- 0x37, 0xf5, 0xd2, 0xca, 0xc4, 0x25, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8, |
|
| 5290 |
- 0x89, 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xb9, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4d, 0x52, |
|
| 5291 |
- 0x0a, 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43, |
|
| 5292 |
- 0x21, 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x29, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16, |
|
| 5293 |
- 0x89, 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0xa8, |
|
| 5294 |
- 0x0e, 0xb6, 0xd6, 0x85, 0xf0, 0xc7, 0x21, 0x09, 0x9c, 0xfb, 0x3a, 0x1f, 0xa0, 0x5c, 0x03, 0xed, |
|
| 5295 |
- 0xa9, 0x7e, 0x06, 0x76, 0x4a, 0xee, 0x4a, 0xa6, 0x57, 0x74, 0x6e, 0x6e, 0x90, 0x1a, 0xaa, 0x35, |
|
| 5296 |
- 0xcd, 0x48, 0x10, 0x27, 0xcd, 0x5e, 0x09, 0x27, 0xc6, 0xaf, 0xb7, 0x9e, 0xe4, 0xab, 0x87, 0x60, |
|
| 5297 |
- 0xa7, 0x8e, 0x1d, 0x7d, 0x08, 0x7b, 0x9c, 0x8e, 0x7d, 0x21, 0xf9, 0x7c, 0x48, 0x62, 0x39, 0x71, |
|
| 5298 |
- 0x7e, 0xa7, 0x09, 0xe5, 0xa5, 0xb3, 0x1d, 0xcb, 0x49, 0x75, 0x08, 0xeb, 0xec, 0xa1, 0x06, 0xd8, |
|
| 5299 |
- 0xea, 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x76, 0x29, 0xf5, 0x08, 0x4a, |
|
| 5300 |
- 0xb8, 0x3b, 0xd1, 0x97, 0xb7, 0x84, 0x8d, 0xa5, 0x6e, 0xe3, 0x52, 0xa2, 0xe6, 0x36, 0x1a, 0xb3, |
|
| 5301 |
- 0xf9, 0xdf, 0x3c, 0x94, 0xd3, 0xef, 0x22, 0x3a, 0x4a, 0xde, 0x33, 0xbd, 0xa5, 0x3b, 0x87, 0x07, |
|
| 5302 |
- 0x6f, 0x7a, 0x47, 0xf5, 0xeb, 0x11, 0xc4, 0x2a, 0xd8, 0x99, 0x6a, 0x61, 0x35, 0x19, 0xfd, 0x12, |
|
| 5303 |
- 0x76, 0x22, 0xc6, 0xe5, 0xb2, 0x86, 0xd4, 0x32, 0x2b, 0x3e, 0xe3, 0xcb, 0x6a, 0x9b, 0x80, 0x9b, |
|
| 5304 |
- 0x13, 0xb8, 0xb3, 0x19, 0x0d, 0x3d, 0x86, 0xed, 0x17, 0x27, 0xfd, 0x4a, 0xae, 0xfa, 0xf0, 0xd5, |
|
| 5305 |
- 0x75, 0xe3, 0x87, 0x9b, 0x1f, 0x5f, 0xf8, 0x5c, 0xc6, 0x24, 0x38, 0xe9, 0xa3, 0x9f, 0xc2, 0x4e, |
|
| 5306 |
- 0xf7, 0x7c, 0x80, 0x71, 0x25, 0x5f, 0xad, 0xbf, 0xba, 0x6e, 0x3c, 0xdc, 0xc4, 0xa9, 0x4f, 0x2c, |
|
| 5307 |
- 0x0e, 0x3d, 0xcc, 0x46, 0xab, 0x76, 0xee, 0x9f, 0x5b, 0x60, 0x9b, 0xd2, 0xfa, 0xae, 0x3b, 0xfe, |
|
| 5308 |
- 0xbd, 0xe4, 0xb5, 0x5a, 0xde, 0x99, 0xad, 0x37, 0x3e, 0x5a, 0xe5, 0x84, 0x60, 0xce, 0xf8, 0x11, |
|
| 5309 |
- 0x94, 0xfd, 0x68, 0xf6, 0xe9, 0x90, 0x86, 0x64, 0x14, 0x98, 0xce, 0xce, 0xc2, 0xb6, 0xf2, 0xf5, |
|
| 5310 |
- 0x12, 0x97, 0xba, 0xb0, 0x7e, 0x28, 0x29, 0x0f, 0x4d, 0xcf, 0x66, 0xe1, 0x95, 0x8d, 0x3e, 0x87, |
|
| 5311 |
- 0x82, 0x1f, 0x91, 0xa9, 0x79, 0x69, 0x33, 0x77, 0x70, 0xd2, 0x6f, 0x9f, 0x19, 0x0d, 0x76, 0xac, |
|
| 5312 |
- 0xc5, 0x4d, 0xbd, 0xa0, 0x1c, 0x58, 0xd3, 0x50, 0x6d, 0xf9, 0xd8, 0xa9, 0x99, 0x74, 0xf1, 0xb5, |
|
| 5313 |
- 0x70, 0xca, 0xd3, 0xfc, 0x5f, 0x01, 0xec, 0xa3, 0x20, 0x16, 0xd2, 0x3c, 0x21, 0xef, 0x2c, 0x6f, |
|
| 5314 |
- 0x2f, 0xe1, 0x1e, 0xd1, 0xcd, 0x3f, 0x09, 0x55, 0x3d, 0xd6, 0x4d, 0x84, 0xc9, 0xdd, 0xe3, 0xcc, |
|
| 5315 |
- 0x70, 0x2b, 0x70, 0xd2, 0x70, 0x74, 0x8a, 0x2a, 0xa6, 0x93, 0xc7, 0x15, 0xf2, 0x8d, 0x2f, 0x68, |
|
| 5316 |
- 0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52, 0xc5, 0x4d, 0xb3, 0x9c, 0xf9, 0x1b, 0xf5, 0x3c, |
|
| 5317 |
- 0x0d, 0x34, 0x25, 0x2c, 0x59, 0xed, 0x66, 0x0c, 0xf4, 0x04, 0x0a, 0x9c, 0x5c, 0x2e, 0x1b, 0xa2, |
|
| 5318 |
- 0x4c, 0x7d, 0x63, 0x72, 0x29, 0x37, 0x42, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48, |
|
| 5319 |
- 0x77, 0x42, 0xb9, 0x39, 0xa7, 0xcc, 0x2d, 0x76, 0x57, 0xa8, 0x8d, 0x28, 0x29, 0x36, 0x3a, 0x85, |
|
| 5320 |
- 0x92, 0x4b, 0x96, 0x4a, 0x2b, 0xde, 0xfe, 0x07, 0x71, 0xd4, 0x36, 0x21, 0x2a, 0x2a, 0xc4, 0xe2, |
|
| 5321 |
- 0xa6, 0x6e, 0x2d, 0x3d, 0xd8, 0x72, 0x89, 0x51, 0xde, 0x29, 0xec, 0xa9, 0x3f, 0x8b, 0xa1, 0x47, |
|
| 5322 |
- 0x2f, 0x49, 0x1c, 0x48, 0xa1, 0x1f, 0xda, 0x5b, 0x4a, 0xb2, 0x6a, 0x53, 0xbb, 0x06, 0x67, 0xd6, |
|
| 5323 |
- 0x55, 0x96, 0x29, 0x1f, 0xfa, 0x13, 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x67, 0xcb, 0x15, 0x5a, |
|
| 5324 |
- 0xb7, 0x6f, 0xb6, 0xb7, 0x02, 0x6f, 0x6c, 0xb6, 0x42, 0xbf, 0xe1, 0x6f, 0xfa, 0x00, 0xc9, 0x23, |
|
| 5325 |
- 0xf7, 0x6e, 0xf5, 0x87, 0xa0, 0xe0, 0x11, 0x49, 0xb4, 0xe4, 0xca, 0x58, 0x8f, 0x3b, 0xce, 0xeb, |
|
| 5326 |
- 0xaf, 0x6b, 0xb9, 0x7f, 0x7f, 0x5d, 0xcb, 0xfd, 0x65, 0x51, 0xcb, 0xbf, 0x5e, 0xd4, 0xf2, 0xff, |
|
| 5327 |
- 0x5a, 0xd4, 0xf2, 0xff, 0x59, 0xd4, 0xf2, 0xa3, 0xa2, 0x6e, 0x0d, 0x7e, 0xf1, 0xff, 0x00, 0x00, |
|
| 5328 |
- 0x00, 0xff, 0xff, 0xed, 0xbe, 0x26, 0xe6, 0x9a, 0x10, 0x00, 0x00, |
|
| 5226 |
+ 0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0x7f, 0xc6, 0xb7, 0xf6, 0xd8, 0x93, |
|
| 5227 |
+ 0xa6, 0xe1, 0x5f, 0xe8, 0x0f, 0x68, 0x07, 0x58, 0x2c, 0xb9, 0x4c, 0x56, 0xb1, 0x67, 0xe2, 0x1b, |
|
| 5228 |
+ 0xde, 0xdb, 0xef, 0x7b, 0x00, 0x1e, 0x3e, 0x3c, 0xbc, 0x05, 0x5b, 0x44, 0xd4, 0x15, 0xad, 0x88, |
|
| 5229 |
+ 0x33, 0xc9, 0x10, 0xf2, 0x98, 0x7b, 0x45, 0x79, 0x4b, 0x7c, 0x49, 0xf8, 0xf4, 0xca, 0x97, 0xad, |
|
| 5230 |
+ 0xd9, 0xcf, 0x6b, 0xb6, 0x9c, 0x47, 0xd4, 0x00, 0x6a, 0xf7, 0xc7, 0x6c, 0xcc, 0xf4, 0xf0, 0x40, |
|
| 5231 |
+ 0x8d, 0x8c, 0xb7, 0x3e, 0x66, 0x6c, 0x1c, 0xd0, 0x03, 0x6d, 0x8d, 0xe2, 0xcb, 0x03, 0x2f, 0xe6, |
|
| 5232 |
+ 0x44, 0xfa, 0x2c, 0x4c, 0xbe, 0x37, 0xaf, 0x8b, 0x60, 0x9d, 0x33, 0x8f, 0x0e, 0x22, 0xea, 0xa2, |
|
| 5233 |
+ 0x63, 0xb0, 0x49, 0x18, 0x32, 0xa9, 0x01, 0xc2, 0x29, 0x34, 0x0a, 0xfb, 0xf6, 0xe1, 0x5e, 0xeb, |
|
| 5234 |
+ 0xdb, 0x33, 0xb7, 0xda, 0x2b, 0x58, 0xa7, 0xf8, 0xfa, 0x66, 0x6f, 0x03, 0x67, 0x99, 0xe8, 0xb7, |
|
| 5235 |
+ 0x50, 0xf1, 0xa8, 0xf0, 0x39, 0xf5, 0x86, 0x9c, 0x05, 0xd4, 0xd9, 0x6c, 0x14, 0xf6, 0xef, 0x1c, |
|
| 5236 |
+ 0xfe, 0x28, 0x2f, 0x92, 0x9a, 0x1c, 0xb3, 0x80, 0x62, 0xdb, 0x30, 0x94, 0x81, 0x8e, 0x01, 0xa6, |
|
| 5237 |
+ 0x74, 0x3a, 0xa2, 0x5c, 0x4c, 0xfc, 0xc8, 0xd9, 0xd2, 0xf4, 0x9f, 0xdc, 0x46, 0x57, 0x6b, 0x6f, |
|
| 5238 |
+ 0x9d, 0x2d, 0xe1, 0x38, 0x43, 0x45, 0x67, 0x50, 0x21, 0x33, 0xe2, 0x07, 0x64, 0xe4, 0x07, 0xbe, |
|
| 5239 |
+ 0x9c, 0x3b, 0x45, 0x1d, 0xea, 0xe3, 0xef, 0x0c, 0xd5, 0xce, 0x10, 0xf0, 0x1a, 0xbd, 0xe9, 0x01, |
|
| 5240 |
+ 0xac, 0x26, 0x42, 0x1f, 0xc1, 0x4e, 0xbf, 0x77, 0xde, 0x3d, 0x39, 0x3f, 0xae, 0x6e, 0xd4, 0x1e, |
|
| 5241 |
+ 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x00, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb, 0x60, |
|
| 5242 |
+ 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x5a, 0xa8, 0xd5, 0x5e, 0x5d, 0x37, 0xde, 0x5f, |
|
| 5243 |
+ 0x07, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xb5, 0xe2, 0x57, 0xff, 0xa8, 0x6f, 0x34, 0xbf, 0x2a, |
|
| 5244 |
+ 0x40, 0x25, 0xbb, 0x08, 0xf4, 0x11, 0x94, 0xda, 0x47, 0x17, 0x27, 0x2f, 0x7a, 0xd5, 0x8d, 0x15, |
|
| 5245 |
+ 0x3d, 0x8b, 0x68, 0xbb, 0xd2, 0x9f, 0x51, 0xf4, 0x18, 0xb6, 0xfb, 0xed, 0x2f, 0x06, 0xbd, 0x6a, |
|
| 5246 |
+ 0x61, 0xb5, 0x9c, 0x2c, 0xac, 0x4f, 0x62, 0xa1, 0x51, 0x5d, 0xdc, 0x3e, 0x39, 0xaf, 0x6e, 0xe6, |
|
| 5247 |
+ 0xa3, 0xba, 0x9c, 0xf8, 0xa1, 0x59, 0xca, 0xdf, 0x8b, 0x60, 0x0f, 0x28, 0x9f, 0xf9, 0xee, 0x3b, |
|
| 5248 |
+ 0x96, 0xc8, 0xa7, 0x50, 0x94, 0x44, 0x5c, 0x69, 0x69, 0xd8, 0xf9, 0xd2, 0xb8, 0x20, 0xe2, 0x4a, |
|
| 5249 |
+ 0x4d, 0x6a, 0xe8, 0x1a, 0xaf, 0x94, 0xc1, 0x69, 0x14, 0xf8, 0x2e, 0x91, 0xd4, 0xd3, 0xca, 0xb0, |
|
| 5250 |
+ 0x0f, 0x7f, 0x9c, 0xc7, 0xc6, 0x4b, 0x94, 0x59, 0xff, 0xb3, 0x0d, 0x9c, 0xa1, 0xa2, 0xa7, 0x50, |
|
| 5251 |
+ 0x1a, 0x07, 0x6c, 0x44, 0x02, 0xad, 0x09, 0xfb, 0xf0, 0x51, 0x5e, 0x90, 0x63, 0x8d, 0x58, 0x05, |
|
| 5252 |
+ 0x30, 0x14, 0xf4, 0x04, 0x4a, 0x71, 0xe4, 0x11, 0x49, 0x9d, 0x92, 0x26, 0x37, 0xf2, 0xc8, 0x5f, |
|
| 5253 |
+ 0x68, 0xc4, 0x11, 0x0b, 0x2f, 0xfd, 0x31, 0x36, 0x78, 0x74, 0x0a, 0x56, 0x48, 0xe5, 0x97, 0x8c, |
|
| 5254 |
+ 0x5f, 0x09, 0x67, 0xa7, 0xb1, 0xb5, 0x6f, 0x1f, 0x7e, 0x92, 0x2b, 0xc6, 0x04, 0xd3, 0x96, 0x92, |
|
| 5255 |
+ 0xb8, 0x93, 0x29, 0x0d, 0x65, 0x12, 0xa6, 0xb3, 0xe9, 0x14, 0xf0, 0x32, 0x00, 0xfa, 0x0d, 0x58, |
|
| 5256 |
+ 0x34, 0xf4, 0x22, 0xe6, 0x87, 0xd2, 0xb1, 0x6e, 0x5f, 0x48, 0xcf, 0x60, 0x54, 0x32, 0xf1, 0x92, |
|
| 5257 |
+ 0xa1, 0xd8, 0x9c, 0x05, 0xc1, 0x88, 0xb8, 0x57, 0x4e, 0xf9, 0x2d, 0xb7, 0xb1, 0x64, 0x74, 0x4a, |
|
| 5258 |
+ 0x50, 0x9c, 0x32, 0x8f, 0x36, 0x0f, 0xe0, 0xde, 0xb7, 0x52, 0x8d, 0x6a, 0x60, 0x99, 0x54, 0x27, |
|
| 5259 |
+ 0x1a, 0x29, 0xe2, 0xa5, 0xdd, 0xbc, 0x0b, 0xbb, 0x6b, 0x69, 0x6d, 0xfe, 0xb5, 0x08, 0x56, 0x7a, |
|
| 5260 |
+ 0xd6, 0xa8, 0x0d, 0x65, 0x97, 0x85, 0x92, 0xf8, 0x21, 0xe5, 0x46, 0x5e, 0xb9, 0x27, 0x73, 0x94, |
|
| 5261 |
+ 0x82, 0x14, 0xeb, 0xd9, 0x06, 0x5e, 0xb1, 0xd0, 0xef, 0xa1, 0xcc, 0xa9, 0x60, 0x31, 0x77, 0xa9, |
|
| 5262 |
+ 0x30, 0xfa, 0xda, 0xcf, 0x57, 0x48, 0x02, 0xc2, 0xf4, 0xcf, 0xb1, 0xcf, 0xa9, 0xca, 0xb2, 0xc0, |
|
| 5263 |
+ 0x2b, 0x2a, 0x7a, 0x0a, 0x3b, 0x9c, 0x0a, 0x49, 0xb8, 0xfc, 0x2e, 0x89, 0xe0, 0x04, 0xd2, 0x67, |
|
| 5264 |
+ 0x81, 0xef, 0xce, 0x71, 0xca, 0x40, 0x4f, 0xa1, 0x1c, 0x05, 0xc4, 0xd5, 0x51, 0x9d, 0x6d, 0x4d, |
|
| 5265 |
+ 0xff, 0x20, 0x8f, 0xde, 0x4f, 0x41, 0x78, 0x85, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4, 0xb8, |
|
| 5266 |
+ 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x96, 0xc7, 0xee, 0x6a, 0x04, 0x2e, 0x07, 0x6c, 0x9c, 0x0c, 0xd1, |
|
| 5267 |
+ 0xf1, 0xf7, 0xd2, 0x57, 0x46, 0x5b, 0xa7, 0x00, 0x64, 0xf9, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7, 0x0a, |
|
| 5268 |
+ 0x65, 0x4e, 0x24, 0x43, 0x47, 0x8f, 0xa0, 0x72, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94, 0xb5, |
|
| 5269 |
+ 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc3, 0x0e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3, 0x14, |
|
| 5270 |
+ 0xde, 0xcb, 0x0d, 0x8a, 0x0e, 0xa1, 0xb2, 0x3c, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3b, 0x77, |
|
| 5271 |
+ 0x17, 0x37, 0x7b, 0xf6, 0x52, 0x0f, 0x27, 0x5d, 0x6c, 0x2f, 0x41, 0x27, 0x5e, 0xf3, 0x6f, 0x16, |
|
| 5272 |
+ 0xec, 0xae, 0x89, 0x05, 0xdd, 0x87, 0x6d, 0x7f, 0x4a, 0xc6, 0x34, 0xa1, 0xe3, 0xc4, 0x40, 0x3d, |
|
| 5273 |
+ 0x28, 0x05, 0x64, 0x44, 0x03, 0x25, 0x19, 0x95, 0xb6, 0x9f, 0xbd, 0x51, 0x75, 0xad, 0x3f, 0x6a, |
|
| 5274 |
+ 0x7c, 0x2f, 0x94, 0x7c, 0x8e, 0x0d, 0x19, 0x39, 0xb0, 0xe3, 0xb2, 0xe9, 0x94, 0x84, 0xaa, 0x38, |
|
| 5275 |
+ 0x6d, 0xed, 0x97, 0x71, 0x6a, 0x22, 0x04, 0x45, 0xc2, 0xc7, 0xc2, 0x29, 0x6a, 0xb7, 0x1e, 0xa3, |
|
| 5276 |
+ 0x2a, 0x6c, 0xd1, 0x70, 0xe6, 0x6c, 0x6b, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e, 0xc6, |
|
| 5277 |
+ 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd1, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xd2, 0x94, 0xc5, |
|
| 5278 |
+ 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6f, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81, 0xa3, |
|
| 5279 |
+ 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33, 0x97, |
|
| 5280 |
+ 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0x69, 0xaf, 0xd0, 0xea, 0x9a, 0x5e, 0x01, 0xdf, 0x55, 0x9c, |
|
| 5281 |
+ 0x63, 0x45, 0xe9, 0x6b, 0x06, 0xea, 0x43, 0x25, 0x8a, 0x83, 0x60, 0xc8, 0xa2, 0xe4, 0x1d, 0x00, |
|
| 5282 |
+ 0x1d, 0xe1, 0x2d, 0x52, 0xd6, 0x8f, 0x83, 0xe0, 0x79, 0x42, 0xc2, 0x76, 0xb4, 0x32, 0xd0, 0xfb, |
|
| 5283 |
+ 0x50, 0x1a, 0x73, 0x16, 0x47, 0xc2, 0xb1, 0x75, 0x32, 0x8c, 0x85, 0x3e, 0x87, 0x1d, 0x41, 0x5d, |
|
| 5284 |
+ 0x4e, 0xa5, 0x70, 0x2a, 0x7a, 0xab, 0x1f, 0xe6, 0x4d, 0x32, 0xd0, 0x10, 0x4c, 0x2f, 0x29, 0xa7, |
|
| 5285 |
+ 0xa1, 0x4b, 0x71, 0xca, 0x41, 0x0f, 0x60, 0x4b, 0xca, 0xb9, 0xb3, 0xdb, 0x28, 0xec, 0x5b, 0x9d, |
|
| 5286 |
+ 0x9d, 0xc5, 0xcd, 0xde, 0xd6, 0xc5, 0xc5, 0x4b, 0xac, 0x7c, 0xaa, 0x46, 0x4d, 0x98, 0x90, 0x21, |
|
| 5287 |
+ 0x99, 0x52, 0xe7, 0x8e, 0xce, 0xed, 0xd2, 0x46, 0x2f, 0x01, 0xbc, 0x50, 0x0c, 0x5d, 0x7d, 0x29, |
|
| 5288 |
+ 0x9c, 0xbb, 0x7a, 0x77, 0x9f, 0xbc, 0x79, 0x77, 0xdd, 0xf3, 0x81, 0xa9, 0xd3, 0xbb, 0x8b, 0x9b, |
|
| 5289 |
+ 0xbd, 0xf2, 0xd2, 0xc4, 0x65, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8, 0x89, |
|
| 5290 |
+ 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xbd, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4b, 0x52, 0x0a, |
|
| 5291 |
+ 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43, 0x21, |
|
| 5292 |
+ 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x2b, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16, 0x89, |
|
| 5293 |
+ 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0x68, 0x0f, |
|
| 5294 |
+ 0x6c, 0xad, 0x0b, 0xe1, 0x8f, 0x43, 0x12, 0x38, 0xf7, 0x75, 0x3e, 0x40, 0xb9, 0x06, 0xda, 0x53, |
|
| 5295 |
+ 0xfb, 0x0c, 0xec, 0x8c, 0xdc, 0x95, 0x4c, 0xaf, 0xe8, 0xdc, 0xdc, 0x20, 0x35, 0x54, 0x6b, 0x9a, |
|
| 5296 |
+ 0x91, 0x20, 0x4e, 0x9a, 0xbd, 0x32, 0x4e, 0x8c, 0x5f, 0x6f, 0x3e, 0x29, 0xd4, 0x0e, 0xc1, 0xce, |
|
| 5297 |
+ 0x1c, 0x3b, 0xfa, 0x10, 0x76, 0x39, 0x1d, 0xfb, 0x42, 0xf2, 0xf9, 0x90, 0xc4, 0x72, 0xe2, 0xfc, |
|
| 5298 |
+ 0x4e, 0x13, 0x2a, 0xa9, 0xb3, 0x1d, 0xcb, 0x49, 0x6d, 0x08, 0xab, 0xec, 0xa1, 0x06, 0xd8, 0xea, |
|
| 5299 |
+ 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x75, 0x29, 0xf5, 0x08, 0x4a, 0xb8, |
|
| 5300 |
+ 0x3b, 0xd1, 0x97, 0xb7, 0x8c, 0x8d, 0xa5, 0x6e, 0x63, 0x2a, 0x51, 0x73, 0x1b, 0x8d, 0xd9, 0xfc, |
|
| 5301 |
+ 0x6f, 0x01, 0x2a, 0xd9, 0x77, 0x11, 0x1d, 0x25, 0xef, 0x99, 0xde, 0xd2, 0x9d, 0xc3, 0x83, 0x37, |
|
| 5302 |
+ 0xbd, 0xa3, 0xfa, 0xf5, 0x08, 0x62, 0x15, 0xec, 0x4c, 0xb5, 0xb0, 0x9a, 0x8c, 0x7e, 0x09, 0xdb, |
|
| 5303 |
+ 0x11, 0xe3, 0x32, 0xad, 0x21, 0xf5, 0xdc, 0x8a, 0xcf, 0x78, 0x5a, 0x6d, 0x13, 0x70, 0x73, 0x02, |
|
| 5304 |
+ 0x77, 0xd6, 0xa3, 0xa1, 0xc7, 0xb0, 0xf5, 0xe2, 0xa4, 0x5f, 0xdd, 0xa8, 0x3d, 0x7c, 0x75, 0xdd, |
|
| 5305 |
+ 0xf8, 0xe1, 0xfa, 0xc7, 0x17, 0x3e, 0x97, 0x31, 0x09, 0x4e, 0xfa, 0xe8, 0xa7, 0xb0, 0xdd, 0x3d, |
|
| 5306 |
+ 0x1f, 0x60, 0x5c, 0x2d, 0xd4, 0xf6, 0x5e, 0x5d, 0x37, 0x1e, 0xae, 0xe3, 0xd4, 0x27, 0x16, 0x87, |
|
| 5307 |
+ 0x1e, 0x66, 0xa3, 0x65, 0x3b, 0xf7, 0xcf, 0x4d, 0xb0, 0x4d, 0x69, 0x7d, 0xd7, 0x1d, 0xff, 0x6e, |
|
| 5308 |
+ 0xf2, 0x5a, 0xa5, 0x77, 0x66, 0xf3, 0x8d, 0x8f, 0x56, 0x25, 0x21, 0x98, 0x33, 0x7e, 0x04, 0x15, |
|
| 5309 |
+ 0x3f, 0x9a, 0x7d, 0x3a, 0xa4, 0x21, 0x19, 0x05, 0xa6, 0xb3, 0xb3, 0xb0, 0xad, 0x7c, 0xbd, 0xc4, |
|
| 5310 |
+ 0xa5, 0x2e, 0xac, 0x1f, 0x4a, 0xca, 0x43, 0xd3, 0xb3, 0x59, 0x78, 0x69, 0xa3, 0xcf, 0xa1, 0xe8, |
|
| 5311 |
+ 0x47, 0x64, 0x6a, 0x5e, 0xda, 0xdc, 0x1d, 0x9c, 0xf4, 0xdb, 0x67, 0x46, 0x83, 0x1d, 0x6b, 0x71, |
|
| 5312 |
+ 0xb3, 0x57, 0x54, 0x0e, 0xac, 0x69, 0xa8, 0x9e, 0x3e, 0x76, 0x6a, 0x26, 0x5d, 0x7c, 0x2d, 0x9c, |
|
| 5313 |
+ 0xf1, 0x28, 0x1d, 0xf9, 0xe1, 0x98, 0x53, 0x21, 0x74, 0x19, 0xb6, 0x70, 0x6a, 0x36, 0xff, 0x57, |
|
| 5314 |
+ 0x04, 0xfb, 0x28, 0x88, 0x85, 0x34, 0x8f, 0xcb, 0x3b, 0xcb, 0xe8, 0x4b, 0xb8, 0x47, 0xf4, 0x6f, |
|
| 5315 |
+ 0x01, 0x09, 0x55, 0xa5, 0xd6, 0xed, 0x85, 0xc9, 0xea, 0xe3, 0xdc, 0x70, 0x4b, 0x70, 0xd2, 0x8a, |
|
| 5316 |
+ 0x74, 0x4a, 0x2a, 0xa6, 0x53, 0xc0, 0x55, 0xf2, 0x8d, 0x2f, 0x68, 0x00, 0xbb, 0x8c, 0xbb, 0x13, |
|
| 5317 |
+ 0x2a, 0x64, 0x52, 0xdf, 0x4d, 0x1b, 0x9d, 0xfb, 0x83, 0xf5, 0x3c, 0x0b, 0x34, 0xc5, 0x2d, 0x59, |
|
| 5318 |
+ 0xed, 0x7a, 0x0c, 0xf4, 0x04, 0x8a, 0x9c, 0x5c, 0xa6, 0xad, 0x52, 0xae, 0xf2, 0x31, 0xb9, 0x94, |
|
| 5319 |
+ 0x6b, 0x21, 0x34, 0x03, 0xfd, 0x01, 0xc0, 0xf3, 0x45, 0x44, 0xa4, 0x3b, 0xa1, 0xdc, 0x9c, 0x60, |
|
| 5320 |
+ 0xee, 0x16, 0xbb, 0x4b, 0xd4, 0x5a, 0x94, 0x0c, 0x1b, 0x9d, 0x42, 0xd9, 0x25, 0xa9, 0x06, 0x4b, |
|
| 5321 |
+ 0xb7, 0xff, 0x5b, 0x1c, 0xb5, 0x4d, 0x88, 0xaa, 0x0a, 0xb1, 0xb8, 0xd9, 0xb3, 0x52, 0x0f, 0xb6, |
|
| 5322 |
+ 0x5c, 0x62, 0x34, 0x79, 0x0a, 0xbb, 0xea, 0x9f, 0x63, 0xe8, 0xd1, 0x4b, 0x12, 0x07, 0x32, 0x39, |
|
| 5323 |
+ 0xfb, 0x5b, 0x8a, 0xb5, 0x6a, 0x60, 0xbb, 0x06, 0x67, 0xd6, 0x55, 0x91, 0x19, 0x1f, 0xfa, 0x13, |
|
| 5324 |
+ 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x81, 0xe9, 0x0a, 0xad, 0xdb, 0x37, 0xdb, 0x5b, 0x82, 0xd7, |
|
| 5325 |
+ 0x36, 0x5b, 0xa5, 0xdf, 0xf0, 0x37, 0x7d, 0x80, 0xe4, 0xf9, 0x7b, 0xb7, 0xfa, 0x43, 0x50, 0xf4, |
|
| 5326 |
+ 0x88, 0x24, 0x5a, 0x72, 0x15, 0xac, 0xc7, 0x1d, 0xe7, 0xf5, 0xd7, 0xf5, 0x8d, 0x7f, 0x7f, 0x5d, |
|
| 5327 |
+ 0xdf, 0xf8, 0xcb, 0xa2, 0x5e, 0x78, 0xbd, 0xa8, 0x17, 0xfe, 0xb5, 0xa8, 0x17, 0xfe, 0xb3, 0xa8, |
|
| 5328 |
+ 0x17, 0x46, 0x25, 0xdd, 0x34, 0xfc, 0xe2, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x64, 0x44, 0x1e, |
|
| 5329 |
+ 0x4f, 0xb4, 0x10, 0x00, 0x00, |
|
| 5329 | 5330 |
} |
| ... | ... |
@@ -321,6 +321,12 @@ message NetworkSpec {
|
| 321 | 321 |
// enabled(default case) no manual attachment to this network |
| 322 | 322 |
// can happen. |
| 323 | 323 |
bool attachable = 6; |
| 324 |
+ |
|
| 325 |
+ // Ingress indicates this network will provide the routing-mesh. |
|
| 326 |
+ // In older versions, the network providing the routing mesh was |
|
| 327 |
+ // swarm internally created only and it was identified by the name |
|
| 328 |
+ // "ingress" and the label "com.docker.swarm.internal": "true". |
|
| 329 |
+ bool ingress = 7; |
|
| 324 | 330 |
} |
| 325 | 331 |
|
| 326 | 332 |
// ClusterSpec specifies global cluster settings. |
| ... | ... |
@@ -113,12 +113,39 @@ type LocalSigner struct {
|
| 113 | 113 |
Key []byte |
| 114 | 114 |
} |
| 115 | 115 |
|
| 116 |
-// RootCA is the representation of everything we need to sign certificates |
|
| 116 |
+// RootCA is the representation of everything we need to sign certificates and/or to verify certificates |
|
| 117 |
+// |
|
| 118 |
+// RootCA.Cert: [signing CA cert][CA cert1][CA cert2] |
|
| 119 |
+// RootCA.Intermediates: [intermediate CA1][intermediate CA2][intermediate CA3] |
|
| 120 |
+// RootCA.Signer.Key: [signing CA key] |
|
| 121 |
+// |
|
| 122 |
+// Requirements: |
|
| 123 |
+// |
|
| 124 |
+// - [signing CA key] must be the private key for [signing CA cert] |
|
| 125 |
+// - [signing CA cert] must be the first cert in RootCA.Cert |
|
| 126 |
+// |
|
| 127 |
+// - [intermediate CA1] must have the same public key and subject as [signing CA cert], because otherwise when |
|
| 128 |
+// appended to a leaf certificate, the intermediates will not form a chain (because [intermediate CA1] won't because |
|
| 129 |
+// the signer of the leaf certificate) |
|
| 130 |
+// - [intermediate CA1] must be signed by [intermediate CA2], which must be signed by [intermediate CA3] |
|
| 131 |
+// |
|
| 132 |
+// - When we issue a certificate, the intermediates will be appended so that the certificate looks like: |
|
| 133 |
+// [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] |
|
| 134 |
+// - [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] is guaranteed to form a |
|
| 135 |
+// valid chain from [leaf signed by signing CA cert] to one of the root certs ([signing CA cert], [CA cert1], [CA cert2]) |
|
| 136 |
+// using zero or more of the intermediate certs ([intermediate CA1][intermediate CA2][intermediate CA3]) as intermediates |
|
| 137 |
+// |
|
| 117 | 138 |
type RootCA struct {
|
| 118 | 139 |
// Cert contains a bundle of PEM encoded Certificate for the Root CA, the first one of which |
| 119 | 140 |
// must correspond to the key in the local signer, if provided |
| 120 | 141 |
Cert []byte |
| 121 | 142 |
|
| 143 |
+ // Intermediates contains a bundle of PEM encoded intermediate CA certificates to append to any |
|
| 144 |
+ // issued TLS (leaf) certificates. The first one must have the same public key and subject as the |
|
| 145 |
+ // signing root certificate, and the rest must form a chain, each one certifying the one above it, |
|
| 146 |
+ // as per RFC5246 section 7.4.2. |
|
| 147 |
+ Intermediates []byte |
|
| 148 |
+ |
|
| 122 | 149 |
// Pool is the root pool used to validate TLS certificates |
| 123 | 150 |
Pool *x509.CertPool |
| 124 | 151 |
|
| ... | ... |
@@ -306,7 +333,7 @@ func (rca *RootCA) ParseValidateAndSignCSR(csrBytes []byte, cn, ou, org string) |
| 306 | 306 |
return nil, errors.Wrap(err, "failed to sign node certificate") |
| 307 | 307 |
} |
| 308 | 308 |
|
| 309 |
- return cert, nil |
|
| 309 |
+ return append(cert, rca.Intermediates...), nil |
|
| 310 | 310 |
} |
| 311 | 311 |
|
| 312 | 312 |
// CrossSignCACertificate takes a CA root certificate and generates an intermediate CA from it signed with the current root signer |
| ... | ... |
@@ -348,7 +375,7 @@ func (rca *RootCA) CrossSignCACertificate(otherCAPEM []byte) ([]byte, error) {
|
| 348 | 348 |
// NewRootCA creates a new RootCA object from unparsed PEM cert bundle and key byte |
| 349 | 349 |
// slices. key may be nil, and in this case NewRootCA will return a RootCA |
| 350 | 350 |
// without a signer. |
| 351 |
-func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, error) {
|
|
| 351 |
+func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration, intermediates []byte) (RootCA, error) {
|
|
| 352 | 352 |
// Parse all the certificates in the cert bundle |
| 353 | 353 |
parsedCerts, err := helpers.ParseCertificatesPEM(certBytes) |
| 354 | 354 |
if err != nil {
|
| ... | ... |
@@ -368,7 +395,6 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er |
| 368 | 368 |
default: |
| 369 | 369 |
return RootCA{}, fmt.Errorf("unsupported signature algorithm: %s", cert.SignatureAlgorithm.String())
|
| 370 | 370 |
} |
| 371 |
- |
|
| 372 | 371 |
// Check to see if all of the certificates are valid, self-signed root CA certs |
| 373 | 372 |
selfpool := x509.NewCertPool() |
| 374 | 373 |
selfpool.AddCert(cert) |
| ... | ... |
@@ -381,9 +407,28 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er |
| 381 | 381 |
// Calculate the digest for our Root CA bundle |
| 382 | 382 |
digest := digest.FromBytes(certBytes) |
| 383 | 383 |
|
| 384 |
+ // We do not yet support arbitrary chains of intermediates (e.g. the case of an offline root, and the swarm CA is an |
|
| 385 |
+ // intermediate CA). We currently only intermediates for which the first intermediate is cross-signed version of the |
|
| 386 |
+ // CA signing cert (the first cert of the root certs) for the purposes of root rotation. If we wanted to support |
|
| 387 |
+ // offline roots, we'd have to separate the CA signing cert from the self-signed root certs, but this intermediate |
|
| 388 |
+ // validation logic should remain the same. Either the first intermediate would BE the intermediate CA we sign with |
|
| 389 |
+ // (in which case it'd have the same subject and public key), or it would be a cross-signed intermediate with the |
|
| 390 |
+ // same subject and public key as our signing cert (which could be either an intermediate cert or a self-signed root |
|
| 391 |
+ // cert). |
|
| 392 |
+ if len(intermediates) > 0 {
|
|
| 393 |
+ parsedIntermediates, err := ValidateCertChain(pool, intermediates, false) |
|
| 394 |
+ if err != nil {
|
|
| 395 |
+ return RootCA{}, errors.Wrap(err, "invalid intermediate chain")
|
|
| 396 |
+ } |
|
| 397 |
+ if !bytes.Equal(parsedIntermediates[0].RawSubject, parsedCerts[0].RawSubject) || |
|
| 398 |
+ !bytes.Equal(parsedIntermediates[0].RawSubjectPublicKeyInfo, parsedCerts[0].RawSubjectPublicKeyInfo) {
|
|
| 399 |
+ return RootCA{}, errors.New("invalid intermediate chain - the first intermediate must have the same subject and public key as the root")
|
|
| 400 |
+ } |
|
| 401 |
+ } |
|
| 402 |
+ |
|
| 384 | 403 |
if len(keyBytes) == 0 {
|
| 385 |
- // This RootCA does not have a valid signer. |
|
| 386 |
- return RootCA{Cert: certBytes, Digest: digest, Pool: pool}, nil
|
|
| 404 |
+ // This RootCA does not have a valid signer |
|
| 405 |
+ return RootCA{Cert: certBytes, Intermediates: intermediates, Digest: digest, Pool: pool}, nil
|
|
| 387 | 406 |
} |
| 388 | 407 |
|
| 389 | 408 |
var ( |
| ... | ... |
@@ -434,7 +479,7 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er |
| 434 | 434 |
} |
| 435 | 435 |
} |
| 436 | 436 |
|
| 437 |
- return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Digest: digest, Cert: certBytes, Pool: pool}, nil
|
|
| 437 |
+ return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Intermediates: intermediates, Digest: digest, Cert: certBytes, Pool: pool}, nil
|
|
| 438 | 438 |
} |
| 439 | 439 |
|
| 440 | 440 |
// ValidateCertChain checks checks that the certificates provided chain up to the root pool provided. In addition |
| ... | ... |
@@ -586,7 +631,7 @@ func GetLocalRootCA(paths CertPaths) (RootCA, error) {
|
| 586 | 586 |
key = nil |
| 587 | 587 |
} |
| 588 | 588 |
|
| 589 |
- return NewRootCA(cert, key, DefaultNodeCertExpiration) |
|
| 589 |
+ return NewRootCA(cert, key, DefaultNodeCertExpiration, nil) |
|
| 590 | 590 |
} |
| 591 | 591 |
|
| 592 | 592 |
func getGRPCConnection(creds credentials.TransportCredentials, connBroker *connectionbroker.Broker, forceRemote bool) (*connectionbroker.Conn, error) {
|
| ... | ... |
@@ -641,7 +686,7 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbro |
| 641 | 641 |
|
| 642 | 642 |
// NewRootCA will validate that the certificates are otherwise valid and create a RootCA object. |
| 643 | 643 |
// Since there is no key, the certificate expiry does not matter and will not be used. |
| 644 |
- return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration) |
|
| 644 |
+ return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration, nil) |
|
| 645 | 645 |
} |
| 646 | 646 |
|
| 647 | 647 |
// CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially |
| ... | ... |
@@ -660,7 +705,7 @@ func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) {
|
| 660 | 660 |
return RootCA{}, err
|
| 661 | 661 |
} |
| 662 | 662 |
|
| 663 |
- rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration) |
|
| 663 |
+ rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration, nil) |
|
| 664 | 664 |
if err != nil {
|
| 665 | 665 |
return RootCA{}, err
|
| 666 | 666 |
} |
| ... | ... |
@@ -132,7 +132,7 @@ func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration |
| 132 | 132 |
s.mu.Lock() |
| 133 | 133 |
defer s.mu.Unlock() |
| 134 | 134 |
|
| 135 |
- rootCA, err := NewRootCA(cert, key, certExpiry) |
|
| 135 |
+ rootCA, err := NewRootCA(cert, key, certExpiry, nil) |
|
| 136 | 136 |
if err != nil {
|
| 137 | 137 |
return err |
| 138 | 138 |
} |
| ... | ... |
@@ -96,7 +96,7 @@ func (eca *ExternalCA) Sign(ctx context.Context, req signer.SignRequest) (cert [ |
| 96 | 96 |
for _, url := range urls {
|
| 97 | 97 |
cert, err = makeExternalSignRequest(ctx, client, url, csrJSON) |
| 98 | 98 |
if err == nil {
|
| 99 |
- return cert, err |
|
| 99 |
+ return append(cert, eca.rootCA.Intermediates...), err |
|
| 100 | 100 |
} |
| 101 | 101 |
logrus.Debugf("unable to proxy certificate signing request to %s: %s", url, err)
|
| 102 | 102 |
} |
| ... | ... |
@@ -6,7 +6,6 @@ import ( |
| 6 | 6 |
|
| 7 | 7 |
"github.com/docker/go-events" |
| 8 | 8 |
"github.com/docker/swarmkit/api" |
| 9 |
- "github.com/docker/swarmkit/identity" |
|
| 10 | 9 |
"github.com/docker/swarmkit/log" |
| 11 | 10 |
"github.com/docker/swarmkit/manager/allocator/networkallocator" |
| 12 | 11 |
"github.com/docker/swarmkit/manager/state" |
| ... | ... |
@@ -18,42 +17,18 @@ import ( |
| 18 | 18 |
|
| 19 | 19 |
const ( |
| 20 | 20 |
// Network allocator Voter ID for task allocation vote. |
| 21 |
- networkVoter = "network" |
|
| 22 |
- |
|
| 23 |
- ingressNetworkName = "ingress" |
|
| 24 |
- ingressSubnet = "10.255.0.0/16" |
|
| 25 |
- |
|
| 21 |
+ networkVoter = "network" |
|
| 26 | 22 |
allocatedStatusMessage = "pending task scheduling" |
| 27 | 23 |
) |
| 28 | 24 |
|
| 29 | 25 |
var ( |
| 26 |
+ // ErrNoIngress is returned when no ingress network is found in store |
|
| 27 |
+ ErrNoIngress = errors.New("no ingress network found")
|
|
| 30 | 28 |
errNoChanges = errors.New("task unchanged")
|
| 31 | 29 |
|
| 32 | 30 |
retryInterval = 5 * time.Minute |
| 33 | 31 |
) |
| 34 | 32 |
|
| 35 |
-func newIngressNetwork() *api.Network {
|
|
| 36 |
- return &api.Network{
|
|
| 37 |
- Spec: api.NetworkSpec{
|
|
| 38 |
- Annotations: api.Annotations{
|
|
| 39 |
- Name: ingressNetworkName, |
|
| 40 |
- Labels: map[string]string{
|
|
| 41 |
- "com.docker.swarm.internal": "true", |
|
| 42 |
- }, |
|
| 43 |
- }, |
|
| 44 |
- DriverConfig: &api.Driver{},
|
|
| 45 |
- IPAM: &api.IPAMOptions{
|
|
| 46 |
- Driver: &api.Driver{},
|
|
| 47 |
- Configs: []*api.IPAMConfig{
|
|
| 48 |
- {
|
|
| 49 |
- Subnet: ingressSubnet, |
|
| 50 |
- }, |
|
| 51 |
- }, |
|
| 52 |
- }, |
|
| 53 |
- }, |
|
| 54 |
- } |
|
| 55 |
-} |
|
| 56 |
- |
|
| 57 | 33 |
// Network context information which is used throughout the network allocation code. |
| 58 | 34 |
type networkContext struct {
|
| 59 | 35 |
ingressNetwork *api.Network |
| ... | ... |
@@ -97,7 +72,6 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
|
| 97 | 97 |
unallocatedTasks: make(map[string]*api.Task), |
| 98 | 98 |
unallocatedServices: make(map[string]*api.Service), |
| 99 | 99 |
unallocatedNetworks: make(map[string]*api.Network), |
| 100 |
- ingressNetwork: newIngressNetwork(), |
|
| 101 | 100 |
lastRetry: time.Now(), |
| 102 | 101 |
} |
| 103 | 102 |
a.netCtx = nc |
| ... | ... |
@@ -108,63 +82,38 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
|
| 108 | 108 |
} |
| 109 | 109 |
}() |
| 110 | 110 |
|
| 111 |
- // Check if we have the ingress network. If not found create |
|
| 112 |
- // it before reading all network objects for allocation. |
|
| 113 |
- var networks []*api.Network |
|
| 114 |
- a.store.View(func(tx store.ReadTx) {
|
|
| 115 |
- networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName)) |
|
| 116 |
- if len(networks) > 0 {
|
|
| 117 |
- nc.ingressNetwork = networks[0] |
|
| 118 |
- } |
|
| 119 |
- }) |
|
| 120 |
- if err != nil {
|
|
| 121 |
- return errors.Wrap(err, "failed to find ingress network during init") |
|
| 122 |
- } |
|
| 123 |
- |
|
| 124 |
- // If ingress network is not found, create one right away |
|
| 125 |
- // using the predefined template. |
|
| 126 |
- if len(networks) == 0 {
|
|
| 127 |
- if err := a.store.Update(func(tx store.Tx) error {
|
|
| 128 |
- nc.ingressNetwork.ID = identity.NewID() |
|
| 129 |
- if err := store.CreateNetwork(tx, nc.ingressNetwork); err != nil {
|
|
| 130 |
- return err |
|
| 131 |
- } |
|
| 132 |
- |
|
| 133 |
- return nil |
|
| 134 |
- }); err != nil {
|
|
| 135 |
- return errors.Wrap(err, "failed to create ingress network") |
|
| 136 |
- } |
|
| 137 |
- |
|
| 138 |
- a.store.View(func(tx store.ReadTx) {
|
|
| 139 |
- networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName)) |
|
| 140 |
- if len(networks) > 0 {
|
|
| 141 |
- nc.ingressNetwork = networks[0] |
|
| 142 |
- } |
|
| 143 |
- }) |
|
| 144 |
- if err != nil {
|
|
| 145 |
- return errors.Wrap(err, "failed to find ingress network after creating it") |
|
| 146 |
- } |
|
| 147 |
- |
|
| 148 |
- } |
|
| 149 |
- |
|
| 150 |
- // Try to complete ingress network allocation before anything else so |
|
| 151 |
- // that the we can get the preferred subnet for ingress |
|
| 152 |
- // network. |
|
| 153 |
- if !na.IsAllocated(nc.ingressNetwork) {
|
|
| 154 |
- if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
|
|
| 155 |
- log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
|
|
| 156 |
- } else if _, err := a.store.Batch(func(batch *store.Batch) error {
|
|
| 157 |
- if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
|
|
| 111 |
+ // Ingress network is now created at cluster's first time creation. |
|
| 112 |
+ // Check if we have the ingress network. If found, make sure it is |
|
| 113 |
+ // allocated, before reading all network objects for allocation. |
|
| 114 |
+ // If not found, it means it was removed by user, nothing to do here. |
|
| 115 |
+ ingressNetwork, err := GetIngressNetwork(a.store) |
|
| 116 |
+ switch err {
|
|
| 117 |
+ case nil: |
|
| 118 |
+ // Try to complete ingress network allocation before anything else so |
|
| 119 |
+ // that the we can get the preferred subnet for ingress network. |
|
| 120 |
+ nc.ingressNetwork = ingressNetwork |
|
| 121 |
+ if !na.IsAllocated(nc.ingressNetwork) {
|
|
| 122 |
+ if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
|
|
| 123 |
+ log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
|
|
| 124 |
+ } else if _, err := a.store.Batch(func(batch *store.Batch) error {
|
|
| 125 |
+ if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
|
|
| 126 |
+ log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
|
|
| 127 |
+ } |
|
| 128 |
+ return nil |
|
| 129 |
+ }); err != nil {
|
|
| 158 | 130 |
log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
|
| 159 | 131 |
} |
| 160 |
- return nil |
|
| 161 |
- }); err != nil {
|
|
| 162 |
- log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
|
|
| 163 | 132 |
} |
| 133 |
+ case ErrNoIngress: |
|
| 134 |
+ // Ingress network is not present in store, It means user removed it |
|
| 135 |
+ // and did not create a new one. |
|
| 136 |
+ default: |
|
| 137 |
+ return errors.Wrap(err, "failure while looking for ingress network during init") |
|
| 164 | 138 |
} |
| 165 | 139 |
|
| 166 | 140 |
// Allocate networks in the store so far before we started |
| 167 | 141 |
// watching. |
| 142 |
+ var networks []*api.Network |
|
| 168 | 143 |
a.store.View(func(tx store.ReadTx) {
|
| 169 | 144 |
networks, err = store.FindNetworks(tx, store.All) |
| 170 | 145 |
}) |
| ... | ... |
@@ -196,43 +145,12 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
|
| 196 | 196 |
log.G(ctx).WithError(err).Error("failed committing allocation of networks during init")
|
| 197 | 197 |
} |
| 198 | 198 |
|
| 199 |
- // Allocate nodes in the store so far before we process watched events. |
|
| 200 |
- var nodes []*api.Node |
|
| 201 |
- a.store.View(func(tx store.ReadTx) {
|
|
| 202 |
- nodes, err = store.FindNodes(tx, store.All) |
|
| 203 |
- }) |
|
| 204 |
- if err != nil {
|
|
| 205 |
- return errors.Wrap(err, "error listing all nodes in store while trying to allocate during init") |
|
| 206 |
- } |
|
| 207 |
- |
|
| 208 |
- var allocatedNodes []*api.Node |
|
| 209 |
- for _, node := range nodes {
|
|
| 210 |
- if na.IsNodeAllocated(node) {
|
|
| 211 |
- continue |
|
| 212 |
- } |
|
| 213 |
- |
|
| 214 |
- if node.Attachment == nil {
|
|
| 215 |
- node.Attachment = &api.NetworkAttachment{}
|
|
| 216 |
- } |
|
| 217 |
- |
|
| 218 |
- node.Attachment.Network = nc.ingressNetwork.Copy() |
|
| 219 |
- if err := a.allocateNode(ctx, node); err != nil {
|
|
| 220 |
- log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID)
|
|
| 221 |
- continue |
|
| 222 |
- } |
|
| 223 |
- |
|
| 224 |
- allocatedNodes = append(allocatedNodes, node) |
|
| 225 |
- } |
|
| 226 |
- |
|
| 227 |
- if _, err := a.store.Batch(func(batch *store.Batch) error {
|
|
| 228 |
- for _, node := range allocatedNodes {
|
|
| 229 |
- if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
|
|
| 230 |
- log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s during init", node.ID)
|
|
| 231 |
- } |
|
| 199 |
+ // Allocate nodes in the store so far before we process watched events, |
|
| 200 |
+ // if the ingress network is present. |
|
| 201 |
+ if nc.ingressNetwork != nil {
|
|
| 202 |
+ if err := a.allocateNodes(ctx); err != nil {
|
|
| 203 |
+ return err |
|
| 232 | 204 |
} |
| 233 |
- return nil |
|
| 234 |
- }); err != nil {
|
|
| 235 |
- log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes during init")
|
|
| 236 | 205 |
} |
| 237 | 206 |
|
| 238 | 207 |
// Allocate services in the store so far before we process watched events. |
| ... | ... |
@@ -346,6 +264,12 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
|
| 346 | 346 |
break |
| 347 | 347 |
} |
| 348 | 348 |
|
| 349 |
+ if IsIngressNetwork(n) && nc.ingressNetwork != nil {
|
|
| 350 |
+ log.G(ctx).Errorf("Cannot allocate ingress network %s (%s) because another ingress network is already present: %s (%s)",
|
|
| 351 |
+ n.ID, n.Spec.Annotations.Name, nc.ingressNetwork.ID, nc.ingressNetwork.Spec.Annotations) |
|
| 352 |
+ break |
|
| 353 |
+ } |
|
| 354 |
+ |
|
| 349 | 355 |
if err := a.allocateNetwork(ctx, n); err != nil {
|
| 350 | 356 |
log.G(ctx).WithError(err).Errorf("Failed allocation for network %s", n.ID)
|
| 351 | 357 |
break |
| ... | ... |
@@ -356,9 +280,24 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
|
| 356 | 356 |
}); err != nil {
|
| 357 | 357 |
log.G(ctx).WithError(err).Errorf("Failed to commit allocation for network %s", n.ID)
|
| 358 | 358 |
} |
| 359 |
+ |
|
| 360 |
+ if IsIngressNetwork(n) {
|
|
| 361 |
+ nc.ingressNetwork = n |
|
| 362 |
+ err := a.allocateNodes(ctx) |
|
| 363 |
+ if err != nil {
|
|
| 364 |
+ log.G(ctx).WithError(err).Error(err) |
|
| 365 |
+ } |
|
| 366 |
+ } |
|
| 359 | 367 |
case state.EventDeleteNetwork: |
| 360 | 368 |
n := v.Network.Copy() |
| 361 | 369 |
|
| 370 |
+ if IsIngressNetwork(n) && nc.ingressNetwork.ID == n.ID {
|
|
| 371 |
+ nc.ingressNetwork = nil |
|
| 372 |
+ if err := a.deallocateNodes(ctx); err != nil {
|
|
| 373 |
+ log.G(ctx).WithError(err).Error(err) |
|
| 374 |
+ } |
|
| 375 |
+ } |
|
| 376 |
+ |
|
| 362 | 377 |
// The assumption here is that all dependent objects |
| 363 | 378 |
// have been cleaned up when we are here so the only |
| 364 | 379 |
// thing that needs to happen is free the network |
| ... | ... |
@@ -467,7 +406,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
|
| 467 | 467 |
return |
| 468 | 468 |
} |
| 469 | 469 |
|
| 470 |
- if !nc.nwkAllocator.IsNodeAllocated(node) {
|
|
| 470 |
+ if !nc.nwkAllocator.IsNodeAllocated(node) && nc.ingressNetwork != nil {
|
|
| 471 | 471 |
if node.Attachment == nil {
|
| 472 | 472 |
node.Attachment = &api.NetworkAttachment{}
|
| 473 | 473 |
} |
| ... | ... |
@@ -486,6 +425,85 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
|
| 486 | 486 |
} |
| 487 | 487 |
} |
| 488 | 488 |
|
| 489 |
+func (a *Allocator) allocateNodes(ctx context.Context) error {
|
|
| 490 |
+ // Allocate nodes in the store so far before we process watched events. |
|
| 491 |
+ var ( |
|
| 492 |
+ allocatedNodes []*api.Node |
|
| 493 |
+ nodes []*api.Node |
|
| 494 |
+ err error |
|
| 495 |
+ nc = a.netCtx |
|
| 496 |
+ ) |
|
| 497 |
+ |
|
| 498 |
+ a.store.View(func(tx store.ReadTx) {
|
|
| 499 |
+ nodes, err = store.FindNodes(tx, store.All) |
|
| 500 |
+ }) |
|
| 501 |
+ if err != nil {
|
|
| 502 |
+ return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources") |
|
| 503 |
+ } |
|
| 504 |
+ |
|
| 505 |
+ for _, node := range nodes {
|
|
| 506 |
+ if nc.nwkAllocator.IsNodeAllocated(node) {
|
|
| 507 |
+ continue |
|
| 508 |
+ } |
|
| 509 |
+ |
|
| 510 |
+ if node.Attachment == nil {
|
|
| 511 |
+ node.Attachment = &api.NetworkAttachment{}
|
|
| 512 |
+ } |
|
| 513 |
+ |
|
| 514 |
+ node.Attachment.Network = nc.ingressNetwork.Copy() |
|
| 515 |
+ if err := a.allocateNode(ctx, node); err != nil {
|
|
| 516 |
+ log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID)
|
|
| 517 |
+ continue |
|
| 518 |
+ } |
|
| 519 |
+ |
|
| 520 |
+ allocatedNodes = append(allocatedNodes, node) |
|
| 521 |
+ } |
|
| 522 |
+ |
|
| 523 |
+ if _, err := a.store.Batch(func(batch *store.Batch) error {
|
|
| 524 |
+ for _, node := range allocatedNodes {
|
|
| 525 |
+ if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
|
|
| 526 |
+ log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s", node.ID)
|
|
| 527 |
+ } |
|
| 528 |
+ } |
|
| 529 |
+ return nil |
|
| 530 |
+ }); err != nil {
|
|
| 531 |
+ log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes")
|
|
| 532 |
+ } |
|
| 533 |
+ |
|
| 534 |
+ return nil |
|
| 535 |
+} |
|
| 536 |
+ |
|
| 537 |
+func (a *Allocator) deallocateNodes(ctx context.Context) error {
|
|
| 538 |
+ var ( |
|
| 539 |
+ nodes []*api.Node |
|
| 540 |
+ nc = a.netCtx |
|
| 541 |
+ err error |
|
| 542 |
+ ) |
|
| 543 |
+ |
|
| 544 |
+ a.store.View(func(tx store.ReadTx) {
|
|
| 545 |
+ nodes, err = store.FindNodes(tx, store.All) |
|
| 546 |
+ }) |
|
| 547 |
+ if err != nil {
|
|
| 548 |
+ return fmt.Errorf("error listing all nodes in store while trying to free network resources")
|
|
| 549 |
+ } |
|
| 550 |
+ |
|
| 551 |
+ for _, node := range nodes {
|
|
| 552 |
+ if nc.nwkAllocator.IsNodeAllocated(node) {
|
|
| 553 |
+ if err := nc.nwkAllocator.DeallocateNode(node); err != nil {
|
|
| 554 |
+ log.G(ctx).WithError(err).Errorf("Failed freeing network resources for node %s", node.ID)
|
|
| 555 |
+ } |
|
| 556 |
+ node.Attachment = nil |
|
| 557 |
+ if _, err := a.store.Batch(func(batch *store.Batch) error {
|
|
| 558 |
+ return a.commitAllocatedNode(ctx, batch, node) |
|
| 559 |
+ }); err != nil {
|
|
| 560 |
+ log.G(ctx).WithError(err).Errorf("Failed to commit deallocation of network resources for node %s", node.ID)
|
|
| 561 |
+ } |
|
| 562 |
+ } |
|
| 563 |
+ } |
|
| 564 |
+ |
|
| 565 |
+ return nil |
|
| 566 |
+} |
|
| 567 |
+ |
|
| 489 | 568 |
// taskReadyForNetworkVote checks if the task is ready for a network |
| 490 | 569 |
// vote to move it to PENDING state. |
| 491 | 570 |
func taskReadyForNetworkVote(t *api.Task, s *api.Service, nc *networkContext) bool {
|
| ... | ... |
@@ -711,6 +729,9 @@ func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
|
| 711 | 711 |
// world. Automatically attach the service to the ingress |
| 712 | 712 |
// network only if it is not already done. |
| 713 | 713 |
if isIngressNetworkNeeded(s) {
|
| 714 |
+ if nc.ingressNetwork == nil {
|
|
| 715 |
+ return fmt.Errorf("ingress network is missing")
|
|
| 716 |
+ } |
|
| 714 | 717 |
var found bool |
| 715 | 718 |
for _, vip := range s.Endpoint.VirtualIPs {
|
| 716 | 719 |
if vip.NetworkID == nc.ingressNetwork.ID {
|
| ... | ... |
@@ -1022,3 +1043,36 @@ func updateTaskStatus(t *api.Task, newStatus api.TaskState, message string) {
|
| 1022 | 1022 |
t.Status.Message = message |
| 1023 | 1023 |
t.Status.Timestamp = ptypes.MustTimestampProto(time.Now()) |
| 1024 | 1024 |
} |
| 1025 |
+ |
|
| 1026 |
+// IsIngressNetwork returns whether the passed network is an ingress network. |
|
| 1027 |
+func IsIngressNetwork(nw *api.Network) bool {
|
|
| 1028 |
+ if nw.Spec.Ingress {
|
|
| 1029 |
+ return true |
|
| 1030 |
+ } |
|
| 1031 |
+ // Check if legacy defined ingress network |
|
| 1032 |
+ _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"] |
|
| 1033 |
+ return ok && nw.Spec.Annotations.Name == "ingress" |
|
| 1034 |
+} |
|
| 1035 |
+ |
|
| 1036 |
+// GetIngressNetwork fetches the ingress network from store. |
|
| 1037 |
+// ErrNoIngress will be returned if the ingress network is not present, |
|
| 1038 |
+// nil otherwise. In case of any other failure in accessing the store, |
|
| 1039 |
+// the respective error will be reported as is. |
|
| 1040 |
+func GetIngressNetwork(s *store.MemoryStore) (*api.Network, error) {
|
|
| 1041 |
+ var ( |
|
| 1042 |
+ networks []*api.Network |
|
| 1043 |
+ err error |
|
| 1044 |
+ ) |
|
| 1045 |
+ s.View(func(tx store.ReadTx) {
|
|
| 1046 |
+ networks, err = store.FindNetworks(tx, store.All) |
|
| 1047 |
+ }) |
|
| 1048 |
+ if err != nil {
|
|
| 1049 |
+ return nil, err |
|
| 1050 |
+ } |
|
| 1051 |
+ for _, n := range networks {
|
|
| 1052 |
+ if IsIngressNetwork(n) {
|
|
| 1053 |
+ return n, nil |
|
| 1054 |
+ } |
|
| 1055 |
+ } |
|
| 1056 |
+ return nil, ErrNoIngress |
|
| 1057 |
+} |
| ... | ... |
@@ -1,7 +1,6 @@ |
| 1 | 1 |
package controlapi |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
- "fmt" |
|
| 5 | 4 |
"net" |
| 6 | 5 |
|
| 7 | 6 |
"github.com/docker/docker/pkg/plugingetter" |
| ... | ... |
@@ -9,6 +8,7 @@ import ( |
| 9 | 9 |
"github.com/docker/libnetwork/ipamapi" |
| 10 | 10 |
"github.com/docker/swarmkit/api" |
| 11 | 11 |
"github.com/docker/swarmkit/identity" |
| 12 |
+ "github.com/docker/swarmkit/manager/allocator" |
|
| 12 | 13 |
"github.com/docker/swarmkit/manager/state/store" |
| 13 | 14 |
"golang.org/x/net/context" |
| 14 | 15 |
"google.golang.org/grpc" |
| ... | ... |
@@ -75,6 +75,14 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er |
| 75 | 75 |
return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) |
| 76 | 76 |
} |
| 77 | 77 |
|
| 78 |
+ if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" {
|
|
| 79 |
+ return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network") |
|
| 80 |
+ } |
|
| 81 |
+ |
|
| 82 |
+ if spec.Attachable && spec.Ingress {
|
|
| 83 |
+ return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable") |
|
| 84 |
+ } |
|
| 85 |
+ |
|
| 78 | 86 |
if err := validateAnnotations(spec.Annotations); err != nil {
|
| 79 | 87 |
return err |
| 80 | 88 |
} |
| ... | ... |
@@ -94,16 +102,10 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er |
| 94 | 94 |
// - Returns `InvalidArgument` if the NetworkSpec is malformed. |
| 95 | 95 |
// - Returns an error if the creation fails. |
| 96 | 96 |
func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRequest) (*api.CreateNetworkResponse, error) {
|
| 97 |
- // if you change this function, you have to change createInternalNetwork in |
|
| 98 |
- // the tests to match it (except the part where we check the label). |
|
| 99 | 97 |
if err := validateNetworkSpec(request.Spec, s.pg); err != nil {
|
| 100 | 98 |
return nil, err |
| 101 | 99 |
} |
| 102 | 100 |
|
| 103 |
- if _, ok := request.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
|
|
| 104 |
- return nil, grpc.Errorf(codes.PermissionDenied, "label com.docker.swarm.internal is for predefined internal networks and cannot be applied by users") |
|
| 105 |
- } |
|
| 106 |
- |
|
| 107 | 101 |
// TODO(mrjana): Consider using `Name` as a primary key to handle |
| 108 | 102 |
// duplicate creations. See #65 |
| 109 | 103 |
n := &api.Network{
|
| ... | ... |
@@ -112,6 +114,13 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe |
| 112 | 112 |
} |
| 113 | 113 |
|
| 114 | 114 |
err := s.store.Update(func(tx store.Tx) error {
|
| 115 |
+ if request.Spec.Ingress {
|
|
| 116 |
+ if n, err := allocator.GetIngressNetwork(s.store); err == nil {
|
|
| 117 |
+ return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID) |
|
| 118 |
+ } else if err != allocator.ErrNoIngress {
|
|
| 119 |
+ return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err) |
|
| 120 |
+ } |
|
| 121 |
+ } |
|
| 115 | 122 |
return store.CreateNetwork(tx, n) |
| 116 | 123 |
}) |
| 117 | 124 |
if err != nil {
|
| ... | ... |
@@ -152,44 +161,70 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe |
| 152 | 152 |
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error()) |
| 153 | 153 |
} |
| 154 | 154 |
|
| 155 |
- err := s.store.Update(func(tx store.Tx) error {
|
|
| 156 |
- services, err := store.FindServices(tx, store.ByReferencedNetworkID(request.NetworkID)) |
|
| 155 |
+ var ( |
|
| 156 |
+ n *api.Network |
|
| 157 |
+ rm = s.removeNetwork |
|
| 158 |
+ ) |
|
| 159 |
+ |
|
| 160 |
+ s.store.View(func(tx store.ReadTx) {
|
|
| 161 |
+ n = store.GetNetwork(tx, request.NetworkID) |
|
| 162 |
+ }) |
|
| 163 |
+ if n == nil {
|
|
| 164 |
+ return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) |
|
| 165 |
+ } |
|
| 166 |
+ |
|
| 167 |
+ if allocator.IsIngressNetwork(n) {
|
|
| 168 |
+ rm = s.removeIngressNetwork |
|
| 169 |
+ } |
|
| 170 |
+ |
|
| 171 |
+ if err := rm(n.ID); err != nil {
|
|
| 172 |
+ if err == store.ErrNotExist {
|
|
| 173 |
+ return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) |
|
| 174 |
+ } |
|
| 175 |
+ return nil, err |
|
| 176 |
+ } |
|
| 177 |
+ return &api.RemoveNetworkResponse{}, nil
|
|
| 178 |
+} |
|
| 179 |
+ |
|
| 180 |
+func (s *Server) removeNetwork(id string) error {
|
|
| 181 |
+ return s.store.Update(func(tx store.Tx) error {
|
|
| 182 |
+ services, err := store.FindServices(tx, store.ByReferencedNetworkID(id)) |
|
| 157 | 183 |
if err != nil {
|
| 158 |
- return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", request.NetworkID, err) |
|
| 184 |
+ return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) |
|
| 159 | 185 |
} |
| 160 | 186 |
|
| 161 | 187 |
if len(services) != 0 {
|
| 162 |
- return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", request.NetworkID, services[0].ID) |
|
| 188 |
+ return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID) |
|
| 163 | 189 |
} |
| 164 | 190 |
|
| 165 |
- tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(request.NetworkID)) |
|
| 191 |
+ tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id)) |
|
| 166 | 192 |
if err != nil {
|
| 167 |
- return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", request.NetworkID, err) |
|
| 193 |
+ return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err) |
|
| 168 | 194 |
} |
| 169 | 195 |
|
| 170 | 196 |
for _, t := range tasks {
|
| 171 | 197 |
if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning {
|
| 172 |
- return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", request.NetworkID, t.ID) |
|
| 198 |
+ return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID) |
|
| 173 | 199 |
} |
| 174 | 200 |
} |
| 175 | 201 |
|
| 176 |
- nw := store.GetNetwork(tx, request.NetworkID) |
|
| 177 |
- if _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
|
|
| 178 |
- networkDescription := nw.ID |
|
| 179 |
- if nw.Spec.Annotations.Name != "" {
|
|
| 180 |
- networkDescription = fmt.Sprintf("%s (%s)", nw.Spec.Annotations.Name, nw.ID)
|
|
| 202 |
+ return store.DeleteNetwork(tx, id) |
|
| 203 |
+ }) |
|
| 204 |
+} |
|
| 205 |
+ |
|
| 206 |
+func (s *Server) removeIngressNetwork(id string) error {
|
|
| 207 |
+ return s.store.Update(func(tx store.Tx) error {
|
|
| 208 |
+ services, err := store.FindServices(tx, store.All) |
|
| 209 |
+ if err != nil {
|
|
| 210 |
+ return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err) |
|
| 211 |
+ } |
|
| 212 |
+ for _, srv := range services {
|
|
| 213 |
+ if doesServiceNeedIngress(srv) {
|
|
| 214 |
+ return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID) |
|
| 181 | 215 |
} |
| 182 |
- return grpc.Errorf(codes.PermissionDenied, "%s is a pre-defined network and cannot be removed", networkDescription) |
|
| 183 | 216 |
} |
| 184 |
- return store.DeleteNetwork(tx, request.NetworkID) |
|
| 217 |
+ return store.DeleteNetwork(tx, id) |
|
| 185 | 218 |
}) |
| 186 |
- if err != nil {
|
|
| 187 |
- if err == store.ErrNotExist {
|
|
| 188 |
- return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID) |
|
| 189 |
- } |
|
| 190 |
- return nil, err |
|
| 191 |
- } |
|
| 192 |
- return &api.RemoveNetworkResponse{}, nil
|
|
| 193 | 219 |
} |
| 194 | 220 |
|
| 195 | 221 |
func filterNetworks(candidates []*api.Network, filters ...func(*api.Network) bool) []*api.Network {
|
| ... | ... |
@@ -11,6 +11,7 @@ import ( |
| 11 | 11 |
"github.com/docker/distribution/reference" |
| 12 | 12 |
"github.com/docker/swarmkit/api" |
| 13 | 13 |
"github.com/docker/swarmkit/identity" |
| 14 |
+ "github.com/docker/swarmkit/manager/allocator" |
|
| 14 | 15 |
"github.com/docker/swarmkit/manager/constraint" |
| 15 | 16 |
"github.com/docker/swarmkit/manager/state/store" |
| 16 | 17 |
"github.com/docker/swarmkit/protobuf/ptypes" |
| ... | ... |
@@ -288,7 +289,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error |
| 288 | 288 |
if network == nil {
|
| 289 | 289 |
continue |
| 290 | 290 |
} |
| 291 |
- if _, ok := network.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
|
|
| 291 |
+ if network.Spec.Internal {
|
|
| 292 | 292 |
return grpc.Errorf(codes.InvalidArgument, |
| 293 | 293 |
"Service cannot be explicitly attached to %q network which is a swarm internal network", |
| 294 | 294 |
network.Spec.Annotations.Name) |
| ... | ... |
@@ -424,6 +425,36 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error |
| 424 | 424 |
return nil |
| 425 | 425 |
} |
| 426 | 426 |
|
| 427 |
+func doesServiceNeedIngress(srv *api.Service) bool {
|
|
| 428 |
+ // Only VIP mode with target ports needs routing mesh. |
|
| 429 |
+ // If no endpoint is specified, it defaults to VIP mode but no target ports |
|
| 430 |
+ // are specified, so the service does not need the routing mesh. |
|
| 431 |
+ if srv.Spec.Endpoint == nil || srv.Spec.Endpoint.Mode != api.ResolutionModeVirtualIP {
|
|
| 432 |
+ return false |
|
| 433 |
+ } |
|
| 434 |
+ // Go through the ports' config |
|
| 435 |
+ for _, p := range srv.Spec.Endpoint.Ports {
|
|
| 436 |
+ if p.PublishMode != api.PublishModeIngress {
|
|
| 437 |
+ continue |
|
| 438 |
+ } |
|
| 439 |
+ if p.PublishedPort != 0 {
|
|
| 440 |
+ return true |
|
| 441 |
+ } |
|
| 442 |
+ } |
|
| 443 |
+ // Go through the ports' state |
|
| 444 |
+ if srv.Endpoint != nil {
|
|
| 445 |
+ for _, p := range srv.Endpoint.Ports {
|
|
| 446 |
+ if p.PublishMode != api.PublishModeIngress {
|
|
| 447 |
+ continue |
|
| 448 |
+ } |
|
| 449 |
+ if p.PublishedPort != 0 {
|
|
| 450 |
+ return true |
|
| 451 |
+ } |
|
| 452 |
+ } |
|
| 453 |
+ } |
|
| 454 |
+ return false |
|
| 455 |
+} |
|
| 456 |
+ |
|
| 427 | 457 |
// CreateService creates and returns a Service based on the provided ServiceSpec. |
| 428 | 458 |
// - Returns `InvalidArgument` if the ServiceSpec is malformed. |
| 429 | 459 |
// - Returns `Unimplemented` if the ServiceSpec references unimplemented features. |
| ... | ... |
@@ -449,6 +480,12 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe |
| 449 | 449 |
Spec: *request.Spec, |
| 450 | 450 |
} |
| 451 | 451 |
|
| 452 |
+ if doesServiceNeedIngress(service) {
|
|
| 453 |
+ if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
|
|
| 454 |
+ return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") |
|
| 455 |
+ } |
|
| 456 |
+ } |
|
| 457 |
+ |
|
| 452 | 458 |
err := s.store.Update(func(tx store.Tx) error {
|
| 453 | 459 |
// Check to see if all the secrets being added exist as objects |
| 454 | 460 |
// in our datastore |
| ... | ... |
@@ -578,6 +615,12 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe |
| 578 | 578 |
service.UpdateStatus = nil |
| 579 | 579 |
} |
| 580 | 580 |
|
| 581 |
+ if doesServiceNeedIngress(service) {
|
|
| 582 |
+ if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
|
|
| 583 |
+ return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present") |
|
| 584 |
+ } |
|
| 585 |
+ } |
|
| 586 |
+ |
|
| 581 | 587 |
return store.UpdateService(tx, service) |
| 582 | 588 |
}) |
| 583 | 589 |
if err != nil {
|
| ... | ... |
@@ -47,7 +47,7 @@ func (m *Manager) IsStateDirty() (bool, error) {
|
| 47 | 47 |
if structField.Type.Kind() != reflect.Slice {
|
| 48 | 48 |
panic("unexpected field type in StoreSnapshot")
|
| 49 | 49 |
} |
| 50 |
- if structField.Name != "Nodes" && structField.Name != "Clusters" && field.Len() != 0 {
|
|
| 50 |
+ if structField.Name != "Nodes" && structField.Name != "Clusters" && structField.Name != "Networks" && field.Len() != 0 {
|
|
| 51 | 51 |
// One of the other data types has an entry |
| 52 | 52 |
return true, nil |
| 53 | 53 |
} |
| ... | ... |
@@ -19,6 +19,7 @@ import ( |
| 19 | 19 |
"github.com/docker/swarmkit/api" |
| 20 | 20 |
"github.com/docker/swarmkit/ca" |
| 21 | 21 |
"github.com/docker/swarmkit/connectionbroker" |
| 22 |
+ "github.com/docker/swarmkit/identity" |
|
| 22 | 23 |
"github.com/docker/swarmkit/log" |
| 23 | 24 |
"github.com/docker/swarmkit/manager/allocator" |
| 24 | 25 |
"github.com/docker/swarmkit/manager/controlapi" |
| ... | ... |
@@ -892,7 +893,18 @@ func (m *Manager) becomeLeader(ctx context.Context) {
|
| 892 | 892 |
rootCA)) |
| 893 | 893 |
// Add Node entry for ourself, if one |
| 894 | 894 |
// doesn't exist already. |
| 895 |
- store.CreateNode(tx, managerNode(nodeID, m.config.Availability)) |
|
| 895 |
+ freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability)) |
|
| 896 |
+ |
|
| 897 |
+ if freshCluster {
|
|
| 898 |
+ // This is a fresh swarm cluster. Add to store now any initial |
|
| 899 |
+ // cluster resource, like the default ingress network which |
|
| 900 |
+ // provides the routing mesh for this cluster. |
|
| 901 |
+ log.G(ctx).Info("Creating default ingress network")
|
|
| 902 |
+ if err := store.CreateNetwork(tx, newIngressNetwork()); err != nil {
|
|
| 903 |
+ log.G(ctx).WithError(err).Error("failed to create default ingress network")
|
|
| 904 |
+ } |
|
| 905 |
+ } |
|
| 906 |
+ |
|
| 896 | 907 |
return nil |
| 897 | 908 |
}) |
| 898 | 909 |
|
| ... | ... |
@@ -1084,3 +1096,28 @@ func managerNode(nodeID string, availability api.NodeSpec_Availability) *api.Nod |
| 1084 | 1084 |
}, |
| 1085 | 1085 |
} |
| 1086 | 1086 |
} |
| 1087 |
+ |
|
| 1088 |
+// newIngressNetwork returns the network object for the default ingress |
|
| 1089 |
+// network, the network which provides the routing mesh. Caller will save to |
|
| 1090 |
+// store this object once, at fresh cluster creation. It is expected to |
|
| 1091 |
+// call this function inside a store update transaction. |
|
| 1092 |
+func newIngressNetwork() *api.Network {
|
|
| 1093 |
+ return &api.Network{
|
|
| 1094 |
+ ID: identity.NewID(), |
|
| 1095 |
+ Spec: api.NetworkSpec{
|
|
| 1096 |
+ Ingress: true, |
|
| 1097 |
+ Annotations: api.Annotations{
|
|
| 1098 |
+ Name: "ingress", |
|
| 1099 |
+ }, |
|
| 1100 |
+ DriverConfig: &api.Driver{},
|
|
| 1101 |
+ IPAM: &api.IPAMOptions{
|
|
| 1102 |
+ Driver: &api.Driver{},
|
|
| 1103 |
+ Configs: []*api.IPAMConfig{
|
|
| 1104 |
+ {
|
|
| 1105 |
+ Subnet: "10.255.0.0/16", |
|
| 1106 |
+ }, |
|
| 1107 |
+ }, |
|
| 1108 |
+ }, |
|
| 1109 |
+ }, |
|
| 1110 |
+ } |
|
| 1111 |
+} |
| ... | ... |
@@ -111,6 +111,10 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference |
| 111 | 111 |
tree = next |
| 112 | 112 |
} |
| 113 | 113 |
|
| 114 |
+ if node.ActiveTasksCountByService != nil {
|
|
| 115 |
+ tree.tasks += node.ActiveTasksCountByService[serviceID] |
|
| 116 |
+ } |
|
| 117 |
+ |
|
| 114 | 118 |
if tree.nodeHeap.lessFunc == nil {
|
| 115 | 119 |
tree.nodeHeap.lessFunc = nodeLess |
| 116 | 120 |
} |
| ... | ... |
@@ -1,7 +1,6 @@ |
| 1 | 1 |
package scheduler |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
- "container/list" |
|
| 5 | 4 |
"time" |
| 6 | 5 |
|
| 7 | 6 |
"github.com/docker/swarmkit/api" |
| ... | ... |
@@ -30,7 +29,7 @@ type schedulingDecision struct {
|
| 30 | 30 |
// Scheduler assigns tasks to nodes. |
| 31 | 31 |
type Scheduler struct {
|
| 32 | 32 |
store *store.MemoryStore |
| 33 |
- unassignedTasks *list.List |
|
| 33 |
+ unassignedTasks map[string]*api.Task |
|
| 34 | 34 |
// preassignedTasks already have NodeID, need resource validation |
| 35 | 35 |
preassignedTasks map[string]*api.Task |
| 36 | 36 |
nodeSet nodeSet |
| ... | ... |
@@ -47,7 +46,7 @@ type Scheduler struct {
|
| 47 | 47 |
func New(store *store.MemoryStore) *Scheduler {
|
| 48 | 48 |
return &Scheduler{
|
| 49 | 49 |
store: store, |
| 50 |
- unassignedTasks: list.New(), |
|
| 50 |
+ unassignedTasks: make(map[string]*api.Task), |
|
| 51 | 51 |
preassignedTasks: make(map[string]*api.Task), |
| 52 | 52 |
allTasks: make(map[string]*api.Task), |
| 53 | 53 |
stopChan: make(chan struct{}),
|
| ... | ... |
@@ -191,7 +190,7 @@ func (s *Scheduler) Stop() {
|
| 191 | 191 |
|
| 192 | 192 |
// enqueue queues a task for scheduling. |
| 193 | 193 |
func (s *Scheduler) enqueue(t *api.Task) {
|
| 194 |
- s.unassignedTasks.PushBack(t) |
|
| 194 |
+ s.unassignedTasks[t.ID] = t |
|
| 195 | 195 |
} |
| 196 | 196 |
|
| 197 | 197 |
func (s *Scheduler) createTask(ctx context.Context, t *api.Task) int {
|
| ... | ... |
@@ -333,15 +332,12 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) {
|
| 333 | 333 |
// tick attempts to schedule the queue. |
| 334 | 334 |
func (s *Scheduler) tick(ctx context.Context) {
|
| 335 | 335 |
tasksByCommonSpec := make(map[string]map[string]*api.Task) |
| 336 |
- schedulingDecisions := make(map[string]schedulingDecision, s.unassignedTasks.Len()) |
|
| 336 |
+ schedulingDecisions := make(map[string]schedulingDecision, len(s.unassignedTasks)) |
|
| 337 | 337 |
|
| 338 |
- var next *list.Element |
|
| 339 |
- for e := s.unassignedTasks.Front(); e != nil; e = next {
|
|
| 340 |
- next = e.Next() |
|
| 341 |
- t := s.allTasks[e.Value.(*api.Task).ID] |
|
| 338 |
+ for taskID, t := range s.unassignedTasks {
|
|
| 342 | 339 |
if t == nil || t.NodeID != "" {
|
| 343 | 340 |
// task deleted or already assigned |
| 344 |
- s.unassignedTasks.Remove(e) |
|
| 341 |
+ delete(s.unassignedTasks, taskID) |
|
| 345 | 342 |
continue |
| 346 | 343 |
} |
| 347 | 344 |
|
| ... | ... |
@@ -362,8 +358,8 @@ func (s *Scheduler) tick(ctx context.Context) {
|
| 362 | 362 |
if tasksByCommonSpec[taskGroupKey] == nil {
|
| 363 | 363 |
tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task) |
| 364 | 364 |
} |
| 365 |
- tasksByCommonSpec[taskGroupKey][t.ID] = t |
|
| 366 |
- s.unassignedTasks.Remove(e) |
|
| 365 |
+ tasksByCommonSpec[taskGroupKey][taskID] = t |
|
| 366 |
+ delete(s.unassignedTasks, taskID) |
|
| 367 | 367 |
} |
| 368 | 368 |
|
| 369 | 369 |
for _, taskGroup := range tasksByCommonSpec {
|
| ... | ... |
@@ -602,6 +598,12 @@ func (s *Scheduler) scheduleNTasksOnNodes(ctx context.Context, n int, taskGroup |
| 602 | 602 |
nodeIter := 0 |
| 603 | 603 |
nodeCount := len(nodes) |
| 604 | 604 |
for taskID, t := range taskGroup {
|
| 605 |
+ // Skip tasks which were already scheduled because they ended |
|
| 606 |
+ // up in two groups at once. |
|
| 607 |
+ if _, exists := schedulingDecisions[taskID]; exists {
|
|
| 608 |
+ continue |
|
| 609 |
+ } |
|
| 610 |
+ |
|
| 605 | 611 |
node := &nodes[nodeIter%nodeCount] |
| 606 | 612 |
|
| 607 | 613 |
log.G(ctx).WithField("task.id", t.ID).Debugf("assigning to node %s", node.ID)
|