Browse code

Vendoring swarmkit @9fdea50

Signed-off-by: Alessandro Boch <aboch@docker.com>

Alessandro Boch authored on 2017/03/14 06:45:06
Showing 13 changed files
... ...
@@ -105,7 +105,7 @@ github.com/docker/containerd 422e31ce907fd9c3833a38d7b8fdd023e5a76e73
105 105
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
106 106
 
107 107
 # cluster
108
-github.com/docker/swarmkit 0e2d9ebcea9d5bbd4a06b3b964fb96356801f880
108
+github.com/docker/swarmkit 9fdea50c14492b6e1f472813849794d36bfef217
109 109
 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
110 110
 github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2
111 111
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
... ...
@@ -591,6 +591,11 @@ type NetworkSpec struct {
591 591
 	// enabled(default case) no manual attachment to this network
592 592
 	// can happen.
593 593
 	Attachable bool `protobuf:"varint,6,opt,name=attachable,proto3" json:"attachable,omitempty"`
594
+	// Ingress indicates this network will provide the routing-mesh.
595
+	// In older versions, the network providing the routing mesh was
596
+	// swarm internally created only and it was identified by the name
597
+	// "ingress" and the label "com.docker.swarm.internal": "true".
598
+	Ingress bool `protobuf:"varint,7,opt,name=ingress,proto3" json:"ingress,omitempty"`
594 599
 }
595 600
 
596 601
 func (m *NetworkSpec) Reset()                    { *m = NetworkSpec{} }
... ...
@@ -1795,6 +1800,16 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
1795 1795
 		}
1796 1796
 		i++
1797 1797
 	}
1798
+	if m.Ingress {
1799
+		dAtA[i] = 0x38
1800
+		i++
1801
+		if m.Ingress {
1802
+			dAtA[i] = 1
1803
+		} else {
1804
+			dAtA[i] = 0
1805
+		}
1806
+		i++
1807
+	}
1798 1808
 	return i, nil
1799 1809
 }
1800 1810
 
... ...
@@ -2255,6 +2270,9 @@ func (m *NetworkSpec) Size() (n int) {
2255 2255
 	if m.Attachable {
2256 2256
 		n += 2
2257 2257
 	}
2258
+	if m.Ingress {
2259
+		n += 2
2260
+	}
2258 2261
 	return n
2259 2262
 }
2260 2263
 
... ...
@@ -2502,6 +2520,7 @@ func (this *NetworkSpec) String() string {
2502 2502
 		`Internal:` + fmt.Sprintf("%v", this.Internal) + `,`,
2503 2503
 		`IPAM:` + strings.Replace(fmt.Sprintf("%v", this.IPAM), "IPAMOptions", "IPAMOptions", 1) + `,`,
2504 2504
 		`Attachable:` + fmt.Sprintf("%v", this.Attachable) + `,`,
2505
+		`Ingress:` + fmt.Sprintf("%v", this.Ingress) + `,`,
2505 2506
 		`}`,
2506 2507
 	}, "")
2507 2508
 	return s
... ...
@@ -4688,6 +4707,26 @@ func (m *NetworkSpec) Unmarshal(dAtA []byte) error {
4688 4688
 				}
4689 4689
 			}
4690 4690
 			m.Attachable = bool(v != 0)
4691
+		case 7:
4692
+			if wireType != 0 {
4693
+				return fmt.Errorf("proto: wrong wireType = %d for field Ingress", wireType)
4694
+			}
4695
+			var v int
4696
+			for shift := uint(0); ; shift += 7 {
4697
+				if shift >= 64 {
4698
+					return ErrIntOverflowSpecs
4699
+				}
4700
+				if iNdEx >= l {
4701
+					return io.ErrUnexpectedEOF
4702
+				}
4703
+				b := dAtA[iNdEx]
4704
+				iNdEx++
4705
+				v |= (int(b) & 0x7F) << shift
4706
+				if b < 0x80 {
4707
+					break
4708
+				}
4709
+			}
4710
+			m.Ingress = bool(v != 0)
4691 4711
 		default:
4692 4712
 			iNdEx = preIndex
4693 4713
 			skippy, err := skipSpecs(dAtA[iNdEx:])
... ...
@@ -5218,112 +5257,113 @@ var (
5218 5218
 func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
5219 5219
 
5220 5220
 var fileDescriptorSpecs = []byte{
5221
-	// 1707 bytes of a gzipped FileDescriptorProto
5221
+	// 1717 bytes of a gzipped FileDescriptorProto
5222 5222
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x41, 0x73, 0x1b, 0xb7,
5223
-	0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3,
5223
+	0x15, 0x16, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x50, 0x34, 0xe3,
5224 5224
 	0xa6, 0x4a, 0x33, 0xa5, 0xa6, 0x6a, 0x27, 0x75, 0xea, 0x66, 0x5a, 0x52, 0x64, 0x65, 0x55, 0x95,
5225 5225
 	0xcc, 0x01, 0x15, 0x77, 0x7c, 0xe2, 0x80, 0xbb, 0x10, 0xb9, 0xa3, 0xe5, 0x62, 0x0b, 0x60, 0x99,
5226
-	0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0xbf, 0xa1, 0xff, 0xc1, 0xc7, 0x1e,
5227
-	0x7b, 0xd2, 0x34, 0xfc, 0x0b, 0xfd, 0x01, 0xed, 0x00, 0x0b, 0x92, 0xcb, 0x64, 0x15, 0x7b, 0x26,
5228
-	0xbe, 0xe1, 0xbd, 0xfd, 0xbe, 0x07, 0xe0, 0xe1, 0xc3, 0xc3, 0x5b, 0xb0, 0x45, 0x44, 0x5d, 0xd1,
5229
-	0x8a, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x57, 0x94, 0xb7, 0xc4, 0x97, 0x84, 0x4f, 0xaf, 0x7c,
5230
-	0xd9, 0x9a, 0xfd, 0xbc, 0x6a, 0xcb, 0x79, 0x44, 0x0d, 0xa0, 0x7a, 0x7f, 0xcc, 0xc6, 0x4c, 0x0f,
5231
-	0x0f, 0xd4, 0xc8, 0x78, 0x6b, 0x63, 0xc6, 0xc6, 0x01, 0x3d, 0xd0, 0xd6, 0x28, 0xbe, 0x3c, 0xf0,
5232
-	0x62, 0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xba, 0x00, 0xd6, 0x39, 0xf3, 0xe8, 0x20, 0xa2,
5233
-	0x2e, 0x3a, 0x06, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0xbf, 0x6f, 0x1f, 0xd6,
5234
-	0x5b, 0xdf, 0x9e, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0x37, 0xf5, 0x1c, 0x4e, 0x33, 0xd1,
5235
-	0x6f, 0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0xef, 0xdf,
5236
-	0x39, 0xfc, 0x51, 0x56, 0x24, 0x35, 0x39, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0x28, 0x03, 0x1d, 0x03,
5237
-	0x4c, 0xe9, 0x74, 0x44, 0xb9, 0x98, 0xf8, 0x91, 0xb3, 0xad, 0xe9, 0x3f, 0xb9, 0x8d, 0xae, 0xd6,
5238
-	0xde, 0x3a, 0x5b, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa0, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f,
5239
-	0x7c, 0x39, 0x77, 0x0a, 0x3a, 0xd4, 0xc7, 0xdf, 0x19, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3,
5240
-	0x03, 0x58, 0x4f, 0x84, 0x3e, 0x82, 0xdd, 0x7e, 0xef, 0xbc, 0x7b, 0x72, 0x7e, 0x5c, 0xc9, 0x55,
5241
-	0x1f, 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb,
5242
-	0x60, 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0x5d, 0x37, 0xde,
5243
-	0xdf, 0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0x57, 0xff, 0xa8, 0xe5, 0x9a, 0x5f,
5244
-	0xe5, 0xa1, 0x9c, 0x5e, 0x04, 0xfa, 0x08, 0x8a, 0xed, 0xa3, 0x8b, 0x93, 0x17, 0xbd, 0x4a, 0x6e,
5245
-	0x4d, 0x4f, 0x23, 0xda, 0xae, 0xf4, 0x67, 0x14, 0x3d, 0x86, 0x9d, 0x7e, 0xfb, 0x8b, 0x41, 0xaf,
5246
-	0x92, 0x5f, 0x2f, 0x27, 0x0d, 0xeb, 0x93, 0x58, 0x68, 0x54, 0x17, 0xb7, 0x4f, 0xce, 0x2b, 0x5b,
5247
-	0xd9, 0xa8, 0x2e, 0x27, 0x7e, 0x68, 0x96, 0xf2, 0xf7, 0x02, 0xd8, 0x03, 0xca, 0x67, 0xbe, 0xfb,
5248
-	0x8e, 0x25, 0xf2, 0x29, 0x14, 0x24, 0x11, 0x57, 0x5a, 0x1a, 0x76, 0xb6, 0x34, 0x2e, 0x88, 0xb8,
5249
-	0x52, 0x93, 0x1a, 0xba, 0xc6, 0x2b, 0x65, 0x70, 0x1a, 0x05, 0xbe, 0x4b, 0x24, 0xf5, 0xb4, 0x32,
5250
-	0xec, 0xc3, 0x1f, 0x67, 0xb1, 0xf1, 0x0a, 0x65, 0xd6, 0xff, 0x2c, 0x87, 0x53, 0x54, 0xf4, 0x14,
5251
-	0x8a, 0xe3, 0x80, 0x8d, 0x48, 0xa0, 0x35, 0x61, 0x1f, 0x3e, 0xca, 0x0a, 0x72, 0xac, 0x11, 0xeb,
5252
-	0x00, 0x86, 0x82, 0x9e, 0x40, 0x31, 0x8e, 0x3c, 0x22, 0xa9, 0x53, 0xd4, 0xe4, 0x46, 0x16, 0xf9,
5253
-	0x0b, 0x8d, 0x38, 0x62, 0xe1, 0xa5, 0x3f, 0xc6, 0x06, 0x8f, 0x4e, 0xc1, 0x0a, 0xa9, 0xfc, 0x92,
5254
-	0xf1, 0x2b, 0xe1, 0xec, 0x36, 0xb6, 0xf7, 0xed, 0xc3, 0x4f, 0x32, 0xc5, 0x98, 0x60, 0xda, 0x52,
5255
-	0x12, 0x77, 0x32, 0xa5, 0xa1, 0x4c, 0xc2, 0x74, 0xb6, 0x9c, 0x3c, 0x5e, 0x05, 0x40, 0xbf, 0x01,
5256
-	0x8b, 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x3a, 0xd6, 0xed, 0x0b, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x5e,
5257
-	0x31, 0x14, 0x9b, 0xb3, 0x20, 0x18, 0x11, 0xf7, 0xca, 0x29, 0xbd, 0xe5, 0x36, 0x56, 0x8c, 0x4e,
5258
-	0x11, 0x0a, 0x53, 0xe6, 0xd1, 0xe6, 0x01, 0xdc, 0xfb, 0x56, 0xaa, 0x51, 0x15, 0x2c, 0x93, 0xea,
5259
-	0x44, 0x23, 0x05, 0xbc, 0xb2, 0x9b, 0x77, 0x61, 0x6f, 0x23, 0xad, 0xcd, 0xbf, 0x16, 0xc0, 0x5a,
5260
-	0x9e, 0x35, 0x6a, 0x43, 0xc9, 0x65, 0xa1, 0x24, 0x7e, 0x48, 0xb9, 0x91, 0x57, 0xe6, 0xc9, 0x1c,
5261
-	0x2d, 0x41, 0x8a, 0xf5, 0x2c, 0x87, 0xd7, 0x2c, 0xf4, 0x7b, 0x28, 0x71, 0x2a, 0x58, 0xcc, 0x5d,
5262
-	0x2a, 0x8c, 0xbe, 0xf6, 0xb3, 0x15, 0x92, 0x80, 0x30, 0xfd, 0x73, 0xec, 0x73, 0xaa, 0xb2, 0x2c,
5263
-	0xf0, 0x9a, 0x8a, 0x9e, 0xc2, 0x2e, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0x4b, 0x22, 0x38, 0x81, 0xf4,
5264
-	0x59, 0xe0, 0xbb, 0x73, 0xbc, 0x64, 0xa0, 0xa7, 0x50, 0x8a, 0x02, 0xe2, 0xea, 0xa8, 0xce, 0x8e,
5265
-	0xa6, 0x7f, 0x90, 0x45, 0xef, 0x2f, 0x41, 0x78, 0x8d, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4,
5266
-	0xb8, 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x9a, 0xc5, 0xee, 0x6a, 0x04, 0x2e, 0x05, 0x6c, 0x9c, 0x0c,
5267
-	0xd1, 0xf1, 0xf7, 0xd2, 0x57, 0x4a, 0x5b, 0xa7, 0x00, 0x64, 0xf5, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7,
5268
-	0x0a, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x8f, 0xa0, 0x7c, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94,
5269
-	0xb4, 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc1, 0x2e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3,
5270
-	0x14, 0xde, 0xcb, 0x0c, 0x8a, 0x0e, 0xa1, 0xbc, 0x3a, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3a,
5271
-	0x77, 0x17, 0x37, 0x75, 0x7b, 0xa5, 0x87, 0x93, 0x2e, 0xb6, 0x57, 0xa0, 0x13, 0xaf, 0xf9, 0x37,
5272
-	0x0b, 0xf6, 0x36, 0xc4, 0x82, 0xee, 0xc3, 0x8e, 0x3f, 0x25, 0x63, 0x9a, 0xd0, 0x71, 0x62, 0xa0,
5273
-	0x1e, 0x14, 0x03, 0x32, 0xa2, 0x81, 0x92, 0x8c, 0x4a, 0xdb, 0xcf, 0xde, 0xa8, 0xba, 0xd6, 0x1f,
5274
-	0x35, 0xbe, 0x17, 0x4a, 0x3e, 0xc7, 0x86, 0x8c, 0x1c, 0xd8, 0x75, 0xd9, 0x74, 0x4a, 0x42, 0x55,
5275
-	0x9c, 0xb6, 0xf7, 0x4b, 0x78, 0x69, 0x22, 0x04, 0x05, 0xc2, 0xc7, 0xc2, 0x29, 0x68, 0xb7, 0x1e,
5276
-	0xa3, 0x0a, 0x6c, 0xd3, 0x70, 0xe6, 0xec, 0x68, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e,
5277
-	0xc2, 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd5, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xe2, 0x94,
5278
-	0xc5, 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6b, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81,
5279
-	0xa3, 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33,
5280
-	0x97, 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0xcb, 0x5e, 0xa1, 0xd5, 0x35, 0xbd, 0x02, 0xbe, 0xab,
5281
-	0x38, 0xc7, 0x8a, 0xd2, 0xd7, 0x0c, 0xd4, 0x87, 0x72, 0x14, 0x07, 0xc1, 0x90, 0x45, 0xc9, 0x3b,
5282
-	0x00, 0x3a, 0xc2, 0x5b, 0xa4, 0xac, 0x1f, 0x07, 0xc1, 0xf3, 0x84, 0x84, 0xed, 0x68, 0x6d, 0xa0,
5283
-	0xf7, 0xa1, 0x38, 0xe6, 0x2c, 0x8e, 0x84, 0x63, 0xeb, 0x64, 0x18, 0x0b, 0x7d, 0x0e, 0xbb, 0x82,
5284
-	0xba, 0x9c, 0x4a, 0xe1, 0x94, 0xf5, 0x56, 0x3f, 0xcc, 0x9a, 0x64, 0xa0, 0x21, 0x98, 0x5e, 0x52,
5285
-	0x4e, 0x43, 0x97, 0xe2, 0x25, 0x07, 0x3d, 0x80, 0x6d, 0x29, 0xe7, 0xce, 0x5e, 0x23, 0xbf, 0x6f,
5286
-	0x75, 0x76, 0x17, 0x37, 0xf5, 0xed, 0x8b, 0x8b, 0x97, 0x58, 0xf9, 0x54, 0x8d, 0x9a, 0x30, 0x21,
5287
-	0x43, 0x32, 0xa5, 0xce, 0x1d, 0x9d, 0xdb, 0x95, 0x8d, 0x5e, 0x02, 0x78, 0xa1, 0x18, 0xba, 0xfa,
5288
-	0x52, 0x38, 0x77, 0xf5, 0xee, 0x3e, 0x79, 0xf3, 0xee, 0xba, 0xe7, 0x03, 0x53, 0xa7, 0xf7, 0x16,
5289
-	0x37, 0xf5, 0xd2, 0xca, 0xc4, 0x25, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8,
5290
-	0x89, 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xb9, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4d, 0x52,
5291
-	0x0a, 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43,
5292
-	0x21, 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x29, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16,
5293
-	0x89, 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0xa8,
5294
-	0x0e, 0xb6, 0xd6, 0x85, 0xf0, 0xc7, 0x21, 0x09, 0x9c, 0xfb, 0x3a, 0x1f, 0xa0, 0x5c, 0x03, 0xed,
5295
-	0xa9, 0x7e, 0x06, 0x76, 0x4a, 0xee, 0x4a, 0xa6, 0x57, 0x74, 0x6e, 0x6e, 0x90, 0x1a, 0xaa, 0x35,
5296
-	0xcd, 0x48, 0x10, 0x27, 0xcd, 0x5e, 0x09, 0x27, 0xc6, 0xaf, 0xb7, 0x9e, 0xe4, 0xab, 0x87, 0x60,
5297
-	0xa7, 0x8e, 0x1d, 0x7d, 0x08, 0x7b, 0x9c, 0x8e, 0x7d, 0x21, 0xf9, 0x7c, 0x48, 0x62, 0x39, 0x71,
5298
-	0x7e, 0xa7, 0x09, 0xe5, 0xa5, 0xb3, 0x1d, 0xcb, 0x49, 0x75, 0x08, 0xeb, 0xec, 0xa1, 0x06, 0xd8,
5299
-	0xea, 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x76, 0x29, 0xf5, 0x08, 0x4a,
5300
-	0xb8, 0x3b, 0xd1, 0x97, 0xb7, 0x84, 0x8d, 0xa5, 0x6e, 0xe3, 0x52, 0xa2, 0xe6, 0x36, 0x1a, 0xb3,
5301
-	0xf9, 0xdf, 0x3c, 0x94, 0xd3, 0xef, 0x22, 0x3a, 0x4a, 0xde, 0x33, 0xbd, 0xa5, 0x3b, 0x87, 0x07,
5302
-	0x6f, 0x7a, 0x47, 0xf5, 0xeb, 0x11, 0xc4, 0x2a, 0xd8, 0x99, 0x6a, 0x61, 0x35, 0x19, 0xfd, 0x12,
5303
-	0x76, 0x22, 0xc6, 0xe5, 0xb2, 0x86, 0xd4, 0x32, 0x2b, 0x3e, 0xe3, 0xcb, 0x6a, 0x9b, 0x80, 0x9b,
5304
-	0x13, 0xb8, 0xb3, 0x19, 0x0d, 0x3d, 0x86, 0xed, 0x17, 0x27, 0xfd, 0x4a, 0xae, 0xfa, 0xf0, 0xd5,
5305
-	0x75, 0xe3, 0x87, 0x9b, 0x1f, 0x5f, 0xf8, 0x5c, 0xc6, 0x24, 0x38, 0xe9, 0xa3, 0x9f, 0xc2, 0x4e,
5306
-	0xf7, 0x7c, 0x80, 0x71, 0x25, 0x5f, 0xad, 0xbf, 0xba, 0x6e, 0x3c, 0xdc, 0xc4, 0xa9, 0x4f, 0x2c,
5307
-	0x0e, 0x3d, 0xcc, 0x46, 0xab, 0x76, 0xee, 0x9f, 0x5b, 0x60, 0x9b, 0xd2, 0xfa, 0xae, 0x3b, 0xfe,
5308
-	0xbd, 0xe4, 0xb5, 0x5a, 0xde, 0x99, 0xad, 0x37, 0x3e, 0x5a, 0xe5, 0x84, 0x60, 0xce, 0xf8, 0x11,
5309
-	0x94, 0xfd, 0x68, 0xf6, 0xe9, 0x90, 0x86, 0x64, 0x14, 0x98, 0xce, 0xce, 0xc2, 0xb6, 0xf2, 0xf5,
5310
-	0x12, 0x97, 0xba, 0xb0, 0x7e, 0x28, 0x29, 0x0f, 0x4d, 0xcf, 0x66, 0xe1, 0x95, 0x8d, 0x3e, 0x87,
5311
-	0x82, 0x1f, 0x91, 0xa9, 0x79, 0x69, 0x33, 0x77, 0x70, 0xd2, 0x6f, 0x9f, 0x19, 0x0d, 0x76, 0xac,
5312
-	0xc5, 0x4d, 0xbd, 0xa0, 0x1c, 0x58, 0xd3, 0x50, 0x6d, 0xf9, 0xd8, 0xa9, 0x99, 0x74, 0xf1, 0xb5,
5313
-	0x70, 0xca, 0xd3, 0xfc, 0x5f, 0x01, 0xec, 0xa3, 0x20, 0x16, 0xd2, 0x3c, 0x21, 0xef, 0x2c, 0x6f,
5314
-	0x2f, 0xe1, 0x1e, 0xd1, 0xcd, 0x3f, 0x09, 0x55, 0x3d, 0xd6, 0x4d, 0x84, 0xc9, 0xdd, 0xe3, 0xcc,
5315
-	0x70, 0x2b, 0x70, 0xd2, 0x70, 0x74, 0x8a, 0x2a, 0xa6, 0x93, 0xc7, 0x15, 0xf2, 0x8d, 0x2f, 0x68,
5316
-	0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52, 0xc5, 0x4d, 0xb3, 0x9c, 0xf9, 0x1b, 0xf5, 0x3c,
5317
-	0x0d, 0x34, 0x25, 0x2c, 0x59, 0xed, 0x66, 0x0c, 0xf4, 0x04, 0x0a, 0x9c, 0x5c, 0x2e, 0x1b, 0xa2,
5318
-	0x4c, 0x7d, 0x63, 0x72, 0x29, 0x37, 0x42, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48,
5319
-	0x77, 0x42, 0xb9, 0x39, 0xa7, 0xcc, 0x2d, 0x76, 0x57, 0xa8, 0x8d, 0x28, 0x29, 0x36, 0x3a, 0x85,
5320
-	0x92, 0x4b, 0x96, 0x4a, 0x2b, 0xde, 0xfe, 0x07, 0x71, 0xd4, 0x36, 0x21, 0x2a, 0x2a, 0xc4, 0xe2,
5321
-	0xa6, 0x6e, 0x2d, 0x3d, 0xd8, 0x72, 0x89, 0x51, 0xde, 0x29, 0xec, 0xa9, 0x3f, 0x8b, 0xa1, 0x47,
5322
-	0x2f, 0x49, 0x1c, 0x48, 0xa1, 0x1f, 0xda, 0x5b, 0x4a, 0xb2, 0x6a, 0x53, 0xbb, 0x06, 0x67, 0xd6,
5323
-	0x55, 0x96, 0x29, 0x1f, 0xfa, 0x13, 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x67, 0xcb, 0x15, 0x5a,
5324
-	0xb7, 0x6f, 0xb6, 0xb7, 0x02, 0x6f, 0x6c, 0xb6, 0x42, 0xbf, 0xe1, 0x6f, 0xfa, 0x00, 0xc9, 0x23,
5325
-	0xf7, 0x6e, 0xf5, 0x87, 0xa0, 0xe0, 0x11, 0x49, 0xb4, 0xe4, 0xca, 0x58, 0x8f, 0x3b, 0xce, 0xeb,
5326
-	0xaf, 0x6b, 0xb9, 0x7f, 0x7f, 0x5d, 0xcb, 0xfd, 0x65, 0x51, 0xcb, 0xbf, 0x5e, 0xd4, 0xf2, 0xff,
5327
-	0x5a, 0xd4, 0xf2, 0xff, 0x59, 0xd4, 0xf2, 0xa3, 0xa2, 0x6e, 0x0d, 0x7e, 0xf1, 0xff, 0x00, 0x00,
5328
-	0x00, 0xff, 0xff, 0xed, 0xbe, 0x26, 0xe6, 0x9a, 0x10, 0x00, 0x00,
5226
+	0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0x7f, 0xc6, 0xb7, 0xf6, 0xd8, 0x93,
5227
+	0xa6, 0xe1, 0x5f, 0xe8, 0x0f, 0x68, 0x07, 0x58, 0x2c, 0xb9, 0x4c, 0x56, 0xb1, 0x67, 0xe2, 0x1b,
5228
+	0xde, 0xdb, 0xef, 0x7b, 0x00, 0x1e, 0x3e, 0x3c, 0xbc, 0x05, 0x5b, 0x44, 0xd4, 0x15, 0xad, 0x88,
5229
+	0x33, 0xc9, 0x10, 0xf2, 0x98, 0x7b, 0x45, 0x79, 0x4b, 0x7c, 0x49, 0xf8, 0xf4, 0xca, 0x97, 0xad,
5230
+	0xd9, 0xcf, 0x6b, 0xb6, 0x9c, 0x47, 0xd4, 0x00, 0x6a, 0xf7, 0xc7, 0x6c, 0xcc, 0xf4, 0xf0, 0x40,
5231
+	0x8d, 0x8c, 0xb7, 0x3e, 0x66, 0x6c, 0x1c, 0xd0, 0x03, 0x6d, 0x8d, 0xe2, 0xcb, 0x03, 0x2f, 0xe6,
5232
+	0x44, 0xfa, 0x2c, 0x4c, 0xbe, 0x37, 0xaf, 0x8b, 0x60, 0x9d, 0x33, 0x8f, 0x0e, 0x22, 0xea, 0xa2,
5233
+	0x63, 0xb0, 0x49, 0x18, 0x32, 0xa9, 0x01, 0xc2, 0x29, 0x34, 0x0a, 0xfb, 0xf6, 0xe1, 0x5e, 0xeb,
5234
+	0xdb, 0x33, 0xb7, 0xda, 0x2b, 0x58, 0xa7, 0xf8, 0xfa, 0x66, 0x6f, 0x03, 0x67, 0x99, 0xe8, 0xb7,
5235
+	0x50, 0xf1, 0xa8, 0xf0, 0x39, 0xf5, 0x86, 0x9c, 0x05, 0xd4, 0xd9, 0x6c, 0x14, 0xf6, 0xef, 0x1c,
5236
+	0xfe, 0x28, 0x2f, 0x92, 0x9a, 0x1c, 0xb3, 0x80, 0x62, 0xdb, 0x30, 0x94, 0x81, 0x8e, 0x01, 0xa6,
5237
+	0x74, 0x3a, 0xa2, 0x5c, 0x4c, 0xfc, 0xc8, 0xd9, 0xd2, 0xf4, 0x9f, 0xdc, 0x46, 0x57, 0x6b, 0x6f,
5238
+	0x9d, 0x2d, 0xe1, 0x38, 0x43, 0x45, 0x67, 0x50, 0x21, 0x33, 0xe2, 0x07, 0x64, 0xe4, 0x07, 0xbe,
5239
+	0x9c, 0x3b, 0x45, 0x1d, 0xea, 0xe3, 0xef, 0x0c, 0xd5, 0xce, 0x10, 0xf0, 0x1a, 0xbd, 0xe9, 0x01,
5240
+	0xac, 0x26, 0x42, 0x1f, 0xc1, 0x4e, 0xbf, 0x77, 0xde, 0x3d, 0x39, 0x3f, 0xae, 0x6e, 0xd4, 0x1e,
5241
+	0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x00, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb, 0x60,
5242
+	0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x5a, 0xa8, 0xd5, 0x5e, 0x5d, 0x37, 0xde, 0x5f,
5243
+	0x07, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xb5, 0xe2, 0x57, 0xff, 0xa8, 0x6f, 0x34, 0xbf, 0x2a,
5244
+	0x40, 0x25, 0xbb, 0x08, 0xf4, 0x11, 0x94, 0xda, 0x47, 0x17, 0x27, 0x2f, 0x7a, 0xd5, 0x8d, 0x15,
5245
+	0x3d, 0x8b, 0x68, 0xbb, 0xd2, 0x9f, 0x51, 0xf4, 0x18, 0xb6, 0xfb, 0xed, 0x2f, 0x06, 0xbd, 0x6a,
5246
+	0x61, 0xb5, 0x9c, 0x2c, 0xac, 0x4f, 0x62, 0xa1, 0x51, 0x5d, 0xdc, 0x3e, 0x39, 0xaf, 0x6e, 0xe6,
5247
+	0xa3, 0xba, 0x9c, 0xf8, 0xa1, 0x59, 0xca, 0xdf, 0x8b, 0x60, 0x0f, 0x28, 0x9f, 0xf9, 0xee, 0x3b,
5248
+	0x96, 0xc8, 0xa7, 0x50, 0x94, 0x44, 0x5c, 0x69, 0x69, 0xd8, 0xf9, 0xd2, 0xb8, 0x20, 0xe2, 0x4a,
5249
+	0x4d, 0x6a, 0xe8, 0x1a, 0xaf, 0x94, 0xc1, 0x69, 0x14, 0xf8, 0x2e, 0x91, 0xd4, 0xd3, 0xca, 0xb0,
5250
+	0x0f, 0x7f, 0x9c, 0xc7, 0xc6, 0x4b, 0x94, 0x59, 0xff, 0xb3, 0x0d, 0x9c, 0xa1, 0xa2, 0xa7, 0x50,
5251
+	0x1a, 0x07, 0x6c, 0x44, 0x02, 0xad, 0x09, 0xfb, 0xf0, 0x51, 0x5e, 0x90, 0x63, 0x8d, 0x58, 0x05,
5252
+	0x30, 0x14, 0xf4, 0x04, 0x4a, 0x71, 0xe4, 0x11, 0x49, 0x9d, 0x92, 0x26, 0x37, 0xf2, 0xc8, 0x5f,
5253
+	0x68, 0xc4, 0x11, 0x0b, 0x2f, 0xfd, 0x31, 0x36, 0x78, 0x74, 0x0a, 0x56, 0x48, 0xe5, 0x97, 0x8c,
5254
+	0x5f, 0x09, 0x67, 0xa7, 0xb1, 0xb5, 0x6f, 0x1f, 0x7e, 0x92, 0x2b, 0xc6, 0x04, 0xd3, 0x96, 0x92,
5255
+	0xb8, 0x93, 0x29, 0x0d, 0x65, 0x12, 0xa6, 0xb3, 0xe9, 0x14, 0xf0, 0x32, 0x00, 0xfa, 0x0d, 0x58,
5256
+	0x34, 0xf4, 0x22, 0xe6, 0x87, 0xd2, 0xb1, 0x6e, 0x5f, 0x48, 0xcf, 0x60, 0x54, 0x32, 0xf1, 0x92,
5257
+	0xa1, 0xd8, 0x9c, 0x05, 0xc1, 0x88, 0xb8, 0x57, 0x4e, 0xf9, 0x2d, 0xb7, 0xb1, 0x64, 0x74, 0x4a,
5258
+	0x50, 0x9c, 0x32, 0x8f, 0x36, 0x0f, 0xe0, 0xde, 0xb7, 0x52, 0x8d, 0x6a, 0x60, 0x99, 0x54, 0x27,
5259
+	0x1a, 0x29, 0xe2, 0xa5, 0xdd, 0xbc, 0x0b, 0xbb, 0x6b, 0x69, 0x6d, 0xfe, 0xb5, 0x08, 0x56, 0x7a,
5260
+	0xd6, 0xa8, 0x0d, 0x65, 0x97, 0x85, 0x92, 0xf8, 0x21, 0xe5, 0x46, 0x5e, 0xb9, 0x27, 0x73, 0x94,
5261
+	0x82, 0x14, 0xeb, 0xd9, 0x06, 0x5e, 0xb1, 0xd0, 0xef, 0xa1, 0xcc, 0xa9, 0x60, 0x31, 0x77, 0xa9,
5262
+	0x30, 0xfa, 0xda, 0xcf, 0x57, 0x48, 0x02, 0xc2, 0xf4, 0xcf, 0xb1, 0xcf, 0xa9, 0xca, 0xb2, 0xc0,
5263
+	0x2b, 0x2a, 0x7a, 0x0a, 0x3b, 0x9c, 0x0a, 0x49, 0xb8, 0xfc, 0x2e, 0x89, 0xe0, 0x04, 0xd2, 0x67,
5264
+	0x81, 0xef, 0xce, 0x71, 0xca, 0x40, 0x4f, 0xa1, 0x1c, 0x05, 0xc4, 0xd5, 0x51, 0x9d, 0x6d, 0x4d,
5265
+	0xff, 0x20, 0x8f, 0xde, 0x4f, 0x41, 0x78, 0x85, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4, 0xb8,
5266
+	0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x96, 0xc7, 0xee, 0x6a, 0x04, 0x2e, 0x07, 0x6c, 0x9c, 0x0c, 0xd1,
5267
+	0xf1, 0xf7, 0xd2, 0x57, 0x46, 0x5b, 0xa7, 0x00, 0x64, 0xf9, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7, 0x0a,
5268
+	0x65, 0x4e, 0x24, 0x43, 0x47, 0x8f, 0xa0, 0x72, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94, 0xb5,
5269
+	0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc3, 0x0e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3, 0x14,
5270
+	0xde, 0xcb, 0x0d, 0x8a, 0x0e, 0xa1, 0xb2, 0x3c, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3b, 0x77,
5271
+	0x17, 0x37, 0x7b, 0xf6, 0x52, 0x0f, 0x27, 0x5d, 0x6c, 0x2f, 0x41, 0x27, 0x5e, 0xf3, 0x6f, 0x16,
5272
+	0xec, 0xae, 0x89, 0x05, 0xdd, 0x87, 0x6d, 0x7f, 0x4a, 0xc6, 0x34, 0xa1, 0xe3, 0xc4, 0x40, 0x3d,
5273
+	0x28, 0x05, 0x64, 0x44, 0x03, 0x25, 0x19, 0x95, 0xb6, 0x9f, 0xbd, 0x51, 0x75, 0xad, 0x3f, 0x6a,
5274
+	0x7c, 0x2f, 0x94, 0x7c, 0x8e, 0x0d, 0x19, 0x39, 0xb0, 0xe3, 0xb2, 0xe9, 0x94, 0x84, 0xaa, 0x38,
5275
+	0x6d, 0xed, 0x97, 0x71, 0x6a, 0x22, 0x04, 0x45, 0xc2, 0xc7, 0xc2, 0x29, 0x6a, 0xb7, 0x1e, 0xa3,
5276
+	0x2a, 0x6c, 0xd1, 0x70, 0xe6, 0x6c, 0x6b, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e, 0xc6,
5277
+	0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd1, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xd2, 0x94, 0xc5,
5278
+	0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6f, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81, 0xa3,
5279
+	0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33, 0x97,
5280
+	0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0x69, 0xaf, 0xd0, 0xea, 0x9a, 0x5e, 0x01, 0xdf, 0x55, 0x9c,
5281
+	0x63, 0x45, 0xe9, 0x6b, 0x06, 0xea, 0x43, 0x25, 0x8a, 0x83, 0x60, 0xc8, 0xa2, 0xe4, 0x1d, 0x00,
5282
+	0x1d, 0xe1, 0x2d, 0x52, 0xd6, 0x8f, 0x83, 0xe0, 0x79, 0x42, 0xc2, 0x76, 0xb4, 0x32, 0xd0, 0xfb,
5283
+	0x50, 0x1a, 0x73, 0x16, 0x47, 0xc2, 0xb1, 0x75, 0x32, 0x8c, 0x85, 0x3e, 0x87, 0x1d, 0x41, 0x5d,
5284
+	0x4e, 0xa5, 0x70, 0x2a, 0x7a, 0xab, 0x1f, 0xe6, 0x4d, 0x32, 0xd0, 0x10, 0x4c, 0x2f, 0x29, 0xa7,
5285
+	0xa1, 0x4b, 0x71, 0xca, 0x41, 0x0f, 0x60, 0x4b, 0xca, 0xb9, 0xb3, 0xdb, 0x28, 0xec, 0x5b, 0x9d,
5286
+	0x9d, 0xc5, 0xcd, 0xde, 0xd6, 0xc5, 0xc5, 0x4b, 0xac, 0x7c, 0xaa, 0x46, 0x4d, 0x98, 0x90, 0x21,
5287
+	0x99, 0x52, 0xe7, 0x8e, 0xce, 0xed, 0xd2, 0x46, 0x2f, 0x01, 0xbc, 0x50, 0x0c, 0x5d, 0x7d, 0x29,
5288
+	0x9c, 0xbb, 0x7a, 0x77, 0x9f, 0xbc, 0x79, 0x77, 0xdd, 0xf3, 0x81, 0xa9, 0xd3, 0xbb, 0x8b, 0x9b,
5289
+	0xbd, 0xf2, 0xd2, 0xc4, 0x65, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8, 0x89,
5290
+	0x3b, 0xa1, 0xee, 0x95, 0x53, 0xbd, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4b, 0x52, 0x0a,
5291
+	0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43, 0x21,
5292
+	0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x2b, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16, 0x89,
5293
+	0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0x68, 0x0f,
5294
+	0x6c, 0xad, 0x0b, 0xe1, 0x8f, 0x43, 0x12, 0x38, 0xf7, 0x75, 0x3e, 0x40, 0xb9, 0x06, 0xda, 0x53,
5295
+	0xfb, 0x0c, 0xec, 0x8c, 0xdc, 0x95, 0x4c, 0xaf, 0xe8, 0xdc, 0xdc, 0x20, 0x35, 0x54, 0x6b, 0x9a,
5296
+	0x91, 0x20, 0x4e, 0x9a, 0xbd, 0x32, 0x4e, 0x8c, 0x5f, 0x6f, 0x3e, 0x29, 0xd4, 0x0e, 0xc1, 0xce,
5297
+	0x1c, 0x3b, 0xfa, 0x10, 0x76, 0x39, 0x1d, 0xfb, 0x42, 0xf2, 0xf9, 0x90, 0xc4, 0x72, 0xe2, 0xfc,
5298
+	0x4e, 0x13, 0x2a, 0xa9, 0xb3, 0x1d, 0xcb, 0x49, 0x6d, 0x08, 0xab, 0xec, 0xa1, 0x06, 0xd8, 0xea,
5299
+	0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x75, 0x29, 0xf5, 0x08, 0x4a, 0xb8,
5300
+	0x3b, 0xd1, 0x97, 0xb7, 0x8c, 0x8d, 0xa5, 0x6e, 0x63, 0x2a, 0x51, 0x73, 0x1b, 0x8d, 0xd9, 0xfc,
5301
+	0x6f, 0x01, 0x2a, 0xd9, 0x77, 0x11, 0x1d, 0x25, 0xef, 0x99, 0xde, 0xd2, 0x9d, 0xc3, 0x83, 0x37,
5302
+	0xbd, 0xa3, 0xfa, 0xf5, 0x08, 0x62, 0x15, 0xec, 0x4c, 0xb5, 0xb0, 0x9a, 0x8c, 0x7e, 0x09, 0xdb,
5303
+	0x11, 0xe3, 0x32, 0xad, 0x21, 0xf5, 0xdc, 0x8a, 0xcf, 0x78, 0x5a, 0x6d, 0x13, 0x70, 0x73, 0x02,
5304
+	0x77, 0xd6, 0xa3, 0xa1, 0xc7, 0xb0, 0xf5, 0xe2, 0xa4, 0x5f, 0xdd, 0xa8, 0x3d, 0x7c, 0x75, 0xdd,
5305
+	0xf8, 0xe1, 0xfa, 0xc7, 0x17, 0x3e, 0x97, 0x31, 0x09, 0x4e, 0xfa, 0xe8, 0xa7, 0xb0, 0xdd, 0x3d,
5306
+	0x1f, 0x60, 0x5c, 0x2d, 0xd4, 0xf6, 0x5e, 0x5d, 0x37, 0x1e, 0xae, 0xe3, 0xd4, 0x27, 0x16, 0x87,
5307
+	0x1e, 0x66, 0xa3, 0x65, 0x3b, 0xf7, 0xcf, 0x4d, 0xb0, 0x4d, 0x69, 0x7d, 0xd7, 0x1d, 0xff, 0x6e,
5308
+	0xf2, 0x5a, 0xa5, 0x77, 0x66, 0xf3, 0x8d, 0x8f, 0x56, 0x25, 0x21, 0x98, 0x33, 0x7e, 0x04, 0x15,
5309
+	0x3f, 0x9a, 0x7d, 0x3a, 0xa4, 0x21, 0x19, 0x05, 0xa6, 0xb3, 0xb3, 0xb0, 0xad, 0x7c, 0xbd, 0xc4,
5310
+	0xa5, 0x2e, 0xac, 0x1f, 0x4a, 0xca, 0x43, 0xd3, 0xb3, 0x59, 0x78, 0x69, 0xa3, 0xcf, 0xa1, 0xe8,
5311
+	0x47, 0x64, 0x6a, 0x5e, 0xda, 0xdc, 0x1d, 0x9c, 0xf4, 0xdb, 0x67, 0x46, 0x83, 0x1d, 0x6b, 0x71,
5312
+	0xb3, 0x57, 0x54, 0x0e, 0xac, 0x69, 0xa8, 0x9e, 0x3e, 0x76, 0x6a, 0x26, 0x5d, 0x7c, 0x2d, 0x9c,
5313
+	0xf1, 0x28, 0x1d, 0xf9, 0xe1, 0x98, 0x53, 0x21, 0x74, 0x19, 0xb6, 0x70, 0x6a, 0x36, 0xff, 0x57,
5314
+	0x04, 0xfb, 0x28, 0x88, 0x85, 0x34, 0x8f, 0xcb, 0x3b, 0xcb, 0xe8, 0x4b, 0xb8, 0x47, 0xf4, 0x6f,
5315
+	0x01, 0x09, 0x55, 0xa5, 0xd6, 0xed, 0x85, 0xc9, 0xea, 0xe3, 0xdc, 0x70, 0x4b, 0x70, 0xd2, 0x8a,
5316
+	0x74, 0x4a, 0x2a, 0xa6, 0x53, 0xc0, 0x55, 0xf2, 0x8d, 0x2f, 0x68, 0x00, 0xbb, 0x8c, 0xbb, 0x13,
5317
+	0x2a, 0x64, 0x52, 0xdf, 0x4d, 0x1b, 0x9d, 0xfb, 0x83, 0xf5, 0x3c, 0x0b, 0x34, 0xc5, 0x2d, 0x59,
5318
+	0xed, 0x7a, 0x0c, 0xf4, 0x04, 0x8a, 0x9c, 0x5c, 0xa6, 0xad, 0x52, 0xae, 0xf2, 0x31, 0xb9, 0x94,
5319
+	0x6b, 0x21, 0x34, 0x03, 0xfd, 0x01, 0xc0, 0xf3, 0x45, 0x44, 0xa4, 0x3b, 0xa1, 0xdc, 0x9c, 0x60,
5320
+	0xee, 0x16, 0xbb, 0x4b, 0xd4, 0x5a, 0x94, 0x0c, 0x1b, 0x9d, 0x42, 0xd9, 0x25, 0xa9, 0x06, 0x4b,
5321
+	0xb7, 0xff, 0x5b, 0x1c, 0xb5, 0x4d, 0x88, 0xaa, 0x0a, 0xb1, 0xb8, 0xd9, 0xb3, 0x52, 0x0f, 0xb6,
5322
+	0x5c, 0x62, 0x34, 0x79, 0x0a, 0xbb, 0xea, 0x9f, 0x63, 0xe8, 0xd1, 0x4b, 0x12, 0x07, 0x32, 0x39,
5323
+	0xfb, 0x5b, 0x8a, 0xb5, 0x6a, 0x60, 0xbb, 0x06, 0x67, 0xd6, 0x55, 0x91, 0x19, 0x1f, 0xfa, 0x13,
5324
+	0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x81, 0xe9, 0x0a, 0xad, 0xdb, 0x37, 0xdb, 0x5b, 0x82, 0xd7,
5325
+	0x36, 0x5b, 0xa5, 0xdf, 0xf0, 0x37, 0x7d, 0x80, 0xe4, 0xf9, 0x7b, 0xb7, 0xfa, 0x43, 0x50, 0xf4,
5326
+	0x88, 0x24, 0x5a, 0x72, 0x15, 0xac, 0xc7, 0x1d, 0xe7, 0xf5, 0xd7, 0xf5, 0x8d, 0x7f, 0x7f, 0x5d,
5327
+	0xdf, 0xf8, 0xcb, 0xa2, 0x5e, 0x78, 0xbd, 0xa8, 0x17, 0xfe, 0xb5, 0xa8, 0x17, 0xfe, 0xb3, 0xa8,
5328
+	0x17, 0x46, 0x25, 0xdd, 0x34, 0xfc, 0xe2, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x64, 0x44, 0x1e,
5329
+	0x4f, 0xb4, 0x10, 0x00, 0x00,
5329 5330
 }
... ...
@@ -321,6 +321,12 @@ message NetworkSpec {
321 321
 	// enabled(default case) no manual attachment to this network
322 322
 	// can happen.
323 323
 	bool attachable = 6;
324
+	
325
+	// Ingress indicates this network will provide the routing-mesh.
326
+	// In older versions, the network providing the routing mesh was
327
+	// swarm internally created only and it was identified by the name
328
+	// "ingress" and the label "com.docker.swarm.internal": "true".
329
+	bool ingress = 7;
324 330
 }
325 331
 
326 332
 // ClusterSpec specifies global cluster settings.
... ...
@@ -113,12 +113,39 @@ type LocalSigner struct {
113 113
 	Key []byte
114 114
 }
115 115
 
116
-// RootCA is the representation of everything we need to sign certificates
116
+// RootCA is the representation of everything we need to sign certificates and/or to verify certificates
117
+//
118
+// RootCA.Cert:          [signing CA cert][CA cert1][CA cert2]
119
+// RootCA.Intermediates: [intermediate CA1][intermediate CA2][intermediate CA3]
120
+// RootCA.Signer.Key:    [signing CA key]
121
+//
122
+// Requirements:
123
+//
124
+// - [signing CA key] must be the private key for [signing CA cert]
125
+// - [signing CA cert] must be the first cert in RootCA.Cert
126
+//
127
+// - [intermediate CA1] must have the same public key and subject as [signing CA cert], because otherwise when
128
+//   appended to a leaf certificate, the intermediates will not form a chain (because [intermediate CA1] won't because
129
+//   the signer of the leaf certificate)
130
+// - [intermediate CA1] must be signed by [intermediate CA2], which must be signed by [intermediate CA3]
131
+//
132
+// - When we issue a certificate, the intermediates will be appended so that the certificate looks like:
133
+//   [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3]
134
+// - [leaf signed by signing CA cert][intermediate CA1][intermediate CA2][intermediate CA3] is guaranteed to form a
135
+//   valid chain from [leaf signed by signing CA cert] to one of the root certs ([signing CA cert], [CA cert1], [CA cert2])
136
+//   using zero or more of the intermediate certs ([intermediate CA1][intermediate CA2][intermediate CA3]) as intermediates
137
+//
117 138
 type RootCA struct {
118 139
 	// Cert contains a bundle of PEM encoded Certificate for the Root CA, the first one of which
119 140
 	// must correspond to the key in the local signer, if provided
120 141
 	Cert []byte
121 142
 
143
+	// Intermediates contains a bundle of PEM encoded intermediate CA certificates to append to any
144
+	// issued TLS (leaf) certificates. The first one must have the same public key and subject as the
145
+	// signing root certificate, and the rest must form a chain, each one certifying the one above it,
146
+	// as per RFC5246 section 7.4.2.
147
+	Intermediates []byte
148
+
122 149
 	// Pool is the root pool used to validate TLS certificates
123 150
 	Pool *x509.CertPool
124 151
 
... ...
@@ -306,7 +333,7 @@ func (rca *RootCA) ParseValidateAndSignCSR(csrBytes []byte, cn, ou, org string)
306 306
 		return nil, errors.Wrap(err, "failed to sign node certificate")
307 307
 	}
308 308
 
309
-	return cert, nil
309
+	return append(cert, rca.Intermediates...), nil
310 310
 }
311 311
 
312 312
 // CrossSignCACertificate takes a CA root certificate and generates an intermediate CA from it signed with the current root signer
... ...
@@ -348,7 +375,7 @@ func (rca *RootCA) CrossSignCACertificate(otherCAPEM []byte) ([]byte, error) {
348 348
 // NewRootCA creates a new RootCA object from unparsed PEM cert bundle and key byte
349 349
 // slices. key may be nil, and in this case NewRootCA will return a RootCA
350 350
 // without a signer.
351
-func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, error) {
351
+func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration, intermediates []byte) (RootCA, error) {
352 352
 	// Parse all the certificates in the cert bundle
353 353
 	parsedCerts, err := helpers.ParseCertificatesPEM(certBytes)
354 354
 	if err != nil {
... ...
@@ -368,7 +395,6 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
368 368
 		default:
369 369
 			return RootCA{}, fmt.Errorf("unsupported signature algorithm: %s", cert.SignatureAlgorithm.String())
370 370
 		}
371
-
372 371
 		// Check to see if all of the certificates are valid, self-signed root CA certs
373 372
 		selfpool := x509.NewCertPool()
374 373
 		selfpool.AddCert(cert)
... ...
@@ -381,9 +407,28 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
381 381
 	// Calculate the digest for our Root CA bundle
382 382
 	digest := digest.FromBytes(certBytes)
383 383
 
384
+	// We do not yet support arbitrary chains of intermediates (e.g. the case of an offline root, and the swarm CA is an
385
+	// intermediate CA). We currently only intermediates for which the first intermediate is cross-signed version of the
386
+	// CA signing cert (the first cert of the root certs) for the purposes of root rotation.  If we wanted to support
387
+	// offline roots, we'd have to separate the CA signing cert from the self-signed root certs, but this intermediate
388
+	// validation logic should remain the same.  Either the first intermediate would BE the intermediate CA we sign with
389
+	// (in which case it'd have the same subject and public key), or it would be a cross-signed intermediate with the
390
+	// same subject and public key as our signing cert (which could be either an intermediate cert or a self-signed root
391
+	// cert).
392
+	if len(intermediates) > 0 {
393
+		parsedIntermediates, err := ValidateCertChain(pool, intermediates, false)
394
+		if err != nil {
395
+			return RootCA{}, errors.Wrap(err, "invalid intermediate chain")
396
+		}
397
+		if !bytes.Equal(parsedIntermediates[0].RawSubject, parsedCerts[0].RawSubject) ||
398
+			!bytes.Equal(parsedIntermediates[0].RawSubjectPublicKeyInfo, parsedCerts[0].RawSubjectPublicKeyInfo) {
399
+			return RootCA{}, errors.New("invalid intermediate chain - the first intermediate must have the same subject and public key as the root")
400
+		}
401
+	}
402
+
384 403
 	if len(keyBytes) == 0 {
385
-		// This RootCA does not have a valid signer.
386
-		return RootCA{Cert: certBytes, Digest: digest, Pool: pool}, nil
404
+		// This RootCA does not have a valid signer
405
+		return RootCA{Cert: certBytes, Intermediates: intermediates, Digest: digest, Pool: pool}, nil
387 406
 	}
388 407
 
389 408
 	var (
... ...
@@ -434,7 +479,7 @@ func NewRootCA(certBytes, keyBytes []byte, certExpiry time.Duration) (RootCA, er
434 434
 		}
435 435
 	}
436 436
 
437
-	return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Digest: digest, Cert: certBytes, Pool: pool}, nil
437
+	return RootCA{Signer: &LocalSigner{Signer: signer, Key: keyBytes}, Intermediates: intermediates, Digest: digest, Cert: certBytes, Pool: pool}, nil
438 438
 }
439 439
 
440 440
 // ValidateCertChain checks checks that the certificates provided chain up to the root pool provided.  In addition
... ...
@@ -586,7 +631,7 @@ func GetLocalRootCA(paths CertPaths) (RootCA, error) {
586 586
 		key = nil
587 587
 	}
588 588
 
589
-	return NewRootCA(cert, key, DefaultNodeCertExpiration)
589
+	return NewRootCA(cert, key, DefaultNodeCertExpiration, nil)
590 590
 }
591 591
 
592 592
 func getGRPCConnection(creds credentials.TransportCredentials, connBroker *connectionbroker.Broker, forceRemote bool) (*connectionbroker.Conn, error) {
... ...
@@ -641,7 +686,7 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbro
641 641
 
642 642
 	// NewRootCA will validate that the certificates are otherwise valid and create a RootCA object.
643 643
 	// Since there is no key, the certificate expiry does not matter and will not be used.
644
-	return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration)
644
+	return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration, nil)
645 645
 }
646 646
 
647 647
 // CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially
... ...
@@ -660,7 +705,7 @@ func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) {
660 660
 		return RootCA{}, err
661 661
 	}
662 662
 
663
-	rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration)
663
+	rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration, nil)
664 664
 	if err != nil {
665 665
 		return RootCA{}, err
666 666
 	}
... ...
@@ -132,7 +132,7 @@ func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration
132 132
 	s.mu.Lock()
133 133
 	defer s.mu.Unlock()
134 134
 
135
-	rootCA, err := NewRootCA(cert, key, certExpiry)
135
+	rootCA, err := NewRootCA(cert, key, certExpiry, nil)
136 136
 	if err != nil {
137 137
 		return err
138 138
 	}
... ...
@@ -96,7 +96,7 @@ func (eca *ExternalCA) Sign(ctx context.Context, req signer.SignRequest) (cert [
96 96
 	for _, url := range urls {
97 97
 		cert, err = makeExternalSignRequest(ctx, client, url, csrJSON)
98 98
 		if err == nil {
99
-			return cert, err
99
+			return append(cert, eca.rootCA.Intermediates...), err
100 100
 		}
101 101
 		logrus.Debugf("unable to proxy certificate signing request to %s: %s", url, err)
102 102
 	}
... ...
@@ -6,7 +6,6 @@ import (
6 6
 
7 7
 	"github.com/docker/go-events"
8 8
 	"github.com/docker/swarmkit/api"
9
-	"github.com/docker/swarmkit/identity"
10 9
 	"github.com/docker/swarmkit/log"
11 10
 	"github.com/docker/swarmkit/manager/allocator/networkallocator"
12 11
 	"github.com/docker/swarmkit/manager/state"
... ...
@@ -18,42 +17,18 @@ import (
18 18
 
19 19
 const (
20 20
 	// Network allocator Voter ID for task allocation vote.
21
-	networkVoter = "network"
22
-
23
-	ingressNetworkName = "ingress"
24
-	ingressSubnet      = "10.255.0.0/16"
25
-
21
+	networkVoter           = "network"
26 22
 	allocatedStatusMessage = "pending task scheduling"
27 23
 )
28 24
 
29 25
 var (
26
+	// ErrNoIngress is returned when no ingress network is found in store
27
+	ErrNoIngress = errors.New("no ingress network found")
30 28
 	errNoChanges = errors.New("task unchanged")
31 29
 
32 30
 	retryInterval = 5 * time.Minute
33 31
 )
34 32
 
35
-func newIngressNetwork() *api.Network {
36
-	return &api.Network{
37
-		Spec: api.NetworkSpec{
38
-			Annotations: api.Annotations{
39
-				Name: ingressNetworkName,
40
-				Labels: map[string]string{
41
-					"com.docker.swarm.internal": "true",
42
-				},
43
-			},
44
-			DriverConfig: &api.Driver{},
45
-			IPAM: &api.IPAMOptions{
46
-				Driver: &api.Driver{},
47
-				Configs: []*api.IPAMConfig{
48
-					{
49
-						Subnet: ingressSubnet,
50
-					},
51
-				},
52
-			},
53
-		},
54
-	}
55
-}
56
-
57 33
 // Network context information which is used throughout the network allocation code.
58 34
 type networkContext struct {
59 35
 	ingressNetwork *api.Network
... ...
@@ -97,7 +72,6 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
97 97
 		unallocatedTasks:    make(map[string]*api.Task),
98 98
 		unallocatedServices: make(map[string]*api.Service),
99 99
 		unallocatedNetworks: make(map[string]*api.Network),
100
-		ingressNetwork:      newIngressNetwork(),
101 100
 		lastRetry:           time.Now(),
102 101
 	}
103 102
 	a.netCtx = nc
... ...
@@ -108,63 +82,38 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
108 108
 		}
109 109
 	}()
110 110
 
111
-	// Check if we have the ingress network. If not found create
112
-	// it before reading all network objects for allocation.
113
-	var networks []*api.Network
114
-	a.store.View(func(tx store.ReadTx) {
115
-		networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
116
-		if len(networks) > 0 {
117
-			nc.ingressNetwork = networks[0]
118
-		}
119
-	})
120
-	if err != nil {
121
-		return errors.Wrap(err, "failed to find ingress network during init")
122
-	}
123
-
124
-	// If ingress network is not found, create one right away
125
-	// using the predefined template.
126
-	if len(networks) == 0 {
127
-		if err := a.store.Update(func(tx store.Tx) error {
128
-			nc.ingressNetwork.ID = identity.NewID()
129
-			if err := store.CreateNetwork(tx, nc.ingressNetwork); err != nil {
130
-				return err
131
-			}
132
-
133
-			return nil
134
-		}); err != nil {
135
-			return errors.Wrap(err, "failed to create ingress network")
136
-		}
137
-
138
-		a.store.View(func(tx store.ReadTx) {
139
-			networks, err = store.FindNetworks(tx, store.ByName(ingressNetworkName))
140
-			if len(networks) > 0 {
141
-				nc.ingressNetwork = networks[0]
142
-			}
143
-		})
144
-		if err != nil {
145
-			return errors.Wrap(err, "failed to find ingress network after creating it")
146
-		}
147
-
148
-	}
149
-
150
-	// Try to complete ingress network allocation before anything else so
151
-	// that the we can get the preferred subnet for ingress
152
-	// network.
153
-	if !na.IsAllocated(nc.ingressNetwork) {
154
-		if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
155
-			log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
156
-		} else if _, err := a.store.Batch(func(batch *store.Batch) error {
157
-			if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
111
+	// Ingress network is now created at cluster's first time creation.
112
+	// Check if we have the ingress network. If found, make sure it is
113
+	// allocated, before reading all network objects for allocation.
114
+	// If not found, it means it was removed by user, nothing to do here.
115
+	ingressNetwork, err := GetIngressNetwork(a.store)
116
+	switch err {
117
+	case nil:
118
+		// Try to complete ingress network allocation before anything else so
119
+		// that the we can get the preferred subnet for ingress network.
120
+		nc.ingressNetwork = ingressNetwork
121
+		if !na.IsAllocated(nc.ingressNetwork) {
122
+			if err := a.allocateNetwork(ctx, nc.ingressNetwork); err != nil {
123
+				log.G(ctx).WithError(err).Error("failed allocating ingress network during init")
124
+			} else if _, err := a.store.Batch(func(batch *store.Batch) error {
125
+				if err := a.commitAllocatedNetwork(ctx, batch, nc.ingressNetwork); err != nil {
126
+					log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
127
+				}
128
+				return nil
129
+			}); err != nil {
158 130
 				log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
159 131
 			}
160
-			return nil
161
-		}); err != nil {
162
-			log.G(ctx).WithError(err).Error("failed committing allocation of ingress network during init")
163 132
 		}
133
+	case ErrNoIngress:
134
+		// Ingress network is not present in store, It means user removed it
135
+		// and did not create a new one.
136
+	default:
137
+		return errors.Wrap(err, "failure while looking for ingress network during init")
164 138
 	}
165 139
 
166 140
 	// Allocate networks in the store so far before we started
167 141
 	// watching.
142
+	var networks []*api.Network
168 143
 	a.store.View(func(tx store.ReadTx) {
169 144
 		networks, err = store.FindNetworks(tx, store.All)
170 145
 	})
... ...
@@ -196,43 +145,12 @@ func (a *Allocator) doNetworkInit(ctx context.Context) (err error) {
196 196
 		log.G(ctx).WithError(err).Error("failed committing allocation of networks during init")
197 197
 	}
198 198
 
199
-	// Allocate nodes in the store so far before we process watched events.
200
-	var nodes []*api.Node
201
-	a.store.View(func(tx store.ReadTx) {
202
-		nodes, err = store.FindNodes(tx, store.All)
203
-	})
204
-	if err != nil {
205
-		return errors.Wrap(err, "error listing all nodes in store while trying to allocate during init")
206
-	}
207
-
208
-	var allocatedNodes []*api.Node
209
-	for _, node := range nodes {
210
-		if na.IsNodeAllocated(node) {
211
-			continue
212
-		}
213
-
214
-		if node.Attachment == nil {
215
-			node.Attachment = &api.NetworkAttachment{}
216
-		}
217
-
218
-		node.Attachment.Network = nc.ingressNetwork.Copy()
219
-		if err := a.allocateNode(ctx, node); err != nil {
220
-			log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s during init", node.ID)
221
-			continue
222
-		}
223
-
224
-		allocatedNodes = append(allocatedNodes, node)
225
-	}
226
-
227
-	if _, err := a.store.Batch(func(batch *store.Batch) error {
228
-		for _, node := range allocatedNodes {
229
-			if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
230
-				log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s during init", node.ID)
231
-			}
199
+	// Allocate nodes in the store so far before we process watched events,
200
+	// if the ingress network is present.
201
+	if nc.ingressNetwork != nil {
202
+		if err := a.allocateNodes(ctx); err != nil {
203
+			return err
232 204
 		}
233
-		return nil
234
-	}); err != nil {
235
-		log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes during init")
236 205
 	}
237 206
 
238 207
 	// Allocate services in the store so far before we process watched events.
... ...
@@ -346,6 +264,12 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
346 346
 			break
347 347
 		}
348 348
 
349
+		if IsIngressNetwork(n) && nc.ingressNetwork != nil {
350
+			log.G(ctx).Errorf("Cannot allocate ingress network %s (%s) because another ingress network is already present: %s (%s)",
351
+				n.ID, n.Spec.Annotations.Name, nc.ingressNetwork.ID, nc.ingressNetwork.Spec.Annotations)
352
+			break
353
+		}
354
+
349 355
 		if err := a.allocateNetwork(ctx, n); err != nil {
350 356
 			log.G(ctx).WithError(err).Errorf("Failed allocation for network %s", n.ID)
351 357
 			break
... ...
@@ -356,9 +280,24 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
356 356
 		}); err != nil {
357 357
 			log.G(ctx).WithError(err).Errorf("Failed to commit allocation for network %s", n.ID)
358 358
 		}
359
+
360
+		if IsIngressNetwork(n) {
361
+			nc.ingressNetwork = n
362
+			err := a.allocateNodes(ctx)
363
+			if err != nil {
364
+				log.G(ctx).WithError(err).Error(err)
365
+			}
366
+		}
359 367
 	case state.EventDeleteNetwork:
360 368
 		n := v.Network.Copy()
361 369
 
370
+		if IsIngressNetwork(n) && nc.ingressNetwork.ID == n.ID {
371
+			nc.ingressNetwork = nil
372
+			if err := a.deallocateNodes(ctx); err != nil {
373
+				log.G(ctx).WithError(err).Error(err)
374
+			}
375
+		}
376
+
362 377
 		// The assumption here is that all dependent objects
363 378
 		// have been cleaned up when we are here so the only
364 379
 		// thing that needs to happen is free the network
... ...
@@ -467,7 +406,7 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
467 467
 		return
468 468
 	}
469 469
 
470
-	if !nc.nwkAllocator.IsNodeAllocated(node) {
470
+	if !nc.nwkAllocator.IsNodeAllocated(node) && nc.ingressNetwork != nil {
471 471
 		if node.Attachment == nil {
472 472
 			node.Attachment = &api.NetworkAttachment{}
473 473
 		}
... ...
@@ -486,6 +425,85 @@ func (a *Allocator) doNodeAlloc(ctx context.Context, ev events.Event) {
486 486
 	}
487 487
 }
488 488
 
489
+func (a *Allocator) allocateNodes(ctx context.Context) error {
490
+	// Allocate nodes in the store so far before we process watched events.
491
+	var (
492
+		allocatedNodes []*api.Node
493
+		nodes          []*api.Node
494
+		err            error
495
+		nc             = a.netCtx
496
+	)
497
+
498
+	a.store.View(func(tx store.ReadTx) {
499
+		nodes, err = store.FindNodes(tx, store.All)
500
+	})
501
+	if err != nil {
502
+		return errors.Wrap(err, "error listing all nodes in store while trying to allocate network resources")
503
+	}
504
+
505
+	for _, node := range nodes {
506
+		if nc.nwkAllocator.IsNodeAllocated(node) {
507
+			continue
508
+		}
509
+
510
+		if node.Attachment == nil {
511
+			node.Attachment = &api.NetworkAttachment{}
512
+		}
513
+
514
+		node.Attachment.Network = nc.ingressNetwork.Copy()
515
+		if err := a.allocateNode(ctx, node); err != nil {
516
+			log.G(ctx).WithError(err).Errorf("Failed to allocate network resources for node %s", node.ID)
517
+			continue
518
+		}
519
+
520
+		allocatedNodes = append(allocatedNodes, node)
521
+	}
522
+
523
+	if _, err := a.store.Batch(func(batch *store.Batch) error {
524
+		for _, node := range allocatedNodes {
525
+			if err := a.commitAllocatedNode(ctx, batch, node); err != nil {
526
+				log.G(ctx).WithError(err).Errorf("Failed to commit allocation of network resources for node %s", node.ID)
527
+			}
528
+		}
529
+		return nil
530
+	}); err != nil {
531
+		log.G(ctx).WithError(err).Error("Failed to commit allocation of network resources for nodes")
532
+	}
533
+
534
+	return nil
535
+}
536
+
537
+func (a *Allocator) deallocateNodes(ctx context.Context) error {
538
+	var (
539
+		nodes []*api.Node
540
+		nc    = a.netCtx
541
+		err   error
542
+	)
543
+
544
+	a.store.View(func(tx store.ReadTx) {
545
+		nodes, err = store.FindNodes(tx, store.All)
546
+	})
547
+	if err != nil {
548
+		return fmt.Errorf("error listing all nodes in store while trying to free network resources")
549
+	}
550
+
551
+	for _, node := range nodes {
552
+		if nc.nwkAllocator.IsNodeAllocated(node) {
553
+			if err := nc.nwkAllocator.DeallocateNode(node); err != nil {
554
+				log.G(ctx).WithError(err).Errorf("Failed freeing network resources for node %s", node.ID)
555
+			}
556
+			node.Attachment = nil
557
+			if _, err := a.store.Batch(func(batch *store.Batch) error {
558
+				return a.commitAllocatedNode(ctx, batch, node)
559
+			}); err != nil {
560
+				log.G(ctx).WithError(err).Errorf("Failed to commit deallocation of network resources for node %s", node.ID)
561
+			}
562
+		}
563
+	}
564
+
565
+	return nil
566
+}
567
+
489 568
 // taskReadyForNetworkVote checks if the task is ready for a network
490 569
 // vote to move it to PENDING state.
491 570
 func taskReadyForNetworkVote(t *api.Task, s *api.Service, nc *networkContext) bool {
... ...
@@ -711,6 +729,9 @@ func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
711 711
 		// world. Automatically attach the service to the ingress
712 712
 		// network only if it is not already done.
713 713
 		if isIngressNetworkNeeded(s) {
714
+			if nc.ingressNetwork == nil {
715
+				return fmt.Errorf("ingress network is missing")
716
+			}
714 717
 			var found bool
715 718
 			for _, vip := range s.Endpoint.VirtualIPs {
716 719
 				if vip.NetworkID == nc.ingressNetwork.ID {
... ...
@@ -1022,3 +1043,36 @@ func updateTaskStatus(t *api.Task, newStatus api.TaskState, message string) {
1022 1022
 	t.Status.Message = message
1023 1023
 	t.Status.Timestamp = ptypes.MustTimestampProto(time.Now())
1024 1024
 }
1025
+
1026
+// IsIngressNetwork returns whether the passed network is an ingress network.
1027
+func IsIngressNetwork(nw *api.Network) bool {
1028
+	if nw.Spec.Ingress {
1029
+		return true
1030
+	}
1031
+	// Check if legacy defined ingress network
1032
+	_, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]
1033
+	return ok && nw.Spec.Annotations.Name == "ingress"
1034
+}
1035
+
1036
+// GetIngressNetwork fetches the ingress network from store.
1037
+// ErrNoIngress will be returned if the ingress network is not present,
1038
+// nil otherwise. In case of any other failure in accessing the store,
1039
+// the respective error will be reported as is.
1040
+func GetIngressNetwork(s *store.MemoryStore) (*api.Network, error) {
1041
+	var (
1042
+		networks []*api.Network
1043
+		err      error
1044
+	)
1045
+	s.View(func(tx store.ReadTx) {
1046
+		networks, err = store.FindNetworks(tx, store.All)
1047
+	})
1048
+	if err != nil {
1049
+		return nil, err
1050
+	}
1051
+	for _, n := range networks {
1052
+		if IsIngressNetwork(n) {
1053
+			return n, nil
1054
+		}
1055
+	}
1056
+	return nil, ErrNoIngress
1057
+}
... ...
@@ -1,7 +1,6 @@
1 1
 package controlapi
2 2
 
3 3
 import (
4
-	"fmt"
5 4
 	"net"
6 5
 
7 6
 	"github.com/docker/docker/pkg/plugingetter"
... ...
@@ -9,6 +8,7 @@ import (
9 9
 	"github.com/docker/libnetwork/ipamapi"
10 10
 	"github.com/docker/swarmkit/api"
11 11
 	"github.com/docker/swarmkit/identity"
12
+	"github.com/docker/swarmkit/manager/allocator"
12 13
 	"github.com/docker/swarmkit/manager/state/store"
13 14
 	"golang.org/x/net/context"
14 15
 	"google.golang.org/grpc"
... ...
@@ -75,6 +75,14 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er
75 75
 		return grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
76 76
 	}
77 77
 
78
+	if spec.Ingress && spec.DriverConfig != nil && spec.DriverConfig.Name != "overlay" {
79
+		return grpc.Errorf(codes.Unimplemented, "only overlay driver is currently supported for ingress network")
80
+	}
81
+
82
+	if spec.Attachable && spec.Ingress {
83
+		return grpc.Errorf(codes.InvalidArgument, "ingress network cannot be attachable")
84
+	}
85
+
78 86
 	if err := validateAnnotations(spec.Annotations); err != nil {
79 87
 		return err
80 88
 	}
... ...
@@ -94,16 +102,10 @@ func validateNetworkSpec(spec *api.NetworkSpec, pg plugingetter.PluginGetter) er
94 94
 // - Returns `InvalidArgument` if the NetworkSpec is malformed.
95 95
 // - Returns an error if the creation fails.
96 96
 func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRequest) (*api.CreateNetworkResponse, error) {
97
-	// if you change this function, you have to change createInternalNetwork in
98
-	// the tests to match it (except the part where we check the label).
99 97
 	if err := validateNetworkSpec(request.Spec, s.pg); err != nil {
100 98
 		return nil, err
101 99
 	}
102 100
 
103
-	if _, ok := request.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
104
-		return nil, grpc.Errorf(codes.PermissionDenied, "label com.docker.swarm.internal is for predefined internal networks and cannot be applied by users")
105
-	}
106
-
107 101
 	// TODO(mrjana): Consider using `Name` as a primary key to handle
108 102
 	// duplicate creations. See #65
109 103
 	n := &api.Network{
... ...
@@ -112,6 +114,13 @@ func (s *Server) CreateNetwork(ctx context.Context, request *api.CreateNetworkRe
112 112
 	}
113 113
 
114 114
 	err := s.store.Update(func(tx store.Tx) error {
115
+		if request.Spec.Ingress {
116
+			if n, err := allocator.GetIngressNetwork(s.store); err == nil {
117
+				return grpc.Errorf(codes.AlreadyExists, "ingress network (%s) is already present", n.ID)
118
+			} else if err != allocator.ErrNoIngress {
119
+				return grpc.Errorf(codes.Internal, "failed ingress network presence check: %v", err)
120
+			}
121
+		}
115 122
 		return store.CreateNetwork(tx, n)
116 123
 	})
117 124
 	if err != nil {
... ...
@@ -152,44 +161,70 @@ func (s *Server) RemoveNetwork(ctx context.Context, request *api.RemoveNetworkRe
152 152
 		return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
153 153
 	}
154 154
 
155
-	err := s.store.Update(func(tx store.Tx) error {
156
-		services, err := store.FindServices(tx, store.ByReferencedNetworkID(request.NetworkID))
155
+	var (
156
+		n  *api.Network
157
+		rm = s.removeNetwork
158
+	)
159
+
160
+	s.store.View(func(tx store.ReadTx) {
161
+		n = store.GetNetwork(tx, request.NetworkID)
162
+	})
163
+	if n == nil {
164
+		return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
165
+	}
166
+
167
+	if allocator.IsIngressNetwork(n) {
168
+		rm = s.removeIngressNetwork
169
+	}
170
+
171
+	if err := rm(n.ID); err != nil {
172
+		if err == store.ErrNotExist {
173
+			return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
174
+		}
175
+		return nil, err
176
+	}
177
+	return &api.RemoveNetworkResponse{}, nil
178
+}
179
+
180
+func (s *Server) removeNetwork(id string) error {
181
+	return s.store.Update(func(tx store.Tx) error {
182
+		services, err := store.FindServices(tx, store.ByReferencedNetworkID(id))
157 183
 		if err != nil {
158
-			return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", request.NetworkID, err)
184
+			return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
159 185
 		}
160 186
 
161 187
 		if len(services) != 0 {
162
-			return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", request.NetworkID, services[0].ID)
188
+			return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by service %s", id, services[0].ID)
163 189
 		}
164 190
 
165
-		tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(request.NetworkID))
191
+		tasks, err := store.FindTasks(tx, store.ByReferencedNetworkID(id))
166 192
 		if err != nil {
167
-			return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", request.NetworkID, err)
193
+			return grpc.Errorf(codes.Internal, "could not find tasks using network %s: %v", id, err)
168 194
 		}
169 195
 
170 196
 		for _, t := range tasks {
171 197
 			if t.DesiredState <= api.TaskStateRunning && t.Status.State <= api.TaskStateRunning {
172
-				return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", request.NetworkID, t.ID)
198
+				return grpc.Errorf(codes.FailedPrecondition, "network %s is in use by task %s", id, t.ID)
173 199
 			}
174 200
 		}
175 201
 
176
-		nw := store.GetNetwork(tx, request.NetworkID)
177
-		if _, ok := nw.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
178
-			networkDescription := nw.ID
179
-			if nw.Spec.Annotations.Name != "" {
180
-				networkDescription = fmt.Sprintf("%s (%s)", nw.Spec.Annotations.Name, nw.ID)
202
+		return store.DeleteNetwork(tx, id)
203
+	})
204
+}
205
+
206
+func (s *Server) removeIngressNetwork(id string) error {
207
+	return s.store.Update(func(tx store.Tx) error {
208
+		services, err := store.FindServices(tx, store.All)
209
+		if err != nil {
210
+			return grpc.Errorf(codes.Internal, "could not find services using network %s: %v", id, err)
211
+		}
212
+		for _, srv := range services {
213
+			if doesServiceNeedIngress(srv) {
214
+				return grpc.Errorf(codes.FailedPrecondition, "ingress network cannot be removed because service %s depends on it", srv.ID)
181 215
 			}
182
-			return grpc.Errorf(codes.PermissionDenied, "%s is a pre-defined network and cannot be removed", networkDescription)
183 216
 		}
184
-		return store.DeleteNetwork(tx, request.NetworkID)
217
+		return store.DeleteNetwork(tx, id)
185 218
 	})
186
-	if err != nil {
187
-		if err == store.ErrNotExist {
188
-			return nil, grpc.Errorf(codes.NotFound, "network %s not found", request.NetworkID)
189
-		}
190
-		return nil, err
191
-	}
192
-	return &api.RemoveNetworkResponse{}, nil
193 219
 }
194 220
 
195 221
 func filterNetworks(candidates []*api.Network, filters ...func(*api.Network) bool) []*api.Network {
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	"github.com/docker/distribution/reference"
12 12
 	"github.com/docker/swarmkit/api"
13 13
 	"github.com/docker/swarmkit/identity"
14
+	"github.com/docker/swarmkit/manager/allocator"
14 15
 	"github.com/docker/swarmkit/manager/constraint"
15 16
 	"github.com/docker/swarmkit/manager/state/store"
16 17
 	"github.com/docker/swarmkit/protobuf/ptypes"
... ...
@@ -288,7 +289,7 @@ func (s *Server) validateNetworks(networks []*api.NetworkAttachmentConfig) error
288 288
 		if network == nil {
289 289
 			continue
290 290
 		}
291
-		if _, ok := network.Spec.Annotations.Labels["com.docker.swarm.internal"]; ok {
291
+		if network.Spec.Internal {
292 292
 			return grpc.Errorf(codes.InvalidArgument,
293 293
 				"Service cannot be explicitly attached to %q network which is a swarm internal network",
294 294
 				network.Spec.Annotations.Name)
... ...
@@ -424,6 +425,36 @@ func (s *Server) checkSecretExistence(tx store.Tx, spec *api.ServiceSpec) error
424 424
 	return nil
425 425
 }
426 426
 
427
+func doesServiceNeedIngress(srv *api.Service) bool {
428
+	// Only VIP mode with target ports needs routing mesh.
429
+	// If no endpoint is specified, it defaults to VIP mode but no target ports
430
+	// are specified, so the service does not need the routing mesh.
431
+	if srv.Spec.Endpoint == nil || srv.Spec.Endpoint.Mode != api.ResolutionModeVirtualIP {
432
+		return false
433
+	}
434
+	// Go through the ports' config
435
+	for _, p := range srv.Spec.Endpoint.Ports {
436
+		if p.PublishMode != api.PublishModeIngress {
437
+			continue
438
+		}
439
+		if p.PublishedPort != 0 {
440
+			return true
441
+		}
442
+	}
443
+	// Go through the ports' state
444
+	if srv.Endpoint != nil {
445
+		for _, p := range srv.Endpoint.Ports {
446
+			if p.PublishMode != api.PublishModeIngress {
447
+				continue
448
+			}
449
+			if p.PublishedPort != 0 {
450
+				return true
451
+			}
452
+		}
453
+	}
454
+	return false
455
+}
456
+
427 457
 // CreateService creates and returns a Service based on the provided ServiceSpec.
428 458
 // - Returns `InvalidArgument` if the ServiceSpec is malformed.
429 459
 // - Returns `Unimplemented` if the ServiceSpec references unimplemented features.
... ...
@@ -449,6 +480,12 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe
449 449
 		Spec: *request.Spec,
450 450
 	}
451 451
 
452
+	if doesServiceNeedIngress(service) {
453
+		if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
454
+			return nil, grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
455
+		}
456
+	}
457
+
452 458
 	err := s.store.Update(func(tx store.Tx) error {
453 459
 		// Check to see if all the secrets being added exist as objects
454 460
 		// in our datastore
... ...
@@ -578,6 +615,12 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe
578 578
 			service.UpdateStatus = nil
579 579
 		}
580 580
 
581
+		if doesServiceNeedIngress(service) {
582
+			if _, err := allocator.GetIngressNetwork(s.store); err == allocator.ErrNoIngress {
583
+				return grpc.Errorf(codes.FailedPrecondition, "service needs ingress network, but no ingress network is present")
584
+			}
585
+		}
586
+
581 587
 		return store.UpdateService(tx, service)
582 588
 	})
583 589
 	if err != nil {
... ...
@@ -47,7 +47,7 @@ func (m *Manager) IsStateDirty() (bool, error) {
47 47
 		if structField.Type.Kind() != reflect.Slice {
48 48
 			panic("unexpected field type in StoreSnapshot")
49 49
 		}
50
-		if structField.Name != "Nodes" && structField.Name != "Clusters" && field.Len() != 0 {
50
+		if structField.Name != "Nodes" && structField.Name != "Clusters" && structField.Name != "Networks" && field.Len() != 0 {
51 51
 			// One of the other data types has an entry
52 52
 			return true, nil
53 53
 		}
... ...
@@ -19,6 +19,7 @@ import (
19 19
 	"github.com/docker/swarmkit/api"
20 20
 	"github.com/docker/swarmkit/ca"
21 21
 	"github.com/docker/swarmkit/connectionbroker"
22
+	"github.com/docker/swarmkit/identity"
22 23
 	"github.com/docker/swarmkit/log"
23 24
 	"github.com/docker/swarmkit/manager/allocator"
24 25
 	"github.com/docker/swarmkit/manager/controlapi"
... ...
@@ -892,7 +893,18 @@ func (m *Manager) becomeLeader(ctx context.Context) {
892 892
 			rootCA))
893 893
 		// Add Node entry for ourself, if one
894 894
 		// doesn't exist already.
895
-		store.CreateNode(tx, managerNode(nodeID, m.config.Availability))
895
+		freshCluster := nil == store.CreateNode(tx, managerNode(nodeID, m.config.Availability))
896
+
897
+		if freshCluster {
898
+			// This is a fresh swarm cluster. Add to store now any initial
899
+			// cluster resource, like the default ingress network which
900
+			// provides the routing mesh for this cluster.
901
+			log.G(ctx).Info("Creating default ingress network")
902
+			if err := store.CreateNetwork(tx, newIngressNetwork()); err != nil {
903
+				log.G(ctx).WithError(err).Error("failed to create default ingress network")
904
+			}
905
+		}
906
+
896 907
 		return nil
897 908
 	})
898 909
 
... ...
@@ -1084,3 +1096,28 @@ func managerNode(nodeID string, availability api.NodeSpec_Availability) *api.Nod
1084 1084
 		},
1085 1085
 	}
1086 1086
 }
1087
+
1088
+// newIngressNetwork returns the network object for the default ingress
1089
+// network, the network which provides the routing mesh. Caller will save to
1090
+// store this object once, at fresh cluster creation. It is expected to
1091
+// call this function inside a store update transaction.
1092
+func newIngressNetwork() *api.Network {
1093
+	return &api.Network{
1094
+		ID: identity.NewID(),
1095
+		Spec: api.NetworkSpec{
1096
+			Ingress: true,
1097
+			Annotations: api.Annotations{
1098
+				Name: "ingress",
1099
+			},
1100
+			DriverConfig: &api.Driver{},
1101
+			IPAM: &api.IPAMOptions{
1102
+				Driver: &api.Driver{},
1103
+				Configs: []*api.IPAMConfig{
1104
+					{
1105
+						Subnet: "10.255.0.0/16",
1106
+					},
1107
+				},
1108
+			},
1109
+		},
1110
+	}
1111
+}
... ...
@@ -111,6 +111,10 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference
111 111
 			tree = next
112 112
 		}
113 113
 
114
+		if node.ActiveTasksCountByService != nil {
115
+			tree.tasks += node.ActiveTasksCountByService[serviceID]
116
+		}
117
+
114 118
 		if tree.nodeHeap.lessFunc == nil {
115 119
 			tree.nodeHeap.lessFunc = nodeLess
116 120
 		}
... ...
@@ -1,7 +1,6 @@
1 1
 package scheduler
2 2
 
3 3
 import (
4
-	"container/list"
5 4
 	"time"
6 5
 
7 6
 	"github.com/docker/swarmkit/api"
... ...
@@ -30,7 +29,7 @@ type schedulingDecision struct {
30 30
 // Scheduler assigns tasks to nodes.
31 31
 type Scheduler struct {
32 32
 	store           *store.MemoryStore
33
-	unassignedTasks *list.List
33
+	unassignedTasks map[string]*api.Task
34 34
 	// preassignedTasks already have NodeID, need resource validation
35 35
 	preassignedTasks map[string]*api.Task
36 36
 	nodeSet          nodeSet
... ...
@@ -47,7 +46,7 @@ type Scheduler struct {
47 47
 func New(store *store.MemoryStore) *Scheduler {
48 48
 	return &Scheduler{
49 49
 		store:            store,
50
-		unassignedTasks:  list.New(),
50
+		unassignedTasks:  make(map[string]*api.Task),
51 51
 		preassignedTasks: make(map[string]*api.Task),
52 52
 		allTasks:         make(map[string]*api.Task),
53 53
 		stopChan:         make(chan struct{}),
... ...
@@ -191,7 +190,7 @@ func (s *Scheduler) Stop() {
191 191
 
192 192
 // enqueue queues a task for scheduling.
193 193
 func (s *Scheduler) enqueue(t *api.Task) {
194
-	s.unassignedTasks.PushBack(t)
194
+	s.unassignedTasks[t.ID] = t
195 195
 }
196 196
 
197 197
 func (s *Scheduler) createTask(ctx context.Context, t *api.Task) int {
... ...
@@ -333,15 +332,12 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) {
333 333
 // tick attempts to schedule the queue.
334 334
 func (s *Scheduler) tick(ctx context.Context) {
335 335
 	tasksByCommonSpec := make(map[string]map[string]*api.Task)
336
-	schedulingDecisions := make(map[string]schedulingDecision, s.unassignedTasks.Len())
336
+	schedulingDecisions := make(map[string]schedulingDecision, len(s.unassignedTasks))
337 337
 
338
-	var next *list.Element
339
-	for e := s.unassignedTasks.Front(); e != nil; e = next {
340
-		next = e.Next()
341
-		t := s.allTasks[e.Value.(*api.Task).ID]
338
+	for taskID, t := range s.unassignedTasks {
342 339
 		if t == nil || t.NodeID != "" {
343 340
 			// task deleted or already assigned
344
-			s.unassignedTasks.Remove(e)
341
+			delete(s.unassignedTasks, taskID)
345 342
 			continue
346 343
 		}
347 344
 
... ...
@@ -362,8 +358,8 @@ func (s *Scheduler) tick(ctx context.Context) {
362 362
 		if tasksByCommonSpec[taskGroupKey] == nil {
363 363
 			tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task)
364 364
 		}
365
-		tasksByCommonSpec[taskGroupKey][t.ID] = t
366
-		s.unassignedTasks.Remove(e)
365
+		tasksByCommonSpec[taskGroupKey][taskID] = t
366
+		delete(s.unassignedTasks, taskID)
367 367
 	}
368 368
 
369 369
 	for _, taskGroup := range tasksByCommonSpec {
... ...
@@ -602,6 +598,12 @@ func (s *Scheduler) scheduleNTasksOnNodes(ctx context.Context, n int, taskGroup
602 602
 	nodeIter := 0
603 603
 	nodeCount := len(nodes)
604 604
 	for taskID, t := range taskGroup {
605
+		// Skip tasks which were already scheduled because they ended
606
+		// up in two groups at once.
607
+		if _, exists := schedulingDecisions[taskID]; exists {
608
+			continue
609
+		}
610
+
605 611
 		node := &nodes[nodeIter%nodeCount]
606 612
 
607 613
 		log.G(ctx).WithField("task.id", t.ID).Debugf("assigning to node %s", node.ID)