Browse code

Vendor swarmkit 46bbd41

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

Aaron Lehmann authored on 2017/02/28 04:51:00
Showing 28 changed files
... ...
@@ -183,7 +183,15 @@ func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) {
183 183
 	status, out, err := d1.SockRequest("POST", url, node.Spec)
184 184
 	c.Assert(err, checker.IsNil)
185 185
 	c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out)))
186
-	c.Assert(string(out), checker.Contains, "last manager of the swarm")
186
+	// The warning specific to demoting the last manager is best-effort and
187
+	// won't appear until the Role field of the demoted manager has been
188
+	// updated.
189
+	// Yes, I know this looks silly, but checker.Matches is broken, since
190
+	// it anchors the regexp contrary to the documentation, and this makes
191
+	// it impossible to match something that includes a line break.
192
+	if !strings.Contains(string(out), "last manager of the swarm") {
193
+		c.Assert(string(out), checker.Contains, "this would result in a loss of quorum")
194
+	}
187 195
 	info, err = d1.SwarmInfo()
188 196
 	c.Assert(err, checker.IsNil)
189 197
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
... ...
@@ -104,7 +104,7 @@ github.com/docker/containerd 665e84e6c28653a9c29a6db601636a92d46896f3
104 104
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
105 105
 
106 106
 # cluster
107
-github.com/docker/swarmkit 6bc357e9c5f0ac2cdf801898a43d08c260b4d5d0
107
+github.com/docker/swarmkit 46bbd41a00b996a13840607772f661a7f5096ca0
108 108
 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
109 109
 github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2
110 110
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a
... ...
@@ -125,6 +125,7 @@ github.com/prometheus/procfs abf152e5f3e97f2fafac028d2cc06c1feb87ffa5
125 125
 bitbucket.org/ww/goautoneg 75cd24fc2f2c2a2088577d12123ddee5f54e0675
126 126
 github.com/matttproud/golang_protobuf_extensions v1.0.0
127 127
 github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9
128
+github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0
128 129
 
129 130
 # cli
130 131
 github.com/spf13/cobra v1.5.1 https://github.com/dnephin/cobra.git
... ...
@@ -2,14 +2,13 @@ package agent
2 2
 
3 3
 import (
4 4
 	"errors"
5
-	"fmt"
6 5
 )
7 6
 
8 7
 var (
9 8
 	// ErrClosed is returned when an operation fails because the resource is closed.
10 9
 	ErrClosed = errors.New("agent: closed")
11 10
 
12
-	errNodeNotRegistered = fmt.Errorf("node not registered")
11
+	errNodeNotRegistered = errors.New("node not registered")
13 12
 
14 13
 	errAgentStarted    = errors.New("agent: already started")
15 14
 	errAgentNotStarted = errors.New("agent: not started")
... ...
@@ -75,7 +75,7 @@ func (NodeSpec_Availability) EnumDescriptor() ([]byte, []int) { return fileDescr
75 75
 
76 76
 // ResolutionMode specifies the mode of resolution to use for
77 77
 // internal loadbalancing between tasks which are all within
78
-// the cluster. This is sometimes calles east-west data path.
78
+// the cluster. This is sometimes calls east-west data path.
79 79
 type EndpointSpec_ResolutionMode int32
80 80
 
81 81
 const (
... ...
@@ -106,7 +106,7 @@ func (x EndpointSpec_ResolutionMode) String() string {
106 106
 	return proto.EnumName(EndpointSpec_ResolutionMode_name, int32(x))
107 107
 }
108 108
 func (EndpointSpec_ResolutionMode) EnumDescriptor() ([]byte, []int) {
109
-	return fileDescriptorSpecs, []int{7, 0}
109
+	return fileDescriptorSpecs, []int{8, 0}
110 110
 }
111 111
 
112 112
 type NodeSpec struct {
... ...
@@ -289,6 +289,7 @@ type TaskSpec struct {
289 289
 	// Types that are valid to be assigned to Runtime:
290 290
 	//	*TaskSpec_Attachment
291 291
 	//	*TaskSpec_Container
292
+	//	*TaskSpec_Plugin
292 293
 	Runtime isTaskSpec_Runtime `protobuf_oneof:"runtime"`
293 294
 	// Resource requirements for the container.
294 295
 	Resources *ResourceRequirements `protobuf:"bytes,2,opt,name=resources" json:"resources,omitempty"`
... ...
@@ -326,9 +327,13 @@ type TaskSpec_Attachment struct {
326 326
 type TaskSpec_Container struct {
327 327
 	Container *ContainerSpec `protobuf:"bytes,1,opt,name=container,oneof"`
328 328
 }
329
+type TaskSpec_Plugin struct {
330
+	Plugin *PluginSpec `protobuf:"bytes,10,opt,name=plugin,oneof"`
331
+}
329 332
 
330 333
 func (*TaskSpec_Attachment) isTaskSpec_Runtime() {}
331 334
 func (*TaskSpec_Container) isTaskSpec_Runtime()  {}
335
+func (*TaskSpec_Plugin) isTaskSpec_Runtime()     {}
332 336
 
333 337
 func (m *TaskSpec) GetRuntime() isTaskSpec_Runtime {
334 338
 	if m != nil {
... ...
@@ -351,11 +356,19 @@ func (m *TaskSpec) GetContainer() *ContainerSpec {
351 351
 	return nil
352 352
 }
353 353
 
354
+func (m *TaskSpec) GetPlugin() *PluginSpec {
355
+	if x, ok := m.GetRuntime().(*TaskSpec_Plugin); ok {
356
+		return x.Plugin
357
+	}
358
+	return nil
359
+}
360
+
354 361
 // XXX_OneofFuncs is for the internal use of the proto package.
355 362
 func (*TaskSpec) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
356 363
 	return _TaskSpec_OneofMarshaler, _TaskSpec_OneofUnmarshaler, _TaskSpec_OneofSizer, []interface{}{
357 364
 		(*TaskSpec_Attachment)(nil),
358 365
 		(*TaskSpec_Container)(nil),
366
+		(*TaskSpec_Plugin)(nil),
359 367
 	}
360 368
 }
361 369
 
... ...
@@ -373,6 +386,11 @@ func _TaskSpec_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
373 373
 		if err := b.EncodeMessage(x.Container); err != nil {
374 374
 			return err
375 375
 		}
376
+	case *TaskSpec_Plugin:
377
+		_ = b.EncodeVarint(10<<3 | proto.WireBytes)
378
+		if err := b.EncodeMessage(x.Plugin); err != nil {
379
+			return err
380
+		}
376 381
 	case nil:
377 382
 	default:
378 383
 		return fmt.Errorf("TaskSpec.Runtime has unexpected type %T", x)
... ...
@@ -399,6 +417,14 @@ func _TaskSpec_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffe
399 399
 		err := b.DecodeMessage(msg)
400 400
 		m.Runtime = &TaskSpec_Container{msg}
401 401
 		return true, err
402
+	case 10: // runtime.plugin
403
+		if wire != proto.WireBytes {
404
+			return true, proto.ErrInternalBadWireType
405
+		}
406
+		msg := new(PluginSpec)
407
+		err := b.DecodeMessage(msg)
408
+		m.Runtime = &TaskSpec_Plugin{msg}
409
+		return true, err
402 410
 	default:
403 411
 		return false, nil
404 412
 	}
... ...
@@ -418,6 +444,11 @@ func _TaskSpec_OneofSizer(msg proto.Message) (n int) {
418 418
 		n += proto.SizeVarint(1<<3 | proto.WireBytes)
419 419
 		n += proto.SizeVarint(uint64(s))
420 420
 		n += s
421
+	case *TaskSpec_Plugin:
422
+		s := proto.Size(x.Plugin)
423
+		n += proto.SizeVarint(10<<3 | proto.WireBytes)
424
+		n += proto.SizeVarint(uint64(s))
425
+		n += s
421 426
 	case nil:
422 427
 	default:
423 428
 		panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
... ...
@@ -428,7 +459,7 @@ func _TaskSpec_OneofSizer(msg proto.Message) (n int) {
428 428
 // NetworkAttachmentSpec specifies runtime parameters required to attach
429 429
 // a container to a network.
430 430
 type NetworkAttachmentSpec struct {
431
-	// ContainerID spcifies a unique ID of the container for which
431
+	// ContainerID specifies a unique ID of the container for which
432 432
 	// this attachment is for.
433 433
 	ContainerID string `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"`
434 434
 }
... ...
@@ -559,6 +590,18 @@ func (m *ContainerSpec_DNSConfig) Reset()                    { *m = ContainerSpe
559 559
 func (*ContainerSpec_DNSConfig) ProtoMessage()               {}
560 560
 func (*ContainerSpec_DNSConfig) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{6, 2} }
561 561
 
562
+// PluginSpec specifies runtime parameters for a plugin.
563
+type PluginSpec struct {
564
+	// image defines the image reference, as specified in the
565
+	// distribution/reference package. This may include a registry host, name,
566
+	// tag or digest.
567
+	Image string `protobuf:"bytes,1,opt,name=image,proto3" json:"image,omitempty"`
568
+}
569
+
570
+func (m *PluginSpec) Reset()                    { *m = PluginSpec{} }
571
+func (*PluginSpec) ProtoMessage()               {}
572
+func (*PluginSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{7} }
573
+
562 574
 // EndpointSpec defines the properties that can be configured to
563 575
 // access and loadbalance the service.
564 576
 type EndpointSpec struct {
... ...
@@ -570,7 +613,7 @@ type EndpointSpec struct {
570 570
 
571 571
 func (m *EndpointSpec) Reset()                    { *m = EndpointSpec{} }
572 572
 func (*EndpointSpec) ProtoMessage()               {}
573
-func (*EndpointSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{7} }
573
+func (*EndpointSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{8} }
574 574
 
575 575
 // NetworkSpec specifies user defined network parameters.
576 576
 type NetworkSpec struct {
... ...
@@ -595,7 +638,7 @@ type NetworkSpec struct {
595 595
 
596 596
 func (m *NetworkSpec) Reset()                    { *m = NetworkSpec{} }
597 597
 func (*NetworkSpec) ProtoMessage()               {}
598
-func (*NetworkSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{8} }
598
+func (*NetworkSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{9} }
599 599
 
600 600
 // ClusterSpec specifies global cluster settings.
601 601
 type ClusterSpec struct {
... ...
@@ -620,7 +663,7 @@ type ClusterSpec struct {
620 620
 
621 621
 func (m *ClusterSpec) Reset()                    { *m = ClusterSpec{} }
622 622
 func (*ClusterSpec) ProtoMessage()               {}
623
-func (*ClusterSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{9} }
623
+func (*ClusterSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{10} }
624 624
 
625 625
 // SecretSpec specifies a user-provided secret.
626 626
 type SecretSpec struct {
... ...
@@ -631,7 +674,7 @@ type SecretSpec struct {
631 631
 
632 632
 func (m *SecretSpec) Reset()                    { *m = SecretSpec{} }
633 633
 func (*SecretSpec) ProtoMessage()               {}
634
-func (*SecretSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{10} }
634
+func (*SecretSpec) Descriptor() ([]byte, []int) { return fileDescriptorSpecs, []int{11} }
635 635
 
636 636
 func init() {
637 637
 	proto.RegisterType((*NodeSpec)(nil), "docker.swarmkit.v1.NodeSpec")
... ...
@@ -643,6 +686,7 @@ func init() {
643 643
 	proto.RegisterType((*ContainerSpec)(nil), "docker.swarmkit.v1.ContainerSpec")
644 644
 	proto.RegisterType((*ContainerSpec_PullOptions)(nil), "docker.swarmkit.v1.ContainerSpec.PullOptions")
645 645
 	proto.RegisterType((*ContainerSpec_DNSConfig)(nil), "docker.swarmkit.v1.ContainerSpec.DNSConfig")
646
+	proto.RegisterType((*PluginSpec)(nil), "docker.swarmkit.v1.PluginSpec")
646 647
 	proto.RegisterType((*EndpointSpec)(nil), "docker.swarmkit.v1.EndpointSpec")
647 648
 	proto.RegisterType((*NetworkSpec)(nil), "docker.swarmkit.v1.NetworkSpec")
648 649
 	proto.RegisterType((*ClusterSpec)(nil), "docker.swarmkit.v1.ClusterSpec")
... ...
@@ -798,6 +842,12 @@ func (m *TaskSpec) CopyFrom(src interface{}) {
798 798
 			}
799 799
 			github_com_docker_swarmkit_api_deepcopy.Copy(v.Container, o.GetContainer())
800 800
 			m.Runtime = &v
801
+		case *TaskSpec_Plugin:
802
+			v := TaskSpec_Plugin{
803
+				Plugin: &PluginSpec{},
804
+			}
805
+			github_com_docker_swarmkit_api_deepcopy.Copy(v.Plugin, o.GetPlugin())
806
+			m.Runtime = &v
801 807
 		}
802 808
 	}
803 809
 
... ...
@@ -941,6 +991,21 @@ func (m *ContainerSpec_DNSConfig) CopyFrom(src interface{}) {
941 941
 
942 942
 }
943 943
 
944
+func (m *PluginSpec) Copy() *PluginSpec {
945
+	if m == nil {
946
+		return nil
947
+	}
948
+	o := &PluginSpec{}
949
+	o.CopyFrom(m)
950
+	return o
951
+}
952
+
953
+func (m *PluginSpec) CopyFrom(src interface{}) {
954
+
955
+	o := src.(*PluginSpec)
956
+	*m = *o
957
+}
958
+
944 959
 func (m *EndpointSpec) Copy() *EndpointSpec {
945 960
 	if m == nil {
946 961
 		return nil
... ...
@@ -1330,6 +1395,20 @@ func (m *TaskSpec_Attachment) MarshalTo(dAtA []byte) (int, error) {
1330 1330
 	}
1331 1331
 	return i, nil
1332 1332
 }
1333
+func (m *TaskSpec_Plugin) MarshalTo(dAtA []byte) (int, error) {
1334
+	i := 0
1335
+	if m.Plugin != nil {
1336
+		dAtA[i] = 0x52
1337
+		i++
1338
+		i = encodeVarintSpecs(dAtA, i, uint64(m.Plugin.Size()))
1339
+		n17, err := m.Plugin.MarshalTo(dAtA[i:])
1340
+		if err != nil {
1341
+			return 0, err
1342
+		}
1343
+		i += n17
1344
+	}
1345
+	return i, nil
1346
+}
1333 1347
 func (m *NetworkAttachmentSpec) Marshal() (dAtA []byte, err error) {
1334 1348
 	size := m.Size()
1335 1349
 	dAtA = make([]byte, size)
... ...
@@ -1465,21 +1544,21 @@ func (m *ContainerSpec) MarshalTo(dAtA []byte) (int, error) {
1465 1465
 		dAtA[i] = 0x4a
1466 1466
 		i++
1467 1467
 		i = encodeVarintSpecs(dAtA, i, uint64(m.StopGracePeriod.Size()))
1468
-		n17, err := m.StopGracePeriod.MarshalTo(dAtA[i:])
1468
+		n18, err := m.StopGracePeriod.MarshalTo(dAtA[i:])
1469 1469
 		if err != nil {
1470 1470
 			return 0, err
1471 1471
 		}
1472
-		i += n17
1472
+		i += n18
1473 1473
 	}
1474 1474
 	if m.PullOptions != nil {
1475 1475
 		dAtA[i] = 0x52
1476 1476
 		i++
1477 1477
 		i = encodeVarintSpecs(dAtA, i, uint64(m.PullOptions.Size()))
1478
-		n18, err := m.PullOptions.MarshalTo(dAtA[i:])
1478
+		n19, err := m.PullOptions.MarshalTo(dAtA[i:])
1479 1479
 		if err != nil {
1480 1480
 			return 0, err
1481 1481
 		}
1482
-		i += n18
1482
+		i += n19
1483 1483
 	}
1484 1484
 	if len(m.Groups) > 0 {
1485 1485
 		for _, s := range m.Groups {
... ...
@@ -1528,11 +1607,11 @@ func (m *ContainerSpec) MarshalTo(dAtA []byte) (int, error) {
1528 1528
 		dAtA[i] = 0x7a
1529 1529
 		i++
1530 1530
 		i = encodeVarintSpecs(dAtA, i, uint64(m.DNSConfig.Size()))
1531
-		n19, err := m.DNSConfig.MarshalTo(dAtA[i:])
1531
+		n20, err := m.DNSConfig.MarshalTo(dAtA[i:])
1532 1532
 		if err != nil {
1533 1533
 			return 0, err
1534 1534
 		}
1535
-		i += n19
1535
+		i += n20
1536 1536
 	}
1537 1537
 	if m.Healthcheck != nil {
1538 1538
 		dAtA[i] = 0x82
... ...
@@ -1540,11 +1619,11 @@ func (m *ContainerSpec) MarshalTo(dAtA []byte) (int, error) {
1540 1540
 		dAtA[i] = 0x1
1541 1541
 		i++
1542 1542
 		i = encodeVarintSpecs(dAtA, i, uint64(m.Healthcheck.Size()))
1543
-		n20, err := m.Healthcheck.MarshalTo(dAtA[i:])
1543
+		n21, err := m.Healthcheck.MarshalTo(dAtA[i:])
1544 1544
 		if err != nil {
1545 1545
 			return 0, err
1546 1546
 		}
1547
-		i += n20
1547
+		i += n21
1548 1548
 	}
1549 1549
 	if len(m.Hosts) > 0 {
1550 1550
 		for _, s := range m.Hosts {
... ...
@@ -1687,6 +1766,30 @@ func (m *ContainerSpec_DNSConfig) MarshalTo(dAtA []byte) (int, error) {
1687 1687
 	return i, nil
1688 1688
 }
1689 1689
 
1690
+func (m *PluginSpec) Marshal() (dAtA []byte, err error) {
1691
+	size := m.Size()
1692
+	dAtA = make([]byte, size)
1693
+	n, err := m.MarshalTo(dAtA)
1694
+	if err != nil {
1695
+		return nil, err
1696
+	}
1697
+	return dAtA[:n], nil
1698
+}
1699
+
1700
+func (m *PluginSpec) MarshalTo(dAtA []byte) (int, error) {
1701
+	var i int
1702
+	_ = i
1703
+	var l int
1704
+	_ = l
1705
+	if len(m.Image) > 0 {
1706
+		dAtA[i] = 0xa
1707
+		i++
1708
+		i = encodeVarintSpecs(dAtA, i, uint64(len(m.Image)))
1709
+		i += copy(dAtA[i:], m.Image)
1710
+	}
1711
+	return i, nil
1712
+}
1713
+
1690 1714
 func (m *EndpointSpec) Marshal() (dAtA []byte, err error) {
1691 1715
 	size := m.Size()
1692 1716
 	dAtA = make([]byte, size)
... ...
@@ -1740,20 +1843,20 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
1740 1740
 	dAtA[i] = 0xa
1741 1741
 	i++
1742 1742
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Annotations.Size()))
1743
-	n21, err := m.Annotations.MarshalTo(dAtA[i:])
1743
+	n22, err := m.Annotations.MarshalTo(dAtA[i:])
1744 1744
 	if err != nil {
1745 1745
 		return 0, err
1746 1746
 	}
1747
-	i += n21
1747
+	i += n22
1748 1748
 	if m.DriverConfig != nil {
1749 1749
 		dAtA[i] = 0x12
1750 1750
 		i++
1751 1751
 		i = encodeVarintSpecs(dAtA, i, uint64(m.DriverConfig.Size()))
1752
-		n22, err := m.DriverConfig.MarshalTo(dAtA[i:])
1752
+		n23, err := m.DriverConfig.MarshalTo(dAtA[i:])
1753 1753
 		if err != nil {
1754 1754
 			return 0, err
1755 1755
 		}
1756
-		i += n22
1756
+		i += n23
1757 1757
 	}
1758 1758
 	if m.Ipv6Enabled {
1759 1759
 		dAtA[i] = 0x18
... ...
@@ -1779,11 +1882,11 @@ func (m *NetworkSpec) MarshalTo(dAtA []byte) (int, error) {
1779 1779
 		dAtA[i] = 0x2a
1780 1780
 		i++
1781 1781
 		i = encodeVarintSpecs(dAtA, i, uint64(m.IPAM.Size()))
1782
-		n23, err := m.IPAM.MarshalTo(dAtA[i:])
1782
+		n24, err := m.IPAM.MarshalTo(dAtA[i:])
1783 1783
 		if err != nil {
1784 1784
 			return 0, err
1785 1785
 		}
1786
-		i += n23
1786
+		i += n24
1787 1787
 	}
1788 1788
 	if m.Attachable {
1789 1789
 		dAtA[i] = 0x30
... ...
@@ -1816,67 +1919,67 @@ func (m *ClusterSpec) MarshalTo(dAtA []byte) (int, error) {
1816 1816
 	dAtA[i] = 0xa
1817 1817
 	i++
1818 1818
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Annotations.Size()))
1819
-	n24, err := m.Annotations.MarshalTo(dAtA[i:])
1819
+	n25, err := m.Annotations.MarshalTo(dAtA[i:])
1820 1820
 	if err != nil {
1821 1821
 		return 0, err
1822 1822
 	}
1823
-	i += n24
1823
+	i += n25
1824 1824
 	dAtA[i] = 0x12
1825 1825
 	i++
1826 1826
 	i = encodeVarintSpecs(dAtA, i, uint64(m.AcceptancePolicy.Size()))
1827
-	n25, err := m.AcceptancePolicy.MarshalTo(dAtA[i:])
1827
+	n26, err := m.AcceptancePolicy.MarshalTo(dAtA[i:])
1828 1828
 	if err != nil {
1829 1829
 		return 0, err
1830 1830
 	}
1831
-	i += n25
1831
+	i += n26
1832 1832
 	dAtA[i] = 0x1a
1833 1833
 	i++
1834 1834
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Orchestration.Size()))
1835
-	n26, err := m.Orchestration.MarshalTo(dAtA[i:])
1835
+	n27, err := m.Orchestration.MarshalTo(dAtA[i:])
1836 1836
 	if err != nil {
1837 1837
 		return 0, err
1838 1838
 	}
1839
-	i += n26
1839
+	i += n27
1840 1840
 	dAtA[i] = 0x22
1841 1841
 	i++
1842 1842
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Raft.Size()))
1843
-	n27, err := m.Raft.MarshalTo(dAtA[i:])
1843
+	n28, err := m.Raft.MarshalTo(dAtA[i:])
1844 1844
 	if err != nil {
1845 1845
 		return 0, err
1846 1846
 	}
1847
-	i += n27
1847
+	i += n28
1848 1848
 	dAtA[i] = 0x2a
1849 1849
 	i++
1850 1850
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Dispatcher.Size()))
1851
-	n28, err := m.Dispatcher.MarshalTo(dAtA[i:])
1851
+	n29, err := m.Dispatcher.MarshalTo(dAtA[i:])
1852 1852
 	if err != nil {
1853 1853
 		return 0, err
1854 1854
 	}
1855
-	i += n28
1855
+	i += n29
1856 1856
 	dAtA[i] = 0x32
1857 1857
 	i++
1858 1858
 	i = encodeVarintSpecs(dAtA, i, uint64(m.CAConfig.Size()))
1859
-	n29, err := m.CAConfig.MarshalTo(dAtA[i:])
1859
+	n30, err := m.CAConfig.MarshalTo(dAtA[i:])
1860 1860
 	if err != nil {
1861 1861
 		return 0, err
1862 1862
 	}
1863
-	i += n29
1863
+	i += n30
1864 1864
 	dAtA[i] = 0x3a
1865 1865
 	i++
1866 1866
 	i = encodeVarintSpecs(dAtA, i, uint64(m.TaskDefaults.Size()))
1867
-	n30, err := m.TaskDefaults.MarshalTo(dAtA[i:])
1867
+	n31, err := m.TaskDefaults.MarshalTo(dAtA[i:])
1868 1868
 	if err != nil {
1869 1869
 		return 0, err
1870 1870
 	}
1871
-	i += n30
1871
+	i += n31
1872 1872
 	dAtA[i] = 0x42
1873 1873
 	i++
1874 1874
 	i = encodeVarintSpecs(dAtA, i, uint64(m.EncryptionConfig.Size()))
1875
-	n31, err := m.EncryptionConfig.MarshalTo(dAtA[i:])
1875
+	n32, err := m.EncryptionConfig.MarshalTo(dAtA[i:])
1876 1876
 	if err != nil {
1877 1877
 		return 0, err
1878 1878
 	}
1879
-	i += n31
1879
+	i += n32
1880 1880
 	return i, nil
1881 1881
 }
1882 1882
 
... ...
@@ -1898,11 +2001,11 @@ func (m *SecretSpec) MarshalTo(dAtA []byte) (int, error) {
1898 1898
 	dAtA[i] = 0xa
1899 1899
 	i++
1900 1900
 	i = encodeVarintSpecs(dAtA, i, uint64(m.Annotations.Size()))
1901
-	n32, err := m.Annotations.MarshalTo(dAtA[i:])
1901
+	n33, err := m.Annotations.MarshalTo(dAtA[i:])
1902 1902
 	if err != nil {
1903 1903
 		return 0, err
1904 1904
 	}
1905
-	i += n32
1905
+	i += n33
1906 1906
 	if len(m.Data) > 0 {
1907 1907
 		dAtA[i] = 0x12
1908 1908
 		i++
... ...
@@ -2073,6 +2176,15 @@ func (m *TaskSpec_Attachment) Size() (n int) {
2073 2073
 	}
2074 2074
 	return n
2075 2075
 }
2076
+func (m *TaskSpec_Plugin) Size() (n int) {
2077
+	var l int
2078
+	_ = l
2079
+	if m.Plugin != nil {
2080
+		l = m.Plugin.Size()
2081
+		n += 1 + l + sovSpecs(uint64(l))
2082
+	}
2083
+	return n
2084
+}
2076 2085
 func (m *NetworkAttachmentSpec) Size() (n int) {
2077 2086
 	var l int
2078 2087
 	_ = l
... ...
@@ -2218,6 +2330,16 @@ func (m *ContainerSpec_DNSConfig) Size() (n int) {
2218 2218
 	return n
2219 2219
 }
2220 2220
 
2221
+func (m *PluginSpec) Size() (n int) {
2222
+	var l int
2223
+	_ = l
2224
+	l = len(m.Image)
2225
+	if l > 0 {
2226
+		n += 1 + l + sovSpecs(uint64(l))
2227
+	}
2228
+	return n
2229
+}
2230
+
2221 2231
 func (m *EndpointSpec) Size() (n int) {
2222 2232
 	var l int
2223 2233
 	_ = l
... ...
@@ -2409,6 +2531,16 @@ func (this *TaskSpec_Attachment) String() string {
2409 2409
 	}, "")
2410 2410
 	return s
2411 2411
 }
2412
+func (this *TaskSpec_Plugin) String() string {
2413
+	if this == nil {
2414
+		return "nil"
2415
+	}
2416
+	s := strings.Join([]string{`&TaskSpec_Plugin{`,
2417
+		`Plugin:` + strings.Replace(fmt.Sprintf("%v", this.Plugin), "PluginSpec", "PluginSpec", 1) + `,`,
2418
+		`}`,
2419
+	}, "")
2420
+	return s
2421
+}
2412 2422
 func (this *NetworkAttachmentSpec) String() string {
2413 2423
 	if this == nil {
2414 2424
 		return "nil"
... ...
@@ -2480,6 +2612,16 @@ func (this *ContainerSpec_DNSConfig) String() string {
2480 2480
 	}, "")
2481 2481
 	return s
2482 2482
 }
2483
+func (this *PluginSpec) String() string {
2484
+	if this == nil {
2485
+		return "nil"
2486
+	}
2487
+	s := strings.Join([]string{`&PluginSpec{`,
2488
+		`Image:` + fmt.Sprintf("%v", this.Image) + `,`,
2489
+		`}`,
2490
+	}, "")
2491
+	return s
2492
+}
2483 2493
 func (this *EndpointSpec) String() string {
2484 2494
 	if this == nil {
2485 2495
 		return "nil"
... ...
@@ -3377,6 +3519,38 @@ func (m *TaskSpec) Unmarshal(dAtA []byte) error {
3377 3377
 					break
3378 3378
 				}
3379 3379
 			}
3380
+		case 10:
3381
+			if wireType != 2 {
3382
+				return fmt.Errorf("proto: wrong wireType = %d for field Plugin", wireType)
3383
+			}
3384
+			var msglen int
3385
+			for shift := uint(0); ; shift += 7 {
3386
+				if shift >= 64 {
3387
+					return ErrIntOverflowSpecs
3388
+				}
3389
+				if iNdEx >= l {
3390
+					return io.ErrUnexpectedEOF
3391
+				}
3392
+				b := dAtA[iNdEx]
3393
+				iNdEx++
3394
+				msglen |= (int(b) & 0x7F) << shift
3395
+				if b < 0x80 {
3396
+					break
3397
+				}
3398
+			}
3399
+			if msglen < 0 {
3400
+				return ErrInvalidLengthSpecs
3401
+			}
3402
+			postIndex := iNdEx + msglen
3403
+			if postIndex > l {
3404
+				return io.ErrUnexpectedEOF
3405
+			}
3406
+			v := &PluginSpec{}
3407
+			if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
3408
+				return err
3409
+			}
3410
+			m.Runtime = &TaskSpec_Plugin{v}
3411
+			iNdEx = postIndex
3380 3412
 		default:
3381 3413
 			iNdEx = preIndex
3382 3414
 			skippy, err := skipSpecs(dAtA[iNdEx:])
... ...
@@ -4403,6 +4577,85 @@ func (m *ContainerSpec_DNSConfig) Unmarshal(dAtA []byte) error {
4403 4403
 	}
4404 4404
 	return nil
4405 4405
 }
4406
+func (m *PluginSpec) Unmarshal(dAtA []byte) error {
4407
+	l := len(dAtA)
4408
+	iNdEx := 0
4409
+	for iNdEx < l {
4410
+		preIndex := iNdEx
4411
+		var wire uint64
4412
+		for shift := uint(0); ; shift += 7 {
4413
+			if shift >= 64 {
4414
+				return ErrIntOverflowSpecs
4415
+			}
4416
+			if iNdEx >= l {
4417
+				return io.ErrUnexpectedEOF
4418
+			}
4419
+			b := dAtA[iNdEx]
4420
+			iNdEx++
4421
+			wire |= (uint64(b) & 0x7F) << shift
4422
+			if b < 0x80 {
4423
+				break
4424
+			}
4425
+		}
4426
+		fieldNum := int32(wire >> 3)
4427
+		wireType := int(wire & 0x7)
4428
+		if wireType == 4 {
4429
+			return fmt.Errorf("proto: PluginSpec: wiretype end group for non-group")
4430
+		}
4431
+		if fieldNum <= 0 {
4432
+			return fmt.Errorf("proto: PluginSpec: illegal tag %d (wire type %d)", fieldNum, wire)
4433
+		}
4434
+		switch fieldNum {
4435
+		case 1:
4436
+			if wireType != 2 {
4437
+				return fmt.Errorf("proto: wrong wireType = %d for field Image", wireType)
4438
+			}
4439
+			var stringLen uint64
4440
+			for shift := uint(0); ; shift += 7 {
4441
+				if shift >= 64 {
4442
+					return ErrIntOverflowSpecs
4443
+				}
4444
+				if iNdEx >= l {
4445
+					return io.ErrUnexpectedEOF
4446
+				}
4447
+				b := dAtA[iNdEx]
4448
+				iNdEx++
4449
+				stringLen |= (uint64(b) & 0x7F) << shift
4450
+				if b < 0x80 {
4451
+					break
4452
+				}
4453
+			}
4454
+			intStringLen := int(stringLen)
4455
+			if intStringLen < 0 {
4456
+				return ErrInvalidLengthSpecs
4457
+			}
4458
+			postIndex := iNdEx + intStringLen
4459
+			if postIndex > l {
4460
+				return io.ErrUnexpectedEOF
4461
+			}
4462
+			m.Image = string(dAtA[iNdEx:postIndex])
4463
+			iNdEx = postIndex
4464
+		default:
4465
+			iNdEx = preIndex
4466
+			skippy, err := skipSpecs(dAtA[iNdEx:])
4467
+			if err != nil {
4468
+				return err
4469
+			}
4470
+			if skippy < 0 {
4471
+				return ErrInvalidLengthSpecs
4472
+			}
4473
+			if (iNdEx + skippy) > l {
4474
+				return io.ErrUnexpectedEOF
4475
+			}
4476
+			iNdEx += skippy
4477
+		}
4478
+	}
4479
+
4480
+	if iNdEx > l {
4481
+		return io.ErrUnexpectedEOF
4482
+	}
4483
+	return nil
4484
+}
4406 4485
 func (m *EndpointSpec) Unmarshal(dAtA []byte) error {
4407 4486
 	l := len(dAtA)
4408 4487
 	iNdEx := 0
... ...
@@ -5218,112 +5471,114 @@ var (
5218 5218
 func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) }
5219 5219
 
5220 5220
 var fileDescriptorSpecs = []byte{
5221
-	// 1707 bytes of a gzipped FileDescriptorProto
5221
+	// 1730 bytes of a gzipped FileDescriptorProto
5222 5222
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x41, 0x73, 0x1b, 0xb7,
5223
-	0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x1a, 0x75, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3,
5223
+	0x15, 0x26, 0x25, 0x8a, 0x5a, 0xbe, 0xa5, 0x6c, 0x0a, 0x4d, 0xd2, 0x35, 0xdd, 0x90, 0x34, 0xe3,
5224 5224
 	0xa6, 0x4a, 0x33, 0xa5, 0xa6, 0x6a, 0x27, 0x75, 0xea, 0x66, 0x5a, 0x52, 0x64, 0x65, 0x55, 0x95,
5225 5225
 	0xcc, 0x01, 0x15, 0x77, 0x7c, 0xe2, 0x80, 0xbb, 0x10, 0xb9, 0xa3, 0xe5, 0x62, 0x0b, 0x60, 0x99,
5226
-	0xe1, 0xad, 0xc7, 0x8c, 0x0f, 0x3d, 0xf5, 0xaa, 0xe9, 0xa1, 0xbf, 0xa1, 0xff, 0xc1, 0xc7, 0x1e,
5227
-	0x7b, 0xd2, 0x34, 0xfc, 0x0b, 0xfd, 0x01, 0xed, 0x00, 0x0b, 0x92, 0xcb, 0x64, 0x15, 0x7b, 0x26,
5228
-	0xbe, 0xe1, 0xbd, 0xfd, 0xbe, 0x07, 0xe0, 0xe1, 0xc3, 0xc3, 0x5b, 0xb0, 0x45, 0x44, 0x5d, 0xd1,
5229
-	0x8a, 0x38, 0x93, 0x0c, 0x21, 0x8f, 0xb9, 0x57, 0x94, 0xb7, 0xc4, 0x97, 0x84, 0x4f, 0xaf, 0x7c,
5230
-	0xd9, 0x9a, 0xfd, 0xbc, 0x6a, 0xcb, 0x79, 0x44, 0x0d, 0xa0, 0x7a, 0x7f, 0xcc, 0xc6, 0x4c, 0x0f,
5231
-	0x0f, 0xd4, 0xc8, 0x78, 0x6b, 0x63, 0xc6, 0xc6, 0x01, 0x3d, 0xd0, 0xd6, 0x28, 0xbe, 0x3c, 0xf0,
5232
-	0x62, 0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xba, 0x00, 0xd6, 0x39, 0xf3, 0xe8, 0x20, 0xa2,
5233
-	0x2e, 0x3a, 0x06, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0xbf, 0x6f, 0x1f, 0xd6,
5234
-	0x5b, 0xdf, 0x9e, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0x37, 0xf5, 0x1c, 0x4e, 0x33, 0xd1,
5235
-	0x6f, 0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0xef, 0xdf,
5236
-	0x39, 0xfc, 0x51, 0x56, 0x24, 0x35, 0x39, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0x28, 0x03, 0x1d, 0x03,
5237
-	0x4c, 0xe9, 0x74, 0x44, 0xb9, 0x98, 0xf8, 0x91, 0xb3, 0xad, 0xe9, 0x3f, 0xb9, 0x8d, 0xae, 0xd6,
5238
-	0xde, 0x3a, 0x5b, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa0, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f,
5239
-	0x7c, 0x39, 0x77, 0x0a, 0x3a, 0xd4, 0xc7, 0xdf, 0x19, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3,
5240
-	0x03, 0x58, 0x4f, 0x84, 0x3e, 0x82, 0xdd, 0x7e, 0xef, 0xbc, 0x7b, 0x72, 0x7e, 0x5c, 0xc9, 0x55,
5241
-	0x1f, 0xbc, 0xba, 0x6e, 0xbc, 0xa7, 0x62, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0xfb,
5242
-	0x60, 0xb5, 0x8f, 0x8e, 0x7a, 0xfd, 0x8b, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0x5d, 0x37, 0xde,
5243
-	0xdf, 0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0x57, 0xff, 0xa8, 0xe5, 0x9a, 0x5f,
5244
-	0xe5, 0xa1, 0x9c, 0x5e, 0x04, 0xfa, 0x08, 0x8a, 0xed, 0xa3, 0x8b, 0x93, 0x17, 0xbd, 0x4a, 0x6e,
5245
-	0x4d, 0x4f, 0x23, 0xda, 0xae, 0xf4, 0x67, 0x14, 0x3d, 0x86, 0x9d, 0x7e, 0xfb, 0x8b, 0x41, 0xaf,
5246
-	0x92, 0x5f, 0x2f, 0x27, 0x0d, 0xeb, 0x93, 0x58, 0x68, 0x54, 0x17, 0xb7, 0x4f, 0xce, 0x2b, 0x5b,
5247
-	0xd9, 0xa8, 0x2e, 0x27, 0x7e, 0x68, 0x96, 0xf2, 0xf7, 0x02, 0xd8, 0x03, 0xca, 0x67, 0xbe, 0xfb,
5248
-	0x8e, 0x25, 0xf2, 0x29, 0x14, 0x24, 0x11, 0x57, 0x5a, 0x1a, 0x76, 0xb6, 0x34, 0x2e, 0x88, 0xb8,
5249
-	0x52, 0x93, 0x1a, 0xba, 0xc6, 0x2b, 0x65, 0x70, 0x1a, 0x05, 0xbe, 0x4b, 0x24, 0xf5, 0xb4, 0x32,
5250
-	0xec, 0xc3, 0x1f, 0x67, 0xb1, 0xf1, 0x0a, 0x65, 0xd6, 0xff, 0x2c, 0x87, 0x53, 0x54, 0xf4, 0x14,
5251
-	0x8a, 0xe3, 0x80, 0x8d, 0x48, 0xa0, 0x35, 0x61, 0x1f, 0x3e, 0xca, 0x0a, 0x72, 0xac, 0x11, 0xeb,
5252
-	0x00, 0x86, 0x82, 0x9e, 0x40, 0x31, 0x8e, 0x3c, 0x22, 0xa9, 0x53, 0xd4, 0xe4, 0x46, 0x16, 0xf9,
5253
-	0x0b, 0x8d, 0x38, 0x62, 0xe1, 0xa5, 0x3f, 0xc6, 0x06, 0x8f, 0x4e, 0xc1, 0x0a, 0xa9, 0xfc, 0x92,
5254
-	0xf1, 0x2b, 0xe1, 0xec, 0x36, 0xb6, 0xf7, 0xed, 0xc3, 0x4f, 0x32, 0xc5, 0x98, 0x60, 0xda, 0x52,
5255
-	0x12, 0x77, 0x32, 0xa5, 0xa1, 0x4c, 0xc2, 0x74, 0xb6, 0x9c, 0x3c, 0x5e, 0x05, 0x40, 0xbf, 0x01,
5256
-	0x8b, 0x86, 0x5e, 0xc4, 0xfc, 0x50, 0x3a, 0xd6, 0xed, 0x0b, 0xe9, 0x19, 0x8c, 0x4a, 0x26, 0x5e,
5257
-	0x31, 0x14, 0x9b, 0xb3, 0x20, 0x18, 0x11, 0xf7, 0xca, 0x29, 0xbd, 0xe5, 0x36, 0x56, 0x8c, 0x4e,
5258
-	0x11, 0x0a, 0x53, 0xe6, 0xd1, 0xe6, 0x01, 0xdc, 0xfb, 0x56, 0xaa, 0x51, 0x15, 0x2c, 0x93, 0xea,
5259
-	0x44, 0x23, 0x05, 0xbc, 0xb2, 0x9b, 0x77, 0x61, 0x6f, 0x23, 0xad, 0xcd, 0xbf, 0x16, 0xc0, 0x5a,
5260
-	0x9e, 0x35, 0x6a, 0x43, 0xc9, 0x65, 0xa1, 0x24, 0x7e, 0x48, 0xb9, 0x91, 0x57, 0xe6, 0xc9, 0x1c,
5261
-	0x2d, 0x41, 0x8a, 0xf5, 0x2c, 0x87, 0xd7, 0x2c, 0xf4, 0x7b, 0x28, 0x71, 0x2a, 0x58, 0xcc, 0x5d,
5262
-	0x2a, 0x8c, 0xbe, 0xf6, 0xb3, 0x15, 0x92, 0x80, 0x30, 0xfd, 0x73, 0xec, 0x73, 0xaa, 0xb2, 0x2c,
5263
-	0xf0, 0x9a, 0x8a, 0x9e, 0xc2, 0x2e, 0xa7, 0x42, 0x12, 0x2e, 0xbf, 0x4b, 0x22, 0x38, 0x81, 0xf4,
5264
-	0x59, 0xe0, 0xbb, 0x73, 0xbc, 0x64, 0xa0, 0xa7, 0x50, 0x8a, 0x02, 0xe2, 0xea, 0xa8, 0xce, 0x8e,
5265
-	0xa6, 0x7f, 0x90, 0x45, 0xef, 0x2f, 0x41, 0x78, 0x8d, 0x47, 0x9f, 0x01, 0x04, 0x6c, 0x3c, 0xf4,
5266
-	0xb8, 0x3f, 0xa3, 0xdc, 0x48, 0xac, 0x9a, 0xc5, 0xee, 0x6a, 0x04, 0x2e, 0x05, 0x6c, 0x9c, 0x0c,
5267
-	0xd1, 0xf1, 0xf7, 0xd2, 0x57, 0x4a, 0x5b, 0xa7, 0x00, 0x64, 0xf5, 0xd5, 0xa8, 0xeb, 0xe3, 0xb7,
5268
-	0x0a, 0x65, 0x4e, 0x24, 0x45, 0x47, 0x8f, 0xa0, 0x7c, 0xc9, 0xb8, 0x4b, 0x87, 0xe6, 0xd6, 0x94,
5269
-	0xb4, 0x26, 0x6c, 0xed, 0x4b, 0xf4, 0xd5, 0x29, 0xc1, 0x2e, 0x8f, 0x43, 0xe9, 0x4f, 0x69, 0xf3,
5270
-	0x14, 0xde, 0xcb, 0x0c, 0x8a, 0x0e, 0xa1, 0xbc, 0x3a, 0xe6, 0xa1, 0xef, 0x69, 0x7d, 0x94, 0x3a,
5271
-	0x77, 0x17, 0x37, 0x75, 0x7b, 0xa5, 0x87, 0x93, 0x2e, 0xb6, 0x57, 0xa0, 0x13, 0xaf, 0xf9, 0x37,
5272
-	0x0b, 0xf6, 0x36, 0xc4, 0x82, 0xee, 0xc3, 0x8e, 0x3f, 0x25, 0x63, 0x9a, 0xd0, 0x71, 0x62, 0xa0,
5273
-	0x1e, 0x14, 0x03, 0x32, 0xa2, 0x81, 0x92, 0x8c, 0x4a, 0xdb, 0xcf, 0xde, 0xa8, 0xba, 0xd6, 0x1f,
5274
-	0x35, 0xbe, 0x17, 0x4a, 0x3e, 0xc7, 0x86, 0x8c, 0x1c, 0xd8, 0x75, 0xd9, 0x74, 0x4a, 0x42, 0x55,
5275
-	0x9c, 0xb6, 0xf7, 0x4b, 0x78, 0x69, 0x22, 0x04, 0x05, 0xc2, 0xc7, 0xc2, 0x29, 0x68, 0xb7, 0x1e,
5276
-	0xa3, 0x0a, 0x6c, 0xd3, 0x70, 0xe6, 0xec, 0x68, 0x97, 0x1a, 0x2a, 0x8f, 0xe7, 0x27, 0x67, 0x5e,
5277
-	0xc2, 0x6a, 0xa8, 0x78, 0xb1, 0xa0, 0xdc, 0xd9, 0xd5, 0x2e, 0x3d, 0x46, 0xbf, 0x82, 0xe2, 0x94,
5278
-	0xc5, 0xa1, 0x14, 0x8e, 0xa5, 0x17, 0xfb, 0x20, 0x6b, 0xb1, 0x67, 0x0a, 0x61, 0x8a, 0xa7, 0x81,
5279
-	0xa3, 0x1e, 0xdc, 0x13, 0x92, 0x45, 0xc3, 0x31, 0x27, 0x2e, 0x1d, 0x46, 0x94, 0xfb, 0xcc, 0x33,
5280
-	0x97, 0xff, 0x41, 0x2b, 0xe9, 0x15, 0x5a, 0xcb, 0x5e, 0xa1, 0xd5, 0x35, 0xbd, 0x02, 0xbe, 0xab,
5281
-	0x38, 0xc7, 0x8a, 0xd2, 0xd7, 0x0c, 0xd4, 0x87, 0x72, 0x14, 0x07, 0xc1, 0x90, 0x45, 0xc9, 0x3b,
5282
-	0x00, 0x3a, 0xc2, 0x5b, 0xa4, 0xac, 0x1f, 0x07, 0xc1, 0xf3, 0x84, 0x84, 0xed, 0x68, 0x6d, 0xa0,
5283
-	0xf7, 0xa1, 0x38, 0xe6, 0x2c, 0x8e, 0x84, 0x63, 0xeb, 0x64, 0x18, 0x0b, 0x7d, 0x0e, 0xbb, 0x82,
5284
-	0xba, 0x9c, 0x4a, 0xe1, 0x94, 0xf5, 0x56, 0x3f, 0xcc, 0x9a, 0x64, 0xa0, 0x21, 0x98, 0x5e, 0x52,
5285
-	0x4e, 0x43, 0x97, 0xe2, 0x25, 0x07, 0x3d, 0x80, 0x6d, 0x29, 0xe7, 0xce, 0x5e, 0x23, 0xbf, 0x6f,
5286
-	0x75, 0x76, 0x17, 0x37, 0xf5, 0xed, 0x8b, 0x8b, 0x97, 0x58, 0xf9, 0x54, 0x8d, 0x9a, 0x30, 0x21,
5287
-	0x43, 0x32, 0xa5, 0xce, 0x1d, 0x9d, 0xdb, 0x95, 0x8d, 0x5e, 0x02, 0x78, 0xa1, 0x18, 0xba, 0xfa,
5288
-	0x52, 0x38, 0x77, 0xf5, 0xee, 0x3e, 0x79, 0xf3, 0xee, 0xba, 0xe7, 0x03, 0x53, 0xa7, 0xf7, 0x16,
5289
-	0x37, 0xf5, 0xd2, 0xca, 0xc4, 0x25, 0x2f, 0x14, 0xc9, 0x10, 0x75, 0xc0, 0x9e, 0x50, 0x12, 0xc8,
5290
-	0x89, 0x3b, 0xa1, 0xee, 0x95, 0x53, 0xb9, 0xbd, 0xf0, 0x3e, 0xd3, 0x30, 0x13, 0x21, 0x4d, 0x52,
5291
-	0x0a, 0x56, 0x4b, 0x15, 0xce, 0x3d, 0x9d, 0xab, 0xc4, 0x40, 0x1f, 0x00, 0xb0, 0x88, 0x86, 0x43,
5292
-	0x21, 0x3d, 0x3f, 0x74, 0x90, 0xda, 0x32, 0x2e, 0x29, 0xcf, 0x40, 0x39, 0xd0, 0x43, 0x55, 0x16,
5293
-	0x89, 0x37, 0x64, 0x61, 0x30, 0x77, 0x7e, 0xa0, 0xbf, 0x5a, 0xca, 0xf1, 0x3c, 0x0c, 0xe6, 0xa8,
5294
-	0x0e, 0xb6, 0xd6, 0x85, 0xf0, 0xc7, 0x21, 0x09, 0x9c, 0xfb, 0x3a, 0x1f, 0xa0, 0x5c, 0x03, 0xed,
5295
-	0xa9, 0x7e, 0x06, 0x76, 0x4a, 0xee, 0x4a, 0xa6, 0x57, 0x74, 0x6e, 0x6e, 0x90, 0x1a, 0xaa, 0x35,
5296
-	0xcd, 0x48, 0x10, 0x27, 0xcd, 0x5e, 0x09, 0x27, 0xc6, 0xaf, 0xb7, 0x9e, 0xe4, 0xab, 0x87, 0x60,
5297
-	0xa7, 0x8e, 0x1d, 0x7d, 0x08, 0x7b, 0x9c, 0x8e, 0x7d, 0x21, 0xf9, 0x7c, 0x48, 0x62, 0x39, 0x71,
5298
-	0x7e, 0xa7, 0x09, 0xe5, 0xa5, 0xb3, 0x1d, 0xcb, 0x49, 0x75, 0x08, 0xeb, 0xec, 0xa1, 0x06, 0xd8,
5299
-	0xea, 0x54, 0x04, 0xe5, 0x33, 0xca, 0xd5, 0x83, 0xa2, 0x36, 0x9d, 0x76, 0x29, 0xf5, 0x08, 0x4a,
5300
-	0xb8, 0x3b, 0xd1, 0x97, 0xb7, 0x84, 0x8d, 0xa5, 0x6e, 0xe3, 0x52, 0xa2, 0xe6, 0x36, 0x1a, 0xb3,
5301
-	0xf9, 0xdf, 0x3c, 0x94, 0xd3, 0xef, 0x22, 0x3a, 0x4a, 0xde, 0x33, 0xbd, 0xa5, 0x3b, 0x87, 0x07,
5302
-	0x6f, 0x7a, 0x47, 0xf5, 0xeb, 0x11, 0xc4, 0x2a, 0xd8, 0x99, 0x6a, 0x61, 0x35, 0x19, 0xfd, 0x12,
5303
-	0x76, 0x22, 0xc6, 0xe5, 0xb2, 0x86, 0xd4, 0x32, 0x2b, 0x3e, 0xe3, 0xcb, 0x6a, 0x9b, 0x80, 0x9b,
5304
-	0x13, 0xb8, 0xb3, 0x19, 0x0d, 0x3d, 0x86, 0xed, 0x17, 0x27, 0xfd, 0x4a, 0xae, 0xfa, 0xf0, 0xd5,
5305
-	0x75, 0xe3, 0x87, 0x9b, 0x1f, 0x5f, 0xf8, 0x5c, 0xc6, 0x24, 0x38, 0xe9, 0xa3, 0x9f, 0xc2, 0x4e,
5306
-	0xf7, 0x7c, 0x80, 0x71, 0x25, 0x5f, 0xad, 0xbf, 0xba, 0x6e, 0x3c, 0xdc, 0xc4, 0xa9, 0x4f, 0x2c,
5307
-	0x0e, 0x3d, 0xcc, 0x46, 0xab, 0x76, 0xee, 0x9f, 0x5b, 0x60, 0x9b, 0xd2, 0xfa, 0xae, 0x3b, 0xfe,
5308
-	0xbd, 0xe4, 0xb5, 0x5a, 0xde, 0x99, 0xad, 0x37, 0x3e, 0x5a, 0xe5, 0x84, 0x60, 0xce, 0xf8, 0x11,
5309
-	0x94, 0xfd, 0x68, 0xf6, 0xe9, 0x90, 0x86, 0x64, 0x14, 0x98, 0xce, 0xce, 0xc2, 0xb6, 0xf2, 0xf5,
5310
-	0x12, 0x97, 0xba, 0xb0, 0x7e, 0x28, 0x29, 0x0f, 0x4d, 0xcf, 0x66, 0xe1, 0x95, 0x8d, 0x3e, 0x87,
5311
-	0x82, 0x1f, 0x91, 0xa9, 0x79, 0x69, 0x33, 0x77, 0x70, 0xd2, 0x6f, 0x9f, 0x19, 0x0d, 0x76, 0xac,
5312
-	0xc5, 0x4d, 0xbd, 0xa0, 0x1c, 0x58, 0xd3, 0x50, 0x6d, 0xf9, 0xd8, 0xa9, 0x99, 0x74, 0xf1, 0xb5,
5313
-	0x70, 0xca, 0xd3, 0xfc, 0x5f, 0x01, 0xec, 0xa3, 0x20, 0x16, 0xd2, 0x3c, 0x21, 0xef, 0x2c, 0x6f,
5314
-	0x2f, 0xe1, 0x1e, 0xd1, 0xcd, 0x3f, 0x09, 0x55, 0x3d, 0xd6, 0x4d, 0x84, 0xc9, 0xdd, 0xe3, 0xcc,
5315
-	0x70, 0x2b, 0x70, 0xd2, 0x70, 0x74, 0x8a, 0x2a, 0xa6, 0x93, 0xc7, 0x15, 0xf2, 0x8d, 0x2f, 0x68,
5316
-	0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52, 0xc5, 0x4d, 0xb3, 0x9c, 0xf9, 0x1b, 0xf5, 0x3c,
5317
-	0x0d, 0x34, 0x25, 0x2c, 0x59, 0xed, 0x66, 0x0c, 0xf4, 0x04, 0x0a, 0x9c, 0x5c, 0x2e, 0x1b, 0xa2,
5318
-	0x4c, 0x7d, 0x63, 0x72, 0x29, 0x37, 0x42, 0x68, 0x06, 0xfa, 0x03, 0x80, 0xe7, 0x8b, 0x88, 0x48,
5319
-	0x77, 0x42, 0xb9, 0x39, 0xa7, 0xcc, 0x2d, 0x76, 0x57, 0xa8, 0x8d, 0x28, 0x29, 0x36, 0x3a, 0x85,
5320
-	0x92, 0x4b, 0x96, 0x4a, 0x2b, 0xde, 0xfe, 0x07, 0x71, 0xd4, 0x36, 0x21, 0x2a, 0x2a, 0xc4, 0xe2,
5321
-	0xa6, 0x6e, 0x2d, 0x3d, 0xd8, 0x72, 0x89, 0x51, 0xde, 0x29, 0xec, 0xa9, 0x3f, 0x8b, 0xa1, 0x47,
5322
-	0x2f, 0x49, 0x1c, 0x48, 0xa1, 0x1f, 0xda, 0x5b, 0x4a, 0xb2, 0x6a, 0x53, 0xbb, 0x06, 0x67, 0xd6,
5323
-	0x55, 0x96, 0x29, 0x1f, 0xfa, 0x13, 0xdc, 0xa3, 0xa1, 0xcb, 0xe7, 0x5a, 0x67, 0xcb, 0x15, 0x5a,
5324
-	0xb7, 0x6f, 0xb6, 0xb7, 0x02, 0x6f, 0x6c, 0xb6, 0x42, 0xbf, 0xe1, 0x6f, 0xfa, 0x00, 0xc9, 0x23,
5325
-	0xf7, 0x6e, 0xf5, 0x87, 0xa0, 0xe0, 0x11, 0x49, 0xb4, 0xe4, 0xca, 0x58, 0x8f, 0x3b, 0xce, 0xeb,
5326
-	0xaf, 0x6b, 0xb9, 0x7f, 0x7f, 0x5d, 0xcb, 0xfd, 0x65, 0x51, 0xcb, 0xbf, 0x5e, 0xd4, 0xf2, 0xff,
5327
-	0x5a, 0xd4, 0xf2, 0xff, 0x59, 0xd4, 0xf2, 0xa3, 0xa2, 0x6e, 0x0d, 0x7e, 0xf1, 0xff, 0x00, 0x00,
5328
-	0x00, 0xff, 0xff, 0xed, 0xbe, 0x26, 0xe6, 0x9a, 0x10, 0x00, 0x00,
5226
+	0xe1, 0xad, 0xc7, 0x8c, 0xcf, 0xbd, 0x6a, 0x7a, 0xe8, 0x6f, 0xe8, 0x7f, 0xf0, 0xb1, 0xc7, 0x9e,
5227
+	0x34, 0x0d, 0xff, 0x42, 0x7f, 0x40, 0x3b, 0xc0, 0x82, 0xe4, 0x32, 0x59, 0xc5, 0x9e, 0xa9, 0x6f,
5228
+	0xc0, 0xdb, 0xef, 0x7b, 0x78, 0x78, 0xf8, 0xf0, 0xf0, 0x16, 0x6c, 0x11, 0x51, 0x57, 0xb4, 0x22,
5229
+	0xce, 0x24, 0x43, 0xc8, 0x63, 0xee, 0x35, 0xe5, 0x2d, 0xf1, 0x15, 0xe1, 0xd3, 0x6b, 0x5f, 0xb6,
5230
+	0x66, 0x3f, 0xaf, 0xda, 0x72, 0x1e, 0x51, 0x03, 0xa8, 0xbe, 0x37, 0x66, 0x63, 0xa6, 0x87, 0x87,
5231
+	0x6a, 0x64, 0xac, 0xb5, 0x31, 0x63, 0xe3, 0x80, 0x1e, 0xea, 0xd9, 0x28, 0xbe, 0x3a, 0xf4, 0x62,
5232
+	0x4e, 0xa4, 0xcf, 0xc2, 0xe4, 0x7b, 0xf3, 0xa6, 0x00, 0xd6, 0x05, 0xf3, 0xe8, 0x20, 0xa2, 0x2e,
5233
+	0x3a, 0x01, 0x9b, 0x84, 0x21, 0x93, 0x1a, 0x20, 0x9c, 0x7c, 0x23, 0x7f, 0x60, 0x1f, 0xd5, 0x5b,
5234
+	0xdf, 0x5d, 0xb9, 0xd5, 0x5e, 0xc3, 0x3a, 0x85, 0xd7, 0xb7, 0xf5, 0x1c, 0x4e, 0x33, 0xd1, 0x6f,
5235
+	0xa1, 0xec, 0x51, 0xe1, 0x73, 0xea, 0x0d, 0x39, 0x0b, 0xa8, 0xb3, 0xd5, 0xc8, 0x1f, 0xdc, 0x3b,
5236
+	0xfa, 0x51, 0x96, 0x27, 0xb5, 0x38, 0x66, 0x01, 0xc5, 0xb6, 0x61, 0xa8, 0x09, 0x3a, 0x01, 0x98,
5237
+	0xd2, 0xe9, 0x88, 0x72, 0x31, 0xf1, 0x23, 0x67, 0x5b, 0xd3, 0x7f, 0x72, 0x17, 0x5d, 0xc5, 0xde,
5238
+	0x3a, 0x5f, 0xc1, 0x71, 0x8a, 0x8a, 0xce, 0xa1, 0x4c, 0x66, 0xc4, 0x0f, 0xc8, 0xc8, 0x0f, 0x7c,
5239
+	0x39, 0x77, 0x0a, 0xda, 0xd5, 0x27, 0xdf, 0xeb, 0xaa, 0x9d, 0x22, 0xe0, 0x0d, 0x7a, 0xd3, 0x03,
5240
+	0x58, 0x2f, 0x84, 0x3e, 0x86, 0xdd, 0x7e, 0xef, 0xa2, 0x7b, 0x7a, 0x71, 0x52, 0xc9, 0x55, 0x1f,
5241
+	0xbc, 0xba, 0x69, 0xbc, 0xaf, 0x7c, 0xac, 0x01, 0x7d, 0x1a, 0x7a, 0x7e, 0x38, 0x46, 0x07, 0x60,
5242
+	0xb5, 0x8f, 0x8f, 0x7b, 0xfd, 0xcb, 0x5e, 0xb7, 0x92, 0xaf, 0x56, 0x5f, 0xdd, 0x34, 0x3e, 0xd8,
5243
+	0x04, 0xb6, 0x5d, 0x97, 0x46, 0x92, 0x7a, 0xd5, 0xc2, 0xd7, 0x7f, 0xaf, 0xe5, 0x9a, 0x5f, 0xe7,
5244
+	0xa1, 0x9c, 0x0e, 0x02, 0x7d, 0x0c, 0xc5, 0xf6, 0xf1, 0xe5, 0xe9, 0x8b, 0x5e, 0x25, 0xb7, 0xa6,
5245
+	0xa7, 0x11, 0x6d, 0x57, 0xfa, 0x33, 0x8a, 0x1e, 0xc3, 0x4e, 0xbf, 0xfd, 0xe5, 0xa0, 0x57, 0xc9,
5246
+	0xaf, 0xc3, 0x49, 0xc3, 0xfa, 0x24, 0x16, 0x1a, 0xd5, 0xc5, 0xed, 0xd3, 0x8b, 0xca, 0x56, 0x36,
5247
+	0xaa, 0xcb, 0x89, 0x1f, 0x9a, 0x50, 0xfe, 0x56, 0x00, 0x7b, 0x40, 0xf9, 0xcc, 0x77, 0xdf, 0xb1,
5248
+	0x44, 0x3e, 0x83, 0x82, 0x24, 0xe2, 0x5a, 0x4b, 0xc3, 0xce, 0x96, 0xc6, 0x25, 0x11, 0xd7, 0x6a,
5249
+	0x51, 0x43, 0xd7, 0x78, 0xa5, 0x0c, 0x4e, 0xa3, 0xc0, 0x77, 0x89, 0xa4, 0x9e, 0x56, 0x86, 0x7d,
5250
+	0xf4, 0xe3, 0x2c, 0x36, 0x5e, 0xa1, 0x4c, 0xfc, 0xcf, 0x72, 0x38, 0x45, 0x45, 0x4f, 0xa1, 0x38,
5251
+	0x0e, 0xd8, 0x88, 0x04, 0x5a, 0x13, 0xf6, 0xd1, 0xa3, 0x2c, 0x27, 0x27, 0x1a, 0xb1, 0x76, 0x60,
5252
+	0x28, 0xe8, 0x09, 0x14, 0xe3, 0xc8, 0x23, 0x92, 0x3a, 0x45, 0x4d, 0x6e, 0x64, 0x91, 0xbf, 0xd4,
5253
+	0x88, 0x63, 0x16, 0x5e, 0xf9, 0x63, 0x6c, 0xf0, 0xe8, 0x0c, 0xac, 0x90, 0xca, 0xaf, 0x18, 0xbf,
5254
+	0x16, 0xce, 0x6e, 0x63, 0xfb, 0xc0, 0x3e, 0xfa, 0x34, 0x53, 0x8c, 0x09, 0xa6, 0x2d, 0x25, 0x71,
5255
+	0x27, 0x53, 0x1a, 0xca, 0xc4, 0x4d, 0x67, 0xcb, 0xc9, 0xe3, 0x95, 0x03, 0xf4, 0x1b, 0xb0, 0x68,
5256
+	0xe8, 0x45, 0xcc, 0x0f, 0xa5, 0x63, 0xdd, 0x1d, 0x48, 0xcf, 0x60, 0x54, 0x32, 0xf1, 0x8a, 0xa1,
5257
+	0xd8, 0x9c, 0x05, 0xc1, 0x88, 0xb8, 0xd7, 0x4e, 0xe9, 0x2d, 0xb7, 0xb1, 0x62, 0x74, 0x8a, 0x50,
5258
+	0x98, 0x32, 0x8f, 0x36, 0x0f, 0x61, 0xff, 0x3b, 0xa9, 0x46, 0x55, 0xb0, 0x4c, 0xaa, 0x13, 0x8d,
5259
+	0x14, 0xf0, 0x6a, 0xde, 0xbc, 0x0f, 0x7b, 0x1b, 0x69, 0x6d, 0xbe, 0x2e, 0x80, 0xb5, 0x3c, 0x6b,
5260
+	0xd4, 0x86, 0x92, 0xcb, 0x42, 0x49, 0xfc, 0x90, 0x72, 0x23, 0xaf, 0xcc, 0x93, 0x39, 0x5e, 0x82,
5261
+	0x14, 0xeb, 0x59, 0x0e, 0xaf, 0x59, 0xe8, 0xf7, 0x50, 0xe2, 0x54, 0xb0, 0x98, 0xbb, 0x54, 0x18,
5262
+	0x7d, 0x1d, 0x64, 0x2b, 0x24, 0x01, 0x61, 0xfa, 0xe7, 0xd8, 0xe7, 0x54, 0x65, 0x59, 0xe0, 0x35,
5263
+	0x15, 0x3d, 0x85, 0x5d, 0x4e, 0x85, 0x24, 0x5c, 0x7e, 0x9f, 0x44, 0x70, 0x02, 0xe9, 0xb3, 0xc0,
5264
+	0x77, 0xe7, 0x78, 0xc9, 0x40, 0x4f, 0xa1, 0x14, 0x05, 0xc4, 0xd5, 0x5e, 0x9d, 0x1d, 0x4d, 0xff,
5265
+	0x30, 0x8b, 0xde, 0x5f, 0x82, 0xf0, 0x1a, 0x8f, 0x3e, 0x07, 0x08, 0xd8, 0x78, 0xe8, 0x71, 0x7f,
5266
+	0x46, 0xb9, 0x91, 0x58, 0x35, 0x8b, 0xdd, 0xd5, 0x08, 0x5c, 0x0a, 0xd8, 0x38, 0x19, 0xa2, 0x93,
5267
+	0xff, 0x4b, 0x5f, 0x29, 0x6d, 0x9d, 0x01, 0x90, 0xd5, 0x57, 0xa3, 0xae, 0x4f, 0xde, 0xca, 0x95,
5268
+	0x39, 0x91, 0x14, 0x1d, 0x3d, 0x82, 0xf2, 0x15, 0xe3, 0x2e, 0x1d, 0x9a, 0x5b, 0x53, 0xd2, 0x9a,
5269
+	0xb0, 0xb5, 0x2d, 0xd1, 0x97, 0xba, 0x52, 0x51, 0x10, 0x8f, 0xfd, 0xd0, 0x01, 0xbd, 0x56, 0x2d,
5270
+	0x3b, 0x5b, 0x0a, 0x61, 0x16, 0x30, 0xf8, 0x4e, 0x09, 0x76, 0x79, 0x1c, 0x4a, 0x7f, 0x4a, 0x9b,
5271
+	0x67, 0xf0, 0x7e, 0x66, 0x38, 0xe8, 0x08, 0xca, 0x2b, 0x81, 0x0c, 0x7d, 0x4f, 0x2b, 0xab, 0xd4,
5272
+	0xb9, 0xbf, 0xb8, 0xad, 0xdb, 0x2b, 0x25, 0x9d, 0x76, 0xb1, 0xbd, 0x02, 0x9d, 0x7a, 0xcd, 0xbf,
5273
+	0x5a, 0xb0, 0xb7, 0x21, 0x33, 0xf4, 0x1e, 0xec, 0xf8, 0x53, 0x32, 0xa6, 0x09, 0x1d, 0x27, 0x13,
5274
+	0xd4, 0x83, 0x62, 0x40, 0x46, 0x34, 0x50, 0x62, 0x53, 0x09, 0xff, 0xd9, 0x1b, 0xf5, 0xda, 0xfa,
5275
+	0xa3, 0xc6, 0xf7, 0x42, 0xc9, 0xe7, 0xd8, 0x90, 0x91, 0x03, 0xbb, 0x2e, 0x9b, 0x4e, 0x49, 0xa8,
5276
+	0xca, 0xda, 0xf6, 0x41, 0x09, 0x2f, 0xa7, 0x08, 0x41, 0x81, 0xf0, 0xb1, 0x70, 0x0a, 0xda, 0xac,
5277
+	0xc7, 0xa8, 0x02, 0xdb, 0x34, 0x9c, 0x39, 0x3b, 0xda, 0xa4, 0x86, 0xca, 0xe2, 0xf9, 0x89, 0x5a,
5278
+	0x4a, 0x58, 0x0d, 0x15, 0x2f, 0x16, 0x94, 0x3b, 0xbb, 0xda, 0xa4, 0xc7, 0xe8, 0x57, 0x50, 0x9c,
5279
+	0xb2, 0x38, 0x94, 0xc2, 0xb1, 0x74, 0xb0, 0x0f, 0xb2, 0x82, 0x3d, 0x57, 0x08, 0x53, 0x76, 0x0d,
5280
+	0x1c, 0xf5, 0x60, 0x5f, 0x48, 0x16, 0x0d, 0xc7, 0x9c, 0xb8, 0x74, 0x18, 0x51, 0xee, 0x33, 0xcf,
5281
+	0x94, 0x8d, 0x07, 0xad, 0xa4, 0xcb, 0x68, 0x2d, 0xbb, 0x8c, 0x56, 0xd7, 0x74, 0x19, 0xf8, 0xbe,
5282
+	0xe2, 0x9c, 0x28, 0x4a, 0x5f, 0x33, 0x50, 0x1f, 0xca, 0x51, 0x1c, 0x04, 0x43, 0x16, 0x25, 0x2f,
5283
+	0x48, 0x72, 0xd8, 0x6f, 0x91, 0xb2, 0x7e, 0x1c, 0x04, 0xcf, 0x13, 0x12, 0xb6, 0xa3, 0xf5, 0x04,
5284
+	0x7d, 0x00, 0xc5, 0x31, 0x67, 0x71, 0x24, 0x1c, 0x5b, 0x27, 0xc3, 0xcc, 0xd0, 0x17, 0xb0, 0x2b,
5285
+	0xa8, 0xcb, 0xa9, 0x14, 0x4e, 0x59, 0x6f, 0xf5, 0xa3, 0xac, 0x45, 0x06, 0x1a, 0x82, 0xe9, 0x15,
5286
+	0xe5, 0x34, 0x74, 0x29, 0x5e, 0x72, 0xd0, 0x03, 0xd8, 0x96, 0x72, 0xee, 0xec, 0x35, 0xf2, 0x07,
5287
+	0x56, 0x67, 0x77, 0x71, 0x5b, 0xdf, 0xbe, 0xbc, 0x7c, 0x89, 0x95, 0x4d, 0x55, 0xb7, 0x09, 0x13,
5288
+	0x32, 0x24, 0x53, 0xea, 0xdc, 0xd3, 0xb9, 0x5d, 0xcd, 0xd1, 0x4b, 0x00, 0x2f, 0x14, 0x43, 0x57,
5289
+	0x5f, 0x27, 0xe7, 0xbe, 0xde, 0xdd, 0xa7, 0x6f, 0xde, 0x5d, 0xf7, 0x62, 0x60, 0x2a, 0xfc, 0xde,
5290
+	0xe2, 0xb6, 0x5e, 0x5a, 0x4d, 0x71, 0xc9, 0x0b, 0x45, 0x32, 0x44, 0x1d, 0xb0, 0x27, 0x94, 0x04,
5291
+	0x72, 0xe2, 0x4e, 0xa8, 0x7b, 0xed, 0x54, 0xee, 0x2e, 0xd9, 0xcf, 0x34, 0xcc, 0x78, 0x48, 0x93,
5292
+	0x94, 0x82, 0x55, 0xa8, 0xc2, 0xd9, 0xd7, 0xb9, 0x4a, 0x26, 0xe8, 0x43, 0x00, 0x16, 0xd1, 0x70,
5293
+	0x28, 0xa4, 0xe7, 0x87, 0x0e, 0x52, 0x5b, 0xc6, 0x25, 0x65, 0x19, 0x28, 0x03, 0x7a, 0xa8, 0x0a,
5294
+	0x2a, 0xf1, 0x86, 0x2c, 0x0c, 0xe6, 0xce, 0x0f, 0xf4, 0x57, 0x4b, 0x19, 0x9e, 0x87, 0xc1, 0x1c,
5295
+	0xd5, 0xc1, 0xd6, 0xba, 0x10, 0xfe, 0x38, 0x24, 0x81, 0xf3, 0x9e, 0xce, 0x07, 0x28, 0xd3, 0x40,
5296
+	0x5b, 0xaa, 0x9f, 0x83, 0x9d, 0x92, 0xbb, 0x92, 0xe9, 0x35, 0x9d, 0x9b, 0x1b, 0xa4, 0x86, 0x2a,
5297
+	0xa6, 0x19, 0x09, 0xe2, 0xa4, 0x4d, 0x2c, 0xe1, 0x64, 0xf2, 0xeb, 0xad, 0x27, 0xf9, 0xea, 0x11,
5298
+	0xd8, 0xa9, 0x63, 0x47, 0x1f, 0xc1, 0x1e, 0xa7, 0x63, 0x5f, 0x48, 0x3e, 0x1f, 0x92, 0x58, 0x4e,
5299
+	0x9c, 0xdf, 0x69, 0x42, 0x79, 0x69, 0x6c, 0xc7, 0x72, 0x52, 0x1d, 0xc2, 0x3a, 0x7b, 0xa8, 0x01,
5300
+	0xb6, 0x3a, 0x15, 0x41, 0xf9, 0x8c, 0x72, 0xf5, 0x14, 0xa9, 0x4d, 0xa7, 0x4d, 0x4a, 0x3d, 0x82,
5301
+	0x12, 0xee, 0x4e, 0xf4, 0xe5, 0x2d, 0x61, 0x33, 0x53, 0xb7, 0x71, 0x29, 0x51, 0x73, 0x1b, 0xcd,
5302
+	0xb4, 0xd9, 0x04, 0x58, 0x97, 0xa1, 0xec, 0x92, 0xd0, 0xfc, 0x4f, 0x1e, 0xca, 0xe9, 0x57, 0x17,
5303
+	0x1d, 0x27, 0xaf, 0xa5, 0x46, 0xdd, 0x3b, 0x3a, 0x7c, 0xd3, 0x2b, 0xad, 0xdf, 0xa6, 0x20, 0x56,
5304
+	0x0b, 0x9e, 0xab, 0x06, 0x59, 0x93, 0xd1, 0x2f, 0x61, 0x27, 0x62, 0x5c, 0x2e, 0xeb, 0x4c, 0x76,
5305
+	0x85, 0x64, 0x7c, 0x59, 0xcb, 0x13, 0x70, 0x73, 0x02, 0xf7, 0x36, 0xbd, 0xa1, 0xc7, 0xb0, 0xfd,
5306
+	0xe2, 0xb4, 0x5f, 0xc9, 0x55, 0x1f, 0xbe, 0xba, 0x69, 0xfc, 0x70, 0xf3, 0xe3, 0x0b, 0x9f, 0xcb,
5307
+	0x98, 0x04, 0xa7, 0x7d, 0xf4, 0x53, 0xd8, 0xe9, 0x5e, 0x0c, 0x30, 0xae, 0xe4, 0xab, 0xf5, 0x57,
5308
+	0x37, 0x8d, 0x87, 0x9b, 0x38, 0xf5, 0x89, 0xc5, 0xa1, 0x87, 0xd9, 0x68, 0xd5, 0x2c, 0xfe, 0x63,
5309
+	0x0b, 0x6c, 0x53, 0x7e, 0xdf, 0xf5, 0xff, 0xc4, 0x5e, 0xf2, 0x16, 0x2e, 0xef, 0xd5, 0xd6, 0x1b,
5310
+	0x9f, 0xc4, 0x72, 0x42, 0x30, 0x3a, 0x78, 0x04, 0x65, 0x3f, 0x9a, 0x7d, 0x36, 0xa4, 0x21, 0x19,
5311
+	0x05, 0xa6, 0x6f, 0xb4, 0xb0, 0xad, 0x6c, 0xbd, 0xc4, 0xa4, 0x2e, 0xb5, 0x1f, 0x4a, 0xca, 0x43,
5312
+	0xd3, 0x11, 0x5a, 0x78, 0x35, 0x47, 0x5f, 0x40, 0xc1, 0x8f, 0xc8, 0xd4, 0xbc, 0xe3, 0x99, 0x3b,
5313
+	0x38, 0xed, 0xb7, 0xcf, 0x8d, 0x4e, 0x3b, 0xd6, 0xe2, 0xb6, 0x5e, 0x50, 0x06, 0xac, 0x69, 0xa8,
5314
+	0xb6, 0x7c, 0x4a, 0xd5, 0x4a, 0xba, 0x40, 0x5b, 0x38, 0x65, 0x69, 0xfe, 0xb7, 0x00, 0xf6, 0x71,
5315
+	0x10, 0x0b, 0x69, 0x9e, 0x99, 0x77, 0x96, 0xb7, 0x97, 0xb0, 0x4f, 0xf4, 0xaf, 0x05, 0x09, 0x55,
5316
+	0xcd, 0xd6, 0x2d, 0x8a, 0xc9, 0xdd, 0xe3, 0x4c, 0x77, 0x2b, 0x70, 0xd2, 0xce, 0x74, 0x8a, 0xca,
5317
+	0xa7, 0x93, 0xc7, 0x15, 0xf2, 0xad, 0x2f, 0x68, 0x00, 0x7b, 0x8c, 0xbb, 0x13, 0x2a, 0x64, 0x52,
5318
+	0xe9, 0x4d, 0x2b, 0x9e, 0xf9, 0x93, 0xf6, 0x3c, 0x0d, 0x34, 0x65, 0x2e, 0x89, 0x76, 0xd3, 0x07,
5319
+	0x7a, 0x02, 0x05, 0x4e, 0xae, 0x96, 0xed, 0x56, 0xa6, 0xbe, 0x31, 0xb9, 0x92, 0x1b, 0x2e, 0x34,
5320
+	0x03, 0xfd, 0x01, 0xc0, 0xf3, 0x45, 0x44, 0xa4, 0x3b, 0xa1, 0xdc, 0x9c, 0x53, 0xe6, 0x16, 0xbb,
5321
+	0x2b, 0xd4, 0x86, 0x97, 0x14, 0x1b, 0x9d, 0x41, 0xc9, 0x25, 0x4b, 0xa5, 0x15, 0xef, 0xfe, 0x3f,
5322
+	0x39, 0x6e, 0x1b, 0x17, 0x15, 0xe5, 0x62, 0x71, 0x5b, 0xb7, 0x96, 0x16, 0x6c, 0xb9, 0xc4, 0x28,
5323
+	0xef, 0x0c, 0xf6, 0xd4, 0x7f, 0xcb, 0xd0, 0xa3, 0x57, 0x24, 0x0e, 0xa4, 0xd0, 0x8f, 0xf1, 0x1d,
5324
+	0x65, 0x5b, 0x35, 0xc1, 0x5d, 0x83, 0x33, 0x71, 0x95, 0x65, 0xca, 0x86, 0xfe, 0x04, 0xfb, 0x34,
5325
+	0x74, 0xf9, 0x5c, 0xeb, 0x6c, 0x19, 0xa1, 0x75, 0xf7, 0x66, 0x7b, 0x2b, 0xf0, 0xc6, 0x66, 0x2b,
5326
+	0xf4, 0x5b, 0xf6, 0xa6, 0x0f, 0x90, 0x3c, 0x84, 0xef, 0x56, 0x7f, 0x08, 0x0a, 0x1e, 0x91, 0x44,
5327
+	0x4b, 0xae, 0x8c, 0xf5, 0xb8, 0xe3, 0xbc, 0xfe, 0xa6, 0x96, 0xfb, 0xd7, 0x37, 0xb5, 0xdc, 0x5f,
5328
+	0x16, 0xb5, 0xfc, 0xeb, 0x45, 0x2d, 0xff, 0xcf, 0x45, 0x2d, 0xff, 0xef, 0x45, 0x2d, 0x3f, 0x2a,
5329
+	0xea, 0xf6, 0xe1, 0x17, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x91, 0xbf, 0x5e, 0xca, 0xf8, 0x10,
5330
+	0x00, 0x00,
5329 5331
 }
... ...
@@ -101,6 +101,7 @@ message TaskSpec {
101 101
 	oneof runtime {
102 102
 		NetworkAttachmentSpec attachment = 8;
103 103
 		ContainerSpec container = 1;
104
+		PluginSpec plugin = 10;
104 105
 	}
105 106
 
106 107
 	// Resource requirements for the container.
... ...
@@ -131,7 +132,7 @@ message TaskSpec {
131 131
 // NetworkAttachmentSpec specifies runtime parameters required to attach
132 132
 // a container to a network.
133 133
 message NetworkAttachmentSpec {
134
-	// ContainerID spcifies a unique ID of the container for which
134
+	// ContainerID specifies a unique ID of the container for which
135 135
 	// this attachment is for.
136 136
 	string container_id = 1;
137 137
 }
... ...
@@ -266,12 +267,20 @@ message ContainerSpec {
266 266
 	HealthConfig healthcheck = 16;
267 267
 }
268 268
 
269
+// PluginSpec specifies runtime parameters for a plugin.
270
+message PluginSpec {
271
+	// image defines the image reference, as specified in the
272
+	// distribution/reference package. This may include a registry host, name,
273
+	// tag or digest.
274
+	string image = 1;
275
+}
276
+
269 277
 // EndpointSpec defines the properties that can be configured to
270 278
 // access and loadbalance the service.
271 279
 message EndpointSpec {
272 280
 	// ResolutionMode specifies the mode of resolution to use for
273 281
 	// internal loadbalancing between tasks which are all within
274
-	// the cluster. This is sometimes calles east-west data path.
282
+	// the cluster. This is sometimes calls east-west data path.
275 283
 	enum ResolutionMode {
276 284
 		option (gogoproto.goproto_enum_prefix) = false;
277 285
 
... ...
@@ -72,6 +72,7 @@
72 72
 		TaskSpec
73 73
 		NetworkAttachmentSpec
74 74
 		ContainerSpec
75
+		PluginSpec
75 76
 		EndpointSpec
76 77
 		NetworkSpec
77 78
 		ClusterSpec
... ...
@@ -54,7 +54,7 @@ const (
54 54
 // RenewTLSExponentialBackoff sets the exponential backoff when trying to renew TLS certificates that have expired
55 55
 var RenewTLSExponentialBackoff = events.ExponentialBackoffConfig{
56 56
 	Base:   time.Second * 5,
57
-	Factor: time.Minute,
57
+	Factor: time.Second * 5,
58 58
 	Max:    1 * time.Hour,
59 59
 }
60 60
 
... ...
@@ -454,7 +454,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
454 454
 	updates := make(chan CertificateUpdate)
455 455
 
456 456
 	go func() {
457
-		var retry time.Duration
457
+		var (
458
+			retry      time.Duration
459
+			forceRetry bool
460
+		)
458 461
 		expBackoff := events.NewExponentialBackoff(RenewTLSExponentialBackoff)
459 462
 		defer close(updates)
460 463
 		for {
... ...
@@ -487,6 +490,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
487 487
 					log.Warn("the current TLS certificate is expired, so an attempt to renew it will be made immediately")
488 488
 					// retry immediately(ish) with exponential backoff
489 489
 					retry = expBackoff.Proceed(nil)
490
+				} else if forceRetry {
491
+					// A forced renewal was requested, but did not succeed yet.
492
+					// retry immediately(ish) with exponential backoff
493
+					retry = expBackoff.Proceed(nil)
490 494
 				} else {
491 495
 					// Random retry time between 50% and 80% of the total time to expiration
492 496
 					retry = calculateRandomExpiry(validFrom, validUntil)
... ...
@@ -501,6 +508,7 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
501 501
 			case <-time.After(retry):
502 502
 				log.Info("renewing certificate")
503 503
 			case <-renew:
504
+				forceRetry = true
504 505
 				log.Info("forced certificate renewal")
505 506
 			case <-ctx.Done():
506 507
 				log.Info("shutting down certificate renewal routine")
... ...
@@ -515,6 +523,7 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
515 515
 			} else {
516 516
 				certUpdate.Role = s.ClientTLSCreds.Role()
517 517
 				expBackoff = events.NewExponentialBackoff(RenewTLSExponentialBackoff)
518
+				forceRetry = false
518 519
 			}
519 520
 
520 521
 			select {
... ...
@@ -715,7 +715,7 @@ func (s *Server) signNodeCert(ctx context.Context, node *api.Node) error {
715 715
 	return nil
716 716
 }
717 717
 
718
-// reconcileNodeCertificates is a helper method that calles evaluateAndSignNodeCert on all the
718
+// reconcileNodeCertificates is a helper method that calls evaluateAndSignNodeCert on all the
719 719
 // nodes.
720 720
 func (s *Server) reconcileNodeCertificates(ctx context.Context, nodes []*api.Node) error {
721 721
 	for _, node := range nodes {
... ...
@@ -8,6 +8,7 @@ import (
8 8
 
9 9
 	"github.com/docker/swarmkit/api"
10 10
 	"github.com/docker/swarmkit/remotes"
11
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
11 12
 	"google.golang.org/grpc"
12 13
 )
13 14
 
... ...
@@ -59,6 +60,10 @@ func (b *Broker) SelectRemote(dialOpts ...grpc.DialOption) (*Conn, error) {
59 59
 		return nil, err
60 60
 	}
61 61
 
62
+	dialOpts = append(dialOpts,
63
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
64
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor))
65
+
62 66
 	cc, err := grpc.Dial(peer.Addr, dialOpts...)
63 67
 	if err != nil {
64 68
 		b.remotes.ObserveIfExists(peer, -remotes.DefaultObservationWeight)
... ...
@@ -682,7 +682,7 @@ func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, str
682 682
 func (na *NetworkAllocator) loadDriver(name string) error {
683 683
 	pg := na.drvRegistry.GetPluginGetter()
684 684
 	if pg == nil {
685
-		return fmt.Errorf("plugin store is unintialized")
685
+		return fmt.Errorf("plugin store is uninitialized")
686 686
 	}
687 687
 	_, err := pg.Get(name, driverapi.NetworkPluginEndpointType, plugingetter.Lookup)
688 688
 	return err
... ...
@@ -117,7 +117,29 @@ func validateUpdate(uc *api.UpdateConfig) error {
117 117
 	return nil
118 118
 }
119 119
 
120
-func validateContainerSpec(container *api.ContainerSpec) error {
120
+func validateContainerSpec(taskSpec api.TaskSpec) error {
121
+	// Building a empty/dummy Task to validate the templating and
122
+	// the resulting container spec as well. This is a *best effort*
123
+	// validation.
124
+	container, err := template.ExpandContainerSpec(&api.Task{
125
+		Spec:      taskSpec,
126
+		ServiceID: "serviceid",
127
+		Slot:      1,
128
+		NodeID:    "nodeid",
129
+		Networks:  []*api.NetworkAttachment{},
130
+		Annotations: api.Annotations{
131
+			Name: "taskname",
132
+		},
133
+		ServiceAnnotations: api.Annotations{
134
+			Name: "servicename",
135
+		},
136
+		Endpoint:  &api.Endpoint{},
137
+		LogDriver: taskSpec.LogDriver,
138
+	})
139
+	if err != nil {
140
+		return grpc.Errorf(codes.InvalidArgument, err.Error())
141
+	}
142
+
121 143
 	if container == nil {
122 144
 		return grpc.Errorf(codes.InvalidArgument, "ContainerSpec: missing in service spec")
123 145
 	}
... ...
@@ -141,6 +163,18 @@ func validateContainerSpec(container *api.ContainerSpec) error {
141 141
 	return nil
142 142
 }
143 143
 
144
+func validatePluginSpec(plugin *api.PluginSpec) error {
145
+	if plugin.Image == "" {
146
+		return grpc.Errorf(codes.InvalidArgument, "PluginSpec: image reference must be provided")
147
+	}
148
+
149
+	if _, err := reference.ParseNormalizedNamed(plugin.Image); err != nil {
150
+		return grpc.Errorf(codes.InvalidArgument, "PluginSpec: %q is not a valid repository/tag", plugin.Image)
151
+	}
152
+
153
+	return nil
154
+}
155
+
144 156
 func validateTaskSpec(taskSpec api.TaskSpec) error {
145 157
 	if err := validateResourceRequirements(taskSpec.Resources); err != nil {
146 158
 		return err
... ...
@@ -163,36 +197,18 @@ func validateTaskSpec(taskSpec api.TaskSpec) error {
163 163
 		return grpc.Errorf(codes.InvalidArgument, "TaskSpec: missing runtime")
164 164
 	}
165 165
 
166
-	_, ok := taskSpec.GetRuntime().(*api.TaskSpec_Container)
167
-	if !ok {
166
+	switch taskSpec.GetRuntime().(type) {
167
+	case *api.TaskSpec_Container:
168
+		if err := validateContainerSpec(taskSpec); err != nil {
169
+			return err
170
+		}
171
+	case *api.TaskSpec_Plugin:
172
+		if err := validatePluginSpec(taskSpec.GetPlugin()); err != nil {
173
+			return err
174
+		}
175
+	default:
168 176
 		return grpc.Errorf(codes.Unimplemented, "RuntimeSpec: unimplemented runtime in service spec")
169 177
 	}
170
-
171
-	// Building a empty/dummy Task to validate the templating and
172
-	// the resulting container spec as well. This is a *best effort*
173
-	// validation.
174
-	preparedSpec, err := template.ExpandContainerSpec(&api.Task{
175
-		Spec:      taskSpec,
176
-		ServiceID: "serviceid",
177
-		Slot:      1,
178
-		NodeID:    "nodeid",
179
-		Networks:  []*api.NetworkAttachment{},
180
-		Annotations: api.Annotations{
181
-			Name: "taskname",
182
-		},
183
-		ServiceAnnotations: api.Annotations{
184
-			Name: "servicename",
185
-		},
186
-		Endpoint:  &api.Endpoint{},
187
-		LogDriver: taskSpec.LogDriver,
188
-	})
189
-	if err != nil {
190
-		return grpc.Errorf(codes.InvalidArgument, err.Error())
191
-	}
192
-	if err := validateContainerSpec(preparedSpec); err != nil {
193
-		return err
194
-	}
195
-
196 178
 	return nil
197 179
 }
198 180
 
... ...
@@ -38,6 +38,7 @@ import (
38 38
 	"github.com/docker/swarmkit/remotes"
39 39
 	"github.com/docker/swarmkit/xnet"
40 40
 	gogotypes "github.com/gogo/protobuf/types"
41
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
41 42
 	"github.com/pkg/errors"
42 43
 	"golang.org/x/net/context"
43 44
 	"google.golang.org/grpc"
... ...
@@ -201,7 +202,10 @@ func New(config *Config) (*Manager, error) {
201 201
 	raftNode := raft.NewNode(newNodeOpts)
202 202
 
203 203
 	opts := []grpc.ServerOption{
204
-		grpc.Creds(config.SecurityConfig.ServerTLSCreds)}
204
+		grpc.Creds(config.SecurityConfig.ServerTLSCreds),
205
+		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
206
+		grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
207
+	}
205 208
 
206 209
 	m := &Manager{
207 210
 		config:          *config,
... ...
@@ -458,6 +462,7 @@ func (m *Manager) Run(parent context.Context) error {
458 458
 	api.RegisterLogBrokerServer(m.server, proxyLogBrokerAPI)
459 459
 	api.RegisterResourceAllocatorServer(m.server, proxyResourceAPI)
460 460
 	api.RegisterDispatcherServer(m.server, proxyDispatcherAPI)
461
+	grpc_prometheus.Register(m.server)
461 462
 
462 463
 	api.RegisterControlServer(m.localserver, localProxyControlAPI)
463 464
 	api.RegisterLogsServer(m.localserver, localProxyLogsAPI)
... ...
@@ -467,6 +472,7 @@ func (m *Manager) Run(parent context.Context) error {
467 467
 	api.RegisterNodeCAServer(m.localserver, localProxyNodeCAAPI)
468 468
 	api.RegisterResourceAllocatorServer(m.localserver, localProxyResourceAPI)
469 469
 	api.RegisterLogBrokerServer(m.localserver, localProxyLogBrokerAPI)
470
+	grpc_prometheus.Register(m.localserver)
470 471
 
471 472
 	healthServer.SetServingStatus("Raft", api.HealthCheckResponse_NOT_SERVING)
472 473
 	localHealthServer.SetServingStatus("ControlAPI", api.HealthCheckResponse_NOT_SERVING)
... ...
@@ -648,6 +654,8 @@ func (m *Manager) updateKEK(ctx context.Context, cluster *api.Cluster) error {
648 648
 
649 649
 			conn, err := grpc.Dial(
650 650
 				m.config.ControlAPI,
651
+				grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
652
+				grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
651 653
 				grpc.WithTransportCredentials(insecureCreds),
652 654
 				grpc.WithDialer(
653 655
 					func(addr string, timeout time.Duration) (net.Conn, error) {
... ...
@@ -21,7 +21,7 @@ import (
21 21
 	gogotypes "github.com/gogo/protobuf/types"
22 22
 )
23 23
 
24
-const defaultMonitor = 30 * time.Second
24
+const defaultMonitor = 5 * time.Second
25 25
 
26 26
 // Supervisor supervises a set of updates. It's responsible for keeping track of updates,
27 27
 // shutting them down and replacing them.
... ...
@@ -157,7 +157,7 @@ func (u *Updater) Run(ctx context.Context, slots []orchestrator.Slot) {
157 157
 	}
158 158
 
159 159
 	var (
160
-		parallelism            int
160
+		parallelism            = 1
161 161
 		delay                  time.Duration
162 162
 		failureAction          = api.UpdateConfig_PAUSE
163 163
 		allowedFailureFraction = float32(0)
... ...
@@ -354,7 +354,7 @@ func (u *Updater) worker(ctx context.Context, queue <-chan orchestrator.Slot, de
354 354
 
355 355
 		if delay != 0 {
356 356
 			select {
357
-			case <-time.After(u.newService.Spec.Update.Delay):
357
+			case <-time.After(delay):
358 358
 			case <-u.stopChan:
359 359
 				return
360 360
 			}
... ...
@@ -136,11 +136,21 @@ func (rm *roleManager) reconcileRole(node *api.Node) {
136 136
 			rmCtx, rmCancel := context.WithTimeout(rm.ctx, 5*time.Second)
137 137
 			defer rmCancel()
138 138
 
139
+			if member.RaftID == rm.raft.Config.ID {
140
+				// Don't use rmCtx, because we expect to lose
141
+				// leadership, which will cancel this context.
142
+				log.L.Info("demoted; transferring leadership")
143
+				err := rm.raft.TransferLeadership(context.Background())
144
+				if err == nil {
145
+					return
146
+				}
147
+				log.L.WithError(err).Info("failed to transfer leadership")
148
+			}
139 149
 			if err := rm.raft.RemoveMember(rmCtx, member.RaftID); err != nil {
140 150
 				// TODO(aaronl): Retry later
141 151
 				log.L.WithError(err).Debugf("can't demote node %s at this time", node.ID)
142
-				return
143 152
 			}
153
+			return
144 154
 		}
145 155
 
146 156
 		err := rm.store.Update(func(tx store.Tx) error {
... ...
@@ -11,10 +11,10 @@ import (
11 11
 // NodeInfo contains a node and some additional metadata.
12 12
 type NodeInfo struct {
13 13
 	*api.Node
14
-	Tasks                             map[string]*api.Task
15
-	DesiredRunningTasksCount          int
16
-	DesiredRunningTasksCountByService map[string]int
17
-	AvailableResources                api.Resources
14
+	Tasks                     map[string]*api.Task
15
+	ActiveTasksCount          int
16
+	ActiveTasksCountByService map[string]int
17
+	AvailableResources        api.Resources
18 18
 
19 19
 	// recentFailures is a map from service ID to the timestamps of the
20 20
 	// most recent failures the node has experienced from replicas of that
... ...
@@ -28,9 +28,9 @@ func newNodeInfo(n *api.Node, tasks map[string]*api.Task, availableResources api
28 28
 	nodeInfo := NodeInfo{
29 29
 		Node:  n,
30 30
 		Tasks: make(map[string]*api.Task),
31
-		DesiredRunningTasksCountByService: make(map[string]int),
32
-		AvailableResources:                availableResources,
33
-		recentFailures:                    make(map[string][]time.Time),
31
+		ActiveTasksCountByService: make(map[string]int),
32
+		AvailableResources:        availableResources,
33
+		recentFailures:            make(map[string][]time.Time),
34 34
 	}
35 35
 
36 36
 	for _, t := range tasks {
... ...
@@ -48,9 +48,9 @@ func (nodeInfo *NodeInfo) removeTask(t *api.Task) bool {
48 48
 	}
49 49
 
50 50
 	delete(nodeInfo.Tasks, t.ID)
51
-	if oldTask.DesiredState == api.TaskStateRunning {
52
-		nodeInfo.DesiredRunningTasksCount--
53
-		nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]--
51
+	if oldTask.DesiredState <= api.TaskStateRunning {
52
+		nodeInfo.ActiveTasksCount--
53
+		nodeInfo.ActiveTasksCountByService[t.ServiceID]--
54 54
 	}
55 55
 
56 56
 	reservations := taskReservations(t.Spec)
... ...
@@ -65,15 +65,15 @@ func (nodeInfo *NodeInfo) removeTask(t *api.Task) bool {
65 65
 func (nodeInfo *NodeInfo) addTask(t *api.Task) bool {
66 66
 	oldTask, ok := nodeInfo.Tasks[t.ID]
67 67
 	if ok {
68
-		if t.DesiredState == api.TaskStateRunning && oldTask.DesiredState != api.TaskStateRunning {
68
+		if t.DesiredState <= api.TaskStateRunning && oldTask.DesiredState > api.TaskStateRunning {
69 69
 			nodeInfo.Tasks[t.ID] = t
70
-			nodeInfo.DesiredRunningTasksCount++
71
-			nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]++
70
+			nodeInfo.ActiveTasksCount++
71
+			nodeInfo.ActiveTasksCountByService[t.ServiceID]++
72 72
 			return true
73
-		} else if t.DesiredState != api.TaskStateRunning && oldTask.DesiredState == api.TaskStateRunning {
73
+		} else if t.DesiredState > api.TaskStateRunning && oldTask.DesiredState <= api.TaskStateRunning {
74 74
 			nodeInfo.Tasks[t.ID] = t
75
-			nodeInfo.DesiredRunningTasksCount--
76
-			nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]--
75
+			nodeInfo.ActiveTasksCount--
76
+			nodeInfo.ActiveTasksCountByService[t.ServiceID]--
77 77
 			return true
78 78
 		}
79 79
 		return false
... ...
@@ -84,9 +84,9 @@ func (nodeInfo *NodeInfo) addTask(t *api.Task) bool {
84 84
 	nodeInfo.AvailableResources.MemoryBytes -= reservations.MemoryBytes
85 85
 	nodeInfo.AvailableResources.NanoCPUs -= reservations.NanoCPUs
86 86
 
87
-	if t.DesiredState == api.TaskStateRunning {
88
-		nodeInfo.DesiredRunningTasksCount++
89
-		nodeInfo.DesiredRunningTasksCountByService[t.ServiceID]++
87
+	if t.DesiredState <= api.TaskStateRunning {
88
+		nodeInfo.ActiveTasksCount++
89
+		nodeInfo.ActiveTasksCountByService[t.ServiceID]++
90 90
 	}
91 91
 
92 92
 	return true
... ...
@@ -35,8 +35,8 @@ func (ns *nodeSet) addOrUpdateNode(n NodeInfo) {
35 35
 	if n.Tasks == nil {
36 36
 		n.Tasks = make(map[string]*api.Task)
37 37
 	}
38
-	if n.DesiredRunningTasksCountByService == nil {
39
-		n.DesiredRunningTasksCountByService = make(map[string]int)
38
+	if n.ActiveTasksCountByService == nil {
39
+		n.ActiveTasksCountByService = make(map[string]int)
40 40
 	}
41 41
 	if n.recentFailures == nil {
42 42
 		n.recentFailures = make(map[string][]time.Time)
... ...
@@ -96,8 +96,8 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference
96 96
 			// sure that the tree structure is not affected by
97 97
 			// which properties nodes have and don't have.
98 98
 
99
-			if node.DesiredRunningTasksCountByService != nil {
100
-				tree.tasks += node.DesiredRunningTasksCountByService[serviceID]
99
+			if node.ActiveTasksCountByService != nil {
100
+				tree.tasks += node.ActiveTasksCountByService[serviceID]
101 101
 			}
102 102
 
103 103
 			if tree.next == nil {
... ...
@@ -517,8 +517,8 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]
517 517
 			}
518 518
 		}
519 519
 
520
-		tasksByServiceA := a.DesiredRunningTasksCountByService[t.ServiceID]
521
-		tasksByServiceB := b.DesiredRunningTasksCountByService[t.ServiceID]
520
+		tasksByServiceA := a.ActiveTasksCountByService[t.ServiceID]
521
+		tasksByServiceB := b.ActiveTasksCountByService[t.ServiceID]
522 522
 
523 523
 		if tasksByServiceA < tasksByServiceB {
524 524
 			return true
... ...
@@ -528,7 +528,7 @@ func (s *Scheduler) scheduleTaskGroup(ctx context.Context, taskGroup map[string]
528 528
 		}
529 529
 
530 530
 		// Total number of tasks breaks ties.
531
-		return a.DesiredRunningTasksCount < b.DesiredRunningTasksCount
531
+		return a.ActiveTasksCount < b.ActiveTasksCount
532 532
 	}
533 533
 
534 534
 	var prefs []*api.PlacementPreference
... ...
@@ -412,7 +412,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) {
412 412
 	defer conn.Close()
413 413
 	client := api.NewRaftMembershipClient(conn)
414 414
 
415
-	joinCtx, joinCancel := context.WithTimeout(ctx, 10*time.Second)
415
+	joinCtx, joinCancel := context.WithTimeout(ctx, n.reqTimeout())
416 416
 	defer joinCancel()
417 417
 	resp, err := client.Join(joinCtx, &api.JoinRequest{
418 418
 		Addr: n.opts.Addr,
... ...
@@ -1030,6 +1030,10 @@ func (n *Node) UpdateNode(id uint64, addr string) {
1030 1030
 // from a member who is willing to leave its raft
1031 1031
 // membership to an active member of the raft
1032 1032
 func (n *Node) Leave(ctx context.Context, req *api.LeaveRequest) (*api.LeaveResponse, error) {
1033
+	if req.Node == nil {
1034
+		return nil, grpc.Errorf(codes.InvalidArgument, "no node information provided")
1035
+	}
1036
+
1033 1037
 	nodeInfo, err := ca.RemoteNode(ctx)
1034 1038
 	if err != nil {
1035 1039
 		return nil, err
... ...
@@ -1100,18 +1104,58 @@ func (n *Node) removeMember(ctx context.Context, id uint64) error {
1100 1100
 
1101 1101
 	n.membershipLock.Lock()
1102 1102
 	defer n.membershipLock.Unlock()
1103
-	if n.CanRemoveMember(id) {
1104
-		cc := raftpb.ConfChange{
1105
-			ID:      id,
1106
-			Type:    raftpb.ConfChangeRemoveNode,
1107
-			NodeID:  id,
1108
-			Context: []byte(""),
1109
-		}
1110
-		err := n.configure(ctx, cc)
1111
-		return err
1103
+	if !n.CanRemoveMember(id) {
1104
+		return ErrCannotRemoveMember
1112 1105
 	}
1113 1106
 
1114
-	return ErrCannotRemoveMember
1107
+	cc := raftpb.ConfChange{
1108
+		ID:      id,
1109
+		Type:    raftpb.ConfChangeRemoveNode,
1110
+		NodeID:  id,
1111
+		Context: []byte(""),
1112
+	}
1113
+	return n.configure(ctx, cc)
1114
+}
1115
+
1116
+// TransferLeadership attempts to transfer leadership to a different node,
1117
+// and wait for the transfer to happen.
1118
+func (n *Node) TransferLeadership(ctx context.Context) error {
1119
+	ctx, cancelTransfer := context.WithTimeout(ctx, n.reqTimeout())
1120
+	defer cancelTransfer()
1121
+
1122
+	n.stopMu.RLock()
1123
+	defer n.stopMu.RUnlock()
1124
+
1125
+	if !n.IsMember() {
1126
+		return ErrNoRaftMember
1127
+	}
1128
+
1129
+	if !n.isLeader() {
1130
+		return ErrLostLeadership
1131
+	}
1132
+
1133
+	transferee, err := n.transport.LongestActive()
1134
+	if err != nil {
1135
+		return errors.Wrap(err, "failed to get longest-active member")
1136
+	}
1137
+	start := time.Now()
1138
+	n.raftNode.TransferLeadership(ctx, n.Config.ID, transferee)
1139
+	ticker := time.NewTicker(n.opts.TickInterval / 10)
1140
+	defer ticker.Stop()
1141
+	var leader uint64
1142
+	for {
1143
+		leader = n.leader()
1144
+		if leader != raft.None && leader != n.Config.ID {
1145
+			break
1146
+		}
1147
+		select {
1148
+		case <-ctx.Done():
1149
+			return ctx.Err()
1150
+		case <-ticker.C:
1151
+		}
1152
+	}
1153
+	log.G(ctx).Infof("raft: transfer leadership %x -> %x finished in %v", n.Config.ID, leader, time.Since(start))
1154
+	return nil
1115 1155
 }
1116 1156
 
1117 1157
 // RemoveMember submits a configuration change to remove a member from the raft cluster
... ...
@@ -1726,23 +1770,12 @@ func (n *Node) applyRemoveNode(ctx context.Context, cc raftpb.ConfChange) (err e
1726 1726
 	}
1727 1727
 
1728 1728
 	if cc.NodeID == n.Config.ID {
1729
-		// wait the commit ack to be sent before closing connection
1729
+		// wait for the commit ack to be sent before closing connection
1730 1730
 		n.asyncTasks.Wait()
1731 1731
 
1732 1732
 		n.NodeRemoved()
1733
-		// if there are only 2 nodes in the cluster, and leader is leaving
1734
-		// before closing the connection, leader has to ensure that follower gets
1735
-		// noticed about this raft conf change commit. Otherwise, follower would
1736
-		// assume there are still 2 nodes in the cluster and won't get elected
1737
-		// into the leader by acquiring the majority (2 nodes)
1738
-
1739
-		// while n.asyncTasks.Wait() could be helpful in this case
1740
-		// it's the best-effort strategy, because this send could be fail due to some errors (such as time limit exceeds)
1741
-		// TODO(Runshen Zhu): use leadership transfer to solve this case, after vendoring raft 3.0+
1742
-	} else {
1743
-		if err := n.transport.RemovePeer(cc.NodeID); err != nil {
1744
-			return err
1745
-		}
1733
+	} else if err := n.transport.RemovePeer(cc.NodeID); err != nil {
1734
+		return err
1746 1735
 	}
1747 1736
 
1748 1737
 	return n.cluster.RemoveMember(cc.NodeID)
... ...
@@ -1852,3 +1885,7 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 {
1852 1852
 	}
1853 1853
 	return sids
1854 1854
 }
1855
+
1856
+func (n *Node) reqTimeout() time.Duration {
1857
+	return 5*time.Second + 2*time.Duration(n.Config.ElectionTick)*n.opts.TickInterval
1858
+}
... ...
@@ -134,7 +134,7 @@ func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error {
134 134
 		// force commit newly appended entries
135 135
 		err := n.raftLogger.SaveEntries(st, toAppEnts)
136 136
 		if err != nil {
137
-			log.G(ctx).WithError(err).Fatalf("failed to save WAL while forcing new cluster")
137
+			log.G(ctx).WithError(err).Fatal("failed to save WAL while forcing new cluster")
138 138
 		}
139 139
 		if len(toAppEnts) != 0 {
140 140
 			st.Commit = toAppEnts[len(toAppEnts)-1].Index
... ...
@@ -8,6 +8,7 @@ import (
8 8
 
9 9
 	"golang.org/x/net/context"
10 10
 
11
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
11 12
 	"google.golang.org/grpc"
12 13
 	"google.golang.org/grpc/credentials"
13 14
 
... ...
@@ -295,6 +296,19 @@ func (t *Transport) Active(id uint64) bool {
295 295
 	return active
296 296
 }
297 297
 
298
+// LongestActive returns the ID of the peer that has been active for the longest
299
+// length of time.
300
+func (t *Transport) LongestActive() (uint64, error) {
301
+	p, err := t.longestActive()
302
+	if err != nil {
303
+		return 0, err
304
+	}
305
+
306
+	return p.id, nil
307
+}
308
+
309
+// longestActive returns the peer that has been active for the longest length of
310
+// time.
298 311
 func (t *Transport) longestActive() (*peer, error) {
299 312
 	var longest *peer
300 313
 	var longestTime time.Time
... ...
@@ -322,6 +336,8 @@ func (t *Transport) longestActive() (*peer, error) {
322 322
 
323 323
 func (t *Transport) dial(addr string) (*grpc.ClientConn, error) {
324 324
 	grpcOptions := []grpc.DialOption{
325
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
326
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
325 327
 		grpc.WithBackoffMaxDelay(8 * time.Second),
326 328
 	}
327 329
 	if t.config.Credentials != nil {
... ...
@@ -8,6 +8,7 @@ import (
8 8
 	"github.com/docker/swarmkit/api"
9 9
 	"github.com/docker/swarmkit/manager/state"
10 10
 	"github.com/docker/swarmkit/manager/state/store"
11
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
11 12
 	"google.golang.org/grpc"
12 13
 	"google.golang.org/grpc/credentials"
13 14
 )
... ...
@@ -17,6 +18,8 @@ func dial(addr string, protocol string, creds credentials.TransportCredentials,
17 17
 	grpcOptions := []grpc.DialOption{
18 18
 		grpc.WithBackoffMaxDelay(2 * time.Second),
19 19
 		grpc.WithTransportCredentials(creds),
20
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
21
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
20 22
 	}
21 23
 
22 24
 	if timeout != 0 {
... ...
@@ -26,6 +26,7 @@ import (
26 26
 	"github.com/docker/swarmkit/manager/encryption"
27 27
 	"github.com/docker/swarmkit/remotes"
28 28
 	"github.com/docker/swarmkit/xnet"
29
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
29 30
 	"github.com/pkg/errors"
30 31
 	"golang.org/x/net/context"
31 32
 	"google.golang.org/grpc"
... ...
@@ -640,7 +641,10 @@ func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, erro
640 640
 }
641 641
 
642 642
 func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
643
-	opts := []grpc.DialOption{}
643
+	opts := []grpc.DialOption{
644
+		grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
645
+		grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
646
+	}
644 647
 	insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
645 648
 	opts = append(opts, grpc.WithTransportCredentials(insecureCreds))
646 649
 	addr := n.config.ListenControlAPI
647 650
new file mode 100644
... ...
@@ -0,0 +1,201 @@
0
+                 Apache License
1
+                           Version 2.0, January 2004
2
+                        http://www.apache.org/licenses/
3
+
4
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
5
+
6
+   1. Definitions.
7
+
8
+      "License" shall mean the terms and conditions for use, reproduction,
9
+      and distribution as defined by Sections 1 through 9 of this document.
10
+
11
+      "Licensor" shall mean the copyright owner or entity authorized by
12
+      the copyright owner that is granting the License.
13
+
14
+      "Legal Entity" shall mean the union of the acting entity and all
15
+      other entities that control, are controlled by, or are under common
16
+      control with that entity. For the purposes of this definition,
17
+      "control" means (i) the power, direct or indirect, to cause the
18
+      direction or management of such entity, whether by contract or
19
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
20
+      outstanding shares, or (iii) beneficial ownership of such entity.
21
+
22
+      "You" (or "Your") shall mean an individual or Legal Entity
23
+      exercising permissions granted by this License.
24
+
25
+      "Source" form shall mean the preferred form for making modifications,
26
+      including but not limited to software source code, documentation
27
+      source, and configuration files.
28
+
29
+      "Object" form shall mean any form resulting from mechanical
30
+      transformation or translation of a Source form, including but
31
+      not limited to compiled object code, generated documentation,
32
+      and conversions to other media types.
33
+
34
+      "Work" shall mean the work of authorship, whether in Source or
35
+      Object form, made available under the License, as indicated by a
36
+      copyright notice that is included in or attached to the work
37
+      (an example is provided in the Appendix below).
38
+
39
+      "Derivative Works" shall mean any work, whether in Source or Object
40
+      form, that is based on (or derived from) the Work and for which the
41
+      editorial revisions, annotations, elaborations, or other modifications
42
+      represent, as a whole, an original work of authorship. For the purposes
43
+      of this License, Derivative Works shall not include works that remain
44
+      separable from, or merely link (or bind by name) to the interfaces of,
45
+      the Work and Derivative Works thereof.
46
+
47
+      "Contribution" shall mean any work of authorship, including
48
+      the original version of the Work and any modifications or additions
49
+      to that Work or Derivative Works thereof, that is intentionally
50
+      submitted to Licensor for inclusion in the Work by the copyright owner
51
+      or by an individual or Legal Entity authorized to submit on behalf of
52
+      the copyright owner. For the purposes of this definition, "submitted"
53
+      means any form of electronic, verbal, or written communication sent
54
+      to the Licensor or its representatives, including but not limited to
55
+      communication on electronic mailing lists, source code control systems,
56
+      and issue tracking systems that are managed by, or on behalf of, the
57
+      Licensor for the purpose of discussing and improving the Work, but
58
+      excluding communication that is conspicuously marked or otherwise
59
+      designated in writing by the copyright owner as "Not a Contribution."
60
+
61
+      "Contributor" shall mean Licensor and any individual or Legal Entity
62
+      on behalf of whom a Contribution has been received by Licensor and
63
+      subsequently incorporated within the Work.
64
+
65
+   2. Grant of Copyright License. Subject to the terms and conditions of
66
+      this License, each Contributor hereby grants to You a perpetual,
67
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
68
+      copyright license to reproduce, prepare Derivative Works of,
69
+      publicly display, publicly perform, sublicense, and distribute the
70
+      Work and such Derivative Works in Source or Object form.
71
+
72
+   3. Grant of Patent License. Subject to the terms and conditions of
73
+      this License, each Contributor hereby grants to You a perpetual,
74
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
75
+      (except as stated in this section) patent license to make, have made,
76
+      use, offer to sell, sell, import, and otherwise transfer the Work,
77
+      where such license applies only to those patent claims licensable
78
+      by such Contributor that are necessarily infringed by their
79
+      Contribution(s) alone or by combination of their Contribution(s)
80
+      with the Work to which such Contribution(s) was submitted. If You
81
+      institute patent litigation against any entity (including a
82
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
83
+      or a Contribution incorporated within the Work constitutes direct
84
+      or contributory patent infringement, then any patent licenses
85
+      granted to You under this License for that Work shall terminate
86
+      as of the date such litigation is filed.
87
+
88
+   4. Redistribution. You may reproduce and distribute copies of the
89
+      Work or Derivative Works thereof in any medium, with or without
90
+      modifications, and in Source or Object form, provided that You
91
+      meet the following conditions:
92
+
93
+      (a) You must give any other recipients of the Work or
94
+          Derivative Works a copy of this License; and
95
+
96
+      (b) You must cause any modified files to carry prominent notices
97
+          stating that You changed the files; and
98
+
99
+      (c) You must retain, in the Source form of any Derivative Works
100
+          that You distribute, all copyright, patent, trademark, and
101
+          attribution notices from the Source form of the Work,
102
+          excluding those notices that do not pertain to any part of
103
+          the Derivative Works; and
104
+
105
+      (d) If the Work includes a "NOTICE" text file as part of its
106
+          distribution, then any Derivative Works that You distribute must
107
+          include a readable copy of the attribution notices contained
108
+          within such NOTICE file, excluding those notices that do not
109
+          pertain to any part of the Derivative Works, in at least one
110
+          of the following places: within a NOTICE text file distributed
111
+          as part of the Derivative Works; within the Source form or
112
+          documentation, if provided along with the Derivative Works; or,
113
+          within a display generated by the Derivative Works, if and
114
+          wherever such third-party notices normally appear. The contents
115
+          of the NOTICE file are for informational purposes only and
116
+          do not modify the License. You may add Your own attribution
117
+          notices within Derivative Works that You distribute, alongside
118
+          or as an addendum to the NOTICE text from the Work, provided
119
+          that such additional attribution notices cannot be construed
120
+          as modifying the License.
121
+
122
+      You may add Your own copyright statement to Your modifications and
123
+      may provide additional or different license terms and conditions
124
+      for use, reproduction, or distribution of Your modifications, or
125
+      for any such Derivative Works as a whole, provided Your use,
126
+      reproduction, and distribution of the Work otherwise complies with
127
+      the conditions stated in this License.
128
+
129
+   5. Submission of Contributions. Unless You explicitly state otherwise,
130
+      any Contribution intentionally submitted for inclusion in the Work
131
+      by You to the Licensor shall be under the terms and conditions of
132
+      this License, without any additional terms or conditions.
133
+      Notwithstanding the above, nothing herein shall supersede or modify
134
+      the terms of any separate license agreement you may have executed
135
+      with Licensor regarding such Contributions.
136
+
137
+   6. Trademarks. This License does not grant permission to use the trade
138
+      names, trademarks, service marks, or product names of the Licensor,
139
+      except as required for reasonable and customary use in describing the
140
+      origin of the Work and reproducing the content of the NOTICE file.
141
+
142
+   7. Disclaimer of Warranty. Unless required by applicable law or
143
+      agreed to in writing, Licensor provides the Work (and each
144
+      Contributor provides its Contributions) on an "AS IS" BASIS,
145
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
146
+      implied, including, without limitation, any warranties or conditions
147
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
148
+      PARTICULAR PURPOSE. You are solely responsible for determining the
149
+      appropriateness of using or redistributing the Work and assume any
150
+      risks associated with Your exercise of permissions under this License.
151
+
152
+   8. Limitation of Liability. In no event and under no legal theory,
153
+      whether in tort (including negligence), contract, or otherwise,
154
+      unless required by applicable law (such as deliberate and grossly
155
+      negligent acts) or agreed to in writing, shall any Contributor be
156
+      liable to You for damages, including any direct, indirect, special,
157
+      incidental, or consequential damages of any character arising as a
158
+      result of this License or out of the use or inability to use the
159
+      Work (including but not limited to damages for loss of goodwill,
160
+      work stoppage, computer failure or malfunction, or any and all
161
+      other commercial damages or losses), even if such Contributor
162
+      has been advised of the possibility of such damages.
163
+
164
+   9. Accepting Warranty or Additional Liability. While redistributing
165
+      the Work or Derivative Works thereof, You may choose to offer,
166
+      and charge a fee for, acceptance of support, warranty, indemnity,
167
+      or other liability obligations and/or rights consistent with this
168
+      License. However, in accepting such obligations, You may act only
169
+      on Your own behalf and on Your sole responsibility, not on behalf
170
+      of any other Contributor, and only if You agree to indemnify,
171
+      defend, and hold each Contributor harmless for any liability
172
+      incurred by, or claims asserted against, such Contributor by reason
173
+      of your accepting any such warranty or additional liability.
174
+
175
+   END OF TERMS AND CONDITIONS
176
+
177
+   APPENDIX: How to apply the Apache License to your work.
178
+
179
+      To apply the Apache License to your work, attach the following
180
+      boilerplate notice, with the fields enclosed by brackets "[]"
181
+      replaced with your own identifying information. (Don't include
182
+      the brackets!)  The text should be enclosed in the appropriate
183
+      comment syntax for the file format. We also recommend that a
184
+      file or class name and description of purpose be included on the
185
+      same "printed page" as the copyright notice for easier
186
+      identification within third-party archives.
187
+
188
+   Copyright [yyyy] [name of copyright owner]
189
+
190
+   Licensed under the Apache License, Version 2.0 (the "License");
191
+   you may not use this file except in compliance with the License.
192
+   You may obtain a copy of the License at
193
+
194
+       http://www.apache.org/licenses/LICENSE-2.0
195
+
196
+   Unless required by applicable law or agreed to in writing, software
197
+   distributed under the License is distributed on an "AS IS" BASIS,
198
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
199
+   See the License for the specific language governing permissions and
200
+   limitations under the License.
0 201
\ No newline at end of file
1 202
new file mode 100644
... ...
@@ -0,0 +1,72 @@
0
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
1
+// See LICENSE for licensing terms.
2
+
3
+// gRPC Prometheus monitoring interceptors for client-side gRPC.
4
+
5
+package grpc_prometheus
6
+
7
+import (
8
+	"io"
9
+
10
+	"golang.org/x/net/context"
11
+	"google.golang.org/grpc"
12
+	"google.golang.org/grpc/codes"
13
+)
14
+
15
+// UnaryClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Unary RPCs.
16
+func UnaryClientInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
17
+	monitor := newClientReporter(Unary, method)
18
+	monitor.SentMessage()
19
+	err := invoker(ctx, method, req, reply, cc, opts...)
20
+	if err != nil {
21
+		monitor.ReceivedMessage()
22
+	}
23
+	monitor.Handled(grpc.Code(err))
24
+	return err
25
+}
26
+
27
+// StreamServerInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Streaming RPCs.
28
+func StreamClientInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
29
+	monitor := newClientReporter(clientStreamType(desc), method)
30
+	clientStream, err := streamer(ctx, desc, cc, method, opts...)
31
+	if err != nil {
32
+		monitor.Handled(grpc.Code(err))
33
+		return nil, err
34
+	}
35
+	return &monitoredClientStream{clientStream, monitor}, nil
36
+}
37
+
38
+func clientStreamType(desc *grpc.StreamDesc) grpcType {
39
+	if desc.ClientStreams && !desc.ServerStreams {
40
+		return ClientStream
41
+	} else if !desc.ClientStreams && desc.ServerStreams {
42
+		return ServerStream
43
+	}
44
+	return BidiStream
45
+}
46
+
47
+// monitoredClientStream wraps grpc.ClientStream allowing each Sent/Recv of message to increment counters.
48
+type monitoredClientStream struct {
49
+	grpc.ClientStream
50
+	monitor *clientReporter
51
+}
52
+
53
+func (s *monitoredClientStream) SendMsg(m interface{}) error {
54
+	err := s.ClientStream.SendMsg(m)
55
+	if err == nil {
56
+		s.monitor.SentMessage()
57
+	}
58
+	return err
59
+}
60
+
61
+func (s *monitoredClientStream) RecvMsg(m interface{}) error {
62
+	err := s.ClientStream.RecvMsg(m)
63
+	if err == nil {
64
+		s.monitor.ReceivedMessage()
65
+	} else if err == io.EOF {
66
+		s.monitor.Handled(codes.OK)
67
+	} else {
68
+		s.monitor.Handled(grpc.Code(err))
69
+	}
70
+	return err
71
+}
0 72
new file mode 100644
... ...
@@ -0,0 +1,111 @@
0
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
1
+// See LICENSE for licensing terms.
2
+
3
+package grpc_prometheus
4
+
5
+import (
6
+	"time"
7
+
8
+	"google.golang.org/grpc/codes"
9
+
10
+	prom "github.com/prometheus/client_golang/prometheus"
11
+)
12
+
13
+var (
14
+	clientStartedCounter = prom.NewCounterVec(
15
+		prom.CounterOpts{
16
+			Namespace: "grpc",
17
+			Subsystem: "client",
18
+			Name:      "started_total",
19
+			Help:      "Total number of RPCs started on the client.",
20
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
21
+
22
+	clientHandledCounter = prom.NewCounterVec(
23
+		prom.CounterOpts{
24
+			Namespace: "grpc",
25
+			Subsystem: "client",
26
+			Name:      "handled_total",
27
+			Help:      "Total number of RPCs completed by the client, regardless of success or failure.",
28
+		}, []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"})
29
+
30
+	clientStreamMsgReceived = prom.NewCounterVec(
31
+		prom.CounterOpts{
32
+			Namespace: "grpc",
33
+			Subsystem: "client",
34
+			Name:      "msg_received_total",
35
+			Help:      "Total number of RPC stream messages received by the client.",
36
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
37
+
38
+	clientStreamMsgSent = prom.NewCounterVec(
39
+		prom.CounterOpts{
40
+			Namespace: "grpc",
41
+			Subsystem: "client",
42
+			Name:      "msg_sent_total",
43
+			Help:      "Total number of gRPC stream messages sent by the client.",
44
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
45
+
46
+	clientHandledHistogramEnabled = false
47
+	clientHandledHistogramOpts    = prom.HistogramOpts{
48
+		Namespace: "grpc",
49
+		Subsystem: "client",
50
+		Name:      "handling_seconds",
51
+		Help:      "Histogram of response latency (seconds) of the gRPC until it is finished by the application.",
52
+		Buckets:   prom.DefBuckets,
53
+	}
54
+	clientHandledHistogram *prom.HistogramVec
55
+)
56
+
57
+func init() {
58
+	prom.MustRegister(clientStartedCounter)
59
+	prom.MustRegister(clientHandledCounter)
60
+	prom.MustRegister(clientStreamMsgReceived)
61
+	prom.MustRegister(clientStreamMsgSent)
62
+}
63
+
64
+// EnableClientHandlingTimeHistogram turns on recording of handling time of RPCs.
65
+// Histogram metrics can be very expensive for Prometheus to retain and query.
66
+func EnableClientHandlingTimeHistogram(opts ...HistogramOption) {
67
+	for _, o := range opts {
68
+		o(&clientHandledHistogramOpts)
69
+	}
70
+	if !clientHandledHistogramEnabled {
71
+		clientHandledHistogram = prom.NewHistogramVec(
72
+			clientHandledHistogramOpts,
73
+			[]string{"grpc_type", "grpc_service", "grpc_method"},
74
+		)
75
+		prom.Register(clientHandledHistogram)
76
+	}
77
+	clientHandledHistogramEnabled = true
78
+}
79
+
80
+type clientReporter struct {
81
+	rpcType     grpcType
82
+	serviceName string
83
+	methodName  string
84
+	startTime   time.Time
85
+}
86
+
87
+func newClientReporter(rpcType grpcType, fullMethod string) *clientReporter {
88
+	r := &clientReporter{rpcType: rpcType}
89
+	if clientHandledHistogramEnabled {
90
+		r.startTime = time.Now()
91
+	}
92
+	r.serviceName, r.methodName = splitMethodName(fullMethod)
93
+	clientStartedCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
94
+	return r
95
+}
96
+
97
+func (r *clientReporter) ReceivedMessage() {
98
+	clientStreamMsgReceived.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
99
+}
100
+
101
+func (r *clientReporter) SentMessage() {
102
+	clientStreamMsgSent.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
103
+}
104
+
105
+func (r *clientReporter) Handled(code codes.Code) {
106
+	clientHandledCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName, code.String()).Inc()
107
+	if clientHandledHistogramEnabled {
108
+		clientHandledHistogram.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Observe(time.Since(r.startTime).Seconds())
109
+	}
110
+}
0 111
new file mode 100644
... ...
@@ -0,0 +1,74 @@
0
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
1
+// See LICENSE for licensing terms.
2
+
3
+// gRPC Prometheus monitoring interceptors for server-side gRPC.
4
+
5
+package grpc_prometheus
6
+
7
+import (
8
+	"golang.org/x/net/context"
9
+	"google.golang.org/grpc"
10
+)
11
+
12
+// PreregisterServices takes a gRPC server and pre-initializes all counters to 0.
13
+// This allows for easier monitoring in Prometheus (no missing metrics), and should be called *after* all services have
14
+// been registered with the server.
15
+func Register(server *grpc.Server) {
16
+	serviceInfo := server.GetServiceInfo()
17
+	for serviceName, info := range serviceInfo {
18
+		for _, mInfo := range info.Methods {
19
+			preRegisterMethod(serviceName, &mInfo)
20
+		}
21
+	}
22
+}
23
+
24
+// UnaryServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Unary RPCs.
25
+func UnaryServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
26
+	monitor := newServerReporter(Unary, info.FullMethod)
27
+	monitor.ReceivedMessage()
28
+	resp, err := handler(ctx, req)
29
+	monitor.Handled(grpc.Code(err))
30
+	if err == nil {
31
+		monitor.SentMessage()
32
+	}
33
+	return resp, err
34
+}
35
+
36
+// StreamServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Streaming RPCs.
37
+func StreamServerInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
38
+	monitor := newServerReporter(streamRpcType(info), info.FullMethod)
39
+	err := handler(srv, &monitoredServerStream{ss, monitor})
40
+	monitor.Handled(grpc.Code(err))
41
+	return err
42
+}
43
+
44
+func streamRpcType(info *grpc.StreamServerInfo) grpcType {
45
+	if info.IsClientStream && !info.IsServerStream {
46
+		return ClientStream
47
+	} else if !info.IsClientStream && info.IsServerStream {
48
+		return ServerStream
49
+	}
50
+	return BidiStream
51
+}
52
+
53
+// monitoredStream wraps grpc.ServerStream allowing each Sent/Recv of message to increment counters.
54
+type monitoredServerStream struct {
55
+	grpc.ServerStream
56
+	monitor *serverReporter
57
+}
58
+
59
+func (s *monitoredServerStream) SendMsg(m interface{}) error {
60
+	err := s.ServerStream.SendMsg(m)
61
+	if err == nil {
62
+		s.monitor.SentMessage()
63
+	}
64
+	return err
65
+}
66
+
67
+func (s *monitoredServerStream) RecvMsg(m interface{}) error {
68
+	err := s.ServerStream.RecvMsg(m)
69
+	if err == nil {
70
+		s.monitor.ReceivedMessage()
71
+	}
72
+	return err
73
+}
0 74
new file mode 100644
... ...
@@ -0,0 +1,157 @@
0
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
1
+// See LICENSE for licensing terms.
2
+
3
+package grpc_prometheus
4
+
5
+import (
6
+	"time"
7
+
8
+	"google.golang.org/grpc/codes"
9
+
10
+	prom "github.com/prometheus/client_golang/prometheus"
11
+	"google.golang.org/grpc"
12
+)
13
+
14
+type grpcType string
15
+
16
+const (
17
+	Unary        grpcType = "unary"
18
+	ClientStream grpcType = "client_stream"
19
+	ServerStream grpcType = "server_stream"
20
+	BidiStream   grpcType = "bidi_stream"
21
+)
22
+
23
+var (
24
+	serverStartedCounter = prom.NewCounterVec(
25
+		prom.CounterOpts{
26
+			Namespace: "grpc",
27
+			Subsystem: "server",
28
+			Name:      "started_total",
29
+			Help:      "Total number of RPCs started on the server.",
30
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
31
+
32
+	serverHandledCounter = prom.NewCounterVec(
33
+		prom.CounterOpts{
34
+			Namespace: "grpc",
35
+			Subsystem: "server",
36
+			Name:      "handled_total",
37
+			Help:      "Total number of RPCs completed on the server, regardless of success or failure.",
38
+		}, []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"})
39
+
40
+	serverStreamMsgReceived = prom.NewCounterVec(
41
+		prom.CounterOpts{
42
+			Namespace: "grpc",
43
+			Subsystem: "server",
44
+			Name:      "msg_received_total",
45
+			Help:      "Total number of RPC stream messages received on the server.",
46
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
47
+
48
+	serverStreamMsgSent = prom.NewCounterVec(
49
+		prom.CounterOpts{
50
+			Namespace: "grpc",
51
+			Subsystem: "server",
52
+			Name:      "msg_sent_total",
53
+			Help:      "Total number of gRPC stream messages sent by the server.",
54
+		}, []string{"grpc_type", "grpc_service", "grpc_method"})
55
+
56
+	serverHandledHistogramEnabled = false
57
+	serverHandledHistogramOpts    = prom.HistogramOpts{
58
+		Namespace: "grpc",
59
+		Subsystem: "server",
60
+		Name:      "handling_seconds",
61
+		Help:      "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.",
62
+		Buckets:   prom.DefBuckets,
63
+	}
64
+	serverHandledHistogram *prom.HistogramVec
65
+)
66
+
67
+func init() {
68
+	prom.MustRegister(serverStartedCounter)
69
+	prom.MustRegister(serverHandledCounter)
70
+	prom.MustRegister(serverStreamMsgReceived)
71
+	prom.MustRegister(serverStreamMsgSent)
72
+}
73
+
74
+type HistogramOption func(*prom.HistogramOpts)
75
+
76
+// WithHistogramBuckets allows you to specify custom bucket ranges for histograms if EnableHandlingTimeHistogram is on.
77
+func WithHistogramBuckets(buckets []float64) HistogramOption {
78
+	return func(o *prom.HistogramOpts) { o.Buckets = buckets }
79
+}
80
+
81
+// EnableHandlingTimeHistogram turns on recording of handling time of RPCs for server-side interceptors.
82
+// Histogram metrics can be very expensive for Prometheus to retain and query.
83
+func EnableHandlingTimeHistogram(opts ...HistogramOption) {
84
+	for _, o := range opts {
85
+		o(&serverHandledHistogramOpts)
86
+	}
87
+	if !serverHandledHistogramEnabled {
88
+		serverHandledHistogram = prom.NewHistogramVec(
89
+			serverHandledHistogramOpts,
90
+			[]string{"grpc_type", "grpc_service", "grpc_method"},
91
+		)
92
+		prom.Register(serverHandledHistogram)
93
+	}
94
+	serverHandledHistogramEnabled = true
95
+}
96
+
97
+type serverReporter struct {
98
+	rpcType     grpcType
99
+	serviceName string
100
+	methodName  string
101
+	startTime   time.Time
102
+}
103
+
104
+func newServerReporter(rpcType grpcType, fullMethod string) *serverReporter {
105
+	r := &serverReporter{rpcType: rpcType}
106
+	if serverHandledHistogramEnabled {
107
+		r.startTime = time.Now()
108
+	}
109
+	r.serviceName, r.methodName = splitMethodName(fullMethod)
110
+	serverStartedCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
111
+	return r
112
+}
113
+
114
+func (r *serverReporter) ReceivedMessage() {
115
+	serverStreamMsgReceived.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
116
+}
117
+
118
+func (r *serverReporter) SentMessage() {
119
+	serverStreamMsgSent.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc()
120
+}
121
+
122
+func (r *serverReporter) Handled(code codes.Code) {
123
+	serverHandledCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName, code.String()).Inc()
124
+	if serverHandledHistogramEnabled {
125
+		serverHandledHistogram.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Observe(time.Since(r.startTime).Seconds())
126
+	}
127
+}
128
+
129
+// preRegisterMethod is invoked on Register of a Server, allowing all gRPC services labels to be pre-populated.
130
+func preRegisterMethod(serviceName string, mInfo *grpc.MethodInfo) {
131
+	methodName := mInfo.Name
132
+	methodType := string(typeFromMethodInfo(mInfo))
133
+	// These are just references (no increments), as just referencing will create the labels but not set values.
134
+	serverStartedCounter.GetMetricWithLabelValues(methodType, serviceName, methodName)
135
+	serverStreamMsgReceived.GetMetricWithLabelValues(methodType, serviceName, methodName)
136
+	serverStreamMsgSent.GetMetricWithLabelValues(methodType, serviceName, methodName)
137
+	if serverHandledHistogramEnabled {
138
+		serverHandledHistogram.GetMetricWithLabelValues(methodType, serviceName, methodName)
139
+	}
140
+	for _, code := range allCodes {
141
+		serverHandledCounter.GetMetricWithLabelValues(methodType, serviceName, methodName, code.String())
142
+	}
143
+}
144
+
145
+func typeFromMethodInfo(mInfo *grpc.MethodInfo) grpcType {
146
+	if mInfo.IsClientStream == false && mInfo.IsServerStream == false {
147
+		return Unary
148
+	}
149
+	if mInfo.IsClientStream == true && mInfo.IsServerStream == false {
150
+		return ClientStream
151
+	}
152
+	if mInfo.IsClientStream == false && mInfo.IsServerStream == true {
153
+		return ServerStream
154
+	}
155
+	return BidiStream
156
+}
0 157
new file mode 100644
... ...
@@ -0,0 +1,27 @@
0
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
1
+// See LICENSE for licensing terms.
2
+
3
+package grpc_prometheus
4
+
5
+import (
6
+	"strings"
7
+
8
+	"google.golang.org/grpc/codes"
9
+)
10
+
11
+var (
12
+	allCodes = []codes.Code{
13
+		codes.OK, codes.Canceled, codes.Unknown, codes.InvalidArgument, codes.DeadlineExceeded, codes.NotFound,
14
+		codes.AlreadyExists, codes.PermissionDenied, codes.Unauthenticated, codes.ResourceExhausted,
15
+		codes.FailedPrecondition, codes.Aborted, codes.OutOfRange, codes.Unimplemented, codes.Internal,
16
+		codes.Unavailable, codes.DataLoss,
17
+	}
18
+)
19
+
20
+func splitMethodName(fullMethodName string) (string, string) {
21
+	fullMethodName = strings.TrimPrefix(fullMethodName, "/") // remove leading slash
22
+	if i := strings.Index(fullMethodName, "/"); i >= 0 {
23
+		return fullMethodName[:i], fullMethodName[i+1:]
24
+	}
25
+	return "unknown", "unknown"
26
+}