This fix cherry-pick SwarmKit update to 0af40501a9cc98cd3e9425d2e4246dd3eff5526e
for branch 1.13.x. The following has been added:
- [docker/swarmkit#1909]: raft: Disable address change detection
- [docker/swarmkit#1910]: Fix issue in service update of published ports in host mode
This fix fixes #30199 in docker.
Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
| ... | ... |
@@ -100,7 +100,7 @@ github.com/docker/containerd 03e5862ec0d8d3b3f750e19fca3ee367e13c090e |
| 100 | 100 |
github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4 |
| 101 | 101 |
|
| 102 | 102 |
# cluster |
| 103 |
-github.com/docker/swarmkit 335561b66a44bf214224afe879e4368204e7fa45 |
|
| 103 |
+github.com/docker/swarmkit 0af40501a9cc98cd3e9425d2e4246dd3eff5526e |
|
| 104 | 104 |
github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 |
| 105 | 105 |
github.com/gogo/protobuf v0.3 |
| 106 | 106 |
github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a |
| ... | ... |
@@ -371,12 +371,15 @@ func (a *Allocator) doNetworkAlloc(ctx context.Context, ev events.Event) {
|
| 371 | 371 |
s := v.Service.Copy() |
| 372 | 372 |
|
| 373 | 373 |
if nc.nwkAllocator.IsServiceAllocated(s) {
|
| 374 |
- break |
|
| 375 |
- } |
|
| 376 |
- |
|
| 377 |
- if err := a.allocateService(ctx, s); err != nil {
|
|
| 378 |
- log.G(ctx).WithError(err).Errorf("Failed allocation during update of service %s", s.ID)
|
|
| 379 |
- break |
|
| 374 |
+ if nc.nwkAllocator.PortsAllocatedInHostPublishMode(s) {
|
|
| 375 |
+ break |
|
| 376 |
+ } |
|
| 377 |
+ updatePortsInHostPublishMode(s) |
|
| 378 |
+ } else {
|
|
| 379 |
+ if err := a.allocateService(ctx, s); err != nil {
|
|
| 380 |
+ log.G(ctx).WithError(err).Errorf("Failed allocation during update of service %s", s.ID)
|
|
| 381 |
+ break |
|
| 382 |
+ } |
|
| 380 | 383 |
} |
| 381 | 384 |
|
| 382 | 385 |
if _, err := a.store.Batch(func(batch *store.Batch) error {
|
| ... | ... |
@@ -641,6 +644,36 @@ func (a *Allocator) commitAllocatedNode(ctx context.Context, batch *store.Batch, |
| 641 | 641 |
return nil |
| 642 | 642 |
} |
| 643 | 643 |
|
| 644 |
+// This function prepares the service object for being updated when the change regards |
|
| 645 |
+// the published ports in host mode: It resets the runtime state ports (s.Endpoint.Ports) |
|
| 646 |
+// to the current ingress mode runtime state ports plus the newly configured publish mode ports, |
|
| 647 |
+// so that the service allocation invoked on this new service object will trigger the deallocation |
|
| 648 |
+// of any old publish mode port and allocation of any new one. |
|
| 649 |
+func updatePortsInHostPublishMode(s *api.Service) {
|
|
| 650 |
+ if s.Endpoint != nil {
|
|
| 651 |
+ var portConfigs []*api.PortConfig |
|
| 652 |
+ for _, portConfig := range s.Endpoint.Ports {
|
|
| 653 |
+ if portConfig.PublishMode == api.PublishModeIngress {
|
|
| 654 |
+ portConfigs = append(portConfigs, portConfig) |
|
| 655 |
+ } |
|
| 656 |
+ } |
|
| 657 |
+ s.Endpoint.Ports = portConfigs |
|
| 658 |
+ } |
|
| 659 |
+ |
|
| 660 |
+ if s.Spec.Endpoint != nil {
|
|
| 661 |
+ if s.Endpoint == nil {
|
|
| 662 |
+ s.Endpoint = &api.Endpoint{}
|
|
| 663 |
+ } |
|
| 664 |
+ for _, portConfig := range s.Spec.Endpoint.Ports {
|
|
| 665 |
+ if portConfig.PublishMode == api.PublishModeIngress {
|
|
| 666 |
+ continue |
|
| 667 |
+ } |
|
| 668 |
+ s.Endpoint.Ports = append(s.Endpoint.Ports, portConfig.Copy()) |
|
| 669 |
+ } |
|
| 670 |
+ s.Endpoint.Spec = s.Spec.Endpoint.Copy() |
|
| 671 |
+ } |
|
| 672 |
+} |
|
| 673 |
+ |
|
| 644 | 674 |
func (a *Allocator) allocateService(ctx context.Context, s *api.Service) error {
|
| 645 | 675 |
nc := a.netCtx |
| 646 | 676 |
|
| ... | ... |
@@ -283,6 +283,12 @@ func (na *NetworkAllocator) IsTaskAllocated(t *api.Task) bool {
|
| 283 | 283 |
return true |
| 284 | 284 |
} |
| 285 | 285 |
|
| 286 |
+// PortsAllocatedInHostPublishMode returns if the passed service has its published ports in |
|
| 287 |
+// host (non ingress) mode allocated |
|
| 288 |
+func (na *NetworkAllocator) PortsAllocatedInHostPublishMode(s *api.Service) bool {
|
|
| 289 |
+ return na.portAllocator.portsAllocatedInHostPublishMode(s) |
|
| 290 |
+} |
|
| 291 |
+ |
|
| 286 | 292 |
// IsServiceAllocated returns if the passed service has its network resources allocated or not. |
| 287 | 293 |
func (na *NetworkAllocator) IsServiceAllocated(s *api.Service) bool {
|
| 288 | 294 |
// If endpoint mode is VIP and allocator does not have the |
| ... | ... |
@@ -38,6 +38,71 @@ type portSpace struct {
|
| 38 | 38 |
dynamicPortSpace *idm.Idm |
| 39 | 39 |
} |
| 40 | 40 |
|
| 41 |
+type allocatedPorts map[api.PortConfig]map[uint32]*api.PortConfig |
|
| 42 |
+ |
|
| 43 |
+// addState add the state of an allocated port to the collection. |
|
| 44 |
+// `allocatedPorts` is a map of portKey:publishedPort:portState. |
|
| 45 |
+// In case the value of the portKey is missing, the map |
|
| 46 |
+// publishedPort:portState is created automatically |
|
| 47 |
+func (ps allocatedPorts) addState(p *api.PortConfig) {
|
|
| 48 |
+ portKey := getPortConfigKey(p) |
|
| 49 |
+ if _, ok := ps[portKey]; !ok {
|
|
| 50 |
+ ps[portKey] = make(map[uint32]*api.PortConfig) |
|
| 51 |
+ } |
|
| 52 |
+ ps[portKey][p.PublishedPort] = p |
|
| 53 |
+} |
|
| 54 |
+ |
|
| 55 |
+// delState delete the state of an allocated port from the collection. |
|
| 56 |
+// `allocatedPorts` is a map of portKey:publishedPort:portState. |
|
| 57 |
+// |
|
| 58 |
+// If publishedPort is non-zero, then it is user defined. We will try to |
|
| 59 |
+// remove the portState from `allocatedPorts` directly and return |
|
| 60 |
+// the portState (or nil if no portState exists) |
|
| 61 |
+// |
|
| 62 |
+// If publishedPort is zero, then it is dynamically allocated. We will try |
|
| 63 |
+// to remove the portState from `allocatedPorts`, as long as there is |
|
| 64 |
+// a portState associated with a non-zero publishedPort. |
|
| 65 |
+// Note multiple dynamically allocated ports might exists. In this case, |
|
| 66 |
+// we will remove only at a time so both allocated ports are tracked. |
|
| 67 |
+// |
|
| 68 |
+// Note because of the potential co-existence of user-defined and dynamically |
|
| 69 |
+// allocated ports, delState has to be called for user-defined port first. |
|
| 70 |
+// dynamically allocated ports should be removed later. |
|
| 71 |
+func (ps allocatedPorts) delState(p *api.PortConfig) *api.PortConfig {
|
|
| 72 |
+ portKey := getPortConfigKey(p) |
|
| 73 |
+ |
|
| 74 |
+ portStateMap, ok := ps[portKey] |
|
| 75 |
+ |
|
| 76 |
+ // If name, port, protocol values don't match then we |
|
| 77 |
+ // are not allocated. |
|
| 78 |
+ if !ok {
|
|
| 79 |
+ return nil |
|
| 80 |
+ } |
|
| 81 |
+ |
|
| 82 |
+ if p.PublishedPort != 0 {
|
|
| 83 |
+ // If SwarmPort was user defined but the port state |
|
| 84 |
+ // SwarmPort doesn't match we are not allocated. |
|
| 85 |
+ v := portStateMap[p.PublishedPort] |
|
| 86 |
+ |
|
| 87 |
+ // Delete state from allocatedPorts |
|
| 88 |
+ delete(portStateMap, p.PublishedPort) |
|
| 89 |
+ |
|
| 90 |
+ return v |
|
| 91 |
+ } |
|
| 92 |
+ |
|
| 93 |
+ // If PublishedPort == 0 and we don't have non-zero port |
|
| 94 |
+ // then we are not allocated |
|
| 95 |
+ for publishedPort, v := range portStateMap {
|
|
| 96 |
+ if publishedPort != 0 {
|
|
| 97 |
+ // Delete state from allocatedPorts |
|
| 98 |
+ delete(portStateMap, publishedPort) |
|
| 99 |
+ return v |
|
| 100 |
+ } |
|
| 101 |
+ } |
|
| 102 |
+ |
|
| 103 |
+ return nil |
|
| 104 |
+} |
|
| 105 |
+ |
|
| 41 | 106 |
func newPortAllocator() (*portAllocator, error) {
|
| 42 | 107 |
portSpaces := make(map[api.PortConfig_Protocol]*portSpace) |
| 43 | 108 |
for _, protocol := range []api.PortConfig_Protocol{api.ProtocolTCP, api.ProtocolUDP} {
|
| ... | ... |
@@ -191,6 +256,33 @@ func (pa *portAllocator) serviceDeallocatePorts(s *api.Service) {
|
| 191 | 191 |
s.Endpoint.Ports = nil |
| 192 | 192 |
} |
| 193 | 193 |
|
| 194 |
+func (pa *portAllocator) portsAllocatedInHostPublishMode(s *api.Service) bool {
|
|
| 195 |
+ if s.Endpoint == nil && s.Spec.Endpoint == nil {
|
|
| 196 |
+ return true |
|
| 197 |
+ } |
|
| 198 |
+ |
|
| 199 |
+ portStates := allocatedPorts{}
|
|
| 200 |
+ if s.Endpoint != nil {
|
|
| 201 |
+ for _, portState := range s.Endpoint.Ports {
|
|
| 202 |
+ if portState.PublishMode == api.PublishModeHost {
|
|
| 203 |
+ portStates.addState(portState) |
|
| 204 |
+ } |
|
| 205 |
+ } |
|
| 206 |
+ } |
|
| 207 |
+ |
|
| 208 |
+ if s.Spec.Endpoint != nil {
|
|
| 209 |
+ for _, portConfig := range s.Spec.Endpoint.Ports {
|
|
| 210 |
+ if portConfig.PublishMode == api.PublishModeHost {
|
|
| 211 |
+ if portStates.delState(portConfig) == nil {
|
|
| 212 |
+ return false |
|
| 213 |
+ } |
|
| 214 |
+ } |
|
| 215 |
+ } |
|
| 216 |
+ } |
|
| 217 |
+ |
|
| 218 |
+ return true |
|
| 219 |
+} |
|
| 220 |
+ |
|
| 194 | 221 |
func (pa *portAllocator) isPortsAllocated(s *api.Service) bool {
|
| 195 | 222 |
// If service has no user-defined endpoint and allocated endpoint, |
| 196 | 223 |
// we assume it is allocated and return true. |
| ... | ... |
@@ -1419,11 +1419,15 @@ func (n *Node) sendToMember(ctx context.Context, members map[uint64]*membership. |
| 1419 | 1419 |
officialHost, officialPort, _ := net.SplitHostPort(conn.Addr) |
| 1420 | 1420 |
if officialHost != lastSeenHost {
|
| 1421 | 1421 |
reconnectAddr := net.JoinHostPort(lastSeenHost, officialPort) |
| 1422 |
- log.G(ctx).Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr)
|
|
| 1423 |
- if err := n.handleAddressChange(ctx, conn, reconnectAddr); err != nil {
|
|
| 1424 |
- log.G(ctx).WithError(err).Error("failed to hande address change")
|
|
| 1425 |
- } |
|
| 1426 |
- return |
|
| 1422 |
+ log.G(ctx).Debugf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr)
|
|
| 1423 |
+ // TODO(aaronl): Address changes are temporarily disabled. |
|
| 1424 |
+ // See https://github.com/docker/docker/issues/30455. |
|
| 1425 |
+ // This should be reenabled in the future with additional |
|
| 1426 |
+ // safeguards (perhaps storing multiple addresses per node). |
|
| 1427 |
+ //if err := n.handleAddressChange(ctx, conn, reconnectAddr); err != nil {
|
|
| 1428 |
+ // log.G(ctx).WithError(err).Error("failed to hande address change")
|
|
| 1429 |
+ //} |
|
| 1430 |
+ //return |
|
| 1427 | 1431 |
} |
| 1428 | 1432 |
} |
| 1429 | 1433 |
|