Browse code

backport ConstraintEnforcer fix

Revendors docker/swarmkit to backport fixes made to the
ConstraintEnforcer (see docker/swarmkit#2857)

Signed-off-by: Drew Erny <drew.erny@docker.com>

Drew Erny authored on 2019/05/21 03:30:00
Showing 2 changed files
... ...
@@ -130,7 +130,7 @@ github.com/containerd/ttrpc                         f02858b1457c5ca3aaec3a0803eb
130 130
 github.com/gogo/googleapis                          d31c731455cb061f42baff3bda55bad0118b126b # v1.2.0
131 131
 
132 132
 # cluster
133
-github.com/docker/swarmkit                          59163bf75df38489d4a10392265d27156dc473c5
133
+github.com/docker/swarmkit                          48eb1828ce81be20b25d647f6ca8f33d599f705c
134 134
 github.com/gogo/protobuf                            ba06b47c162d49f2af050fb4c75bcbc86a159d5c # v1.2.1
135 135
 github.com/cloudflare/cfssl                         5d63dbd981b5c408effbb58c442d54761ff94fbd # 1.3.2
136 136
 github.com/fernet/fernet-go                         1b2437bc582b3cfbb341ee5a29f8ef5b42912ff2
... ...
@@ -76,8 +76,22 @@ func (ce *ConstraintEnforcer) rejectNoncompliantTasks(node *api.Node) {
76 76
 		err   error
77 77
 	)
78 78
 
79
+	services := map[string]*api.Service{}
79 80
 	ce.store.View(func(tx store.ReadTx) {
80 81
 		tasks, err = store.FindTasks(tx, store.ByNodeID(node.ID))
82
+		if err != nil {
83
+			return
84
+		}
85
+
86
+		// Deduplicate service IDs using the services map. It's okay for the
87
+		// values to be nil for now, we will look them up from the store next.
88
+		for _, task := range tasks {
89
+			services[task.ServiceID] = nil
90
+		}
91
+
92
+		for serviceID := range services {
93
+			services[serviceID] = store.GetService(tx, serviceID)
94
+		}
81 95
 	})
82 96
 
83 97
 	if err != nil {
... ...
@@ -105,10 +119,44 @@ loop:
105 105
 			continue
106 106
 		}
107 107
 
108
-		// Ensure that the task still meets scheduling
109
-		// constraints.
110
-		if t.Spec.Placement != nil && len(t.Spec.Placement.Constraints) != 0 {
111
-			constraints, _ := constraint.Parse(t.Spec.Placement.Constraints)
108
+		// Ensure that the node still satisfies placement constraints.
109
+		// NOTE: If the task is associacted with a service then we must use the
110
+		// constraints from the current service spec rather than the
111
+		// constraints from the task spec because they may be outdated. This
112
+		// will happen if the service was previously updated in a way which
113
+		// only changes the placement constraints and the node matched the
114
+		// placement constraints both before and after that update. In the case
115
+		// of such updates, the tasks are not considered "dirty" and are not
116
+		// restarted but it will mean that the task spec's placement
117
+		// constraints are outdated. Consider this example:
118
+		// - A service is created with no constraints and a task is scheduled
119
+		//   to a node.
120
+		// - The node is updated to add a label, this doesn't affect the task
121
+		//   on that node because it has no constraints.
122
+		// - The service is updated to add a node label constraint which
123
+		//   matches the label which was just added to the node. The updater
124
+		//   does not shut down the task because the only the constraints have
125
+		//   changed and the node still matches the updated constraints.
126
+		// - The node is updated to remove the node label. The node no longer
127
+		//   satisfies the placement constraints of the service, so the task
128
+		//   should be shutdown. However, the task's spec still has the
129
+		//   original and outdated constraints (that are still satisfied by
130
+		//   the node). If we used those original constraints then the task
131
+		//   would incorrectly not be removed. This is why the constraints
132
+		//   from the service spec should be used instead.
133
+		var placement *api.Placement
134
+		if service := services[t.ServiceID]; service != nil {
135
+			// This task is associated with a service, so we use the service's
136
+			// current placement constraints.
137
+			placement = service.Spec.Task.Placement
138
+		} else {
139
+			// This task is not associated with a service (or the service no
140
+			// longer exists), so we use the placement constraints from the
141
+			// original task spec.
142
+			placement = t.Spec.Placement
143
+		}
144
+		if placement != nil && len(placement.Constraints) > 0 {
145
+			constraints, _ := constraint.Parse(placement.Constraints)
112 146
 			if !constraint.NodeMatches(constraints, node) {
113 147
 				removeTasks[t.ID] = t
114 148
 				continue