Browse code

Considering existing deployments for deployment configs - If a running/pending/new deployment exists, the config is requeued - If multiple running/previous/new deployments exists, older ones are cancelled

Abhishek Gupta authored on 2015/05/12 04:30:06
Showing 7 changed files
... ...
@@ -183,6 +183,12 @@ const (
183 183
 	// new deployment.
184 184
 	// TODO: This should be made public upstream.
185 185
 	DesiredReplicasAnnotation = "kubectl.kubernetes.io/desired-replicas"
186
+	// DeploymentStatusReasonAnnotation represents the reason for deployment being in a given state
187
+	// Used for specifying the reason for cancellation or failure of a deployment
188
+	DeploymentStatusReasonAnnotation = "openshift.io/deployment.status-reason"
189
+	// DeploymentCancelledAnnotation indicates that the deployment has been cancelled
190
+	// The annotation value does not matter and its mere presence indicates cancellation
191
+	DeploymentCancelledAnnotation = "openshift.io/deployment.cancelled"
186 192
 )
187 193
 
188 194
 // DeploymentConfig represents a configuration for a single deployment (represented as a
... ...
@@ -180,6 +180,12 @@ const (
180 180
 	// DeploymentConfigLabel is the name of a label used to correlate a deployment with the
181 181
 	// DeploymentConfigs on which the deployment is based.
182 182
 	DeploymentConfigLabel = "deploymentconfig"
183
+	// DeploymentStatusReasonAnnotation represents the reason for deployment being in a given state
184
+	// Used for specifying the reason for cancellation or failure of a deployment
185
+	DeploymentStatusReasonAnnotation = "openshift.io/deployment.status-reason"
186
+	// DeploymentCancelledAnnotation indicates that the deployment has been cancelled
187
+	// The annotation value does not matter and its mere presence indicates cancellation
188
+	DeploymentCancelledAnnotation = "openshift.io/deployment.cancelled"
183 189
 )
184 190
 
185 191
 // DeploymentConfig represents a configuration for a single deployment (represented as a
... ...
@@ -147,6 +147,12 @@ const (
147 147
 	// DeploymentConfigLabel is the name of a label used to correlate a deployment with the
148 148
 	// DeploymentConfigs on which the deployment is based.
149 149
 	DeploymentConfigLabel = "deploymentconfig"
150
+	// DeploymentStatusReasonAnnotation represents the reason for deployment being in a given state
151
+	// Used for specifying the reason for cancellation or failure of a deployment
152
+	DeploymentStatusReasonAnnotation = "openshift.io/deployment.status-reason"
153
+	// DeploymentCancelledAnnotation indicates that the deployment has been cancelled
154
+	// The annotation value does not matter and its mere presence indicates cancellation
155
+	DeploymentCancelledAnnotation = "openshift.io/deployment.cancelled"
150 156
 )
151 157
 
152 158
 // DeploymentConfig represents a configuration for a single deployment (represented as a
... ...
@@ -1,11 +1,13 @@
1 1
 package deployerpod
2 2
 
3 3
 import (
4
+	"fmt"
4 5
 	"time"
5 6
 
6 7
 	"github.com/golang/glog"
7 8
 
8 9
 	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
10
+	kerrors "github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
9 11
 	kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client"
10 12
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
11 13
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
... ...
@@ -37,9 +39,6 @@ func (factory *DeployerPodControllerFactory) Create() controller.RunnableControl
37 37
 			return factory.KubeClient.ReplicationControllers(kapi.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
38 38
 		},
39 39
 	}
40
-	deploymentQueue := cache.NewFIFO(cache.MetaNamespaceKeyFunc)
41
-	cache.NewReflector(deploymentLW, &kapi.ReplicationController{}, deploymentQueue, 2*time.Minute).Run()
42
-
43 40
 	deploymentStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
44 41
 	cache.NewReflector(deploymentLW, &kapi.ReplicationController{}, deploymentStore, 2*time.Minute).Run()
45 42
 
... ...
@@ -85,13 +84,14 @@ func (factory *DeployerPodControllerFactory) Create() controller.RunnableControl
85 85
 
86 86
 // pollPods lists all pods associated with pending or running deployments and returns
87 87
 // a cache.Enumerator suitable for use with a cache.Poller.
88
-func pollPods(deploymentStore cache.Store, kClient kclient.PodsNamespacer) (cache.Enumerator, error) {
88
+func pollPods(deploymentStore cache.Store, kClient kclient.Interface) (cache.Enumerator, error) {
89 89
 	list := &kapi.PodList{}
90 90
 
91 91
 	for _, obj := range deploymentStore.List() {
92 92
 		deployment := obj.(*kapi.ReplicationController)
93
+		currentStatus := deployutil.DeploymentStatusFor(deployment)
93 94
 
94
-		switch deployutil.DeploymentStatusFor(deployment) {
95
+		switch currentStatus {
95 96
 		case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning:
96 97
 			// Validate the correlating pod annotation
97 98
 			podID := deployutil.DeployerPodNameFor(deployment)
... ...
@@ -103,6 +103,20 @@ func pollPods(deploymentStore cache.Store, kClient kclient.PodsNamespacer) (cach
103 103
 			pod, err := kClient.Pods(deployment.Namespace).Get(podID)
104 104
 			if err != nil {
105 105
 				glog.V(2).Infof("Couldn't find pod %s for deployment %s: %#v", podID, deployment.Name, err)
106
+
107
+				// if the deployer pod doesn't exist, update the deployment status to failed
108
+				// TODO: This update should be moved the controller
109
+				// once this poll is changed in favor of pod status updates.
110
+				if kerrors.IsNotFound(err) {
111
+					nextStatus := deployapi.DeploymentStatusFailed
112
+					deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
113
+					deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = fmt.Sprintf("Couldn't find pod %s for deployment %s", podID, deployment.Name)
114
+
115
+					if _, err := kClient.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil {
116
+						glog.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
117
+					}
118
+					glog.V(2).Infof("Updated deployment %s status from %s to %s", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus)
119
+				}
106 120
 				continue
107 121
 			}
108 122
 
... ...
@@ -53,35 +53,50 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
53 53
 		return nil
54 54
 	}
55 55
 
56
-	// Check if the latest deployment already exists
57
-	if deployment, err := c.deploymentClient.getDeployment(config.Namespace, deployutil.LatestDeploymentNameForConfig(config)); err != nil {
58
-		if !errors.IsNotFound(err) {
59
-			return fmt.Errorf("couldn't get deployment for config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
60
-		}
61
-	} else {
62
-		// If there's an existing deployment, nothing needs to be done.
63
-		if deployment != nil {
64
-			return nil
65
-		}
66
-	}
67
-
68
-	// Check if any previous deployment is still running (any non-terminal state).
56
+	// Check if any existing inflight deployments (any non-terminal state).
69 57
 	existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name)
70 58
 	if err != nil {
71 59
 		return fmt.Errorf("couldn't list deployments for config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
72 60
 	}
61
+	var inflightDeployment *kapi.ReplicationController
73 62
 	for _, deployment := range existingDeployments.Items {
74 63
 		deploymentStatus := deployutil.DeploymentStatusFor(&deployment)
75 64
 		switch deploymentStatus {
76 65
 		case deployapi.DeploymentStatusFailed,
77 66
 			deployapi.DeploymentStatusComplete:
78 67
 			// Previous deployment in terminal state - can ignore
79
-			// Ignoring specific deployment states so that any new
68
+			// Ignoring specific deployment states so that any newly introduced
80 69
 			// deployment state will not be ignored
81 70
 		default:
82
-			glog.V(4).Infof("Found previous deployment %s (status %s) - will requeue", deployutil.LabelForDeployment(&deployment), deploymentStatus)
83
-			return transientError(fmt.Sprintf("found previous deployment (state: %s) for %s - requeuing", deploymentStatus, deployutil.LabelForDeploymentConfig(config)))
71
+			if inflightDeployment == nil {
72
+				inflightDeployment = &deployment
73
+				continue
74
+			}
75
+			var deploymentForCancellation *kapi.ReplicationController
76
+			if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) {
77
+				deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment
78
+			} else {
79
+				deploymentForCancellation = &deployment
80
+			}
81
+
82
+			deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = "true"
83
+			if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil {
84
+				glog.Errorf("couldn't cancel deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err)
85
+			}
86
+			glog.V(2).Infof("Cancelled deployment %s for config %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config))
87
+		}
88
+	}
89
+
90
+	// check to see if there are inflight deployments
91
+	if inflightDeployment != nil {
92
+		// check if this is the latest and only deployment
93
+		// if so, nothing needs to be done
94
+		if deployutil.DeploymentVersionFor(inflightDeployment) == config.LatestVersion {
95
+			return nil
84 96
 		}
97
+		// if this is an earlier deployment, raise a transientError so that the deployment config can be re-queued
98
+		glog.V(4).Infof("Found previous inflight deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config))
99
+		return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)))
85 100
 	}
86 101
 
87 102
 	// Try and build a deployment for the config.
... ...
@@ -128,22 +143,18 @@ func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig)
128 128
 
129 129
 // deploymentClient abstracts access to deployments.
130 130
 type deploymentClient interface {
131
-	getDeployment(namespace, name string) (*kapi.ReplicationController, error)
132 131
 	createDeployment(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error)
133 132
 	// listDeploymentsForConfig should return deployments associated with the
134 133
 	// provided config.
135 134
 	listDeploymentsForConfig(namespace, configName string) (*kapi.ReplicationControllerList, error)
135
+	updateDeployment(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error)
136 136
 }
137 137
 
138 138
 // deploymentClientImpl is a pluggable deploymentClient.
139 139
 type deploymentClientImpl struct {
140
-	getDeploymentFunc            func(namespace, name string) (*kapi.ReplicationController, error)
141 140
 	createDeploymentFunc         func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error)
142 141
 	listDeploymentsForConfigFunc func(namespace, configName string) (*kapi.ReplicationControllerList, error)
143
-}
144
-
145
-func (i *deploymentClientImpl) getDeployment(namespace, name string) (*kapi.ReplicationController, error) {
146
-	return i.getDeploymentFunc(namespace, name)
142
+	updateDeploymentFunc         func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error)
147 143
 }
148 144
 
149 145
 func (i *deploymentClientImpl) createDeployment(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
... ...
@@ -153,3 +164,7 @@ func (i *deploymentClientImpl) createDeployment(namespace string, deployment *ka
153 153
 func (i *deploymentClientImpl) listDeploymentsForConfig(namespace, configName string) (*kapi.ReplicationControllerList, error) {
154 154
 	return i.listDeploymentsForConfigFunc(namespace, configName)
155 155
 }
156
+
157
+func (i *deploymentClientImpl) updateDeployment(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
158
+	return i.updateDeploymentFunc(namespace, deployment)
159
+}
... ...
@@ -3,6 +3,7 @@ package deploymentconfig
3 3
 import (
4 4
 	"fmt"
5 5
 	"reflect"
6
+	"sort"
6 7
 	"strconv"
7 8
 	"testing"
8 9
 
... ...
@@ -24,10 +25,6 @@ func TestHandle_initialOk(t *testing.T) {
24 24
 			return deployutil.MakeDeployment(config, api.Codec)
25 25
 		},
26 26
 		deploymentClient: &deploymentClientImpl{
27
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
28
-				t.Fatalf("unexpected call with name %s", name)
29
-				return nil, nil
30
-			},
31 27
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
32 28
 				t.Fatalf("unexpected call with deployment %v", deployment)
33 29
 				return nil, nil
... ...
@@ -36,6 +33,10 @@ func TestHandle_initialOk(t *testing.T) {
36 36
 				t.Fatalf("unexpected call to list deployments")
37 37
 				return nil, nil
38 38
 			},
39
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
40
+				t.Fatalf("unexpected update call with deployment %v", deployment)
41
+				return nil, nil
42
+			},
39 43
 		},
40 44
 		recorder: &record.FakeRecorder{},
41 45
 	}
... ...
@@ -62,9 +63,6 @@ func TestHandle_updateOk(t *testing.T) {
62 62
 			return deployutil.MakeDeployment(config, api.Codec)
63 63
 		},
64 64
 		deploymentClient: &deploymentClientImpl{
65
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
66
-				return nil, kerrors.NewNotFound("ReplicationController", name)
67
-			},
68 65
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
69 66
 				deployed = deployment
70 67
 				return deployment, nil
... ...
@@ -72,6 +70,10 @@ func TestHandle_updateOk(t *testing.T) {
72 72
 			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
73 73
 				return existingDeployments, nil
74 74
 			},
75
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
76
+				t.Fatalf("unexpected update call with deployment %v", deployment)
77
+				return nil, nil
78
+			},
75 79
 		},
76 80
 		recorder: &record.FakeRecorder{},
77 81
 	}
... ...
@@ -132,15 +134,15 @@ func TestHandle_nonfatalLookupError(t *testing.T) {
132 132
 			return deployutil.MakeDeployment(config, api.Codec)
133 133
 		},
134 134
 		deploymentClient: &deploymentClientImpl{
135
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
136
-				return nil, kerrors.NewInternalError(fmt.Errorf("fatal test error"))
137
-			},
138 135
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
139 136
 				t.Fatalf("unexpected call with deployment %v", deployment)
140 137
 				return nil, nil
141 138
 			},
142 139
 			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
143
-				t.Fatalf("unexpected call to list deployments")
140
+				return nil, kerrors.NewInternalError(fmt.Errorf("fatal test error"))
141
+			},
142
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
143
+				t.Fatalf("unexpected update call with deployment %v", deployment)
144 144
 				return nil, nil
145 145
 			},
146 146
 		},
... ...
@@ -166,16 +168,18 @@ func TestHandle_configAlreadyDeployed(t *testing.T) {
166 166
 			return deployutil.MakeDeployment(config, api.Codec)
167 167
 		},
168 168
 		deploymentClient: &deploymentClientImpl{
169
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
170
-				deployment, _ := deployutil.MakeDeployment(deploymentConfig, kapi.Codec)
171
-				return deployment, nil
172
-			},
173 169
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
174 170
 				t.Fatalf("unexpected call to to create deployment: %v", deployment)
175 171
 				return nil, nil
176 172
 			},
177 173
 			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
178
-				t.Fatalf("unexpected call to list deployments")
174
+				existingDeployments := []kapi.ReplicationController{}
175
+				deployment, _ := deployutil.MakeDeployment(deploymentConfig, kapi.Codec)
176
+				existingDeployments = append(existingDeployments, *deployment)
177
+				return &kapi.ReplicationControllerList{Items: existingDeployments}, nil
178
+			},
179
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
180
+				t.Fatalf("unexpected update call with deployment %v", deployment)
179 181
 				return nil, nil
180 182
 			},
181 183
 		},
... ...
@@ -196,15 +200,16 @@ func TestHandle_nonfatalCreateError(t *testing.T) {
196 196
 			return deployutil.MakeDeployment(config, api.Codec)
197 197
 		},
198 198
 		deploymentClient: &deploymentClientImpl{
199
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
200
-				return nil, kerrors.NewNotFound("ReplicationController", name)
201
-			},
202 199
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
203 200
 				return nil, kerrors.NewInternalError(fmt.Errorf("test error"))
204 201
 			},
205 202
 			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
206 203
 				return &kapi.ReplicationControllerList{}, nil
207 204
 			},
205
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
206
+				t.Fatalf("unexpected update call with deployment %v", deployment)
207
+				return nil, nil
208
+			},
208 209
 		},
209 210
 		recorder: &record.FakeRecorder{},
210 211
 	}
... ...
@@ -226,9 +231,6 @@ func TestHandle_fatalError(t *testing.T) {
226 226
 			return nil, fmt.Errorf("couldn't make deployment")
227 227
 		},
228 228
 		deploymentClient: &deploymentClientImpl{
229
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
230
-				return nil, kerrors.NewNotFound("ReplicationController", name)
231
-			},
232 229
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
233 230
 				t.Fatalf("unexpected call to create")
234 231
 				return nil, kerrors.NewInternalError(fmt.Errorf("test error"))
... ...
@@ -236,6 +238,10 @@ func TestHandle_fatalError(t *testing.T) {
236 236
 			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
237 237
 				return &kapi.ReplicationControllerList{}, nil
238 238
 			},
239
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
240
+				t.Fatalf("unexpected update call with deployment %v", deployment)
241
+				return nil, nil
242
+			},
239 243
 		},
240 244
 	}
241 245
 
... ...
@@ -252,6 +258,7 @@ func TestHandle_fatalError(t *testing.T) {
252 252
 // new deployment for a config that has existing deployments succeeds of fails
253 253
 // depending upon the state of the existing deployments
254 254
 func TestHandle_existingDeployments(t *testing.T) {
255
+	var updatedDeployments []kapi.ReplicationController
255 256
 	var (
256 257
 		config              *deployapi.DeploymentConfig
257 258
 		deployed            *kapi.ReplicationController
... ...
@@ -263,9 +270,6 @@ func TestHandle_existingDeployments(t *testing.T) {
263 263
 			return deployutil.MakeDeployment(config, api.Codec)
264 264
 		},
265 265
 		deploymentClient: &deploymentClientImpl{
266
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
267
-				return nil, kerrors.NewNotFound("ReplicationController", name)
268
-			},
269 266
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
270 267
 				deployed = deployment
271 268
 				return deployment, nil
... ...
@@ -273,13 +277,19 @@ func TestHandle_existingDeployments(t *testing.T) {
273 273
 			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
274 274
 				return existingDeployments, nil
275 275
 			},
276
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
277
+				updatedDeployments = append(updatedDeployments, *deployment)
278
+				//t.Fatalf("unexpected update call with deployment %v", deployment)
279
+				return deployment, nil
280
+			},
276 281
 		},
277 282
 		recorder: &record.FakeRecorder{},
278 283
 	}
279 284
 
280 285
 	type existing struct {
281
-		version int
282
-		status  deployapi.DeploymentStatus
286
+		version      int
287
+		status       deployapi.DeploymentStatus
288
+		shouldCancel bool
283 289
 	}
284 290
 
285 291
 	type scenario struct {
... ...
@@ -293,25 +303,32 @@ func TestHandle_existingDeployments(t *testing.T) {
293 293
 		// No existing deployments
294 294
 		{1, []existing{}, nil},
295 295
 		// A single existing completed deployment
296
-		{2, []existing{{1, deployapi.DeploymentStatusComplete}}, nil},
296
+		{2, []existing{{1, deployapi.DeploymentStatusComplete, false}}, nil},
297 297
 		// A single existing failed deployment
298
-		{2, []existing{{1, deployapi.DeploymentStatusFailed}}, nil},
298
+		{2, []existing{{1, deployapi.DeploymentStatusFailed, false}}, nil},
299 299
 		// Multiple existing completed/failed deployments
300
-		{3, []existing{{2, deployapi.DeploymentStatusFailed}, {1, deployapi.DeploymentStatusComplete}}, nil},
300
+		{3, []existing{{2, deployapi.DeploymentStatusFailed, false}, {1, deployapi.DeploymentStatusComplete, false}}, nil},
301 301
 
302 302
 		// A single existing deployment in the default state
303
-		{2, []existing{{1, ""}}, transientErrorType},
303
+		{2, []existing{{1, "", false}}, transientErrorType},
304 304
 		// A single existing new deployment
305
-		{2, []existing{{1, deployapi.DeploymentStatusNew}}, transientErrorType},
305
+		{2, []existing{{1, deployapi.DeploymentStatusNew, false}}, transientErrorType},
306 306
 		// A single existing pending deployment
307
-		{2, []existing{{1, deployapi.DeploymentStatusPending}}, transientErrorType},
307
+		{2, []existing{{1, deployapi.DeploymentStatusPending, false}}, transientErrorType},
308 308
 		// A single existing running deployment
309
-		{2, []existing{{1, deployapi.DeploymentStatusRunning}}, transientErrorType},
309
+		{2, []existing{{1, deployapi.DeploymentStatusRunning, false}}, transientErrorType},
310 310
 		// Multiple existing deployments with one in new/pending/running
311
-		{4, []existing{{3, deployapi.DeploymentStatusRunning}, {2, deployapi.DeploymentStatusComplete}, {1, deployapi.DeploymentStatusFailed}}, transientErrorType},
311
+		{4, []existing{{3, deployapi.DeploymentStatusRunning, false}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusFailed, false}}, transientErrorType},
312
+
313
+		// Multiple existing deployments with more than one in new/pending/running
314
+		{4, []existing{{3, deployapi.DeploymentStatusNew, false}, {2, deployapi.DeploymentStatusRunning, true}, {1, deployapi.DeploymentStatusFailed, false}}, transientErrorType},
315
+		// Multiple existing deployments with more than one in new/pending/running
316
+		// Latest deployment has already failed
317
+		{6, []existing{{5, deployapi.DeploymentStatusFailed, false}, {4, deployapi.DeploymentStatusRunning, false}, {3, deployapi.DeploymentStatusNew, true}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusNew, true}}, transientErrorType},
312 318
 	}
313 319
 
314 320
 	for _, scenario := range scenarios {
321
+		updatedDeployments = []kapi.ReplicationController{}
315 322
 		deployed = nil
316 323
 		config = deploytest.OkDeploymentConfig(scenario.version)
317 324
 		existingDeployments = &kapi.ReplicationControllerList{}
... ...
@@ -339,5 +356,22 @@ func TestHandle_existingDeployments(t *testing.T) {
339 339
 				t.Fatalf("error expected: %s, got: %s", scenario.errorType, reflect.TypeOf(err))
340 340
 			}
341 341
 		}
342
+
343
+		expectedCancellations := []int{}
344
+		actualCancellations := []int{}
345
+		for _, e := range scenario.existing {
346
+			if e.shouldCancel {
347
+				expectedCancellations = append(expectedCancellations, e.version)
348
+			}
349
+		}
350
+		for _, d := range updatedDeployments {
351
+			actualCancellations = append(actualCancellations, deployutil.DeploymentVersionFor(&d))
352
+		}
353
+
354
+		sort.Ints(actualCancellations)
355
+		sort.Ints(expectedCancellations)
356
+		if !reflect.DeepEqual(actualCancellations, expectedCancellations) {
357
+			t.Fatalf("expected cancellations: %v, actual: %v", expectedCancellations, actualCancellations)
358
+		}
342 359
 	}
343 360
 }
... ...
@@ -49,9 +49,6 @@ func (factory *DeploymentConfigControllerFactory) Create() controller.RunnableCo
49 49
 
50 50
 	configController := &DeploymentConfigController{
51 51
 		deploymentClient: &deploymentClientImpl{
52
-			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
53
-				return factory.KubeClient.ReplicationControllers(namespace).Get(name)
54
-			},
55 52
 			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
56 53
 				return factory.KubeClient.ReplicationControllers(namespace).Create(deployment)
57 54
 			},
... ...
@@ -62,6 +59,9 @@ func (factory *DeploymentConfigControllerFactory) Create() controller.RunnableCo
62 62
 				}
63 63
 				return factory.KubeClient.ReplicationControllers(namespace).List(selector)
64 64
 			},
65
+			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
66
+				return factory.KubeClient.ReplicationControllers(namespace).Update(deployment)
67
+			},
65 68
 		},
66 69
 		makeDeployment: func(config *deployapi.DeploymentConfig) (*kapi.ReplicationController, error) {
67 70
 			return deployutil.MakeDeployment(config, factory.Codec)