Browse code

Bug 1314270: force dc reconcilation on canceled deployments

Force a deploymentconfig reconcilation when its running deployment
is canceled instead of relying on the deploymentconfig cache sync
interval for rolling back.

kargakis authored on 2016/03/23 22:27:01
Showing 5 changed files
... ...
@@ -305,8 +305,9 @@ func (c *MasterConfig) RunDeploymentController() {
305 305
 
306 306
 // RunDeployerPodController starts the deployer pod controller process.
307 307
 func (c *MasterConfig) RunDeployerPodController() {
308
-	_, kclient := c.DeployerPodControllerClients()
308
+	osclient, kclient := c.DeployerPodControllerClients()
309 309
 	factory := deployerpodcontroller.DeployerPodControllerFactory{
310
+		Client:     osclient,
310 311
 		KubeClient: kclient,
311 312
 		Codec:      c.EtcdHelper.Codec(),
312 313
 	}
... ...
@@ -2,6 +2,7 @@ package deployerpod
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"strconv"
5 6
 
6 7
 	"github.com/golang/glog"
7 8
 
... ...
@@ -10,6 +11,7 @@ import (
10 10
 	"k8s.io/kubernetes/pkg/client/cache"
11 11
 	kclient "k8s.io/kubernetes/pkg/client/unversioned"
12 12
 
13
+	osclient "github.com/openshift/origin/pkg/client"
13 14
 	deployapi "github.com/openshift/origin/pkg/deploy/api"
14 15
 	deployutil "github.com/openshift/origin/pkg/deploy/util"
15 16
 )
... ...
@@ -20,6 +22,7 @@ import (
20 20
 // Use the DeployerPodControllerFactory to create this controller.
21 21
 type DeployerPodController struct {
22 22
 	store   cache.Store
23
+	client  osclient.Interface
23 24
 	kClient kclient.Interface
24 25
 
25 26
 	// decodeConfig knows how to decode the deploymentConfig from a deployment's annotations.
... ...
@@ -94,14 +97,8 @@ func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
94 94
 			nextStatus = deployapi.DeploymentStatusRunning
95 95
 		}
96 96
 	case kapi.PodSucceeded:
97
-		// Detect failure based on the container state
98 97
 		nextStatus = deployapi.DeploymentStatusComplete
99
-		for _, info := range pod.Status.ContainerStatuses {
100
-			if info.State.Terminated != nil && info.State.Terminated.ExitCode != 0 {
101
-				nextStatus = deployapi.DeploymentStatusFailed
102
-				break
103
-			}
104
-		}
98
+
105 99
 		// Sync the internal replica annotation with the target so that we can
106 100
 		// distinguish deployer updates from other scaling events.
107 101
 		deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation]
... ...
@@ -131,6 +128,27 @@ func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
131 131
 			return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
132 132
 		}
133 133
 		glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas)
134
+
135
+		// If the deployment was canceled, trigger a reconcilation of its deployment config
136
+		// so that the latest complete deployment can immediately rollback in place of the
137
+		// canceled deployment.
138
+		if nextStatus == deployapi.DeploymentStatusFailed && deployutil.IsDeploymentCancelled(deployment) {
139
+			// If we are unable to get the deployment config, then the deploymentconfig controller will
140
+			// perform its duties once the resync interval forces the deploymentconfig to be reconciled.
141
+			name := deployutil.DeploymentConfigNameFor(deployment)
142
+			kclient.RetryOnConflict(kclient.DefaultRetry, func() error {
143
+				config, err := c.client.DeploymentConfigs(deployment.Namespace).Get(name)
144
+				if err != nil {
145
+					return err
146
+				}
147
+				if config.Annotations == nil {
148
+					config.Annotations = make(map[string]string)
149
+				}
150
+				config.Annotations[deployapi.DeploymentCancelledAnnotation] = strconv.Itoa(config.Status.LatestVersion)
151
+				_, err = c.client.DeploymentConfigs(config.Namespace).Update(config)
152
+				return err
153
+			})
154
+		}
134 155
 	}
135 156
 
136 157
 	return nil
... ...
@@ -10,12 +10,65 @@ import (
10 10
 	"k8s.io/kubernetes/pkg/runtime"
11 11
 	"k8s.io/kubernetes/pkg/util/sets"
12 12
 
13
+	"github.com/openshift/origin/pkg/client/testclient"
13 14
 	deployapi "github.com/openshift/origin/pkg/deploy/api"
14 15
 	_ "github.com/openshift/origin/pkg/deploy/api/install"
15 16
 	deploytest "github.com/openshift/origin/pkg/deploy/api/test"
16 17
 	deployutil "github.com/openshift/origin/pkg/deploy/util"
17 18
 )
18 19
 
20
+func okPod(deployment *kapi.ReplicationController) *kapi.Pod {
21
+	return &kapi.Pod{
22
+		ObjectMeta: kapi.ObjectMeta{
23
+			Name: deployutil.DeployerPodNameForDeployment(deployment.Name),
24
+			Labels: map[string]string{
25
+				deployapi.DeployerPodForDeploymentLabel: deployment.Name,
26
+			},
27
+			Annotations: map[string]string{
28
+				deployapi.DeploymentAnnotation: deployment.Name,
29
+			},
30
+		},
31
+		Status: kapi.PodStatus{
32
+			ContainerStatuses: []kapi.ContainerStatus{
33
+				{},
34
+			},
35
+		},
36
+	}
37
+}
38
+
39
+func succeededPod(deployment *kapi.ReplicationController) *kapi.Pod {
40
+	p := okPod(deployment)
41
+	p.Status.Phase = kapi.PodSucceeded
42
+	return p
43
+}
44
+
45
+func failedPod(deployment *kapi.ReplicationController) *kapi.Pod {
46
+	p := okPod(deployment)
47
+	p.Status.Phase = kapi.PodFailed
48
+	p.Status.ContainerStatuses = []kapi.ContainerStatus{
49
+		{
50
+			State: kapi.ContainerState{
51
+				Terminated: &kapi.ContainerStateTerminated{
52
+					ExitCode: 1,
53
+				},
54
+			},
55
+		},
56
+	}
57
+	return p
58
+}
59
+
60
+func terminatedPod(deployment *kapi.ReplicationController) *kapi.Pod {
61
+	p := okPod(deployment)
62
+	p.Status.Phase = kapi.PodFailed
63
+	return p
64
+}
65
+
66
+func runningPod(deployment *kapi.ReplicationController) *kapi.Pod {
67
+	p := okPod(deployment)
68
+	p.Status.Phase = kapi.PodRunning
69
+	return p
70
+}
71
+
19 72
 // TestHandle_uncorrelatedPod ensures that pods uncorrelated with a deployment
20 73
 // are ignored.
21 74
 func TestHandle_uncorrelatedPod(t *testing.T) {
... ...
@@ -355,54 +408,63 @@ func TestHandle_cleanupDesiredReplicasAnnotation(t *testing.T) {
355 355
 	}
356 356
 }
357 357
 
358
-func okPod(deployment *kapi.ReplicationController) *kapi.Pod {
359
-	return &kapi.Pod{
360
-		ObjectMeta: kapi.ObjectMeta{
361
-			Name: deployutil.DeployerPodNameForDeployment(deployment.Name),
362
-			Labels: map[string]string{
363
-				deployapi.DeployerPodForDeploymentLabel: deployment.Name,
364
-			},
365
-			Annotations: map[string]string{
366
-				deployapi.DeploymentAnnotation: deployment.Name,
367
-			},
368
-		},
369
-		Status: kapi.PodStatus{
370
-			ContainerStatuses: []kapi.ContainerStatus{
371
-				{},
372
-			},
358
+// TestHandle_canceledDeploymentTrigger ensures that a canceled deployment
359
+// will trigger a reconcilation of its deploymentconfig (via an annotation
360
+// update) so that rolling back can happen on the spot and not rely on the
361
+// deploymentconfig cache resync interval.
362
+func TestHandle_canceledDeploymentTriggerTest(t *testing.T) {
363
+	var (
364
+		updatedDeployment *kapi.ReplicationController
365
+		updatedConfig     *deployapi.DeploymentConfig
366
+	)
367
+
368
+	initial := deploytest.OkDeploymentConfig(1)
369
+	// Canceled deployment
370
+	deployment, _ := deployutil.MakeDeployment(deploytest.TestDeploymentConfig(initial), kapi.Codecs.LegacyCodec(deployapi.SchemeGroupVersion))
371
+	deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
372
+
373
+	kFake := &ktestclient.Fake{}
374
+	kFake.PrependReactor("get", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
375
+		return true, deployment, nil
376
+	})
377
+	kFake.PrependReactor("update", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
378
+		updatedDeployment = deployment
379
+		return true, deployment, nil
380
+	})
381
+	fake := &testclient.Fake{}
382
+	fake.PrependReactor("get", "deploymentconfigs", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
383
+		config := initial
384
+		return true, config, nil
385
+	})
386
+	fake.PrependReactor("update", "deploymentconfigs", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
387
+		updated := action.(ktestclient.UpdateAction).GetObject().(*deployapi.DeploymentConfig)
388
+		updatedConfig = updated
389
+		return true, updated, nil
390
+	})
391
+
392
+	controller := &DeployerPodController{
393
+		decodeConfig: func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error) {
394
+			return deployutil.DecodeDeploymentConfig(deployment, kapi.Codecs.UniversalDecoder())
373 395
 		},
396
+		store:   cache.NewStore(cache.MetaNamespaceKeyFunc),
397
+		client:  fake,
398
+		kClient: kFake,
374 399
 	}
375
-}
376 400
 
377
-func succeededPod(deployment *kapi.ReplicationController) *kapi.Pod {
378
-	p := okPod(deployment)
379
-	p.Status.Phase = kapi.PodSucceeded
380
-	return p
381
-}
401
+	err := controller.Handle(terminatedPod(deployment))
402
+	if err != nil {
403
+		t.Fatalf("unexpected error: %v", err)
404
+	}
382 405
 
383
-func failedPod(deployment *kapi.ReplicationController) *kapi.Pod {
384
-	p := okPod(deployment)
385
-	p.Status.Phase = kapi.PodFailed
386
-	p.Status.ContainerStatuses = []kapi.ContainerStatus{
387
-		{
388
-			State: kapi.ContainerState{
389
-				Terminated: &kapi.ContainerStateTerminated{
390
-					ExitCode: 1,
391
-				},
392
-			},
393
-		},
406
+	if updatedDeployment == nil {
407
+		t.Fatalf("expected deployment update")
394 408
 	}
395
-	return p
396
-}
397 409
 
398
-func terminatedPod(deployment *kapi.ReplicationController) *kapi.Pod {
399
-	p := okPod(deployment)
400
-	p.Status.Phase = kapi.PodFailed
401
-	return p
402
-}
410
+	if e, a := deployapi.DeploymentStatusFailed, deployutil.DeploymentStatusFor(updatedDeployment); e != a {
411
+		t.Fatalf("expected updated deployment status %s, got %s", e, a)
412
+	}
403 413
 
404
-func runningPod(deployment *kapi.ReplicationController) *kapi.Pod {
405
-	p := okPod(deployment)
406
-	p.Status.Phase = kapi.PodRunning
407
-	return p
414
+	if updatedConfig == nil {
415
+		t.Fatalf("expected config update")
416
+	}
408 417
 }
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
12 12
 	"k8s.io/kubernetes/pkg/watch"
13 13
 
14
+	osclient "github.com/openshift/origin/pkg/client"
14 15
 	controller "github.com/openshift/origin/pkg/controller"
15 16
 	deployapi "github.com/openshift/origin/pkg/deploy/api"
16 17
 	deployutil "github.com/openshift/origin/pkg/deploy/util"
... ...
@@ -19,6 +20,8 @@ import (
19 19
 // DeployerPodControllerFactory can create a DeployerPodController which
20 20
 // handles processing deployer pods.
21 21
 type DeployerPodControllerFactory struct {
22
+	// Client is an OpenShift client.
23
+	Client osclient.Interface
22 24
 	// KubeClient is a Kubernetes client.
23 25
 	KubeClient kclient.Interface
24 26
 	// Codec is used for encoding/decoding.
... ...
@@ -58,6 +61,7 @@ func (factory *DeployerPodControllerFactory) Create() controller.RunnableControl
58 58
 
59 59
 	podController := &DeployerPodController{
60 60
 		store:   deploymentStore,
61
+		client:  factory.Client,
61 62
 		kClient: factory.KubeClient,
62 63
 		decodeConfig: func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error) {
63 64
 			return deployutil.DecodeDeploymentConfig(deployment, factory.Codec)
... ...
@@ -96,7 +96,7 @@ func (factory *DeploymentControllerFactory) Create() controller.RunnableControll
96 96
 		decodeConfig: func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error) {
97 97
 			return deployutil.DecodeDeploymentConfig(deployment, factory.Codec)
98 98
 		},
99
-		recorder: eventBroadcaster.NewRecorder(kapi.EventSource{Component: "deployer"}),
99
+		recorder: eventBroadcaster.NewRecorder(kapi.EventSource{Component: "deployment-controller"}),
100 100
 	}
101 101
 
102 102
 	return &controller.RetryController{