Force a deploymentconfig reconcilation when its running deployment
is canceled instead of relying on the deploymentconfig cache sync
interval for rolling back.
| ... | ... |
@@ -305,8 +305,9 @@ func (c *MasterConfig) RunDeploymentController() {
|
| 305 | 305 |
|
| 306 | 306 |
// RunDeployerPodController starts the deployer pod controller process. |
| 307 | 307 |
func (c *MasterConfig) RunDeployerPodController() {
|
| 308 |
- _, kclient := c.DeployerPodControllerClients() |
|
| 308 |
+ osclient, kclient := c.DeployerPodControllerClients() |
|
| 309 | 309 |
factory := deployerpodcontroller.DeployerPodControllerFactory{
|
| 310 |
+ Client: osclient, |
|
| 310 | 311 |
KubeClient: kclient, |
| 311 | 312 |
Codec: c.EtcdHelper.Codec(), |
| 312 | 313 |
} |
| ... | ... |
@@ -2,6 +2,7 @@ package deployerpod |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"fmt" |
| 5 |
+ "strconv" |
|
| 5 | 6 |
|
| 6 | 7 |
"github.com/golang/glog" |
| 7 | 8 |
|
| ... | ... |
@@ -10,6 +11,7 @@ import ( |
| 10 | 10 |
"k8s.io/kubernetes/pkg/client/cache" |
| 11 | 11 |
kclient "k8s.io/kubernetes/pkg/client/unversioned" |
| 12 | 12 |
|
| 13 |
+ osclient "github.com/openshift/origin/pkg/client" |
|
| 13 | 14 |
deployapi "github.com/openshift/origin/pkg/deploy/api" |
| 14 | 15 |
deployutil "github.com/openshift/origin/pkg/deploy/util" |
| 15 | 16 |
) |
| ... | ... |
@@ -20,6 +22,7 @@ import ( |
| 20 | 20 |
// Use the DeployerPodControllerFactory to create this controller. |
| 21 | 21 |
type DeployerPodController struct {
|
| 22 | 22 |
store cache.Store |
| 23 |
+ client osclient.Interface |
|
| 23 | 24 |
kClient kclient.Interface |
| 24 | 25 |
|
| 25 | 26 |
// decodeConfig knows how to decode the deploymentConfig from a deployment's annotations. |
| ... | ... |
@@ -94,14 +97,8 @@ func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
|
| 94 | 94 |
nextStatus = deployapi.DeploymentStatusRunning |
| 95 | 95 |
} |
| 96 | 96 |
case kapi.PodSucceeded: |
| 97 |
- // Detect failure based on the container state |
|
| 98 | 97 |
nextStatus = deployapi.DeploymentStatusComplete |
| 99 |
- for _, info := range pod.Status.ContainerStatuses {
|
|
| 100 |
- if info.State.Terminated != nil && info.State.Terminated.ExitCode != 0 {
|
|
| 101 |
- nextStatus = deployapi.DeploymentStatusFailed |
|
| 102 |
- break |
|
| 103 |
- } |
|
| 104 |
- } |
|
| 98 |
+ |
|
| 105 | 99 |
// Sync the internal replica annotation with the target so that we can |
| 106 | 100 |
// distinguish deployer updates from other scaling events. |
| 107 | 101 |
deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation] |
| ... | ... |
@@ -131,6 +128,27 @@ func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
|
| 131 | 131 |
return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
|
| 132 | 132 |
} |
| 133 | 133 |
glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas)
|
| 134 |
+ |
|
| 135 |
+ // If the deployment was canceled, trigger a reconcilation of its deployment config |
|
| 136 |
+ // so that the latest complete deployment can immediately rollback in place of the |
|
| 137 |
+ // canceled deployment. |
|
| 138 |
+ if nextStatus == deployapi.DeploymentStatusFailed && deployutil.IsDeploymentCancelled(deployment) {
|
|
| 139 |
+ // If we are unable to get the deployment config, then the deploymentconfig controller will |
|
| 140 |
+ // perform its duties once the resync interval forces the deploymentconfig to be reconciled. |
|
| 141 |
+ name := deployutil.DeploymentConfigNameFor(deployment) |
|
| 142 |
+ kclient.RetryOnConflict(kclient.DefaultRetry, func() error {
|
|
| 143 |
+ config, err := c.client.DeploymentConfigs(deployment.Namespace).Get(name) |
|
| 144 |
+ if err != nil {
|
|
| 145 |
+ return err |
|
| 146 |
+ } |
|
| 147 |
+ if config.Annotations == nil {
|
|
| 148 |
+ config.Annotations = make(map[string]string) |
|
| 149 |
+ } |
|
| 150 |
+ config.Annotations[deployapi.DeploymentCancelledAnnotation] = strconv.Itoa(config.Status.LatestVersion) |
|
| 151 |
+ _, err = c.client.DeploymentConfigs(config.Namespace).Update(config) |
|
| 152 |
+ return err |
|
| 153 |
+ }) |
|
| 154 |
+ } |
|
| 134 | 155 |
} |
| 135 | 156 |
|
| 136 | 157 |
return nil |
| ... | ... |
@@ -10,12 +10,65 @@ import ( |
| 10 | 10 |
"k8s.io/kubernetes/pkg/runtime" |
| 11 | 11 |
"k8s.io/kubernetes/pkg/util/sets" |
| 12 | 12 |
|
| 13 |
+ "github.com/openshift/origin/pkg/client/testclient" |
|
| 13 | 14 |
deployapi "github.com/openshift/origin/pkg/deploy/api" |
| 14 | 15 |
_ "github.com/openshift/origin/pkg/deploy/api/install" |
| 15 | 16 |
deploytest "github.com/openshift/origin/pkg/deploy/api/test" |
| 16 | 17 |
deployutil "github.com/openshift/origin/pkg/deploy/util" |
| 17 | 18 |
) |
| 18 | 19 |
|
| 20 |
+func okPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 21 |
+ return &kapi.Pod{
|
|
| 22 |
+ ObjectMeta: kapi.ObjectMeta{
|
|
| 23 |
+ Name: deployutil.DeployerPodNameForDeployment(deployment.Name), |
|
| 24 |
+ Labels: map[string]string{
|
|
| 25 |
+ deployapi.DeployerPodForDeploymentLabel: deployment.Name, |
|
| 26 |
+ }, |
|
| 27 |
+ Annotations: map[string]string{
|
|
| 28 |
+ deployapi.DeploymentAnnotation: deployment.Name, |
|
| 29 |
+ }, |
|
| 30 |
+ }, |
|
| 31 |
+ Status: kapi.PodStatus{
|
|
| 32 |
+ ContainerStatuses: []kapi.ContainerStatus{
|
|
| 33 |
+ {},
|
|
| 34 |
+ }, |
|
| 35 |
+ }, |
|
| 36 |
+ } |
|
| 37 |
+} |
|
| 38 |
+ |
|
| 39 |
+func succeededPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 40 |
+ p := okPod(deployment) |
|
| 41 |
+ p.Status.Phase = kapi.PodSucceeded |
|
| 42 |
+ return p |
|
| 43 |
+} |
|
| 44 |
+ |
|
| 45 |
+func failedPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 46 |
+ p := okPod(deployment) |
|
| 47 |
+ p.Status.Phase = kapi.PodFailed |
|
| 48 |
+ p.Status.ContainerStatuses = []kapi.ContainerStatus{
|
|
| 49 |
+ {
|
|
| 50 |
+ State: kapi.ContainerState{
|
|
| 51 |
+ Terminated: &kapi.ContainerStateTerminated{
|
|
| 52 |
+ ExitCode: 1, |
|
| 53 |
+ }, |
|
| 54 |
+ }, |
|
| 55 |
+ }, |
|
| 56 |
+ } |
|
| 57 |
+ return p |
|
| 58 |
+} |
|
| 59 |
+ |
|
| 60 |
+func terminatedPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 61 |
+ p := okPod(deployment) |
|
| 62 |
+ p.Status.Phase = kapi.PodFailed |
|
| 63 |
+ return p |
|
| 64 |
+} |
|
| 65 |
+ |
|
| 66 |
+func runningPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 67 |
+ p := okPod(deployment) |
|
| 68 |
+ p.Status.Phase = kapi.PodRunning |
|
| 69 |
+ return p |
|
| 70 |
+} |
|
| 71 |
+ |
|
| 19 | 72 |
// TestHandle_uncorrelatedPod ensures that pods uncorrelated with a deployment |
| 20 | 73 |
// are ignored. |
| 21 | 74 |
func TestHandle_uncorrelatedPod(t *testing.T) {
|
| ... | ... |
@@ -355,54 +408,63 @@ func TestHandle_cleanupDesiredReplicasAnnotation(t *testing.T) {
|
| 355 | 355 |
} |
| 356 | 356 |
} |
| 357 | 357 |
|
| 358 |
-func okPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 359 |
- return &kapi.Pod{
|
|
| 360 |
- ObjectMeta: kapi.ObjectMeta{
|
|
| 361 |
- Name: deployutil.DeployerPodNameForDeployment(deployment.Name), |
|
| 362 |
- Labels: map[string]string{
|
|
| 363 |
- deployapi.DeployerPodForDeploymentLabel: deployment.Name, |
|
| 364 |
- }, |
|
| 365 |
- Annotations: map[string]string{
|
|
| 366 |
- deployapi.DeploymentAnnotation: deployment.Name, |
|
| 367 |
- }, |
|
| 368 |
- }, |
|
| 369 |
- Status: kapi.PodStatus{
|
|
| 370 |
- ContainerStatuses: []kapi.ContainerStatus{
|
|
| 371 |
- {},
|
|
| 372 |
- }, |
|
| 358 |
+// TestHandle_canceledDeploymentTrigger ensures that a canceled deployment |
|
| 359 |
+// will trigger a reconcilation of its deploymentconfig (via an annotation |
|
| 360 |
+// update) so that rolling back can happen on the spot and not rely on the |
|
| 361 |
+// deploymentconfig cache resync interval. |
|
| 362 |
+func TestHandle_canceledDeploymentTriggerTest(t *testing.T) {
|
|
| 363 |
+ var ( |
|
| 364 |
+ updatedDeployment *kapi.ReplicationController |
|
| 365 |
+ updatedConfig *deployapi.DeploymentConfig |
|
| 366 |
+ ) |
|
| 367 |
+ |
|
| 368 |
+ initial := deploytest.OkDeploymentConfig(1) |
|
| 369 |
+ // Canceled deployment |
|
| 370 |
+ deployment, _ := deployutil.MakeDeployment(deploytest.TestDeploymentConfig(initial), kapi.Codecs.LegacyCodec(deployapi.SchemeGroupVersion)) |
|
| 371 |
+ deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue |
|
| 372 |
+ |
|
| 373 |
+ kFake := &ktestclient.Fake{}
|
|
| 374 |
+ kFake.PrependReactor("get", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
|
|
| 375 |
+ return true, deployment, nil |
|
| 376 |
+ }) |
|
| 377 |
+ kFake.PrependReactor("update", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
|
|
| 378 |
+ updatedDeployment = deployment |
|
| 379 |
+ return true, deployment, nil |
|
| 380 |
+ }) |
|
| 381 |
+ fake := &testclient.Fake{}
|
|
| 382 |
+ fake.PrependReactor("get", "deploymentconfigs", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
|
|
| 383 |
+ config := initial |
|
| 384 |
+ return true, config, nil |
|
| 385 |
+ }) |
|
| 386 |
+ fake.PrependReactor("update", "deploymentconfigs", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
|
|
| 387 |
+ updated := action.(ktestclient.UpdateAction).GetObject().(*deployapi.DeploymentConfig) |
|
| 388 |
+ updatedConfig = updated |
|
| 389 |
+ return true, updated, nil |
|
| 390 |
+ }) |
|
| 391 |
+ |
|
| 392 |
+ controller := &DeployerPodController{
|
|
| 393 |
+ decodeConfig: func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error) {
|
|
| 394 |
+ return deployutil.DecodeDeploymentConfig(deployment, kapi.Codecs.UniversalDecoder()) |
|
| 373 | 395 |
}, |
| 396 |
+ store: cache.NewStore(cache.MetaNamespaceKeyFunc), |
|
| 397 |
+ client: fake, |
|
| 398 |
+ kClient: kFake, |
|
| 374 | 399 |
} |
| 375 |
-} |
|
| 376 | 400 |
|
| 377 |
-func succeededPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 378 |
- p := okPod(deployment) |
|
| 379 |
- p.Status.Phase = kapi.PodSucceeded |
|
| 380 |
- return p |
|
| 381 |
-} |
|
| 401 |
+ err := controller.Handle(terminatedPod(deployment)) |
|
| 402 |
+ if err != nil {
|
|
| 403 |
+ t.Fatalf("unexpected error: %v", err)
|
|
| 404 |
+ } |
|
| 382 | 405 |
|
| 383 |
-func failedPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 384 |
- p := okPod(deployment) |
|
| 385 |
- p.Status.Phase = kapi.PodFailed |
|
| 386 |
- p.Status.ContainerStatuses = []kapi.ContainerStatus{
|
|
| 387 |
- {
|
|
| 388 |
- State: kapi.ContainerState{
|
|
| 389 |
- Terminated: &kapi.ContainerStateTerminated{
|
|
| 390 |
- ExitCode: 1, |
|
| 391 |
- }, |
|
| 392 |
- }, |
|
| 393 |
- }, |
|
| 406 |
+ if updatedDeployment == nil {
|
|
| 407 |
+ t.Fatalf("expected deployment update")
|
|
| 394 | 408 |
} |
| 395 |
- return p |
|
| 396 |
-} |
|
| 397 | 409 |
|
| 398 |
-func terminatedPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 399 |
- p := okPod(deployment) |
|
| 400 |
- p.Status.Phase = kapi.PodFailed |
|
| 401 |
- return p |
|
| 402 |
-} |
|
| 410 |
+ if e, a := deployapi.DeploymentStatusFailed, deployutil.DeploymentStatusFor(updatedDeployment); e != a {
|
|
| 411 |
+ t.Fatalf("expected updated deployment status %s, got %s", e, a)
|
|
| 412 |
+ } |
|
| 403 | 413 |
|
| 404 |
-func runningPod(deployment *kapi.ReplicationController) *kapi.Pod {
|
|
| 405 |
- p := okPod(deployment) |
|
| 406 |
- p.Status.Phase = kapi.PodRunning |
|
| 407 |
- return p |
|
| 414 |
+ if updatedConfig == nil {
|
|
| 415 |
+ t.Fatalf("expected config update")
|
|
| 416 |
+ } |
|
| 408 | 417 |
} |
| ... | ... |
@@ -11,6 +11,7 @@ import ( |
| 11 | 11 |
utilruntime "k8s.io/kubernetes/pkg/util/runtime" |
| 12 | 12 |
"k8s.io/kubernetes/pkg/watch" |
| 13 | 13 |
|
| 14 |
+ osclient "github.com/openshift/origin/pkg/client" |
|
| 14 | 15 |
controller "github.com/openshift/origin/pkg/controller" |
| 15 | 16 |
deployapi "github.com/openshift/origin/pkg/deploy/api" |
| 16 | 17 |
deployutil "github.com/openshift/origin/pkg/deploy/util" |
| ... | ... |
@@ -19,6 +20,8 @@ import ( |
| 19 | 19 |
// DeployerPodControllerFactory can create a DeployerPodController which |
| 20 | 20 |
// handles processing deployer pods. |
| 21 | 21 |
type DeployerPodControllerFactory struct {
|
| 22 |
+ // Client is an OpenShift client. |
|
| 23 |
+ Client osclient.Interface |
|
| 22 | 24 |
// KubeClient is a Kubernetes client. |
| 23 | 25 |
KubeClient kclient.Interface |
| 24 | 26 |
// Codec is used for encoding/decoding. |
| ... | ... |
@@ -58,6 +61,7 @@ func (factory *DeployerPodControllerFactory) Create() controller.RunnableControl |
| 58 | 58 |
|
| 59 | 59 |
podController := &DeployerPodController{
|
| 60 | 60 |
store: deploymentStore, |
| 61 |
+ client: factory.Client, |
|
| 61 | 62 |
kClient: factory.KubeClient, |
| 62 | 63 |
decodeConfig: func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error) {
|
| 63 | 64 |
return deployutil.DecodeDeploymentConfig(deployment, factory.Codec) |
| ... | ... |
@@ -96,7 +96,7 @@ func (factory *DeploymentControllerFactory) Create() controller.RunnableControll |
| 96 | 96 |
decodeConfig: func(deployment *kapi.ReplicationController) (*deployapi.DeploymentConfig, error) {
|
| 97 | 97 |
return deployutil.DecodeDeploymentConfig(deployment, factory.Codec) |
| 98 | 98 |
}, |
| 99 |
- recorder: eventBroadcaster.NewRecorder(kapi.EventSource{Component: "deployer"}),
|
|
| 99 |
+ recorder: eventBroadcaster.NewRecorder(kapi.EventSource{Component: "deployment-controller"}),
|
|
| 100 | 100 |
} |
| 101 | 101 |
|
| 102 | 102 |
return &controller.RetryController{
|