package deployerpod import ( "fmt" "strconv" "testing" kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" kerrors "github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors" kutil "github.com/GoogleCloudPlatform/kubernetes/pkg/util" deployapi "github.com/openshift/origin/pkg/deploy/api" deploytest "github.com/openshift/origin/pkg/deploy/api/test" deployutil "github.com/openshift/origin/pkg/deploy/util" ) // TestHandle_uncorrelatedPod ensures that pods uncorrelated with a deployment // are ignored. func TestHandle_uncorrelatedPod(t *testing.T) { controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { t.Fatalf("unexpected deployment update") return nil, nil }, }, } // Verify no-op deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) pod := runningPod(deployment) pod.Annotations = make(map[string]string) err := controller.Handle(pod) if err != nil { t.Fatalf("unexpected err: %v", err) } } // TestHandle_orphanedPod ensures that deployer pods associated with a non- // existent deployment results in all deployer pods being deleted. func TestHandle_orphanedPod(t *testing.T) { deleted := kutil.NewStringSet() controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { t.Fatalf("Unexpected deployment update") return nil, nil }, getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) { return nil, kerrors.NewNotFound("ReplicationController", name) }, }, deployerPodsFor: func(namespace, name string) (*kapi.PodList, error) { mkpod := func(suffix string) kapi.Pod { deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) p := okPod(deployment) p.Name = p.Name + suffix return *p } return &kapi.PodList{ Items: []kapi.Pod{ mkpod(""), mkpod("-prehook"), mkpod("-posthook"), }, }, nil }, deletePod: func(namespace, name string) error { deleted.Insert(name) return nil }, } deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) err := controller.Handle(runningPod(deployment)) if err != nil { t.Fatalf("unexpected error: %v", err) } deployerName := deployutil.DeployerPodNameForDeployment(deployment.Name) if !deleted.HasAll(deployerName, deployerName+"-prehook", deployerName+"-posthook") { t.Fatalf("unexpected deleted names: %v", deleted.List()) } } // TestHandle_runningPod ensures that a running deployer pod results in a // transition of the deployment's status to running. func TestHandle_runningPod(t *testing.T) { deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusPending) var updatedDeployment *kapi.ReplicationController controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) { return deployment, nil }, updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { updatedDeployment = deployment return deployment, nil }, }, } err := controller.Handle(runningPod(deployment)) if err != nil { t.Fatalf("unexpected error: %v", err) } if updatedDeployment == nil { t.Fatalf("expected deployment update") } if e, a := deployapi.DeploymentStatusRunning, deployutil.DeploymentStatusFor(updatedDeployment); e != a { t.Fatalf("expected updated deployment status %s, got %s", e, a) } } // TestHandle_podTerminatedOk ensures that a successfully completed deployer // pod results in a transition of the deployment's status to complete. func TestHandle_podTerminatedOk(t *testing.T) { deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusRunning) var updatedDeployment *kapi.ReplicationController controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) { return deployment, nil }, updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { updatedDeployment = deployment return deployment, nil }, }, } err := controller.Handle(succeededPod(deployment)) if err != nil { t.Fatalf("unexpected error: %v", err) } if updatedDeployment == nil { t.Fatalf("expected deployment update") } if e, a := deployapi.DeploymentStatusComplete, deployutil.DeploymentStatusFor(updatedDeployment); e != a { t.Fatalf("expected updated deployment status %s, got %s", e, a) } } // TestHandle_podTerminatedFailNoContainerStatus ensures that a failed // deployer pod with no container status results in a transition of the // deployment's status to failed. func TestHandle_podTerminatedFailNoContainerStatus(t *testing.T) { var updatedDeployment *kapi.ReplicationController deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) // since we do not set the desired replicas annotation, // this also tests that the error is just logged and not result in a failure deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusRunning) controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) { return deployment, nil }, updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { updatedDeployment = deployment return deployment, nil }, listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) { return &kapi.ReplicationControllerList{Items: []kapi.ReplicationController{*deployment}}, nil }, }, } err := controller.Handle(terminatedPod(deployment)) if err != nil { t.Fatalf("unexpected error: %v", err) } if updatedDeployment == nil { t.Fatalf("expected deployment update") } if e, a := deployapi.DeploymentStatusFailed, deployutil.DeploymentStatusFor(updatedDeployment); e != a { t.Fatalf("expected updated deployment status %s, got %s", e, a) } } // TestHandle_deploymentCleanupTransientError ensures that a failure // to clean up a failed deployment results in a transient error // and the deployment status is not set to Failed. func TestHandle_deploymentCleanupTransientError(t *testing.T) { completedDeployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) completedDeployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusComplete) currentDeployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(2), kapi.Codec) currentDeployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusRunning) currentDeployment.Annotations[deployapi.DesiredReplicasAnnotation] = "2" controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) { return currentDeployment, nil }, updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { // simulate failure ONLY for the completed deployment if deployutil.DeploymentStatusFor(deployment) == deployapi.DeploymentStatusComplete { return nil, fmt.Errorf("test failure in updating completed deployment") } return deployment, nil }, listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) { return &kapi.ReplicationControllerList{Items: []kapi.ReplicationController{*currentDeployment, *completedDeployment}}, nil }, }, } err := controller.Handle(terminatedPod(currentDeployment)) if err == nil { t.Fatalf("unexpected error: %v", err) } if _, isTransient := err.(transientError); !isTransient { t.Fatalf("expected transientError on failure to update deployment") } if e, a := deployapi.DeploymentStatusRunning, deployutil.DeploymentStatusFor(currentDeployment); e != a { t.Fatalf("expected updated deployment status to remain %s, got %s", e, a) } } // TestHandle_cleanupDeploymentFailure ensures that clean up happens // for the deployment if the deployer pod fails. // - failed deployment is scaled down // - the last completed deployment is scaled back up func TestHandle_cleanupDeploymentFailure(t *testing.T) { var existingDeployments *kapi.ReplicationControllerList var failedDeployment *kapi.ReplicationController // map of deployment-version to updated replicas var updatedDeployments map[int]*kapi.ReplicationController controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) { return failedDeployment, nil }, updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { if _, found := updatedDeployments[deployutil.DeploymentVersionFor(deployment)]; found { t.Fatalf("unexpected multiple updates for deployment #%d", deployutil.DeploymentVersionFor(deployment)) } updatedDeployments[deployutil.DeploymentVersionFor(deployment)] = deployment return deployment, nil }, listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) { return existingDeployments, nil }, }, } type existing struct { version int status deployapi.DeploymentStatus initialReplicas int updatedReplicas int } type scenario struct { name string // this is the deployment that is passed to Handle version int // this is the target replicas for the deployment that failed desiredReplicas int // existing deployments also include the one being handled currently existing []existing } // existing deployments intentionally placed un-ordered // in order to verify sorting scenarios := []scenario{ {"No previous deployments", 1, 3, []existing{ {1, deployapi.DeploymentStatusRunning, 3, 0}, }}, {"Multiple existing deployments - none in complete state", 3, 2, []existing{ {1, deployapi.DeploymentStatusFailed, 2, 2}, {2, deployapi.DeploymentStatusFailed, 0, 0}, {3, deployapi.DeploymentStatusRunning, 2, 0}, }}, {"Failed deployment is already at 0 replicas", 3, 2, []existing{ {1, deployapi.DeploymentStatusFailed, 2, 2}, {2, deployapi.DeploymentStatusFailed, 0, 0}, {3, deployapi.DeploymentStatusRunning, 0, 0}, }}, {"Multiple existing completed deployments", 4, 2, []existing{ {3, deployapi.DeploymentStatusComplete, 0, 2}, {2, deployapi.DeploymentStatusComplete, 0, 0}, {4, deployapi.DeploymentStatusRunning, 1, 0}, {1, deployapi.DeploymentStatusFailed, 0, 0}, }}, // A deployment already exists after the current failed deployment // only the current deployment is marked as failed // the completed deployment is not scaled up {"Deployment exists after current failed", 4, 2, []existing{ {3, deployapi.DeploymentStatusComplete, 1, 1}, {2, deployapi.DeploymentStatusComplete, 0, 0}, {4, deployapi.DeploymentStatusRunning, 2, 0}, {5, deployapi.DeploymentStatusNew, 0, 0}, {1, deployapi.DeploymentStatusFailed, 0, 0}, }}, } for _, scenario := range scenarios { t.Logf("running scenario: %s", scenario.name) updatedDeployments = make(map[int]*kapi.ReplicationController) failedDeployment = nil existingDeployments = &kapi.ReplicationControllerList{} for _, e := range scenario.existing { d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), kapi.Codec) d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status) d.Spec.Replicas = e.initialReplicas // if this is the deployment passed to Handle, set the desired replica annotation if e.version == scenario.version { d.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(scenario.desiredReplicas) failedDeployment = d } existingDeployments.Items = append(existingDeployments.Items, *d) } associatedDeployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(scenario.version), kapi.Codec) err := controller.Handle(terminatedPod(associatedDeployment)) if err != nil { t.Fatalf("unexpected error: %v", err) } // only the failed and the last completed deployment should be updated if len(updatedDeployments) > 2 { t.Fatalf("expected to update only the failed and last completed deployment") } for _, existing := range scenario.existing { updatedDeployment, ok := updatedDeployments[existing.version] if existing.initialReplicas != existing.updatedReplicas { if !ok { t.Fatalf("expected deployment #%d to be updated", existing.version) } if e, a := existing.updatedReplicas, updatedDeployment.Spec.Replicas; e != a { t.Fatalf("expected deployment #%d to be scaled to %d, got %d", existing.version, e, a) } } else if ok && existing.version != scenario.version { t.Fatalf("unexpected update for deployment #%d; replicas %d; status: %s", existing.version, updatedDeployment.Spec.Replicas, deployutil.DeploymentStatusFor(updatedDeployment)) } } if deployutil.DeploymentStatusFor(updatedDeployments[scenario.version]) != deployapi.DeploymentStatusFailed { t.Fatalf("status for deployment #%d expected to be updated to failed; got %s", scenario.version, deployutil.DeploymentStatusFor(updatedDeployments[scenario.version])) } if updatedDeployments[scenario.version].Spec.Replicas != 0 { t.Fatalf("deployment #%d expected to be scaled down to 0; got %d", scenario.version, updatedDeployments[scenario.version].Spec.Replicas) } } } // TestHandle_cleanupDesiredReplicasAnnotation ensures that the desired replicas annotation // will be cleaned up in a complete deployment and stay around in a failed deployment func TestHandle_cleanupDesiredReplicasAnnotation(t *testing.T) { deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec) tests := []struct { name string pod *kapi.Pod expected bool }{ { name: "complete deployment - cleaned up annotation", pod: succeededPod(deployment), expected: false, }, { name: "failed deployment - annotation stays", pod: terminatedPod(deployment), expected: true, }, } for _, test := range tests { var updatedDeployment *kapi.ReplicationController deployment.Annotations[deployapi.DesiredReplicasAnnotation] = "1" controller := &DeployerPodController{ deploymentClient: &deploymentClientImpl{ getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) { return deployment, nil }, updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) { updatedDeployment = deployment return deployment, nil }, listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) { return &kapi.ReplicationControllerList{Items: []kapi.ReplicationController{*deployment}}, nil }, }, } if err := controller.Handle(test.pod); err != nil { t.Errorf("%s: unexpected error: %v", test.name, err) continue } if updatedDeployment == nil { t.Errorf("%s: expected deployment update", test.name) continue } if _, got := updatedDeployment.Annotations[deployapi.DesiredReplicasAnnotation]; got != test.expected { t.Errorf("%s: expected annotation: %t, got %t", test.name, test.expected, got) } } } func okPod(deployment *kapi.ReplicationController) *kapi.Pod { return &kapi.Pod{ ObjectMeta: kapi.ObjectMeta{ Name: deployutil.DeployerPodNameForDeployment(deployment.Name), Annotations: map[string]string{ deployapi.DeploymentAnnotation: deployment.Name, }, }, Status: kapi.PodStatus{ ContainerStatuses: []kapi.ContainerStatus{ {}, }, }, } } func succeededPod(deployment *kapi.ReplicationController) *kapi.Pod { p := okPod(deployment) p.Status.Phase = kapi.PodSucceeded return p } func failedPod(deployment *kapi.ReplicationController) *kapi.Pod { p := okPod(deployment) p.Status.Phase = kapi.PodFailed p.Status.ContainerStatuses = []kapi.ContainerStatus{ { State: kapi.ContainerState{ Terminated: &kapi.ContainerStateTerminated{ ExitCode: 1, }, }, }, } return p } func terminatedPod(deployment *kapi.ReplicationController) *kapi.Pod { p := okPod(deployment) p.Status.Phase = kapi.PodFailed return p } func runningPod(deployment *kapi.ReplicationController) *kapi.Pod { p := okPod(deployment) p.Status.Phase = kapi.PodRunning return p }