package controller import ( "encoding/json" "fmt" "testing" "time" unidlingapi "github.com/openshift/origin/pkg/unidling/api" deployapi "github.com/openshift/origin/pkg/deploy/api" deployfake "github.com/openshift/origin/pkg/deploy/client/clientset_generated/internalclientset/fake" kapi "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/errors" kunversioned "k8s.io/kubernetes/pkg/api/unversioned" kextapi "k8s.io/kubernetes/pkg/apis/extensions" kfake "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" ktestingcore "k8s.io/kubernetes/pkg/client/testing/core" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/types" // install the APIs we need for the codecs to run correctly in order to build patches _ "github.com/openshift/origin/pkg/api/install" ) type fakeResults struct { resMap map[unidlingapi.CrossGroupObjectReference]kextapi.Scale resEndpoints *kapi.Endpoints } func prepFakeClient(t *testing.T, nowTime time.Time, scales ...kextapi.Scale) (*kfake.Clientset, *deployfake.Clientset, *fakeResults) { fakeClient := &kfake.Clientset{} fakeDeployClient := &deployfake.Clientset{} nowTimeStr := nowTime.Format(time.RFC3339) targets := make([]unidlingapi.RecordedScaleReference, len(scales)) for i, scale := range scales { targets[i] = unidlingapi.RecordedScaleReference{ CrossGroupObjectReference: unidlingapi.CrossGroupObjectReference{ Name: scale.Name, Kind: scale.Kind, }, Replicas: 2, } } targetsAnnotation, err := json.Marshal(targets) if err != nil { t.Fatalf("unexpected error: %v", err) } endpointsObj := kapi.Endpoints{ ObjectMeta: kapi.ObjectMeta{ Name: "somesvc", Annotations: map[string]string{ unidlingapi.IdledAtAnnotation: nowTimeStr, unidlingapi.UnidleTargetAnnotation: string(targetsAnnotation), }, }, } fakeClient.PrependReactor("get", "endpoints", func(action ktestingcore.Action) (bool, runtime.Object, error) { if action.(ktestingcore.GetAction).GetName() == endpointsObj.Name { return true, &endpointsObj, nil } return false, nil, nil }) fakeDeployClient.PrependReactor("get", "deploymentconfigs", func(action ktestingcore.Action) (bool, runtime.Object, error) { objName := action.(ktestingcore.GetAction).GetName() for _, scale := range scales { if scale.Kind == "DeploymentConfig" && objName == scale.Name { return true, &deployapi.DeploymentConfig{ ObjectMeta: kapi.ObjectMeta{ Name: objName, }, Spec: deployapi.DeploymentConfigSpec{ Replicas: scale.Spec.Replicas, }, }, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), objName) }) fakeClient.PrependReactor("get", "replicationcontrollers", func(action ktestingcore.Action) (bool, runtime.Object, error) { objName := action.(ktestingcore.GetAction).GetName() for _, scale := range scales { if scale.Kind == "ReplicationController" && objName == scale.Name { return true, &kapi.ReplicationController{ ObjectMeta: kapi.ObjectMeta{ Name: objName, }, Spec: kapi.ReplicationControllerSpec{ Replicas: scale.Spec.Replicas, }, }, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), objName) }) res := &fakeResults{ resMap: make(map[unidlingapi.CrossGroupObjectReference]kextapi.Scale), } fakeDeployClient.PrependReactor("update", "deploymentconfigs", func(action ktestingcore.Action) (bool, runtime.Object, error) { obj := action.(ktestingcore.UpdateAction).GetObject().(*deployapi.DeploymentConfig) for _, scale := range scales { if scale.Kind == "DeploymentConfig" && obj.Name == scale.Name { newScale := scale newScale.Spec.Replicas = obj.Spec.Replicas res.resMap[unidlingapi.CrossGroupObjectReference{Name: obj.Name, Kind: "DeploymentConfig"}] = newScale return true, &deployapi.DeploymentConfig{}, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), obj.Name) }) fakeClient.PrependReactor("update", "replicationcontrollers", func(action ktestingcore.Action) (bool, runtime.Object, error) { obj := action.(ktestingcore.UpdateAction).GetObject().(*kapi.ReplicationController) for _, scale := range scales { if scale.Kind == "ReplicationController" && obj.Name == scale.Name { newScale := scale newScale.Spec.Replicas = obj.Spec.Replicas res.resMap[unidlingapi.CrossGroupObjectReference{Name: obj.Name, Kind: "ReplicationController"}] = newScale return true, &kapi.ReplicationController{}, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), obj.Name) }) fakeDeployClient.PrependReactor("patch", "deploymentconfigs", func(action ktestingcore.Action) (bool, runtime.Object, error) { patchAction := action.(ktestingcore.PatchActionImpl) var patch deployapi.DeploymentConfig json.Unmarshal(patchAction.GetPatch(), &patch) for _, scale := range scales { if scale.Kind == "DeploymentConfig" && patchAction.GetName() == scale.Name { newScale := scale newScale.Spec.Replicas = patch.Spec.Replicas res.resMap[unidlingapi.CrossGroupObjectReference{Name: patchAction.GetName(), Kind: "DeploymentConfig"}] = newScale return true, &deployapi.DeploymentConfig{}, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), patchAction.GetName()) }) fakeClient.PrependReactor("patch", "replicationcontrollers", func(action ktestingcore.Action) (bool, runtime.Object, error) { patchAction := action.(ktestingcore.PatchActionImpl) var patch kapi.ReplicationController json.Unmarshal(patchAction.GetPatch(), &patch) for _, scale := range scales { if scale.Kind == "ReplicationController" && patchAction.GetName() == scale.Name { newScale := scale newScale.Spec.Replicas = patch.Spec.Replicas res.resMap[unidlingapi.CrossGroupObjectReference{Name: patchAction.GetName(), Kind: "ReplicationController"}] = newScale return true, &kapi.ReplicationController{}, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), patchAction.GetName()) }) fakeClient.AddReactor("*", "endpoints", func(action ktestingcore.Action) (bool, runtime.Object, error) { obj := action.(ktestingcore.UpdateAction).GetObject().(*kapi.Endpoints) if obj.Name != endpointsObj.Name { return false, nil, nil } res.resEndpoints = obj return true, obj, nil }) return fakeClient, fakeDeployClient, res } func TestControllerHandlesStaleEvents(t *testing.T) { nowTime := time.Now().Truncate(time.Second) fakeClient, fakeDeployClient, res := prepFakeClient(t, nowTime) controller := &UnidlingController{ scaleNamespacer: fakeClient.Extensions(), endpointsNamespacer: fakeClient.Core(), rcNamespacer: fakeClient.Core(), dcNamespacer: fakeDeployClient.Core(), } retry, err := controller.handleRequest(types.NamespacedName{ Namespace: "somens", Name: "somesvc", }, nowTime.Add(-10*time.Second)) if err != nil { t.Fatalf("Unable to unidle: unexpected error (retry: %v): %v", retry, err) } if len(res.resMap) != 0 { t.Errorf("Did not expect to have anything scaled, but got %v", res.resMap) } if res.resEndpoints != nil { t.Errorf("Did not expect to have endpoints object updated, but got %v", res.resEndpoints) } } func TestControllerIgnoresAlreadyScaledObjects(t *testing.T) { // truncate to avoid conversion comparison issues nowTime := time.Now().Truncate(time.Second) baseScales := []kextapi.Scale{ { ObjectMeta: kapi.ObjectMeta{ Name: "somerc", }, TypeMeta: kunversioned.TypeMeta{ Kind: "ReplicationController", }, Spec: kextapi.ScaleSpec{ Replicas: 0, }, }, { ObjectMeta: kapi.ObjectMeta{ Name: "somedc", }, TypeMeta: kunversioned.TypeMeta{ Kind: "DeploymentConfig", }, Spec: kextapi.ScaleSpec{ Replicas: 5, }, }, } idledTime := nowTime.Add(-10 * time.Second) fakeClient, fakeDeployClient, res := prepFakeClient(t, idledTime, baseScales...) controller := &UnidlingController{ scaleNamespacer: fakeClient.Extensions(), endpointsNamespacer: fakeClient.Core(), rcNamespacer: fakeClient.Core(), dcNamespacer: fakeDeployClient.Core(), } retry, err := controller.handleRequest(types.NamespacedName{ Namespace: "somens", Name: "somesvc", }, nowTime) if err != nil { t.Fatalf("Unable to unidle: unexpected error (retry: %v): %v", retry, err) } if len(res.resMap) != 1 { t.Errorf("Incorrect unidling results: got %v, expected to end up with 1 objects scaled to 1", res.resMap) } stillPresent := make(map[unidlingapi.CrossGroupObjectReference]struct{}) for _, scale := range baseScales { scaleRef := unidlingapi.CrossGroupObjectReference{Kind: scale.Kind, Name: scale.Name} resScale, ok := res.resMap[scaleRef] if scale.Spec.Replicas != 0 { stillPresent[scaleRef] = struct{}{} if ok { t.Errorf("Expected to %s %q to not have been scaled, but it was scaled to %v", scale.Kind, scale.Name, resScale.Spec.Replicas) } continue } else if !ok { t.Errorf("Expected to %s %q to have been scaled, but it was not", scale.Kind, scale.Name) continue } if resScale.Spec.Replicas != 2 { t.Errorf("Expected %s %q to have been scaled to 2, but it was scaled to %v", scale.Kind, scale.Name, resScale.Spec.Replicas) } } if res.resEndpoints == nil { t.Fatalf("Expected endpoints object to be updated, but it was not") } resTargetsRaw, hadTargets := res.resEndpoints.Annotations[unidlingapi.UnidleTargetAnnotation] resIdledTimeRaw, hadIdledTime := res.resEndpoints.Annotations[unidlingapi.IdledAtAnnotation] if !hadTargets { t.Errorf("Expected targets annotation to still be present, but it was not") } var resTargets []unidlingapi.RecordedScaleReference if err = json.Unmarshal([]byte(resTargetsRaw), &resTargets); err != nil { t.Fatalf("Unexpected error: %v", err) } if len(resTargets) != len(stillPresent) { t.Errorf("Expected the new target list to contain the unscaled scalables only, but it was %v", resTargets) } for _, target := range resTargets { if _, ok := stillPresent[target.CrossGroupObjectReference]; !ok { t.Errorf("Expected new target list to contain the unscaled scalables only, but it was %v", resTargets) } } if !hadIdledTime { t.Errorf("Expected idled-at annotation to still be present, but it was not") } resIdledTime, err := time.Parse(time.RFC3339, resIdledTimeRaw) if err != nil { t.Fatalf("Unexpected error: %v", err) } if !resIdledTime.Equal(idledTime) { t.Errorf("Expected output idled time annotation to be %s, but was changed to %s", idledTime, resIdledTime) } } func TestControllerUnidlesProperly(t *testing.T) { nowTime := time.Now().Truncate(time.Second) baseScales := []kextapi.Scale{ { ObjectMeta: kapi.ObjectMeta{ Name: "somerc", }, TypeMeta: kunversioned.TypeMeta{ Kind: "ReplicationController", }, Spec: kextapi.ScaleSpec{ Replicas: 0, }, }, { ObjectMeta: kapi.ObjectMeta{ Name: "somedc", }, TypeMeta: kunversioned.TypeMeta{ Kind: "DeploymentConfig", }, Spec: kextapi.ScaleSpec{ Replicas: 0, }, }, } fakeClient, fakeDeployClient, res := prepFakeClient(t, nowTime.Add(-10*time.Second), baseScales...) controller := &UnidlingController{ scaleNamespacer: fakeClient.Extensions(), endpointsNamespacer: fakeClient.Core(), rcNamespacer: fakeClient.Core(), dcNamespacer: fakeDeployClient.Core(), } retry, err := controller.handleRequest(types.NamespacedName{ Namespace: "somens", Name: "somesvc", }, nowTime) if err != nil { t.Fatalf("Unable to unidle: unexpected error (retry: %v): %v", retry, err) } if len(res.resMap) != len(baseScales) { t.Errorf("Incorrect unidling results: got %v, expected to end up with %v objects scaled to 1", res.resMap, len(baseScales)) } for _, scale := range baseScales { resScale, ok := res.resMap[unidlingapi.CrossGroupObjectReference{Kind: scale.Kind, Name: scale.Name}] if !ok { t.Errorf("Expected to %s %q to have been scaled, but it was not", scale.Kind, scale.Name) continue } if resScale.Spec.Replicas != 2 { t.Errorf("Expected %s %q to have been scaled to 2, but it was scaled to %v", scale.Kind, scale.Name, resScale.Spec.Replicas) } } if res.resEndpoints == nil { t.Fatalf("Expected endpoints object to be updated, but it was not") } resTargets, hadTargets := res.resEndpoints.Annotations[unidlingapi.UnidleTargetAnnotation] resIdledTime, hadIdledTime := res.resEndpoints.Annotations[unidlingapi.IdledAtAnnotation] if hadTargets { t.Errorf("Expected targets annotation to be removed, but it was %q", resTargets) } if hadIdledTime { t.Errorf("Expected idled-at annotation to be removed, but it was %q", resIdledTime) } } type failureTestInfo struct { name string endpointsGet *kapi.Endpoints scaleGets []kextapi.Scale scaleUpdatesNotFound []bool preventEndpointsUpdate bool errorExpected bool retryExpected bool annotationsExpected map[string]string } func prepareFakeClientForFailureTest(test failureTestInfo) (*kfake.Clientset, *deployfake.Clientset) { fakeClient := &kfake.Clientset{} fakeDeployClient := &deployfake.Clientset{} fakeClient.PrependReactor("get", "endpoints", func(action ktestingcore.Action) (bool, runtime.Object, error) { objName := action.(ktestingcore.GetAction).GetName() if test.endpointsGet != nil && objName == test.endpointsGet.Name { return true, test.endpointsGet, nil } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), objName) }) fakeDeployClient.PrependReactor("get", "deploymentconfigs", func(action ktestingcore.Action) (bool, runtime.Object, error) { objName := action.(ktestingcore.GetAction).GetName() for _, scale := range test.scaleGets { if scale.Kind == "DeploymentConfig" && objName == scale.Name { return true, &deployapi.DeploymentConfig{ ObjectMeta: kapi.ObjectMeta{ Name: objName, }, Spec: deployapi.DeploymentConfigSpec{ Replicas: scale.Spec.Replicas, }, }, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), objName) }) fakeClient.PrependReactor("get", "replicationcontrollers", func(action ktestingcore.Action) (bool, runtime.Object, error) { objName := action.(ktestingcore.GetAction).GetName() for _, scale := range test.scaleGets { if scale.Kind == "ReplicationController" && objName == scale.Name { return true, &kapi.ReplicationController{ ObjectMeta: kapi.ObjectMeta{ Name: objName, }, Spec: kapi.ReplicationControllerSpec{ Replicas: scale.Spec.Replicas, }, }, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), objName) }) fakeDeployClient.PrependReactor("update", "deploymentconfigs", func(action ktestingcore.Action) (bool, runtime.Object, error) { obj := action.(ktestingcore.UpdateAction).GetObject().(*deployapi.DeploymentConfig) for i, scale := range test.scaleGets { if scale.Kind == "DeploymentConfig" && obj.Name == scale.Name { if test.scaleUpdatesNotFound != nil && test.scaleUpdatesNotFound[i] { return false, nil, nil } return true, &deployapi.DeploymentConfig{}, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), obj.Name) }) fakeClient.PrependReactor("update", "replicationcontrollers", func(action ktestingcore.Action) (bool, runtime.Object, error) { obj := action.(ktestingcore.UpdateAction).GetObject().(*kapi.ReplicationController) for i, scale := range test.scaleGets { if scale.Kind == "ReplicationController" && obj.Name == scale.Name { if test.scaleUpdatesNotFound != nil && test.scaleUpdatesNotFound[i] { return false, nil, nil } return true, &kapi.ReplicationController{}, nil } } return true, nil, errors.NewNotFound(action.GetResource().GroupResource(), obj.Name) }) fakeClient.PrependReactor("update", "endpoints", func(action ktestingcore.Action) (bool, runtime.Object, error) { obj := action.(ktestingcore.UpdateAction).GetObject().(*kapi.Endpoints) if obj.Name != test.endpointsGet.Name { return false, nil, nil } if test.preventEndpointsUpdate { return true, nil, fmt.Errorf("some problem updating the endpoints") } return true, obj, nil }) return fakeClient, fakeDeployClient } func TestControllerPerformsCorrectlyOnFailures(t *testing.T) { nowTime := time.Now().Truncate(time.Second) baseScalables := []unidlingapi.RecordedScaleReference{ { CrossGroupObjectReference: unidlingapi.CrossGroupObjectReference{ Kind: "ReplicationController", Name: "somerc", }, Replicas: 2, }, { CrossGroupObjectReference: unidlingapi.CrossGroupObjectReference{ Kind: "DeploymentConfig", Name: "somedc", }, Replicas: 2, }, } baseScalablesBytes, err := json.Marshal(baseScalables) if err != nil { t.Fatalf("Unexpected error: %v", err) } outScalables := []unidlingapi.RecordedScaleReference{ { CrossGroupObjectReference: unidlingapi.CrossGroupObjectReference{ Kind: "DeploymentConfig", Name: "somedc", }, Replicas: 2, }, } outScalablesBytes, err := json.Marshal(outScalables) if err != nil { t.Fatalf("Unexpected error: %v", err) } tests := []failureTestInfo{ { name: "retry on failed endpoints get", endpointsGet: nil, errorExpected: true, retryExpected: true, }, { name: "not retry on failure to parse time", endpointsGet: &kapi.Endpoints{ ObjectMeta: kapi.ObjectMeta{ Name: "somesvc", Annotations: map[string]string{ unidlingapi.IdledAtAnnotation: "cheddar", }, }, }, errorExpected: true, retryExpected: false, }, { name: "not retry on failure to unmarshal target scalables", endpointsGet: &kapi.Endpoints{ ObjectMeta: kapi.ObjectMeta{ Name: "somesvc", Annotations: map[string]string{ unidlingapi.IdledAtAnnotation: nowTime.Format(time.RFC3339), unidlingapi.UnidleTargetAnnotation: "pecorino romano", }, }, }, errorExpected: true, retryExpected: false, }, { name: "remove a scalable from the list if it cannot be found (while getting)", endpointsGet: &kapi.Endpoints{ ObjectMeta: kapi.ObjectMeta{ Name: "somesvc", Annotations: map[string]string{ unidlingapi.IdledAtAnnotation: nowTime.Format(time.RFC3339), unidlingapi.UnidleTargetAnnotation: string(baseScalablesBytes), }, }, }, scaleGets: []kextapi.Scale{ { TypeMeta: kunversioned.TypeMeta{ Kind: "DeploymentConfig", }, ObjectMeta: kapi.ObjectMeta{ Name: "somedc", }, Spec: kextapi.ScaleSpec{Replicas: 0}, }, }, errorExpected: false, annotationsExpected: map[string]string{ unidlingapi.IdledAtAnnotation: nowTime.Format(time.RFC3339), unidlingapi.UnidleTargetAnnotation: string(outScalablesBytes), }, }, { name: "should remove a scalable from the list if it cannot be found (while updating)", endpointsGet: &kapi.Endpoints{ ObjectMeta: kapi.ObjectMeta{ Name: "somesvc", Annotations: map[string]string{ unidlingapi.IdledAtAnnotation: nowTime.Format(time.RFC3339), unidlingapi.UnidleTargetAnnotation: string(baseScalablesBytes), }, }, }, scaleGets: []kextapi.Scale{ { TypeMeta: kunversioned.TypeMeta{ Kind: "ReplicationController", }, ObjectMeta: kapi.ObjectMeta{ Name: "somerc", }, Spec: kextapi.ScaleSpec{Replicas: 0}, }, { TypeMeta: kunversioned.TypeMeta{ Kind: "DeploymentConfig", }, ObjectMeta: kapi.ObjectMeta{ Name: "somedc", }, Spec: kextapi.ScaleSpec{Replicas: 0}, }, }, scaleUpdatesNotFound: []bool{false, true}, errorExpected: false, annotationsExpected: map[string]string{ unidlingapi.IdledAtAnnotation: nowTime.Format(time.RFC3339), unidlingapi.UnidleTargetAnnotation: string(outScalablesBytes), }, }, { name: "retry on failed endpoints update", endpointsGet: &kapi.Endpoints{ ObjectMeta: kapi.ObjectMeta{ Name: "somesvc", Annotations: map[string]string{ unidlingapi.IdledAtAnnotation: nowTime.Format(time.RFC3339), unidlingapi.UnidleTargetAnnotation: string(baseScalablesBytes), }, }, }, scaleGets: []kextapi.Scale{ { TypeMeta: kunversioned.TypeMeta{ Kind: "ReplicationController", }, ObjectMeta: kapi.ObjectMeta{ Name: "somerc", }, Spec: kextapi.ScaleSpec{Replicas: 0}, }, { TypeMeta: kunversioned.TypeMeta{ Kind: "DeploymentConfig", }, ObjectMeta: kapi.ObjectMeta{ Name: "somedc", }, Spec: kextapi.ScaleSpec{Replicas: 0}, }, }, preventEndpointsUpdate: true, errorExpected: true, retryExpected: true, }, } for _, test := range tests { fakeClient, fakeDeployClient := prepareFakeClientForFailureTest(test) controller := &UnidlingController{ scaleNamespacer: fakeClient.Extensions(), endpointsNamespacer: fakeClient.Core(), rcNamespacer: fakeClient.Core(), dcNamespacer: fakeDeployClient.Core(), } var retry bool retry, err = controller.handleRequest(types.NamespacedName{ Namespace: "somens", Name: "somesvc", }, nowTime.Add(10*time.Second)) if err != nil && !test.errorExpected { t.Errorf("for test 'it should %s': unexpected error while idling: %v", test.name, err) continue } if err == nil && test.errorExpected { t.Errorf("for test 'it should %s': expected error, but did not get one", test.name) continue } if test.errorExpected && (test.retryExpected != retry) { t.Errorf("for test 'it should %s': expected retry to be %v, but it was %v with error %v", test.name, test.retryExpected, retry, err) return } } }