package controller
import (
"fmt"
"github.com/golang/glog"
kapi "k8s.io/kubernetes/pkg/api"
errors "k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/client/cache"
kcoreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned"
"k8s.io/kubernetes/pkg/client/record"
utilruntime "k8s.io/kubernetes/pkg/util/runtime"
builddefaults "github.com/openshift/origin/pkg/build/admission/defaults"
buildoverrides "github.com/openshift/origin/pkg/build/admission/overrides"
buildapi "github.com/openshift/origin/pkg/build/api"
buildclient "github.com/openshift/origin/pkg/build/client"
"github.com/openshift/origin/pkg/build/controller/policy"
strategy "github.com/openshift/origin/pkg/build/controller/strategy"
buildutil "github.com/openshift/origin/pkg/build/util"
imageapi "github.com/openshift/origin/pkg/image/api"
)
// BuildController watches build resources and manages their state
type BuildController struct {
BuildUpdater buildclient.BuildUpdater
BuildLister buildclient.BuildLister
PodManager podManager
BuildStrategy BuildStrategy
ImageStreamClient imageStreamClient
Recorder record.EventRecorder
RunPolicies []policy.RunPolicy
BuildDefaults builddefaults.BuildDefaults
BuildOverrides buildoverrides.BuildOverrides
}
// BuildStrategy knows how to create a pod spec for a pod which can execute a build.
type BuildStrategy interface {
CreateBuildPod(build *buildapi.Build) (*kapi.Pod, error)
}
type podManager interface {
CreatePod(namespace string, pod *kapi.Pod) (*kapi.Pod, error)
DeletePod(namespace string, pod *kapi.Pod) error
GetPod(namespace, name string) (*kapi.Pod, error)
}
type imageStreamClient interface {
GetImageStream(namespace, name string) (*imageapi.ImageStream, error)
}
// CancelBuild updates a build status to Cancelled, after its associated pod is deleted.
func (bc *BuildController) CancelBuild(build *buildapi.Build) error {
if !isBuildCancellable(build) {
glog.V(4).Infof("Build %s/%s can be cancelled only if it has pending/running status, not %s.", build.Namespace, build.Name, build.Status.Phase)
return nil
}
glog.V(4).Infof("Cancelling build %s/%s.", build.Namespace, build.Name)
pod, err := bc.PodManager.GetPod(build.Namespace, buildapi.GetBuildPodName(build))
if err != nil {
if !errors.IsNotFound(err) {
return fmt.Errorf("Failed to get pod for build %s/%s: %v", build.Namespace, build.Name, err)
}
} else {
err := bc.PodManager.DeletePod(build.Namespace, pod)
if err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("Couldn't delete build pod %s/%s: %v", build.Namespace, pod.Name, err)
}
}
build.Status.Phase = buildapi.BuildPhaseCancelled
now := unversioned.Now()
build.Status.CompletionTimestamp = &now
// set the status details for the cancelled build before updating the build
// object.
build.Status.Reason = buildapi.StatusReasonCancelledBuild
build.Status.Message = buildapi.StatusMessageCancelledBuild
if err := bc.BuildUpdater.Update(build.Namespace, build); err != nil {
return fmt.Errorf("Failed to update build %s/%s: %v", build.Namespace, build.Name, err)
}
glog.V(4).Infof("Build %s/%s was successfully cancelled.", build.Namespace, build.Name)
return nil
}
// HandleBuild deletes pods for cancelled builds and takes new builds and puts
// them in the pending state after creating a corresponding pod
func (bc *BuildController) HandleBuild(build *buildapi.Build) error {
// these builds are processed/updated/etc by the jenkins sync plugin
if build.Spec.Strategy.JenkinsPipelineStrategy != nil {
glog.V(4).Infof("Ignoring build with jenkins pipeline strategy")
return nil
}
glog.V(4).Infof("Handling build %s/%s (%s)", build.Namespace, build.Name, build.Status.Phase)
runPolicy := policy.ForBuild(build, bc.RunPolicies)
if runPolicy == nil {
return fmt.Errorf("unable to determine build scheduler for %s/%s", build.Namespace, build.Name)
}
if buildutil.IsBuildComplete(build) {
if err := runPolicy.OnComplete(build); err != nil {
return err
}
return nil
}
// A cancelling event was triggered for the build, delete its pod and update build status.
if build.Status.Cancelled && build.Status.Phase != buildapi.BuildPhaseCancelled {
glog.V(5).Infof("Marking build %s/%s as cancelled", build.Namespace, build.Name)
if err := bc.CancelBuild(build); err != nil {
build.Status.Reason = buildapi.StatusReasonCancelBuildFailed
build.Status.Message = buildapi.StatusMessageCancelBuildFailed
if err = bc.BuildUpdater.Update(build.Namespace, build); err != nil {
utilruntime.HandleError(fmt.Errorf("Failed to update build %s/%s: %v", build.Namespace, build.Name, err))
}
return fmt.Errorf("Failed to cancel build %s/%s: %v, will retry", build.Namespace, build.Name, err)
}
}
// Handle only new builds from this point
if build.Status.Phase != buildapi.BuildPhaseNew {
return nil
}
// The runPolicy decides whether to execute this build or not.
if run, err := runPolicy.IsRunnable(build); err != nil || !run {
return err
}
if err := bc.nextBuildPhase(build); err != nil {
return err
}
if err := bc.BuildUpdater.Update(build.Namespace, build); err != nil {
// This is not a retryable error because the build has been created. The worst case
// outcome of not updating the buildconfig is that we might rerun a build for the
// same "new" imageid change in the future, which is better than guaranteeing we
// run the build 2+ times by retrying it here.
glog.V(2).Infof("Failed to record changes to build %s/%s: %v", build.Namespace, build.Name, err)
}
return nil
}
// nextBuildPhase updates build with any appropriate changes, or returns an error if
// the change cannot occur. When returning nil, be sure to set build.Status and optionally
// build.Message.
func (bc *BuildController) nextBuildPhase(build *buildapi.Build) error {
// If a cancelling event was triggered for the build, update build status.
if build.Status.Cancelled {
glog.V(4).Infof("Cancelling build %s/%s.", build.Namespace, build.Name)
build.Status.Phase = buildapi.BuildPhaseCancelled
return nil
}
// Set the output Docker image reference.
ref, err := bc.resolveOutputDockerImageReference(build)
if err != nil {
build.Status.Reason = buildapi.StatusReasonInvalidOutputReference
build.Status.Message = buildapi.StatusMessageInvalidOutputRef
return err
}
build.Status.OutputDockerImageReference = ref
// Make a copy to avoid mutating the build from this point on.
copy, err := kapi.Scheme.Copy(build)
if err != nil {
return fmt.Errorf("unable to copy build: %v", err)
}
buildCopy := copy.(*buildapi.Build)
// TODO(rhcarvalho)
// The S2I and Docker builders expect build.Spec.Output.To to contain a
// resolved reference to a Docker image. Since build.Spec is immutable, we
// change a copy (that is never persisted) and pass it to
// bc.BuildStrategy.CreateBuildPod. We should make the builders use
// build.Status.OutputDockerImageReference, what will make copying the build
// unnecessary.
if build.Spec.Output.To != nil && len(build.Spec.Output.To.Name) != 0 {
buildCopy.Spec.Output.To = &kapi.ObjectReference{
Kind: "DockerImage",
Name: ref,
}
}
// Invoke the strategy to get a build pod.
podSpec, err := bc.BuildStrategy.CreateBuildPod(buildCopy)
if err != nil {
build.Status.Reason = buildapi.StatusReasonCannotCreateBuildPodSpec
build.Status.Message = buildapi.StatusMessageCannotCreateBuildPodSpec
if strategy.IsFatal(err) {
return strategy.FatalError(fmt.Sprintf("failed to create a build pod spec for build %s/%s: %v", build.Namespace, build.Name, err))
}
return fmt.Errorf("failed to create a build pod spec for build %s/%s: %v", build.Namespace, build.Name, err)
}
if err := bc.BuildDefaults.ApplyDefaults(podSpec); err != nil {
return fmt.Errorf("failed to apply build defaults for build %s/%s: %v", build.Namespace, build.Name, err)
}
if err := bc.BuildOverrides.ApplyOverrides(podSpec); err != nil {
return fmt.Errorf("failed to apply build overrides for build %s/%s: %v", build.Namespace, build.Name, err)
}
glog.V(4).Infof("Pod %s for build %s/%s is about to be created", podSpec.Name, build.Namespace, build.Name)
if _, err := bc.PodManager.CreatePod(build.Namespace, podSpec); err != nil {
if errors.IsAlreadyExists(err) {
bc.Recorder.Eventf(build, kapi.EventTypeWarning, "FailedCreate", "Pod already exists: %s/%s", podSpec.Namespace, podSpec.Name)
glog.V(4).Infof("Build pod already existed: %#v", podSpec)
// If the existing pod was created before this build, switch to Error state.
existingPod, err := bc.PodManager.GetPod(podSpec.Namespace, podSpec.Name)
if err == nil && existingPod.CreationTimestamp.Before(build.CreationTimestamp) {
build.Status.Phase = buildapi.BuildPhaseError
build.Status.Reason = buildapi.StatusReasonBuildPodExists
build.Status.Message = buildapi.StatusMessageBuildPodExists
}
return nil
}
// Log an event if the pod is not created (most likely due to quota denial).
bc.Recorder.Eventf(build, kapi.EventTypeWarning, "FailedCreate", "Error creating: %v", err)
build.Status.Reason = buildapi.StatusReasonCannotCreateBuildPod
build.Status.Message = buildapi.StatusMessageCannotCreateBuildPod
return fmt.Errorf("failed to create build pod: %v", err)
}
setBuildPodNameAnnotation(build, podSpec.Name)
glog.V(4).Infof("Created pod for build: %#v", podSpec)
// Set the build phase, which will be persisted.
build.Status.Phase = buildapi.BuildPhasePending
build.Status.Reason = ""
build.Status.Message = ""
return nil
}
// resolveOutputDockerImageReference returns a reference to a Docker image
// computed from the buid.Spec.Output.To reference.
func (bc *BuildController) resolveOutputDockerImageReference(build *buildapi.Build) (string, error) {
outputTo := build.Spec.Output.To
if outputTo == nil || outputTo.Name == "" {
return "", nil
}
var ref string
switch outputTo.Kind {
case "DockerImage":
ref = outputTo.Name
case "ImageStream", "ImageStreamTag":
// TODO(smarterclayton): security, ensure that the reference image stream is actually visible
namespace := outputTo.Namespace
if len(namespace) == 0 {
namespace = build.Namespace
}
var tag string
streamName := outputTo.Name
if outputTo.Kind == "ImageStreamTag" {
var ok bool
streamName, tag, ok = imageapi.SplitImageStreamTag(streamName)
if !ok {
return "", fmt.Errorf("the referenced image stream tag is invalid: %s", outputTo.Name)
}
tag = ":" + tag
}
stream, err := bc.ImageStreamClient.GetImageStream(namespace, streamName)
if err != nil {
if errors.IsNotFound(err) {
return "", fmt.Errorf("the referenced output image stream %s/%s does not exist", namespace, streamName)
}
return "", fmt.Errorf("the referenced output image stream %s/%s could not be found by build %s/%s: %v", namespace, streamName, build.Namespace, build.Name, err)
}
if len(stream.Status.DockerImageRepository) == 0 {
e := fmt.Errorf("the image stream %s/%s cannot be used as the output for build %s/%s because the integrated Docker registry is not configured and no external registry was defined", namespace, outputTo.Name, build.Namespace, build.Name)
bc.Recorder.Eventf(build, kapi.EventTypeWarning, "invalidOutput", "Error starting build: %v", e)
return "", e
}
ref = fmt.Sprintf("%s%s", stream.Status.DockerImageRepository, tag)
}
return ref, nil
}
// BuildPodController watches pods running builds and manages the build state
type BuildPodController struct {
BuildStore cache.Store
BuildUpdater buildclient.BuildUpdater
SecretClient kcoreclient.SecretsGetter
PodManager podManager
}
// HandlePod updates the state of the build based on the pod state
func (bc *BuildPodController) HandlePod(pod *kapi.Pod) error {
obj, exists, err := bc.BuildStore.Get(buildKey(pod))
if err != nil {
glog.V(4).Infof("Error getting build for pod %s/%s: %v", pod.Namespace, pod.Name, err)
return err
}
if !exists || obj == nil {
glog.V(5).Infof("No build found for pod %s/%s", pod.Namespace, pod.Name)
return nil
}
build := obj.(*buildapi.Build)
nextStatus := build.Status.Phase
currentReason := build.Status.Reason
switch pod.Status.Phase {
case kapi.PodRunning:
// The pod's still running
build.Status.Reason = ""
build.Status.Message = ""
nextStatus = buildapi.BuildPhaseRunning
case kapi.PodPending:
build.Status.Reason = ""
build.Status.Message = ""
nextStatus = buildapi.BuildPhasePending
if secret := build.Spec.Output.PushSecret; secret != nil && currentReason != buildapi.StatusReasonMissingPushSecret {
if _, err := bc.SecretClient.Secrets(build.Namespace).Get(secret.Name); err != nil && errors.IsNotFound(err) {
build.Status.Reason = buildapi.StatusReasonMissingPushSecret
build.Status.Message = buildapi.StatusMessageMissingPushSecret
glog.V(4).Infof("Setting reason for pending build to %q due to missing secret %s/%s", build.Status.Reason, build.Namespace, secret.Name)
}
}
case kapi.PodSucceeded:
build.Status.Reason = ""
build.Status.Message = ""
// Check the exit codes of all the containers in the pod
nextStatus = buildapi.BuildPhaseComplete
if len(pod.Status.ContainerStatuses) == 0 {
// no containers in the pod means something went badly wrong, so the build
// should be failed.
glog.V(2).Infof("Failing build %s/%s because the pod has no containers", build.Namespace, build.Name)
nextStatus = buildapi.BuildPhaseFailed
} else {
for _, info := range pod.Status.ContainerStatuses {
if info.State.Terminated != nil && info.State.Terminated.ExitCode != 0 {
nextStatus = buildapi.BuildPhaseFailed
break
}
}
}
case kapi.PodFailed:
nextStatus = buildapi.BuildPhaseFailed
default:
build.Status.Reason = ""
build.Status.Message = ""
}
// Update the build object when it progress to a next state or the reason for
// the current state changed.
if (!hasBuildPodNameAnnotation(build) || build.Status.Phase != nextStatus) && !buildutil.IsBuildComplete(build) {
setBuildPodNameAnnotation(build, pod.Name)
reason := ""
if len(build.Status.Reason) > 0 {
reason = " (" + string(build.Status.Reason) + ")"
}
glog.V(4).Infof("Updating build %s/%s status %s -> %s%s", build.Namespace, build.Name, build.Status.Phase, nextStatus, reason)
build.Status.Phase = nextStatus
if buildutil.IsBuildComplete(build) {
now := unversioned.Now()
build.Status.CompletionTimestamp = &now
}
if build.Status.Phase == buildapi.BuildPhaseRunning {
now := unversioned.Now()
build.Status.StartTimestamp = &now
}
if err := bc.BuildUpdater.Update(build.Namespace, build); err != nil {
return fmt.Errorf("failed to update build %s/%s: %v", build.Namespace, build.Name, err)
}
glog.V(4).Infof("Build %s/%s status was updated %s -> %s", build.Namespace, build.Name, build.Status.Phase, nextStatus)
}
return nil
}
// isBuildCancellable checks for build status and returns true if the condition is checked.
func isBuildCancellable(build *buildapi.Build) bool {
return build.Status.Phase == buildapi.BuildPhaseNew || build.Status.Phase == buildapi.BuildPhasePending || build.Status.Phase == buildapi.BuildPhaseRunning
}
// BuildPodDeleteController watches pods running builds and updates the build if the pod is deleted
type BuildPodDeleteController struct {
BuildStore cache.Store
BuildUpdater buildclient.BuildUpdater
}
// HandleBuildPodDeletion sets the status of a build to error if the build pod has been deleted
func (bc *BuildPodDeleteController) HandleBuildPodDeletion(pod *kapi.Pod) error {
glog.V(4).Infof("Handling deletion of build pod %s/%s", pod.Namespace, pod.Name)
obj, exists, err := bc.BuildStore.Get(buildKey(pod))
if err != nil {
glog.V(4).Infof("Error getting build for pod %s/%s", pod.Namespace, pod.Name)
return err
}
if !exists || obj == nil {
glog.V(5).Infof("No build found for deleted pod %s/%s", pod.Namespace, pod.Name)
return nil
}
build := obj.(*buildapi.Build)
if build.Spec.Strategy.JenkinsPipelineStrategy != nil {
glog.V(4).Infof("Build %s/%s is a pipeline build, ignoring", build.Namespace, build.Name)
return nil
}
// If build was cancelled, we'll leave HandleBuild to update the build
if build.Status.Cancelled {
glog.V(4).Infof("Cancelation for build %s/%s was already triggered, ignoring", build.Namespace, build.Name)
return nil
}
if buildutil.IsBuildComplete(build) {
glog.V(4).Infof("Pod was deleted but build %s/%s is already completed, so no need to update it.", build.Namespace, build.Name)
return nil
}
nextStatus := buildapi.BuildPhaseError
if build.Status.Phase != nextStatus {
glog.V(4).Infof("Updating build %s/%s status %s -> %s", build.Namespace, build.Name, build.Status.Phase, nextStatus)
build.Status.Phase = nextStatus
build.Status.Reason = buildapi.StatusReasonBuildPodDeleted
build.Status.Message = buildapi.StatusMessageBuildPodDeleted
now := unversioned.Now()
build.Status.CompletionTimestamp = &now
if err := bc.BuildUpdater.Update(build.Namespace, build); err != nil {
return fmt.Errorf("Failed to update build %s/%s: %v", build.Namespace, build.Name, err)
}
}
return nil
}
// BuildDeleteController watches for builds being deleted and cleans up associated pods
type BuildDeleteController struct {
PodManager podManager
}
// HandleBuildDeletion deletes a build pod if the corresponding build has been deleted
func (bc *BuildDeleteController) HandleBuildDeletion(build *buildapi.Build) error {
glog.V(4).Infof("Handling deletion of build %s", build.Name)
if build.Spec.Strategy.JenkinsPipelineStrategy != nil {
glog.V(4).Infof("Ignoring build with jenkins pipeline strategy")
return nil
}
podName := buildapi.GetBuildPodName(build)
pod, err := bc.PodManager.GetPod(build.Namespace, podName)
if err != nil && !errors.IsNotFound(err) {
glog.V(2).Infof("Failed to find pod with name %s for build %s in namespace %s due to error: %v", podName, build.Name, build.Namespace, err)
return err
}
if pod == nil {
glog.V(2).Infof("Did not find pod with name %s for build %s in namespace %s", podName, build.Name, build.Namespace)
return nil
}
if buildName := buildapi.GetBuildName(pod); buildName != build.Name {
glog.V(2).Infof("Not deleting pod %s/%s because the build label %s does not match the build name %s", pod.Namespace, podName, buildName, build.Name)
return nil
}
err = bc.PodManager.DeletePod(build.Namespace, pod)
if err != nil && !errors.IsNotFound(err) {
glog.V(2).Infof("Failed to delete pod %s/%s for build %s due to error: %v", build.Namespace, podName, build.Name, err)
return err
}
return nil
}
// buildKey returns a build object that can be used to lookup a build
// in the cache store, given a pod for the build
func buildKey(pod *kapi.Pod) *buildapi.Build {
return &buildapi.Build{
ObjectMeta: kapi.ObjectMeta{
Name: buildutil.GetBuildName(pod),
Namespace: pod.Namespace,
},
}
}
func hasBuildPodNameAnnotation(build *buildapi.Build) bool {
if build.Annotations == nil {
return false
}
_, hasAnnotation := build.Annotations[buildapi.BuildPodNameAnnotation]
return hasAnnotation
}
func setBuildPodNameAnnotation(build *buildapi.Build, podName string) {
if build.Annotations == nil {
build.Annotations = map[string]string{}
}
build.Annotations[buildapi.BuildPodNameAnnotation] = podName
}