@@ -8,11 +8,14 @@ when that change will happen.

 ## Origin 1.0.x / OSE 3.0.x

 1. Currently all build pods have a label named `build`. This label is being deprecated

-  in favor of `openshift.io/build.name` in Origin 1.0.x / OSE 3.1.x at which point both

-  labels will be supported. All the newly created builds will have just the new label.

-  In Origin 1.y / OSE 3.y the support for the old label (`build`) will be removed entirely.

+  in favor of `openshift.io/build.name` in Origin 1.0.x (OSE 3.0.x) - both are supported.

+  In Origin 1.1 we will only set the new label and remove support for the old label.

   See #3502.

 1. Currently `oc exec` will attempt to `POST` to `pods/podname/exec`, if that fails it will

-fallback to a `GET` to match older policy roles.  In Origin 1.y/ OSE 3.y the support for the

-old `oc exec` endpoint via `GET` will be removed.

\ No newline at end of file

+  fallback to a `GET` to match older policy roles.  In Origin 1.1 (OSE 3.1) the support for the

+  old `oc exec` endpoint via `GET` will be removed.

+

+1. The `pauseControllers` field in `master-config.yaml` is deprecated as of Origin 1.0.4 and will

+  no longer be supported in Origin 1.1. After that, a warning will be printed on startup if it

+  is set to true.

\ No newline at end of file

@@ -116,7 +116,14 @@ type MasterConfig struct {

 	Controllers string

 	// PauseControllers instructs the master to not automatically start controllers, but instead

 	// to wait until a notification to the server is received before launching them.

+	// TODO: will be disabled in function for 1.1.

 	PauseControllers bool

+	// ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire

+	// a lease before controllers start and renewing it within a number of seconds defined by this value.

+	// Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or

+	// omitted) and controller election can be disabled with -1.

+	ControllerLeaseTTL int

+	// TODO: the next field added to controllers must be added to a new controllers struct

 	// Allow to disable OpenShift components

 	DisabledFeatures FeatureList

@@ -99,6 +99,11 @@ type MasterConfig struct {

 	// PauseControllers instructs the master to not automatically start controllers, but instead

 	// to wait until a notification to the server is received before launching them.

 	PauseControllers bool `json:"pauseControllers,omitempty"`

+	// ControllerLeaseTTL enables controller election, instructing the master to attempt to acquire

+	// a lease before controllers start and renewing it within a number of seconds defined by this value.

+	// Setting this value non-negative forces pauseControllers=true. This value defaults off (0, or

+	// omitted) and controller election can be disabled with -1.

+	ControllerLeaseTTL int `json:"controllerLeaseTTL,omitempty"`

 	// DisabledFeatures is a list of features that should not be started.  We

 	// omitempty here because its very unlikely that anyone will want to

@@ -61,6 +61,13 @@ func ValidateMasterConfig(config *api.MasterConfig) ValidationResults {

 		validationResults.AddErrors(urlErrs...)

 	}

+	switch {

+	case config.ControllerLeaseTTL > 300,

+		config.ControllerLeaseTTL < -1,

+		config.ControllerLeaseTTL > 0 && config.ControllerLeaseTTL < 10:

+		validationResults.AddErrors(fielderrors.NewFieldInvalid("controllerLeaseTTL", config.ControllerLeaseTTL, "TTL must be -1 (disabled), 0 (default), or between 10 and 300 seconds"))

+	}

+

 	validationResults.AddErrors(ValidateDisabledFeatures(config.DisabledFeatures, "disabledFeatures")...)

 	if config.AssetConfig != nil {

@@ -6,6 +6,7 @@ import (

 	"errors"

 	"fmt"

 	"net/http"

+	"path"

 	etcdclient "github.com/coreos/go-etcd/etcd"

 	"github.com/golang/glog"

@@ -21,6 +22,7 @@ import (

 	"k8s.io/kubernetes/pkg/storage"

 	etcdstorage "k8s.io/kubernetes/pkg/storage/etcd"

 	kutil "k8s.io/kubernetes/pkg/util"

+	kutilrand "k8s.io/kubernetes/pkg/util/rand"

 	"github.com/openshift/origin/pkg/api/latest"

 	"github.com/openshift/origin/pkg/auth/authenticator"

@@ -58,6 +60,7 @@ import (

 	groupstorage "github.com/openshift/origin/pkg/user/registry/group/etcd"

 	userregistry "github.com/openshift/origin/pkg/user/registry/user"

 	useretcd "github.com/openshift/origin/pkg/user/registry/user/etcd"

+	"github.com/openshift/origin/pkg/util/leaderlease"

 )

 const (

@@ -83,7 +86,8 @@ type MasterConfig struct {

 	TLS bool

-	ControllerPlug plug.Plug

+	ControllerPlug      plug.Plug

+	ControllerPlugStart func()

 	// a function that returns the appropriate image to use for a named component

 	ImageFor func(component string) string

@@ -174,6 +178,8 @@ func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) {

 		return nil, err

 	}

+	plug, plugStart := newControllerPlug(options, client)

+

 	config := &MasterConfig{

 		Options: options,

@@ -191,7 +197,8 @@ func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) {

 		TLS: configapi.UseTLS(options.ServingInfo.ServingInfo),

-		ControllerPlug: plug.NewPlug(!options.PauseControllers),

+		ControllerPlug:      plug,

+		ControllerPlugStart: plugStart,

 		ImageFor:            imageTemplate.ExpandOrDie,

 		EtcdHelper:          etcdHelper,

@@ -213,6 +220,27 @@ func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) {

 	return config, nil

 }

+func newControllerPlug(options configapi.MasterConfig, client *etcdclient.Client) (plug.Plug, func()) {

+	switch {

+	case options.ControllerLeaseTTL > 0:

+		// TODO: replace with future API for leasing from Kube

+		id := fmt.Sprintf("master-%s", kutilrand.String(8))

+		leaser := leaderlease.NewEtcd(

+			client,

+			path.Join(options.EtcdStorageConfig.OpenShiftStoragePrefix, "leases/controllers"),

+			id,

+			uint64(options.ControllerLeaseTTL),

+		)

+		leased := plug.NewLeased(leaser)

+		return leased, func() {

+			glog.V(2).Infof("Attempting to acquire controller lease as %s, renewing every %d seconds", id, options.ControllerLeaseTTL)

+			go leased.Run()

+		}

+	default:

+		return plug.New(!options.PauseControllers), func() {}

+	}

+}

+

 func newServiceAccountTokenGetter(options configapi.MasterConfig, client *etcdclient.Client) (serviceaccount.ServiceAccountTokenGetter, error) {

 	var tokenGetter serviceaccount.ServiceAccountTokenGetter

 	if options.KubernetesMasterConfig == nil {

@@ -437,12 +437,16 @@ func StartControllers(openshiftConfig *origin.MasterConfig, kubeMasterConfig *ku

 	}

 	go func() {

+		openshiftConfig.ControllerPlugStart()

+		// when a manual shutdown (DELETE /controllers) or lease lost occurs, the process should exit

+		// this ensures no code is still running as a controller, and allows a process manager to reset

+		// the controller to come back into a candidate state and compete for the lease

 		openshiftConfig.ControllerPlug.WaitForStop()

-		glog.Fatalf("Master shutdown requested")

+		glog.Fatalf("Controller shutdown requested")

 	}()

 	openshiftConfig.ControllerPlug.WaitForStart()

-	glog.Infof("Master controllers starting (%s)", openshiftConfig.Options.Controllers)

+	glog.Infof("Controllers starting (%s)", openshiftConfig.Options.Controllers)

 	// Start these first, because they provide credentials for other controllers' clients

 	openshiftConfig.RunServiceAccountsController()

@@ -4,14 +4,26 @@ import (

 	"sync"

 )

+// Plug represents a synchronization primitive that holds and releases

+// execution for other objects.

 type Plug interface {

+	// Begins operation of the plug and unblocks WaitForStart().

+	// May be invoked multiple times but only the first invocation has

+	// an effect.

 	Start()

+	// Ends operation of the plug and unblocks WaitForStop()

+	// May be invoked multiple times but only the first invocation has

+	// an effect. Calling Stop() before Start() is undefined.

 	Stop()

+	// Blocks until Start() is invoked

 	WaitForStart()

+	// Blocks until Stop() is invoked

 	WaitForStop()

+	// Returns true if Start() has been invoked

 	IsStarted() bool

 }

+// plug is the default implementation of Plug

 type plug struct {

 	start   sync.Once

 	stop    sync.Once

@@ -19,7 +31,8 @@ type plug struct {

 	stopCh  chan struct{}

 }

-func NewPlug(started bool) Plug {

+// New returns a new plug that can begin in the Started state.

+func New(started bool) Plug {

 	p := &plug{

 		startCh: make(chan struct{}),

 		stopCh:  make(chan struct{}),

@@ -54,3 +67,52 @@ func (p *plug) WaitForStart() {

 func (p *plug) WaitForStop() {

 	<-p.stopCh

 }

+

+// Leaser controls access to a lease

+type Leaser interface {

+	// AcquireAndHold tries to acquire the lease and hold it until it expires, the lease is deleted,

+	// or we observe another party take the lease. The notify channel will be sent a value

+	// when the lease is held, and closed when the lease is lost.

+	AcquireAndHold(chan struct{})

+	Release()

+}

+

+// leased uses a Leaser to control Start and Stop on a Plug

+type Leased struct {

+	Plug

+

+	leaser Leaser

+}

+

+var _ Plug = &Leased{}

+

+// NewLeased creates a Plug that starts when a lease is acquired

+// and stops when it is lost.

+func NewLeased(leaser Leaser) *Leased {

+	return &Leased{

+		Plug:   New(false),

+		leaser: leaser,

+	}

+}

+

+// Stop releases the acquired lease

+func (l *Leased) Stop() {

+	l.leaser.Release()

+	l.Plug.Stop()

+}

+

+// Run tries to acquire and hold a lease, invoking Start()

+// when the lease is held and invoking Stop() when the lease

+// is lost.

+func (l *Leased) Run() {

+	ch := make(chan struct{}, 1)

+	go l.leaser.AcquireAndHold(ch)

+	defer l.Stop()

+	for {

+		_, ok := <-ch

+		if !ok {

+			return

+		}

+		l.Start()

+	}

+}