@@ -85,7 +85,7 @@ function exectest() {

 		exit 0

 	else

 		os::text::print_red "failed  $1"

-		echo "${out}"

+		echo "${out:-}"

 		exit 1

 	fi

@@ -48,7 +48,7 @@ func initControllerRoutes(root *restful.WebService, path string, canStart bool,

 	root.Route(root.DELETE(path).To(func(req *restful.Request, resp *restful.Response) {

 		resp.ResponseWriter.WriteHeader(http.StatusAccepted)

 		fmt.Fprintf(resp, "terminating")

-		plug.Stop()

+		plug.Stop(nil)

 	}).Doc("Stop the master").

 		Returns(http.StatusAccepted, "if the master will stop", nil).

 		Produces(restful.MIME_JSON))

@@ -520,8 +520,10 @@ func startControllers(oc *origin.MasterConfig, kc *kubernetes.MasterConfig) erro

 		// when a manual shutdown (DELETE /controllers) or lease lost occurs, the process should exit

 		// this ensures no code is still running as a controller, and allows a process manager to reset

 		// the controller to come back into a candidate state and compete for the lease

-		oc.ControllerPlug.WaitForStop()

-		glog.Fatalf("Controller shutdown requested")

+		if err := oc.ControllerPlug.WaitForStop(); err != nil {

+			glog.Fatalf("Controller shutdown due to lease being lost: %v", err)

+		}

+		glog.Fatalf("Controller graceful shutdown requested")

 	}()

 	oc.ControllerPlug.WaitForStart()

@@ -13,12 +13,13 @@ type Plug interface {

 	Start()

 	// Ends operation of the plug and unblocks WaitForStop()

 	// May be invoked multiple times but only the first invocation has

-	// an effect. Calling Stop() before Start() is undefined.

-	Stop()

+	// an effect. Calling Stop() before Start() is undefined. An error

+	// may be returned with the stop.

+	Stop(err error)

 	// Blocks until Start() is invoked

 	WaitForStart()

 	// Blocks until Stop() is invoked

-	WaitForStop()

+	WaitForStop() error

 	// Returns true if Start() has been invoked

 	IsStarted() bool

 }

@@ -28,14 +29,14 @@ type plug struct {

 	start   sync.Once

 	stop    sync.Once

 	startCh chan struct{}

-	stopCh  chan struct{}

+	stopCh  chan error

 }

 // New returns a new plug that can begin in the Started state.

 func New(started bool) Plug {

 	p := &plug{

 		startCh: make(chan struct{}),

-		stopCh:  make(chan struct{}),

+		stopCh:  make(chan error, 1),

 	}

 	if started {

 		p.Start()

@@ -47,8 +48,13 @@ func (p *plug) Start() {

 	p.start.Do(func() { close(p.startCh) })

 }

-func (p *plug) Stop() {

-	p.stop.Do(func() { close(p.stopCh) })

+func (p *plug) Stop(err error) {

+	p.stop.Do(func() {

+		if err != nil {

+			p.stopCh <- err

+		}

+		close(p.stopCh)

+	})

 }

 func (p *plug) IsStarted() bool {

@@ -64,16 +70,21 @@ func (p *plug) WaitForStart() {

 	<-p.startCh

 }

-func (p *plug) WaitForStop() {

-	<-p.stopCh

+func (p *plug) WaitForStop() error {

+	err, ok := <-p.stopCh

+	if !ok {

+		return nil

+	}

+	return err

 }

 // Leaser controls access to a lease

 type Leaser interface {

 	// AcquireAndHold tries to acquire the lease and hold it until it expires, the lease is deleted,

-	// or we observe another party take the lease. The notify channel will be sent a value

-	// when the lease is held, and closed when the lease is lost.

-	AcquireAndHold(chan struct{})

+	// or we observe another party take the lease. The notify channel will be sent a nil value

+	// when the lease is held, and closed when the lease is lost. If an error is sent the lease

+	// is also considered lost.

+	AcquireAndHold(chan error)

 	Release()

 }

@@ -96,22 +107,31 @@ func NewLeased(leaser Leaser) *Leased {

 }

 // Stop releases the acquired lease

-func (l *Leased) Stop() {

+func (l *Leased) Stop(err error) {

 	l.leaser.Release()

-	l.Plug.Stop()

+	l.Plug.Stop(err)

 }

 // Run tries to acquire and hold a lease, invoking Start()

 // when the lease is held and invoking Stop() when the lease

-// is lost.

-func (l *Leased) Run() {

-	ch := make(chan struct{}, 1)

+// is lost. If the lease was lost gracefully, nil is returned.

+// If the lease was lost due to an error, the error is returned.

+func (l *Leased) Run() error {

+	ch := make(chan error, 1)

 	go l.leaser.AcquireAndHold(ch)

-	defer l.Stop()

+	var err error

+	defer l.Stop(err)

 	for {

-		_, ok := <-ch

+		var ok bool

+		err, ok = <-ch

 		if !ok {

-			return

+			return nil

+		}

+		if err != nil {

+			for range ch {

+				// read the rest of the channel

+			}

+			return err

 		}

 		l.Start()

 	}

@@ -19,7 +19,7 @@ type Leaser interface {

 	// lease is acquired, and the provided channel will be closed when the lease is lost. If the

 	// function returns true, the lease will be released on exit. If the function returns false,

 	// the lease will be held.

-	AcquireAndHold(chan struct{})

+	AcquireAndHold(chan error)

 	// Release returns any active leases

 	Release()

 }

@@ -54,7 +54,7 @@ func NewEtcd(client *etcdclient.Client, key, value string, ttl uint64) Leaser {

 		value:  value,

 		ttl:    ttl,

-		waitFraction:         0.75,

+		waitFraction:         0.66,

 		pauseInterval:        time.Second,

 		maxRetries:           10,

 		minimumRetryInterval: 100 * time.Millisecond,

@@ -62,7 +62,7 @@ func NewEtcd(client *etcdclient.Client, key, value string, ttl uint64) Leaser {

 }

 // AcquireAndHold implements an acquire and release of a lease.

-func (e *Etcd) AcquireAndHold(notify chan struct{}) {

+func (e *Etcd) AcquireAndHold(notify chan error) {

 	for {

 		ok, ttl, index, err := e.tryAcquire()

 		if err != nil {

@@ -76,12 +76,12 @@ func (e *Etcd) AcquireAndHold(notify chan struct{}) {

 		}

 		// notify

-		notify <- struct{}{}

+		notify <- nil

 		defer close(notify)

 		// hold the lease

 		if err := e.tryHold(ttl, index); err != nil {

-			utilruntime.HandleError(err)

+			notify <- err

 		}

 		break

 	}

@@ -3,6 +3,7 @@

 package integration

 import (

+	"strings"

 	"testing"

 	"time"

@@ -27,12 +28,15 @@ func TestLeaderLeaseAcquire(t *testing.T) {

 	}()

 	lease := leaderlease.NewEtcd(client, key, "holder", 10)

-	ch := make(chan struct{})

+	ch := make(chan error, 1)

 	go lease.AcquireAndHold(ch)

 	<-ch

 	glog.Infof("Lease acquired")

 	close(held)

+	if err, ok := <-ch; err == nil || !ok || !strings.Contains(err.Error(), "the lease has been lost") {

+		t.Errorf("Expected error and open channel when lease was swapped: %v %t", err, ok)

+	}

 	<-ch

 	glog.Infof("Lease lost")

@@ -65,12 +69,15 @@ func TestLeaderLeaseWait(t *testing.T) {

 	}()

 	lease := leaderlease.NewEtcd(client, key, "holder", 10)

-	ch := make(chan struct{})

+	ch := make(chan error, 1)

 	go lease.AcquireAndHold(ch)

 	<-ch

 	glog.Infof("Lease acquired")

 	close(held)

+	if err, ok := <-ch; err == nil || !ok || !strings.Contains(err.Error(), "the lease has been lost") {

+		t.Errorf("Expected error and open channel when lease was swapped: %v %t", err, ok)

+	}

 	<-ch

 	glog.Infof("Lease lost")

@@ -102,12 +109,15 @@ func TestLeaderLeaseSwapWhileWaiting(t *testing.T) {

 	}()

 	lease := leaderlease.NewEtcd(client, key, "other", 10)

-	ch := make(chan struct{})

+	ch := make(chan error, 1)

 	go lease.AcquireAndHold(ch)

 	<-ch

 	glog.Infof("Lease acquired")

 	lease.Release()

+	if err, ok := <-ch; err == nil || !ok || !strings.Contains(err.Error(), "the lease has been lost") {

+		t.Errorf("Expected error and open channel when lease was swapped: %v %t", err, ok)

+	}

 	<-ch

 	glog.Infof("Lease gone")

 }

@@ -131,7 +141,7 @@ func TestLeaderLeaseReacquire(t *testing.T) {

 	}()

 	lease := leaderlease.NewEtcd(client, key, "holder", 1)

-	ch := make(chan struct{})

+	ch := make(chan error, 1)

 	go lease.AcquireAndHold(ch)

 	<-ch

@@ -7,7 +7,6 @@ import (

 	"net/url"

 	"os"

 	"path"

-	"testing"

 	"time"

 	"github.com/golang/glog"

@@ -41,15 +40,6 @@ const ServiceAccountWaitTimeout = 30 * time.Second

 // is available for the admission control cache to catch up and allow pod creation

 const PodCreationWaitTimeout = 10 * time.Second

-// RequireServer verifies if the etcd and the OpenShift server are

-// available and you can successfully connect to them.

-func RequireServer(t *testing.T) {

-	util.RequireEtcd(t)

-	if _, err := util.GetClusterAdminClient(util.KubeConfigPath()); err != nil {

-		os.Exit(1)

-	}

-}

-

 // FindAvailableBindAddress returns a bind address on 127.0.0.1 with a free port in the low-high range.

 // If lowPort is 0, an ephemeral port is allocated.

 func FindAvailableBindAddress(lowPort, highPort int) (string, error) {