Browse code

daemon: fix hanging attaches on initial start failures

Attach can hang forever if there is no data to send. This PR adds notification
of Attach goroutine about container stop.

Signed-off-by: Alexander Morozov <lk4d4@docker.com>

Alexander Morozov authored on 2016/03/09 09:54:33
Showing 4 changed files
... ...
@@ -10,6 +10,8 @@ import (
10 10
 	"syscall"
11 11
 	"time"
12 12
 
13
+	"golang.org/x/net/context"
14
+
13 15
 	"github.com/Sirupsen/logrus"
14 16
 	"github.com/docker/docker/daemon/exec"
15 17
 	"github.com/docker/docker/daemon/execdriver"
... ...
@@ -62,8 +64,9 @@ type CommonContainer struct {
62 62
 	monitor                *containerMonitor
63 63
 	ExecCommands           *exec.Store `json:"-"`
64 64
 	// logDriver for closing
65
-	LogDriver logger.Logger  `json:"-"`
66
-	LogCopier *logger.Copier `json:"-"`
65
+	LogDriver     logger.Logger  `json:"-"`
66
+	LogCopier     *logger.Copier `json:"-"`
67
+	attachContext *attachContext
67 68
 }
68 69
 
69 70
 // NewBaseContainer creates a new container with its
... ...
@@ -71,12 +74,13 @@ type CommonContainer struct {
71 71
 func NewBaseContainer(id, root string) *Container {
72 72
 	return &Container{
73 73
 		CommonContainer: CommonContainer{
74
-			ID:           id,
75
-			State:        NewState(),
76
-			ExecCommands: exec.NewStore(),
77
-			Root:         root,
78
-			MountPoints:  make(map[string]*volume.MountPoint),
79
-			StreamConfig: runconfig.NewStreamConfig(),
74
+			ID:            id,
75
+			State:         NewState(),
76
+			ExecCommands:  exec.NewStore(),
77
+			Root:          root,
78
+			MountPoints:   make(map[string]*volume.MountPoint),
79
+			StreamConfig:  runconfig.NewStreamConfig(),
80
+			attachContext: &attachContext{},
80 81
 		},
81 82
 	}
82 83
 }
... ...
@@ -345,12 +349,13 @@ func (container *Container) GetExecIDs() []string {
345 345
 // Attach connects to the container's TTY, delegating to standard
346 346
 // streams or websockets depending on the configuration.
347 347
 func (container *Container) Attach(stdin io.ReadCloser, stdout io.Writer, stderr io.Writer, keys []byte) chan error {
348
-	return AttachStreams(container.StreamConfig, container.Config.OpenStdin, container.Config.StdinOnce, container.Config.Tty, stdin, stdout, stderr, keys)
348
+	ctx := container.InitAttachContext()
349
+	return AttachStreams(ctx, container.StreamConfig, container.Config.OpenStdin, container.Config.StdinOnce, container.Config.Tty, stdin, stdout, stderr, keys)
349 350
 }
350 351
 
351 352
 // AttachStreams connects streams to a TTY.
352 353
 // Used by exec too. Should this move somewhere else?
353
-func AttachStreams(streamConfig *runconfig.StreamConfig, openStdin, stdinOnce, tty bool, stdin io.ReadCloser, stdout io.Writer, stderr io.Writer, keys []byte) chan error {
354
+func AttachStreams(ctx context.Context, streamConfig *runconfig.StreamConfig, openStdin, stdinOnce, tty bool, stdin io.ReadCloser, stdout io.Writer, stderr io.Writer, keys []byte) chan error {
354 355
 	var (
355 356
 		cStdout, cStderr io.ReadCloser
356 357
 		cStdin           io.WriteCloser
... ...
@@ -379,21 +384,6 @@ func AttachStreams(streamConfig *runconfig.StreamConfig, openStdin, stdinOnce, t
379 379
 			return
380 380
 		}
381 381
 		logrus.Debugf("attach: stdin: begin")
382
-		defer func() {
383
-			if stdinOnce && !tty {
384
-				cStdin.Close()
385
-			} else {
386
-				// No matter what, when stdin is closed (io.Copy unblock), close stdout and stderr
387
-				if cStdout != nil {
388
-					cStdout.Close()
389
-				}
390
-				if cStderr != nil {
391
-					cStderr.Close()
392
-				}
393
-			}
394
-			wg.Done()
395
-			logrus.Debugf("attach: stdin: end")
396
-		}()
397 382
 
398 383
 		var err error
399 384
 		if tty {
... ...
@@ -408,23 +398,26 @@ func AttachStreams(streamConfig *runconfig.StreamConfig, openStdin, stdinOnce, t
408 408
 		if err != nil {
409 409
 			logrus.Errorf("attach: stdin: %s", err)
410 410
 			errors <- err
411
-			return
412 411
 		}
412
+		if stdinOnce && !tty {
413
+			cStdin.Close()
414
+		} else {
415
+			// No matter what, when stdin is closed (io.Copy unblock), close stdout and stderr
416
+			if cStdout != nil {
417
+				cStdout.Close()
418
+			}
419
+			if cStderr != nil {
420
+				cStderr.Close()
421
+			}
422
+		}
423
+		logrus.Debugf("attach: stdin: end")
424
+		wg.Done()
413 425
 	}()
414 426
 
415 427
 	attachStream := func(name string, stream io.Writer, streamPipe io.ReadCloser) {
416 428
 		if stream == nil {
417 429
 			return
418 430
 		}
419
-		defer func() {
420
-			// Make sure stdin gets closed
421
-			if stdin != nil {
422
-				stdin.Close()
423
-			}
424
-			streamPipe.Close()
425
-			wg.Done()
426
-			logrus.Debugf("attach: %s: end", name)
427
-		}()
428 431
 
429 432
 		logrus.Debugf("attach: %s: begin", name)
430 433
 		_, err := io.Copy(stream, streamPipe)
... ...
@@ -435,13 +428,39 @@ func AttachStreams(streamConfig *runconfig.StreamConfig, openStdin, stdinOnce, t
435 435
 			logrus.Errorf("attach: %s: %v", name, err)
436 436
 			errors <- err
437 437
 		}
438
+		// Make sure stdin gets closed
439
+		if stdin != nil {
440
+			stdin.Close()
441
+		}
442
+		streamPipe.Close()
443
+		logrus.Debugf("attach: %s: end", name)
444
+		wg.Done()
438 445
 	}
439 446
 
440 447
 	go attachStream("stdout", stdout, cStdout)
441 448
 	go attachStream("stderr", stderr, cStderr)
442 449
 
443 450
 	return promise.Go(func() error {
444
-		wg.Wait()
451
+		done := make(chan struct{})
452
+		go func() {
453
+			wg.Wait()
454
+			close(done)
455
+		}()
456
+		select {
457
+		case <-done:
458
+		case <-ctx.Done():
459
+			// close all pipes
460
+			if cStdin != nil {
461
+				cStdin.Close()
462
+			}
463
+			if cStdout != nil {
464
+				cStdout.Close()
465
+			}
466
+			if cStderr != nil {
467
+				cStderr.Close()
468
+			}
469
+			<-done
470
+		}
445 471
 		close(errors)
446 472
 		for err := range errors {
447 473
 			if err != nil {
... ...
@@ -597,3 +616,31 @@ func (container *Container) UpdateMonitor(restartPolicy containertypes.RestartPo
597 597
 	}
598 598
 	monitor.mux.Unlock()
599 599
 }
600
+
601
+type attachContext struct {
602
+	ctx    context.Context
603
+	cancel context.CancelFunc
604
+	mu     sync.Mutex
605
+}
606
+
607
+// InitAttachContext initialize or returns existing context for attach calls to
608
+// track container liveness.
609
+func (container *Container) InitAttachContext() context.Context {
610
+	container.attachContext.mu.Lock()
611
+	defer container.attachContext.mu.Unlock()
612
+	if container.attachContext.ctx == nil {
613
+		container.attachContext.ctx, container.attachContext.cancel = context.WithCancel(context.Background())
614
+	}
615
+	return container.attachContext.ctx
616
+}
617
+
618
+// CancelAttachContext cancel attach context. All attach calls should detach
619
+// after this call.
620
+func (container *Container) CancelAttachContext() {
621
+	container.attachContext.mu.Lock()
622
+	if container.attachContext.ctx != nil {
623
+		container.attachContext.cancel()
624
+		container.attachContext.ctx = nil
625
+	}
626
+	container.attachContext.mu.Unlock()
627
+}
... ...
@@ -6,6 +6,8 @@ import (
6 6
 	"strings"
7 7
 	"time"
8 8
 
9
+	"golang.org/x/net/context"
10
+
9 11
 	"github.com/Sirupsen/logrus"
10 12
 	"github.com/docker/docker/container"
11 13
 	"github.com/docker/docker/daemon/exec"
... ...
@@ -181,7 +183,7 @@ func (d *Daemon) ContainerExecStart(name string, stdin io.ReadCloser, stdout io.
181 181
 		ec.NewNopInputPipe()
182 182
 	}
183 183
 
184
-	attachErr := container.AttachStreams(ec.StreamConfig, ec.OpenStdin, true, ec.ProcessConfig.Tty, cStdin, cStdout, cStderr, ec.DetachKeys)
184
+	attachErr := container.AttachStreams(context.Background(), ec.StreamConfig, ec.OpenStdin, true, ec.ProcessConfig.Tty, cStdin, cStdout, cStderr, ec.DetachKeys)
185 185
 
186 186
 	execErr := make(chan error)
187 187
 
... ...
@@ -175,4 +175,5 @@ func (daemon *Daemon) Cleanup(container *container.Container) {
175 175
 	if err := container.UnmountVolumes(false, daemon.LogVolumeEvent); err != nil {
176 176
 		logrus.Warnf("%s cleanup: Failed to umount volumes: %v", container.ID, err)
177 177
 	}
178
+	container.CancelAttachContext()
178 179
 }
... ...
@@ -3,6 +3,7 @@ package main
3 3
 import (
4 4
 	"bufio"
5 5
 	"bytes"
6
+	"encoding/json"
6 7
 	"fmt"
7 8
 	"io/ioutil"
8 9
 	"net"
... ...
@@ -4187,3 +4188,42 @@ func (s *DockerSuite) TestRunNamedVolumesFromNotRemoved(c *check.C) {
4187 4187
 	out, _ := dockerCmd(c, "volume", "ls", "-q")
4188 4188
 	c.Assert(strings.TrimSpace(out), checker.Equals, "test")
4189 4189
 }
4190
+
4191
+func (s *DockerSuite) TestRunAttachFailedNoLeak(c *check.C) {
4192
+	type info struct {
4193
+		NGoroutines int
4194
+	}
4195
+	getNGoroutines := func() int {
4196
+		var i info
4197
+		status, b, err := sockRequest("GET", "/info", nil)
4198
+		c.Assert(err, checker.IsNil)
4199
+		c.Assert(status, checker.Equals, 200)
4200
+		c.Assert(json.Unmarshal(b, &i), checker.IsNil)
4201
+		return i.NGoroutines
4202
+	}
4203
+	nroutines := getNGoroutines()
4204
+
4205
+	runSleepingContainer(c, "--name=test", "-p", "8000:8000")
4206
+
4207
+	out, _, err := dockerCmdWithError("run", "-p", "8000:8000", "busybox", "true")
4208
+	c.Assert(err, checker.NotNil)
4209
+	// check for windows error as well
4210
+	c.Assert(strings.Contains(string(out), "port is already allocated") || strings.Contains(string(out), "were not connected because a duplicate name exists"), checker.Equals, true, check.Commentf("Output: %s", out))
4211
+	dockerCmd(c, "rm", "-f", "test")
4212
+
4213
+	// NGoroutines is not updated right away, so we need to wait before failing
4214
+	t := time.After(30 * time.Second)
4215
+	for {
4216
+		select {
4217
+		case <-t:
4218
+			n := getNGoroutines()
4219
+			c.Assert(n <= nroutines, checker.Equals, true, check.Commentf("leaked goroutines: expected less than or equal to %d, got: %d", nroutines, n))
4220
+
4221
+		default:
4222
+			if n := getNGoroutines(); n <= nroutines {
4223
+				return
4224
+			}
4225
+			time.Sleep(200 * time.Millisecond)
4226
+		}
4227
+	}
4228
+}