Browse code

[graph] Use a pipe for downloads to write progress

The process of pulling an image spawns a new goroutine for each layer in the
image manifest. If any of these downloads fail we would stop everything and
return the error, even though other goroutines would still be running and
writing output through a progress reader which is attached to an http response
writer. Since the request handler had already returned from the first error,
the http server panics when one of these download goroutines makes a write to
the response writer buffer.

This patch prevents this crash in the daemon http server by waiting for all of
the download goroutines to complete, even if one of them fails. Only then does
it return, terminating the request handler.

Docker-DCO-1.1-Signed-off-by: Josh Hawn <josh.hawn@docker.com> (github: jlhawn)

Josh Hawn authored on 2015/08/06 09:47:37
Showing 2 changed files
... ...
@@ -1,6 +1,7 @@
1 1
 package graph
2 2
 
3 3
 import (
4
+	"errors"
4 5
 	"fmt"
5 6
 	"io"
6 7
 	"io/ioutil"
... ...
@@ -108,6 +109,7 @@ type downloadInfo struct {
108 108
 	layer   distribution.ReadSeekCloser
109 109
 	size    int64
110 110
 	err     chan error
111
+	out     io.Writer // Download progress is written here.
111 112
 }
112 113
 
113 114
 type errVerification struct{}
... ...
@@ -117,7 +119,7 @@ func (errVerification) Error() string { return "verification failed" }
117 117
 func (p *v2Puller) download(di *downloadInfo) {
118 118
 	logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID)
119 119
 
120
-	out := p.config.OutStream
120
+	out := di.out
121 121
 
122 122
 	if c, err := p.poolAdd("pull", "img:"+di.img.ID); err != nil {
123 123
 		if c != nil {
... ...
@@ -191,7 +193,7 @@ func (p *v2Puller) download(di *downloadInfo) {
191 191
 	di.err <- nil
192 192
 }
193 193
 
194
-func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) {
194
+func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) {
195 195
 	logrus.Debugf("Pulling tag from V2 registry: %q", tag)
196 196
 	out := p.config.OutStream
197 197
 
... ...
@@ -204,7 +206,7 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) {
204 204
 	if err != nil {
205 205
 		return false, err
206 206
 	}
207
-	verified, err := p.validateManifest(manifest, tag)
207
+	verified, err = p.validateManifest(manifest, tag)
208 208
 	if err != nil {
209 209
 		return false, err
210 210
 	}
... ...
@@ -212,6 +214,27 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) {
212 212
 		logrus.Printf("Image manifest for %s has been verified", taggedName)
213 213
 	}
214 214
 
215
+	// By using a pipeWriter for each of the downloads to write their progress
216
+	// to, we can avoid an issue where this function returns an error but
217
+	// leaves behind running download goroutines. By splitting the writer
218
+	// with a pipe, we can close the pipe if there is any error, consequently
219
+	// causing each download to cancel due to an error writing to this pipe.
220
+	pipeReader, pipeWriter := io.Pipe()
221
+	go func() {
222
+		if _, err := io.Copy(out, pipeReader); err != nil {
223
+			logrus.Errorf("error copying from layer download progress reader: %s", err)
224
+		}
225
+	}()
226
+	defer func() {
227
+		if err != nil {
228
+			// All operations on the pipe are synchronous. This call will wait
229
+			// until all current readers/writers are done using the pipe then
230
+			// set the error. All successive reads/writes will return with this
231
+			// error.
232
+			pipeWriter.CloseWithError(errors.New("download canceled"))
233
+		}
234
+	}()
235
+
215 236
 	out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
216 237
 
217 238
 	downloads := make([]downloadInfo, len(manifest.FSLayers))
... ...
@@ -242,6 +265,7 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) {
242 242
 		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
243 243
 
244 244
 		downloads[i].err = make(chan error)
245
+		downloads[i].out = pipeWriter
245 246
 		go p.download(&downloads[i])
246 247
 	}
247 248
 
... ...
@@ -446,7 +446,7 @@ func (s *DockerRegistrySuite) TestPullFailsWithAlteredManifest(c *check.C) {
446 446
 	imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest)
447 447
 	out, exitStatus, _ := dockerCmdWithError("pull", imageReference)
448 448
 	if exitStatus == 0 {
449
-		c.Fatalf("expected a zero exit status but got %d: %s", exitStatus, out)
449
+		c.Fatalf("expected a non-zero exit status but got %d: %s", exitStatus, out)
450 450
 	}
451 451
 
452 452
 	expectedErrorMsg := fmt.Sprintf("image verification failed for digest %s", manifestDigest)