Browse code

Attempt to resume downloads after certain errors

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

Aaron Lehmann authored on 2016/01/27 04:19:18
Showing 1 changed files
... ...
@@ -17,6 +17,7 @@ import (
17 17
 	"github.com/docker/distribution/manifest/schema2"
18 18
 	"github.com/docker/distribution/registry/api/errcode"
19 19
 	"github.com/docker/distribution/registry/client"
20
+	"github.com/docker/distribution/registry/client/transport"
20 21
 	"github.com/docker/docker/distribution/metadata"
21 22
 	"github.com/docker/docker/distribution/xfer"
22 23
 	"github.com/docker/docker/image"
... ...
@@ -115,6 +116,7 @@ type v2LayerDescriptor struct {
115 115
 	repo              distribution.Repository
116 116
 	V2MetadataService *metadata.V2MetadataService
117 117
 	tmpFile           *os.File
118
+	verifier          digest.Verifier
118 119
 }
119 120
 
120 121
 func (ld *v2LayerDescriptor) Key() string {
... ...
@@ -132,15 +134,33 @@ func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) {
132 132
 func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
133 133
 	logrus.Debugf("pulling blob %q", ld.digest)
134 134
 
135
-	var err error
135
+	var (
136
+		err    error
137
+		offset int64
138
+	)
136 139
 
137 140
 	if ld.tmpFile == nil {
138 141
 		ld.tmpFile, err = createDownloadFile()
142
+		if err != nil {
143
+			return nil, 0, xfer.DoNotRetry{Err: err}
144
+		}
139 145
 	} else {
140
-		_, err = ld.tmpFile.Seek(0, os.SEEK_SET)
141
-	}
142
-	if err != nil {
143
-		return nil, 0, xfer.DoNotRetry{Err: err}
146
+		offset, err = ld.tmpFile.Seek(0, os.SEEK_END)
147
+		if err != nil {
148
+			logrus.Debugf("error seeking to end of download file: %v", err)
149
+			offset = 0
150
+
151
+			ld.tmpFile.Close()
152
+			if err := os.Remove(ld.tmpFile.Name()); err != nil {
153
+				logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
154
+			}
155
+			ld.tmpFile, err = createDownloadFile()
156
+			if err != nil {
157
+				return nil, 0, xfer.DoNotRetry{Err: err}
158
+			}
159
+		} else if offset != 0 {
160
+			logrus.Debugf("attempting to resume download of %q from %d bytes", ld.digest, offset)
161
+		}
144 162
 	}
145 163
 
146 164
 	tmpFile := ld.tmpFile
... ...
@@ -148,13 +168,22 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre
148 148
 
149 149
 	layerDownload, err := blobs.Open(ctx, ld.digest)
150 150
 	if err != nil {
151
-		logrus.Debugf("Error statting layer: %v", err)
151
+		logrus.Debugf("Error initiating layer download: %v", err)
152 152
 		if err == distribution.ErrBlobUnknown {
153 153
 			return nil, 0, xfer.DoNotRetry{Err: err}
154 154
 		}
155 155
 		return nil, 0, retryOnError(err)
156 156
 	}
157 157
 
158
+	if offset != 0 {
159
+		_, err := layerDownload.Seek(offset, os.SEEK_SET)
160
+		if err != nil {
161
+			if err := ld.truncateDownloadFile(); err != nil {
162
+				return nil, 0, xfer.DoNotRetry{Err: err}
163
+			}
164
+			return nil, 0, err
165
+		}
166
+	}
158 167
 	size, err := layerDownload.Seek(0, os.SEEK_END)
159 168
 	if err != nil {
160 169
 		// Seek failed, perhaps because there was no Content-Length
... ...
@@ -162,43 +191,59 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre
162 162
 		// still continue without a progress bar.
163 163
 		size = 0
164 164
 	} else {
165
-		// Restore the seek offset at the beginning of the stream.
166
-		_, err = layerDownload.Seek(0, os.SEEK_SET)
165
+		if size != 0 && offset > size {
166
+			logrus.Debugf("Partial download is larger than full blob. Starting over")
167
+			offset = 0
168
+			if err := ld.truncateDownloadFile(); err != nil {
169
+				return nil, 0, xfer.DoNotRetry{Err: err}
170
+			}
171
+		}
172
+
173
+		// Restore the seek offset either at the beginning of the
174
+		// stream, or just after the last byte we have from previous
175
+		// attempts.
176
+		_, err = layerDownload.Seek(offset, os.SEEK_SET)
167 177
 		if err != nil {
168 178
 			return nil, 0, err
169 179
 		}
170 180
 	}
171 181
 
172
-	reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size, ld.ID(), "Downloading")
182
+	reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size-offset, ld.ID(), "Downloading")
173 183
 	defer reader.Close()
174 184
 
175
-	verifier, err := digest.NewDigestVerifier(ld.digest)
176
-	if err != nil {
177
-		return nil, 0, xfer.DoNotRetry{Err: err}
185
+	if ld.verifier == nil {
186
+		ld.verifier, err = digest.NewDigestVerifier(ld.digest)
187
+		if err != nil {
188
+			return nil, 0, xfer.DoNotRetry{Err: err}
189
+		}
178 190
 	}
179 191
 
180
-	_, err = io.Copy(tmpFile, io.TeeReader(reader, verifier))
192
+	_, err = io.Copy(tmpFile, io.TeeReader(reader, ld.verifier))
181 193
 	if err != nil {
182
-		tmpFile.Close()
183
-		if err := os.Remove(tmpFile.Name()); err != nil {
184
-			logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
194
+		if err == transport.ErrWrongCodeForByteRange {
195
+			if err := ld.truncateDownloadFile(); err != nil {
196
+				return nil, 0, xfer.DoNotRetry{Err: err}
197
+			}
198
+			return nil, 0, err
185 199
 		}
186
-		ld.tmpFile = nil
187 200
 		return nil, 0, retryOnError(err)
188 201
 	}
189 202
 
190 203
 	progress.Update(progressOutput, ld.ID(), "Verifying Checksum")
191 204
 
192
-	if !verifier.Verified() {
205
+	if !ld.verifier.Verified() {
193 206
 		err = fmt.Errorf("filesystem layer verification failed for digest %s", ld.digest)
194 207
 		logrus.Error(err)
195 208
 
196
-		tmpFile.Close()
197
-		if err := os.Remove(tmpFile.Name()); err != nil {
198
-			logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
199
-		}
200
-		ld.tmpFile = nil
209
+		// Allow a retry if this digest verification error happened
210
+		// after a resumed download.
211
+		if offset != 0 {
212
+			if err := ld.truncateDownloadFile(); err != nil {
213
+				return nil, 0, xfer.DoNotRetry{Err: err}
214
+			}
201 215
 
216
+			return nil, 0, err
217
+		}
202 218
 		return nil, 0, xfer.DoNotRetry{Err: err}
203 219
 	}
204 220
 
... ...
@@ -213,6 +258,7 @@ func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progre
213 213
 			logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
214 214
 		}
215 215
 		ld.tmpFile = nil
216
+		ld.verifier = nil
216 217
 		return nil, 0, xfer.DoNotRetry{Err: err}
217 218
 	}
218 219
 	return tmpFile, size, nil
... ...
@@ -227,6 +273,23 @@ func (ld *v2LayerDescriptor) Close() {
227 227
 	}
228 228
 }
229 229
 
230
+func (ld *v2LayerDescriptor) truncateDownloadFile() error {
231
+	// Need a new hash context since we will be redoing the download
232
+	ld.verifier = nil
233
+
234
+	if _, err := ld.tmpFile.Seek(0, os.SEEK_SET); err != nil {
235
+		logrus.Debugf("error seeking to beginning of download file: %v", err)
236
+		return err
237
+	}
238
+
239
+	if err := ld.tmpFile.Truncate(0); err != nil {
240
+		logrus.Debugf("error truncating download file: %v", err)
241
+		return err
242
+	}
243
+
244
+	return nil
245
+}
246
+
230 247
 func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
231 248
 	// Cache mapping from this layer's DiffID to the blobsum
232 249
 	ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.FullName()})