Browse code

Upadte archive.ReplaceFileTarWrapper() to not expect a sorted archive

Improve test coverage of ReplaceFileTarWrapper()

Signed-off-by: Daniel Nephin <dnephin@docker.com>

Daniel Nephin authored on 2017/04/06 07:25:29
Showing 5 changed files
... ...
@@ -218,13 +218,14 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
218 218
 			return errors.Errorf("Error checking context: '%s'.", err)
219 219
 		}
220 220
 
221
-		// If .dockerignore mentions .dockerignore or the Dockerfile
222
-		// then make sure we send both files over to the daemon
223
-		// because Dockerfile is, obviously, needed no matter what, and
224
-		// .dockerignore is needed to know if either one needs to be
225
-		// removed. The daemon will remove them for us, if needed, after it
226
-		// parses the Dockerfile. Ignore errors here, as they will have been
227
-		// caught by validateContextDirectory above.
221
+		// If .dockerignore mentions .dockerignore or the Dockerfile then make
222
+		// sure we send both files over to the daemon because Dockerfile is,
223
+		// obviously, needed no matter what, and .dockerignore is needed to know
224
+		// if either one needs to be removed. The daemon will remove them
225
+		// if necessary, after it parses the Dockerfile. Ignore errors here, as
226
+		// they will have been caught by validateContextDirectory above.
227
+		// Excludes are used instead of includes to maintain the order of files
228
+		// in the archive.
228 229
 		if keep, _ := fileutils.Matches(".dockerignore", excludes); keep {
229 230
 			excludes = append(excludes, "!.dockerignore")
230 231
 		}
... ...
@@ -384,17 +385,16 @@ func addDockerfileToBuildContext(dockerfileCtx io.ReadCloser, buildCtx io.ReadCl
384 384
 			if h == nil {
385 385
 				h = hdrTmpl
386 386
 			}
387
-			extraIgnore := randomName + "\n"
387
+
388 388
 			b := &bytes.Buffer{}
389 389
 			if content != nil {
390
-				_, err := b.ReadFrom(content)
391
-				if err != nil {
390
+				if _, err := b.ReadFrom(content); err != nil {
392 391
 					return nil, nil, err
393 392
 				}
394 393
 			} else {
395
-				extraIgnore += ".dockerignore\n"
394
+				b.WriteString(".dockerignore")
396 395
 			}
397
-			b.Write([]byte("\n" + extraIgnore))
396
+			b.WriteString("\n" + randomName + "\n")
398 397
 			return h, b.Bytes(), nil
399 398
 		},
400 399
 	})
... ...
@@ -2069,34 +2069,40 @@ func (s *DockerSuite) testBuildDockerfileStdinNoExtraFiles(c *check.C, hasDocker
2069 2069
 	name := "stdindockerfilenoextra"
2070 2070
 	tmpDir, err := ioutil.TempDir("", "fake-context")
2071 2071
 	c.Assert(err, check.IsNil)
2072
-	err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("bar"), 0600)
2073
-	c.Assert(err, check.IsNil)
2074
-	if hasDockerignore {
2075
-		// test that this file is removed
2076
-		err = ioutil.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(""), 0600)
2072
+	defer os.RemoveAll(tmpDir)
2073
+
2074
+	writeFile := func(filename, content string) {
2075
+		err = ioutil.WriteFile(filepath.Join(tmpDir, filename), []byte(content), 0600)
2077 2076
 		c.Assert(err, check.IsNil)
2077
+	}
2078
+
2079
+	writeFile("foo", "bar")
2080
+
2081
+	if hasDockerignore {
2082
+		// Add an empty Dockerfile to verify that it is not added to the image
2083
+		writeFile("Dockerfile", "")
2084
+
2078 2085
 		ignores := "Dockerfile\n"
2079 2086
 		if ignoreDockerignore {
2080 2087
 			ignores += ".dockerignore\n"
2081 2088
 		}
2082
-		err = ioutil.WriteFile(filepath.Join(tmpDir, ".dockerignore"), []byte(ignores), 0600)
2083
-		c.Assert(err, check.IsNil)
2089
+		writeFile(".dockerignore", ignores)
2084 2090
 	}
2085 2091
 
2086
-	icmd.RunCmd(icmd.Cmd{
2092
+	result := icmd.RunCmd(icmd.Cmd{
2087 2093
 		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", tmpDir},
2088 2094
 		Stdin: strings.NewReader(
2089 2095
 			`FROM busybox
2090 2096
 COPY . /baz`),
2091
-	}).Assert(c, icmd.Success)
2097
+	})
2098
+	result.Assert(c, icmd.Success)
2092 2099
 
2093
-	out, _ := dockerCmd(c, "run", "--rm", name, "ls", "-A", "/baz")
2100
+	result = cli.DockerCmd(c, "run", "--rm", name, "ls", "-A", "/baz")
2094 2101
 	if hasDockerignore && !ignoreDockerignore {
2095
-		c.Assert(strings.TrimSpace(string(out)), checker.Equals, ".dockerignore\nfoo")
2102
+		c.Assert(result.Stdout(), checker.Equals, ".dockerignore\nfoo\n")
2096 2103
 	} else {
2097
-		c.Assert(strings.TrimSpace(string(out)), checker.Equals, "foo")
2104
+		c.Assert(result.Stdout(), checker.Equals, "foo\n")
2098 2105
 	}
2099
-
2100 2106
 }
2101 2107
 
2102 2108
 func (s *DockerSuite) TestBuildWithVolumeOwnership(c *check.C) {
... ...
@@ -14,7 +14,6 @@ import (
14 14
 	"os/exec"
15 15
 	"path/filepath"
16 16
 	"runtime"
17
-	"sort"
18 17
 	"strings"
19 18
 	"syscall"
20 19
 
... ...
@@ -227,7 +226,10 @@ func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, er
227 227
 }
228 228
 
229 229
 // TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to
230
-// define a modification step for a single path
230
+// modify the contents or header of an entry in the archive. If the file already
231
+// exists in the archive the TarModifierFunc will be called with the Header and
232
+// a reader which will return the files content. If the file does not exist both
233
+// header and content will be nil.
231 234
 type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
232 235
 
233 236
 // ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the
... ...
@@ -235,76 +237,77 @@ type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*
235 235
 func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser {
236 236
 	pipeReader, pipeWriter := io.Pipe()
237 237
 
238
-	modKeys := make([]string, 0, len(mods))
239
-	for key := range mods {
240
-		modKeys = append(modKeys, key)
241
-	}
242
-	sort.Strings(modKeys)
243
-
244 238
 	go func() {
245 239
 		tarReader := tar.NewReader(inputTarStream)
246 240
 		tarWriter := tar.NewWriter(pipeWriter)
247
-
248 241
 		defer inputTarStream.Close()
242
+		defer tarWriter.Close()
249 243
 
250
-	loop0:
251
-		for {
252
-			hdr, err := tarReader.Next()
253
-			for len(modKeys) > 0 && (err == io.EOF || err == nil && hdr.Name >= modKeys[0]) {
254
-				var h *tar.Header
255
-				var rdr io.Reader
256
-				if err == nil && hdr != nil && hdr.Name == modKeys[0] {
257
-					h = hdr
258
-					rdr = tarReader
259
-				}
244
+		modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error {
245
+			header, data, err := modifier(name, original, tarReader)
246
+			switch {
247
+			case err != nil:
248
+				return err
249
+			case header == nil:
250
+				return nil
251
+			}
260 252
 
261
-				h2, dt, err := mods[modKeys[0]](modKeys[0], h, rdr)
262
-				if err != nil {
263
-					pipeWriter.CloseWithError(err)
264
-					return
265
-				}
266
-				if h2 != nil {
267
-					h2.Name = modKeys[0]
268
-					h2.Size = int64(len(dt))
269
-					if err := tarWriter.WriteHeader(h2); err != nil {
270
-						pipeWriter.CloseWithError(err)
271
-						return
272
-					}
273
-					if len(dt) != 0 {
274
-						if _, err := tarWriter.Write(dt); err != nil {
275
-							pipeWriter.CloseWithError(err)
276
-							return
277
-						}
278
-					}
279
-				}
280
-				modKeys = modKeys[1:]
281
-				if h != nil {
282
-					continue loop0
253
+			header.Name = name
254
+			header.Size = int64(len(data))
255
+			if err := tarWriter.WriteHeader(header); err != nil {
256
+				return err
257
+			}
258
+			if len(data) != 0 {
259
+				if _, err := tarWriter.Write(data); err != nil {
260
+					return err
283 261
 				}
284 262
 			}
263
+			return nil
264
+		}
285 265
 
266
+		var err error
267
+		var originalHeader *tar.Header
268
+		for {
269
+			originalHeader, err = tarReader.Next()
286 270
 			if err == io.EOF {
287
-				tarWriter.Close()
288
-				pipeWriter.Close()
289
-				return
271
+				break
290 272
 			}
291
-
292 273
 			if err != nil {
293 274
 				pipeWriter.CloseWithError(err)
294 275
 				return
295 276
 			}
296 277
 
297
-			if err := tarWriter.WriteHeader(hdr); err != nil {
278
+			modifier, ok := mods[originalHeader.Name]
279
+			if !ok {
280
+				// No modifiers for this file, copy the header and data
281
+				if err := tarWriter.WriteHeader(originalHeader); err != nil {
282
+					pipeWriter.CloseWithError(err)
283
+					return
284
+				}
285
+				if _, err := pools.Copy(tarWriter, tarReader); err != nil {
286
+					pipeWriter.CloseWithError(err)
287
+					return
288
+				}
289
+				continue
290
+			}
291
+			delete(mods, originalHeader.Name)
292
+
293
+			if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil {
298 294
 				pipeWriter.CloseWithError(err)
299 295
 				return
300 296
 			}
297
+		}
301 298
 
302
-			if _, err := pools.Copy(tarWriter, tarReader); err != nil {
299
+		// Apply the modifiers that haven't matched any files in the archive
300
+		for name, modifier := range mods {
301
+			if err := modify(name, nil, modifier, nil); err != nil {
303 302
 				pipeWriter.CloseWithError(err)
304 303
 				return
305 304
 			}
306
-
307 305
 		}
306
+
307
+		pipeWriter.Close()
308
+
308 309
 	}()
309 310
 	return pipeReader
310 311
 }
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"archive/tar"
5 5
 	"bytes"
6 6
 	"fmt"
7
+	"github.com/docker/docker/pkg/testutil/assert"
7 8
 	"io"
8 9
 	"io/ioutil"
9 10
 	"os"
... ...
@@ -1161,58 +1162,110 @@ func TestTempArchiveCloseMultipleTimes(t *testing.T) {
1161 1161
 	}
1162 1162
 }
1163 1163
 
1164
-func testReplaceFileTarWrapper(t *testing.T, name string) {
1165
-	srcDir, err := ioutil.TempDir("", "docker-test-srcDir")
1166
-	if err != nil {
1167
-		t.Fatal(err)
1164
+func TestReplaceFileTarWrapper(t *testing.T) {
1165
+	filesInArchive := 20
1166
+	testcases := []struct {
1167
+		doc       string
1168
+		filename  string
1169
+		modifier  TarModifierFunc
1170
+		expected  string
1171
+		fileCount int
1172
+	}{
1173
+		{
1174
+			doc:       "Modifier creates a new file",
1175
+			filename:  "newfile",
1176
+			modifier:  createModifier(t),
1177
+			expected:  "the new content",
1178
+			fileCount: filesInArchive + 1,
1179
+		},
1180
+		{
1181
+			doc:       "Modifier replaces a file",
1182
+			filename:  "file-2",
1183
+			modifier:  createOrReplaceModifier,
1184
+			expected:  "the new content",
1185
+			fileCount: filesInArchive,
1186
+		},
1187
+		{
1188
+			doc:       "Modifier replaces the last file",
1189
+			filename:  fmt.Sprintf("file-%d", filesInArchive-1),
1190
+			modifier:  createOrReplaceModifier,
1191
+			expected:  "the new content",
1192
+			fileCount: filesInArchive,
1193
+		},
1194
+		{
1195
+			doc:       "Modifier appends to a file",
1196
+			filename:  "file-3",
1197
+			modifier:  appendModifier,
1198
+			expected:  "fooo\nnext line",
1199
+			fileCount: filesInArchive,
1200
+		},
1168 1201
 	}
1169
-	defer os.RemoveAll(srcDir)
1170 1202
 
1171
-	destDir, err := ioutil.TempDir("", "docker-test-destDir")
1172
-	if err != nil {
1173
-		t.Fatal(err)
1174
-	}
1175
-	defer os.RemoveAll(destDir)
1203
+	for _, testcase := range testcases {
1204
+		sourceArchive, cleanup := buildSourceArchive(t, filesInArchive)
1205
+		defer cleanup()
1176 1206
 
1177
-	_, err = prepareUntarSourceDirectory(20, srcDir, false)
1178
-	if err != nil {
1179
-		t.Fatal(err)
1180
-	}
1207
+		resultArchive := ReplaceFileTarWrapper(
1208
+			sourceArchive,
1209
+			map[string]TarModifierFunc{testcase.filename: testcase.modifier})
1181 1210
 
1182
-	archive, err := TarWithOptions(srcDir, &TarOptions{})
1183
-	if err != nil {
1184
-		t.Fatal(err)
1211
+		actual := readFileFromArchive(t, resultArchive, testcase.filename, testcase.fileCount, testcase.doc)
1212
+		assert.Equal(t, actual, testcase.expected, testcase.doc)
1185 1213
 	}
1186
-	defer archive.Close()
1214
+}
1187 1215
 
1188
-	archive2 := ReplaceFileTarWrapper(archive, map[string]TarModifierFunc{name: func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
1189
-		return &tar.Header{
1190
-			Mode:     0600,
1191
-			Typeflag: tar.TypeReg,
1192
-		}, []byte("foobar"), nil
1193
-	}})
1216
+func buildSourceArchive(t *testing.T, numberOfFiles int) (io.ReadCloser, func()) {
1217
+	srcDir, err := ioutil.TempDir("", "docker-test-srcDir")
1218
+	assert.NilError(t, err)
1194 1219
 
1195
-	if err := Untar(archive2, destDir, nil); err != nil {
1196
-		t.Fatal(err)
1197
-	}
1220
+	_, err = prepareUntarSourceDirectory(numberOfFiles, srcDir, false)
1221
+	assert.NilError(t, err)
1198 1222
 
1199
-	dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
1200
-	if err != nil {
1201
-		t.Fatal(err)
1202
-	}
1203
-	if expected, actual := "foobar", string(dt); actual != expected {
1204
-		t.Fatalf("file contents mismatch, expected: %q, got %q", expected, actual)
1223
+	sourceArchive, err := TarWithOptions(srcDir, &TarOptions{})
1224
+	assert.NilError(t, err)
1225
+	return sourceArchive, func() {
1226
+		os.RemoveAll(srcDir)
1227
+		sourceArchive.Close()
1205 1228
 	}
1206 1229
 }
1207 1230
 
1208
-func TestReplaceFileTarWrapperNewFile(t *testing.T) {
1209
-	testReplaceFileTarWrapper(t, "abc")
1231
+func createOrReplaceModifier(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
1232
+	return &tar.Header{
1233
+		Mode:     0600,
1234
+		Typeflag: tar.TypeReg,
1235
+	}, []byte("the new content"), nil
1210 1236
 }
1211 1237
 
1212
-func TestReplaceFileTarWrapperReplaceFile(t *testing.T) {
1213
-	testReplaceFileTarWrapper(t, "file-2")
1238
+func createModifier(t *testing.T) TarModifierFunc {
1239
+	return func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
1240
+		assert.Nil(t, content)
1241
+		return createOrReplaceModifier(path, header, content)
1242
+	}
1243
+}
1244
+
1245
+func appendModifier(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
1246
+	buffer := bytes.Buffer{}
1247
+	if content != nil {
1248
+		if _, err := buffer.ReadFrom(content); err != nil {
1249
+			return nil, nil, err
1250
+		}
1251
+	}
1252
+	buffer.WriteString("\nnext line")
1253
+	return &tar.Header{Mode: 0600, Typeflag: tar.TypeReg}, buffer.Bytes(), nil
1214 1254
 }
1215 1255
 
1216
-func TestReplaceFileTarWrapperLastFile(t *testing.T) {
1217
-	testReplaceFileTarWrapper(t, "file-999")
1256
+func readFileFromArchive(t *testing.T, archive io.ReadCloser, name string, expectedCount int, doc string) string {
1257
+	destDir, err := ioutil.TempDir("", "docker-test-destDir")
1258
+	assert.NilError(t, err)
1259
+	defer os.RemoveAll(destDir)
1260
+
1261
+	err = Untar(archive, destDir, nil)
1262
+	assert.NilError(t, err)
1263
+
1264
+	files, _ := ioutil.ReadDir(destDir)
1265
+	assert.Equal(t, len(files), expectedCount, doc)
1266
+
1267
+	content, err := ioutil.ReadFile(filepath.Join(destDir, name))
1268
+	assert.NilError(t, err)
1269
+	return string(content)
1218 1270
 }
... ...
@@ -20,9 +20,9 @@ type TestingT interface {
20 20
 
21 21
 // Equal compare the actual value to the expected value and fails the test if
22 22
 // they are not equal.
23
-func Equal(t TestingT, actual, expected interface{}) {
23
+func Equal(t TestingT, actual, expected interface{}, extra ...string) {
24 24
 	if expected != actual {
25
-		fatal(t, "Expected '%v' (%T) got '%v' (%T)", expected, expected, actual, actual)
25
+		fatalWithExtra(t, extra, "Expected '%v' (%T) got '%v' (%T)", expected, expected, actual, actual)
26 26
 	}
27 27
 }
28 28
 
... ...
@@ -103,10 +103,25 @@ func NotNil(t TestingT, obj interface{}) {
103 103
 	}
104 104
 }
105 105
 
106
+// Nil fails the test if the object is not nil
107
+func Nil(t TestingT, obj interface{}) {
108
+	if obj != nil {
109
+		fatal(t, "Expected nil value, got (%T) %s", obj, obj)
110
+	}
111
+}
112
+
106 113
 func fatal(t TestingT, format string, args ...interface{}) {
107 114
 	t.Fatalf(errorSource()+format, args...)
108 115
 }
109 116
 
117
+func fatalWithExtra(t TestingT, extra []string, format string, args ...interface{}) {
118
+	msg := fmt.Sprintf(errorSource()+format, args...)
119
+	if len(extra) > 0 {
120
+		msg += ": " + strings.Join(extra, ", ")
121
+	}
122
+	t.Fatalf(msg)
123
+}
124
+
110 125
 // See testing.decorate()
111 126
 func errorSource() string {
112 127
 	_, filename, line, ok := runtime.Caller(3)