Browse code

Add support for copy/add with multiple src files

Part one of solution for issue #6820

Signed-off-by: Doug Davis <dug@us.ibm.com>

Doug Davis authored on 2014/09/17 01:58:20
Showing 14 changed files
... ...
@@ -65,8 +65,8 @@ func maintainer(b *Builder, args []string, attributes map[string]bool) error {
65 65
 // exist here. If you do not wish to have this automatic handling, use COPY.
66 66
 //
67 67
 func add(b *Builder, args []string, attributes map[string]bool) error {
68
-	if len(args) != 2 {
69
-		return fmt.Errorf("ADD requires two arguments")
68
+	if len(args) < 2 {
69
+		return fmt.Errorf("ADD requires at least two arguments")
70 70
 	}
71 71
 
72 72
 	return b.runContextCommand(args, true, true, "ADD")
... ...
@@ -77,8 +77,8 @@ func add(b *Builder, args []string, attributes map[string]bool) error {
77 77
 // Same as 'ADD' but without the tar and remote url handling.
78 78
 //
79 79
 func dispatchCopy(b *Builder, args []string, attributes map[string]bool) error {
80
-	if len(args) != 2 {
81
-		return fmt.Errorf("COPY requires two arguments")
80
+	if len(args) < 2 {
81
+		return fmt.Errorf("COPY requires at least two arguments")
82 82
 	}
83 83
 
84 84
 	return b.runContextCommand(args, false, false, "COPY")
... ...
@@ -99,37 +99,117 @@ func (b *Builder) commit(id string, autoCmd []string, comment string) error {
99 99
 	return nil
100 100
 }
101 101
 
102
+type copyInfo struct {
103
+	origPath   string
104
+	destPath   string
105
+	hashPath   string
106
+	decompress bool
107
+	tmpDir     string
108
+}
109
+
102 110
 func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecompression bool, cmdName string) error {
103 111
 	if b.context == nil {
104 112
 		return fmt.Errorf("No context given. Impossible to use %s", cmdName)
105 113
 	}
106 114
 
107
-	if len(args) != 2 {
108
-		return fmt.Errorf("Invalid %s format", cmdName)
115
+	if len(args) < 2 {
116
+		return fmt.Errorf("Invalid %s format - at least two arguments required", cmdName)
117
+	}
118
+
119
+	dest := args[len(args)-1] // last one is always the dest
120
+
121
+	if len(args) > 2 && dest[len(dest)-1] != '/' {
122
+		return fmt.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
109 123
 	}
110 124
 
111
-	orig := args[0]
112
-	dest := args[1]
125
+	copyInfos := make([]copyInfo, len(args)-1)
126
+	hasHash := false
127
+	srcPaths := ""
128
+	origPaths := ""
129
+
130
+	b.Config.Image = b.image
131
+
132
+	defer func() {
133
+		for _, ci := range copyInfos {
134
+			if ci.tmpDir != "" {
135
+				os.RemoveAll(ci.tmpDir)
136
+			}
137
+		}
138
+	}()
139
+
140
+	// Loop through each src file and calculate the info we need to
141
+	// do the copy (e.g. hash value if cached).  Don't actually do
142
+	// the copy until we've looked at all src files
143
+	for i, orig := range args[0 : len(args)-1] {
144
+		ci := &copyInfos[i]
145
+		ci.origPath = orig
146
+		ci.destPath = dest
147
+		ci.decompress = true
148
+
149
+		err := calcCopyInfo(b, cmdName, ci, allowRemote, allowDecompression)
150
+		if err != nil {
151
+			return err
152
+		}
153
+
154
+		origPaths += " " + ci.origPath // will have leading space
155
+		if ci.hashPath == "" {
156
+			srcPaths += " " + ci.origPath // note leading space
157
+		} else {
158
+			srcPaths += " " + ci.hashPath // note leading space
159
+			hasHash = true
160
+		}
161
+	}
113 162
 
114 163
 	cmd := b.Config.Cmd
115
-	b.Config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s %s in %s", cmdName, orig, dest)}
164
+	b.Config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s%s in %s", cmdName, srcPaths, dest)}
116 165
 	defer func(cmd []string) { b.Config.Cmd = cmd }(cmd)
117
-	b.Config.Image = b.image
118 166
 
167
+	hit, err := b.probeCache()
168
+	if err != nil {
169
+		return err
170
+	}
171
+	// If we do not have at least one hash, never use the cache
172
+	if hit && hasHash {
173
+		return nil
174
+	}
175
+
176
+	container, _, err := b.Daemon.Create(b.Config, "")
177
+	if err != nil {
178
+		return err
179
+	}
180
+	b.TmpContainers[container.ID] = struct{}{}
181
+
182
+	if err := container.Mount(); err != nil {
183
+		return err
184
+	}
185
+	defer container.Unmount()
186
+
187
+	for _, ci := range copyInfos {
188
+		if err := b.addContext(container, ci.origPath, ci.destPath, ci.decompress); err != nil {
189
+			return err
190
+		}
191
+	}
192
+
193
+	if err := b.commit(container.ID, cmd, fmt.Sprintf("%s%s in %s", cmdName, origPaths, dest)); err != nil {
194
+		return err
195
+	}
196
+	return nil
197
+}
198
+
199
+func calcCopyInfo(b *Builder, cmdName string, ci *copyInfo, allowRemote bool, allowDecompression bool) error {
119 200
 	var (
120
-		origPath   = orig
121
-		destPath   = dest
122 201
 		remoteHash string
123 202
 		isRemote   bool
124
-		decompress = true
125 203
 	)
126 204
 
127
-	isRemote = utils.IsURL(orig)
205
+	saveOrig := ci.origPath
206
+	isRemote = utils.IsURL(ci.origPath)
207
+
128 208
 	if isRemote && !allowRemote {
129 209
 		return fmt.Errorf("Source can't be an URL for %s", cmdName)
130
-	} else if utils.IsURL(orig) {
210
+	} else if isRemote {
131 211
 		// Initiate the download
132
-		resp, err := utils.Download(orig)
212
+		resp, err := utils.Download(ci.origPath)
133 213
 		if err != nil {
134 214
 			return err
135 215
 		}
... ...
@@ -139,6 +219,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
139 139
 		if err != nil {
140 140
 			return err
141 141
 		}
142
+		ci.tmpDir = tmpDirName
142 143
 
143 144
 		// Create a tmp file within our tmp dir
144 145
 		tmpFileName := path.Join(tmpDirName, "tmp")
... ...
@@ -146,7 +227,6 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
146 146
 		if err != nil {
147 147
 			return err
148 148
 		}
149
-		defer os.RemoveAll(tmpDirName)
150 149
 
151 150
 		// Download and dump result to tmp file
152 151
 		if _, err := io.Copy(tmpFile, utils.ProgressReader(resp.Body, int(resp.ContentLength), b.OutOld, b.StreamFormatter, true, "", "Downloading")); err != nil {
... ...
@@ -161,7 +241,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
161 161
 			return err
162 162
 		}
163 163
 
164
-		origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName))
164
+		ci.origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName))
165 165
 
166 166
 		// Process the checksum
167 167
 		r, err := archive.Tar(tmpFileName, archive.Uncompressed)
... ...
@@ -179,8 +259,8 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
179 179
 		r.Close()
180 180
 
181 181
 		// If the destination is a directory, figure out the filename.
182
-		if strings.HasSuffix(dest, "/") {
183
-			u, err := url.Parse(orig)
182
+		if strings.HasSuffix(ci.destPath, "/") {
183
+			u, err := url.Parse(saveOrig)
184 184
 			if err != nil {
185 185
 				return err
186 186
 			}
... ...
@@ -193,30 +273,29 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
193 193
 			if filename == "" {
194 194
 				return fmt.Errorf("cannot determine filename from url: %s", u)
195 195
 			}
196
-			destPath = dest + filename
196
+			ci.destPath = ci.destPath + filename
197 197
 		}
198 198
 	}
199 199
 
200
-	if err := b.checkPathForAddition(origPath); err != nil {
200
+	if err := b.checkPathForAddition(ci.origPath); err != nil {
201 201
 		return err
202 202
 	}
203 203
 
204 204
 	// Hash path and check the cache
205 205
 	if b.UtilizeCache {
206 206
 		var (
207
-			hash string
208 207
 			sums = b.context.GetSums()
209 208
 		)
210 209
 
211 210
 		if remoteHash != "" {
212
-			hash = remoteHash
213
-		} else if fi, err := os.Stat(path.Join(b.contextPath, origPath)); err != nil {
211
+			ci.hashPath = remoteHash
212
+		} else if fi, err := os.Stat(path.Join(b.contextPath, ci.origPath)); err != nil {
214 213
 			return err
215 214
 		} else if fi.IsDir() {
216 215
 			var subfiles []string
217 216
 			for _, fileInfo := range sums {
218 217
 				absFile := path.Join(b.contextPath, fileInfo.Name())
219
-				absOrigPath := path.Join(b.contextPath, origPath)
218
+				absOrigPath := path.Join(b.contextPath, ci.origPath)
220 219
 				if strings.HasPrefix(absFile, absOrigPath) {
221 220
 					subfiles = append(subfiles, fileInfo.Sum())
222 221
 				}
... ...
@@ -224,49 +303,22 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
224 224
 			sort.Strings(subfiles)
225 225
 			hasher := sha256.New()
226 226
 			hasher.Write([]byte(strings.Join(subfiles, ",")))
227
-			hash = "dir:" + hex.EncodeToString(hasher.Sum(nil))
227
+			ci.hashPath = "dir:" + hex.EncodeToString(hasher.Sum(nil))
228 228
 		} else {
229
-			if origPath[0] == '/' && len(origPath) > 1 {
230
-				origPath = origPath[1:]
229
+			if ci.origPath[0] == '/' && len(ci.origPath) > 1 {
230
+				ci.origPath = ci.origPath[1:]
231 231
 			}
232
-			origPath = strings.TrimPrefix(origPath, "./")
232
+			ci.origPath = strings.TrimPrefix(ci.origPath, "./")
233 233
 			// This will match on the first file in sums of the archive
234
-			if fis := sums.GetFile(origPath); fis != nil {
235
-				hash = "file:" + fis.Sum()
234
+			if fis := sums.GetFile(ci.origPath); fis != nil {
235
+				ci.hashPath = "file:" + fis.Sum()
236 236
 			}
237 237
 		}
238
-		b.Config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s %s in %s", cmdName, hash, dest)}
239
-		hit, err := b.probeCache()
240
-		if err != nil {
241
-			return err
242
-		}
243
-		// If we do not have a hash, never use the cache
244
-		if hit && hash != "" {
245
-			return nil
246
-		}
247
-	}
248 238
 
249
-	// Create the container
250
-	container, _, err := b.Daemon.Create(b.Config, "")
251
-	if err != nil {
252
-		return err
253 239
 	}
254
-	b.TmpContainers[container.ID] = struct{}{}
255
-
256
-	if err := container.Mount(); err != nil {
257
-		return err
258
-	}
259
-	defer container.Unmount()
260 240
 
261 241
 	if !allowDecompression || isRemote {
262
-		decompress = false
263
-	}
264
-	if err := b.addContext(container, origPath, destPath, decompress); err != nil {
265
-		return err
266
-	}
267
-
268
-	if err := b.commit(container.ID, cmd, fmt.Sprintf("%s %s in %s", cmdName, orig, dest)); err != nil {
269
-		return err
242
+		ci.decompress = false
270 243
 	}
271 244
 	return nil
272 245
 }
... ...
@@ -131,12 +131,13 @@ or
131 131
  interactively, as with the following command: **docker run -t -i image bash**
132 132
 
133 133
 **ADD**
134
- --**ADD <src> <dest>** The ADD instruction copies new files from <src> and adds them
135
-  to the filesystem of the container at path <dest>.  <src> must be the path to a
136
-  file or directory relative to the source directory that is being built (the
137
-  context of the build) or a remote file URL.  <dest> is the absolute path to
138
-  which the source is copied inside the target container.  All new files and
139
-  directories are created with mode 0755, with uid and gid 0.
134
+ --**ADD <src>... <dest>** The ADD instruction copies new files, directories
135
+ or remote file URLs to the filesystem of the container at path <dest>.  
136
+ Mutliple <src> resources may be specified but if they are files or directories
137
+ then they must be relative to the source directory that is being built 
138
+ (the context of the build).  <dest> is the absolute path to
139
+ which the source is copied inside the target container.  All new files and
140
+ directories are created with mode 0755, with uid and gid 0.
140 141
 
141 142
 **ENTRYPOINT**
142 143
  --**ENTRYPOINT** has two forms: ENTRYPOINT ["executable", "param1", "param2"]
... ...
@@ -284,13 +284,15 @@ change them using `docker run --env <key>=<value>`.
284 284
 
285 285
 ## ADD
286 286
 
287
-    ADD <src> <dest>
287
+    ADD <src>... <dest>
288 288
 
289
-The `ADD` instruction will copy new files from `<src>` and add them to the
290
-container's filesystem at path `<dest>`.
289
+The `ADD` instruction copies new files,directories or remote file URLs to 
290
+the filesystem of the container  from `<src>` and add them to the at 
291
+path `<dest>`.  
291 292
 
292
-`<src>` must be the path to a file or directory relative to the source directory
293
-being built (also called the *context* of the build) or a remote file URL.
293
+Multiple <src> resource may be specified but if they are files or 
294
+directories then they must be relative to the source directory that is 
295
+being built (the context of the build).
294 296
 
295 297
 `<dest>` is the absolute path to which the source will be copied inside the
296 298
 destination container.
... ...
@@ -353,6 +355,9 @@ The copy obeys the following rules:
353 353
   will be considered a directory and the contents of `<src>` will be written
354 354
   at `<dest>/base(<src>)`.
355 355
 
356
+- If multiple `<src>` resources are specified then `<dest>` must be a
357
+  directory, and it must end with a slash `/`.
358
+
356 359
 - If `<dest>` does not end with a trailing slash, it will be considered a
357 360
   regular file and the contents of `<src>` will be written at `<dest>`.
358 361
 
... ...
@@ -361,13 +366,15 @@ The copy obeys the following rules:
361 361
 
362 362
 ## COPY
363 363
 
364
-    COPY <src> <dest>
364
+    COPY <src>... <dest>
365 365
 
366
-The `COPY` instruction will copy new files from `<src>` and add them to the
367
-container's filesystem at path `<dest>`.
366
+The `COPY` instruction copies new files,directories or remote file URLs to 
367
+the filesystem of the container  from `<src>` and add them to the at 
368
+path `<dest>`. 
368 369
 
369
-`<src>` must be the path to a file or directory relative to the source directory
370
-being built (also called the *context* of the build).
370
+Multiple <src> resource may be specified but if they are files or 
371
+directories then they must be relative to the source directory that is being 
372
+built (the context of the build).
371 373
 
372 374
 `<dest>` is the absolute path to which the source will be copied inside the
373 375
 destination container.
... ...
@@ -393,6 +400,9 @@ The copy obeys the following rules:
393 393
   will be considered a directory and the contents of `<src>` will be written
394 394
   at `<dest>/base(<src>)`.
395 395
 
396
+- If multiple `<src>` resources are specified then `<dest>` must be a
397
+  directory, and it must end with a slash `/`.
398
+
396 399
 - If `<dest>` does not end with a trailing slash, it will be considered a
397 400
   regular file and the contents of `<src>` will be written at `<dest>`.
398 401
 
399 402
new file mode 100644
... ...
@@ -0,0 +1,17 @@
0
+FROM busybox
1
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
2
+RUN echo 'dockerio:x:1001:' >> /etc/group
3
+RUN mkdir /exists
4
+RUN touch /exists/exists_file
5
+RUN chown -R dockerio.dockerio /exists
6
+COPY test_file1 test_file2 /exists/
7
+ADD test_file3 test_file4 https://docker.com/robots.txt /exists/
8
+RUN [ $(ls -l / | grep exists | awk '{print $3":"$4}') = 'dockerio:dockerio' ]
9
+RUN [ $(ls -l /exists/test_file1 | awk '{print $3":"$4}') = 'root:root' ]
10
+RUN [ $(ls -l /exists/test_file2 | awk '{print $3":"$4}') = 'root:root' ]
11
+
12
+RUN [ $(ls -l /exists/test_file3 | awk '{print $3":"$4}') = 'root:root' ]
13
+RUN [ $(ls -l /exists/test_file4 | awk '{print $3":"$4}') = 'root:root' ]
14
+RUN [ $(ls -l /exists/robots.txt | awk '{print $3":"$4}') = 'root:root' ]
15
+
16
+RUN [ $(ls -l /exists/exists_file | awk '{print $3":"$4}') = 'dockerio:dockerio' ]
0 17
new file mode 100644
1 18
new file mode 100644
2 19
new file mode 100644
3 20
new file mode 100644
4 21
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+FROM busybox
1
+RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd
2
+RUN echo 'dockerio:x:1001:' >> /etc/group
3
+RUN mkdir /exists
4
+RUN chown -R dockerio.dockerio /exists
5
+COPY test_file1 /exists/
6
+ADD test_file2 test_file3 /exists/test_file1
0 7
new file mode 100644
1 8
new file mode 100644
2 9
new file mode 100644
... ...
@@ -148,6 +148,66 @@ func TestAddSingleFileToExistDir(t *testing.T) {
148 148
 	logDone("build - add single file to existing dir")
149 149
 }
150 150
 
151
+func TestMultipleFiles(t *testing.T) {
152
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestCopy")
153
+	out, exitCode, err := dockerCmdInDir(t, buildDirectory, "build", "-t", "testaddimg", "MultipleFiles")
154
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
155
+
156
+	if err != nil || exitCode != 0 {
157
+		t.Fatal("failed to build the image")
158
+	}
159
+
160
+	deleteImages("testaddimg")
161
+
162
+	logDone("build - mulitple file copy/add tests")
163
+}
164
+
165
+func TestAddMultipleFilesToFile(t *testing.T) {
166
+	name := "testaddmultiplefilestofile"
167
+	defer deleteImages(name)
168
+	ctx, err := fakeContext(`FROM scratch
169
+	ADD file1.txt file2.txt test
170
+        `,
171
+		map[string]string{
172
+			"file1.txt": "test1",
173
+			"file2.txt": "test1",
174
+		})
175
+	defer ctx.Close()
176
+	if err != nil {
177
+		t.Fatal(err)
178
+	}
179
+
180
+	expected := "When using ADD with more than one source file, the destination must be a directory and end with a /"
181
+	if _, err := buildImageFromContext(name, ctx, true); err == nil || !strings.Contains(err.Error(), expected) {
182
+		t.Fatalf("Wrong error: (should contain \"%s\") got:\n%v", expected, err)
183
+	}
184
+
185
+	logDone("build - multiple add files to file")
186
+}
187
+
188
+func TestCopyMultipleFilesToFile(t *testing.T) {
189
+	name := "testcopymultiplefilestofile"
190
+	defer deleteImages(name)
191
+	ctx, err := fakeContext(`FROM scratch
192
+	COPY file1.txt file2.txt test
193
+        `,
194
+		map[string]string{
195
+			"file1.txt": "test1",
196
+			"file2.txt": "test1",
197
+		})
198
+	defer ctx.Close()
199
+	if err != nil {
200
+		t.Fatal(err)
201
+	}
202
+
203
+	expected := "When using COPY with more than one source file, the destination must be a directory and end with a /"
204
+	if _, err := buildImageFromContext(name, ctx, true); err == nil || !strings.Contains(err.Error(), expected) {
205
+		t.Fatalf("Wrong error: (should contain \"%s\") got:\n%v", expected, err)
206
+	}
207
+
208
+	logDone("build - multiple copy files to file")
209
+}
210
+
151 211
 func TestAddSingleFileToNonExistDir(t *testing.T) {
152 212
 	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestAdd")
153 213
 	out, exitCode, err := dockerCmdInDir(t, buildDirectory, "build", "-t", "testaddimg", "SingleFileToNonExistDir")
... ...
@@ -1059,6 +1119,35 @@ func TestBuildADDLocalFileWithCache(t *testing.T) {
1059 1059
 	logDone("build - add local file with cache")
1060 1060
 }
1061 1061
 
1062
+func TestBuildADDMultipleLocalFileWithCache(t *testing.T) {
1063
+	name := "testbuildaddmultiplelocalfilewithcache"
1064
+	defer deleteImages(name)
1065
+	dockerfile := `
1066
+		FROM busybox
1067
+        MAINTAINER dockerio
1068
+        ADD foo Dockerfile /usr/lib/bla/
1069
+		RUN [ "$(cat /usr/lib/bla/foo)" = "hello" ]`
1070
+	ctx, err := fakeContext(dockerfile, map[string]string{
1071
+		"foo": "hello",
1072
+	})
1073
+	defer ctx.Close()
1074
+	if err != nil {
1075
+		t.Fatal(err)
1076
+	}
1077
+	id1, err := buildImageFromContext(name, ctx, true)
1078
+	if err != nil {
1079
+		t.Fatal(err)
1080
+	}
1081
+	id2, err := buildImageFromContext(name, ctx, true)
1082
+	if err != nil {
1083
+		t.Fatal(err)
1084
+	}
1085
+	if id1 != id2 {
1086
+		t.Fatal("The cache should have been used but hasn't.")
1087
+	}
1088
+	logDone("build - add multiple local files with cache")
1089
+}
1090
+
1062 1091
 func TestBuildADDLocalFileWithoutCache(t *testing.T) {
1063 1092
 	name := "testbuildaddlocalfilewithoutcache"
1064 1093
 	defer deleteImages(name)