Browse code

add wildcard support to copy/add

Signed-off-by: Doug Davis <dug@us.ibm.com>

Doug Davis authored on 2014/09/22 22:41:02
Showing 3 changed files
... ...
@@ -102,7 +102,7 @@ func (b *Builder) commit(id string, autoCmd []string, comment string) error {
102 102
 type copyInfo struct {
103 103
 	origPath   string
104 104
 	destPath   string
105
-	hashPath   string
105
+	hash       string
106 106
 	decompress bool
107 107
 	tmpDir     string
108 108
 }
... ...
@@ -118,14 +118,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
118 118
 
119 119
 	dest := args[len(args)-1] // last one is always the dest
120 120
 
121
-	if len(args) > 2 && dest[len(dest)-1] != '/' {
122
-		return fmt.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
123
-	}
124
-
125
-	copyInfos := make([]copyInfo, len(args)-1)
126
-	hasHash := false
127
-	srcPaths := ""
128
-	origPaths := ""
121
+	copyInfos := []*copyInfo{}
129 122
 
130 123
 	b.Config.Image = b.image
131 124
 
... ...
@@ -140,28 +133,44 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
140 140
 	// Loop through each src file and calculate the info we need to
141 141
 	// do the copy (e.g. hash value if cached).  Don't actually do
142 142
 	// the copy until we've looked at all src files
143
-	for i, orig := range args[0 : len(args)-1] {
144
-		ci := &copyInfos[i]
145
-		ci.origPath = orig
146
-		ci.destPath = dest
147
-		ci.decompress = true
148
-
149
-		err := calcCopyInfo(b, cmdName, ci, allowRemote, allowDecompression)
143
+	for _, orig := range args[0 : len(args)-1] {
144
+		err := calcCopyInfo(b, cmdName, &copyInfos, orig, dest, allowRemote, allowDecompression)
150 145
 		if err != nil {
151 146
 			return err
152 147
 		}
148
+	}
153 149
 
154
-		origPaths += " " + ci.origPath // will have leading space
155
-		if ci.hashPath == "" {
156
-			srcPaths += " " + ci.origPath // note leading space
157
-		} else {
158
-			srcPaths += " " + ci.hashPath // note leading space
159
-			hasHash = true
150
+	if len(copyInfos) == 0 {
151
+		return fmt.Errorf("No source files were specified")
152
+	}
153
+
154
+	if len(copyInfos) > 1 && !strings.HasSuffix(dest, "/") {
155
+		return fmt.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
156
+	}
157
+
158
+	// For backwards compat, if there's just one CI then use it as the
159
+	// cache look-up string, otherwise hash 'em all into one
160
+	var srcHash string
161
+	var origPaths string
162
+
163
+	if len(copyInfos) == 1 {
164
+		srcHash = copyInfos[0].hash
165
+		origPaths = copyInfos[0].origPath
166
+	} else {
167
+		var hashs []string
168
+		var origs []string
169
+		for _, ci := range copyInfos {
170
+			hashs = append(hashs, ci.hash)
171
+			origs = append(origs, ci.origPath)
160 172
 		}
173
+		hasher := sha256.New()
174
+		hasher.Write([]byte(strings.Join(hashs, ",")))
175
+		srcHash = "multi:" + hex.EncodeToString(hasher.Sum(nil))
176
+		origPaths = strings.Join(origs, " ")
161 177
 	}
162 178
 
163 179
 	cmd := b.Config.Cmd
164
-	b.Config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s%s in %s", cmdName, srcPaths, dest)}
180
+	b.Config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s %s in %s", cmdName, srcHash, dest)}
165 181
 	defer func(cmd []string) { b.Config.Cmd = cmd }(cmd)
166 182
 
167 183
 	hit, err := b.probeCache()
... ...
@@ -169,7 +178,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
169 169
 		return err
170 170
 	}
171 171
 	// If we do not have at least one hash, never use the cache
172
-	if hit && hasHash {
172
+	if hit && b.UtilizeCache {
173 173
 		return nil
174 174
 	}
175 175
 
... ...
@@ -190,24 +199,32 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecomp
190 190
 		}
191 191
 	}
192 192
 
193
-	if err := b.commit(container.ID, cmd, fmt.Sprintf("%s%s in %s", cmdName, origPaths, dest)); err != nil {
193
+	if err := b.commit(container.ID, cmd, fmt.Sprintf("%s %s in %s", cmdName, origPaths, dest)); err != nil {
194 194
 		return err
195 195
 	}
196 196
 	return nil
197 197
 }
198 198
 
199
-func calcCopyInfo(b *Builder, cmdName string, ci *copyInfo, allowRemote bool, allowDecompression bool) error {
200
-	var (
201
-		remoteHash string
202
-		isRemote   bool
203
-	)
199
+func calcCopyInfo(b *Builder, cmdName string, cInfos *[]*copyInfo, origPath string, destPath string, allowRemote bool, allowDecompression bool) error {
200
+
201
+	if origPath != "" && origPath[0] == '/' && len(origPath) > 1 {
202
+		origPath = origPath[1:]
203
+	}
204
+	origPath = strings.TrimPrefix(origPath, "./")
204 205
 
205
-	saveOrig := ci.origPath
206
-	isRemote = utils.IsURL(ci.origPath)
206
+	// In the remote/URL case, download it and gen its hashcode
207
+	if utils.IsURL(origPath) {
208
+		if !allowRemote {
209
+			return fmt.Errorf("Source can't be a URL for %s", cmdName)
210
+		}
211
+
212
+		ci := copyInfo{}
213
+		ci.origPath = origPath
214
+		ci.hash = origPath // default to this but can change
215
+		ci.destPath = destPath
216
+		ci.decompress = false
217
+		*cInfos = append(*cInfos, &ci)
207 218
 
208
-	if isRemote && !allowRemote {
209
-		return fmt.Errorf("Source can't be an URL for %s", cmdName)
210
-	} else if isRemote {
211 219
 		// Initiate the download
212 220
 		resp, err := utils.Download(ci.origPath)
213 221
 		if err != nil {
... ...
@@ -243,24 +260,9 @@ func calcCopyInfo(b *Builder, cmdName string, ci *copyInfo, allowRemote bool, al
243 243
 
244 244
 		ci.origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName))
245 245
 
246
-		// Process the checksum
247
-		r, err := archive.Tar(tmpFileName, archive.Uncompressed)
248
-		if err != nil {
249
-			return err
250
-		}
251
-		tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version0)
252
-		if err != nil {
253
-			return err
254
-		}
255
-		if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
256
-			return err
257
-		}
258
-		remoteHash = tarSum.Sum(nil)
259
-		r.Close()
260
-
261 246
 		// If the destination is a directory, figure out the filename.
262 247
 		if strings.HasSuffix(ci.destPath, "/") {
263
-			u, err := url.Parse(saveOrig)
248
+			u, err := url.Parse(origPath)
264 249
 			if err != nil {
265 250
 				return err
266 251
 			}
... ...
@@ -275,62 +277,113 @@ func calcCopyInfo(b *Builder, cmdName string, ci *copyInfo, allowRemote bool, al
275 275
 			}
276 276
 			ci.destPath = ci.destPath + filename
277 277
 		}
278
+
279
+		// Calc the checksum, only if we're using the cache
280
+		if b.UtilizeCache {
281
+			r, err := archive.Tar(tmpFileName, archive.Uncompressed)
282
+			if err != nil {
283
+				return err
284
+			}
285
+			tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version0)
286
+			if err != nil {
287
+				return err
288
+			}
289
+			if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
290
+				return err
291
+			}
292
+			ci.hash = tarSum.Sum(nil)
293
+			r.Close()
294
+		}
295
+
296
+		return nil
297
+	}
298
+
299
+	// Deal with wildcards
300
+	if ContainsWildcards(origPath) {
301
+		for _, fileInfo := range b.context.GetSums() {
302
+			if fileInfo.Name() == "" {
303
+				continue
304
+			}
305
+			match, _ := path.Match(origPath, fileInfo.Name())
306
+			if !match {
307
+				continue
308
+			}
309
+
310
+			calcCopyInfo(b, cmdName, cInfos, fileInfo.Name(), destPath, allowRemote, allowDecompression)
311
+		}
312
+		return nil
278 313
 	}
279 314
 
280
-	if err := b.checkPathForAddition(ci.origPath); err != nil {
315
+	// Must be a dir or a file
316
+
317
+	if err := b.checkPathForAddition(origPath); err != nil {
281 318
 		return err
282 319
 	}
320
+	fi, _ := os.Stat(path.Join(b.contextPath, origPath))
283 321
 
284
-	// Hash path and check the cache
285
-	if b.UtilizeCache {
286
-		var (
287
-			sums = b.context.GetSums()
288
-		)
322
+	ci := copyInfo{}
323
+	ci.origPath = origPath
324
+	ci.hash = origPath
325
+	ci.destPath = destPath
326
+	ci.decompress = allowDecompression
327
+	*cInfos = append(*cInfos, &ci)
289 328
 
290
-		if remoteHash != "" {
291
-			ci.hashPath = remoteHash
292
-		} else if fi, err := os.Stat(path.Join(b.contextPath, ci.origPath)); err != nil {
293
-			return err
294
-		} else if fi.IsDir() {
295
-			var subfiles []string
296
-			absOrigPath := path.Join(b.contextPath, ci.origPath)
297
-
298
-			// Add a trailing / to make sure we only
299
-			// pick up nested files under the dir and
300
-			// not sibling files of the dir that just
301
-			// happen to start with the same chars
302
-			if !strings.HasSuffix(absOrigPath, "/") {
303
-				absOrigPath += "/"
304
-			}
305
-			for _, fileInfo := range sums {
306
-				absFile := path.Join(b.contextPath, fileInfo.Name())
307
-				if strings.HasPrefix(absFile, absOrigPath) {
308
-					subfiles = append(subfiles, fileInfo.Sum())
309
-				}
310
-			}
311
-			sort.Strings(subfiles)
312
-			hasher := sha256.New()
313
-			hasher.Write([]byte(strings.Join(subfiles, ",")))
314
-			ci.hashPath = "dir:" + hex.EncodeToString(hasher.Sum(nil))
315
-		} else {
316
-			if ci.origPath[0] == '/' && len(ci.origPath) > 1 {
317
-				ci.origPath = ci.origPath[1:]
318
-			}
319
-			ci.origPath = strings.TrimPrefix(ci.origPath, "./")
320
-			// This will match on the first file in sums of the archive
321
-			if fis := sums.GetFile(ci.origPath); fis != nil {
322
-				ci.hashPath = "file:" + fis.Sum()
323
-			}
329
+	// If not using cache don't need to do anything else.
330
+	// If we are using a cache then calc the hash for the src file/dir
331
+	if !b.UtilizeCache {
332
+		return nil
333
+	}
334
+
335
+	// Deal with the single file case
336
+	if !fi.IsDir() {
337
+		// This will match first file in sums of the archive
338
+		fis := b.context.GetSums().GetFile(ci.origPath)
339
+		if fis != nil {
340
+			ci.hash = "file:" + fis.Sum()
324 341
 		}
342
+		return nil
343
+	}
344
+
345
+	// Must be a dir
346
+	var subfiles []string
347
+	absOrigPath := path.Join(b.contextPath, ci.origPath)
325 348
 
349
+	// Add a trailing / to make sure we only pick up nested files under
350
+	// the dir and not sibling files of the dir that just happen to
351
+	// start with the same chars
352
+	if !strings.HasSuffix(absOrigPath, "/") {
353
+		absOrigPath += "/"
326 354
 	}
327 355
 
328
-	if !allowDecompression || isRemote {
329
-		ci.decompress = false
356
+	// Need path w/o / too to find matching dir w/o trailing /
357
+	absOrigPathNoSlash := absOrigPath[:len(absOrigPath)-1]
358
+
359
+	for _, fileInfo := range b.context.GetSums() {
360
+		absFile := path.Join(b.contextPath, fileInfo.Name())
361
+		if strings.HasPrefix(absFile, absOrigPath) || absFile == absOrigPathNoSlash {
362
+			subfiles = append(subfiles, fileInfo.Sum())
363
+		}
330 364
 	}
365
+	sort.Strings(subfiles)
366
+	hasher := sha256.New()
367
+	hasher.Write([]byte(strings.Join(subfiles, ",")))
368
+	ci.hash = "dir:" + hex.EncodeToString(hasher.Sum(nil))
369
+
331 370
 	return nil
332 371
 }
333 372
 
373
+func ContainsWildcards(name string) bool {
374
+	for i := 0; i < len(name); i++ {
375
+		ch := name[i]
376
+		if ch == '\\' {
377
+			i++
378
+		} else if ch == '*' || ch == '?' || ch == '[' {
379
+			return true
380
+		}
381
+	}
382
+	return false
383
+}
384
+
334 385
 func (b *Builder) pullImage(name string) (*imagepkg.Image, error) {
335 386
 	remote, tag := parsers.ParseRepositoryTag(name)
336 387
 	if tag == "" {
... ...
@@ -295,11 +295,18 @@ The `ADD` instruction copies new files,directories or remote file URLs to
295 295
 the filesystem of the container  from `<src>` and add them to the at 
296 296
 path `<dest>`.  
297 297
 
298
-Multiple <src> resource may be specified but if they are files or 
298
+Multiple `<src>` resource may be specified but if they are files or 
299 299
 directories then they must be relative to the source directory that is 
300 300
 being built (the context of the build).
301 301
 
302
-`<dest>` is the absolute path to which the source will be copied inside the
302
+Each `<src>` may contain wildcards and matching will be done using Go's
303
+[filepath.Match](http://golang.org/pkg/path/filepath#Match) rules.
304
+For most command line uses this should act as expected, for example:
305
+
306
+    ADD hom* /mydir/        # adds all files starting with "hom"
307
+    ADD hom?.txt /mydir/    # ? is replaced with any single character
308
+
309
+The `<dest>` is the absolute path to which the source will be copied inside the
303 310
 destination container.
304 311
 
305 312
 All new files and directories are created with a UID and GID of 0.
... ...
@@ -360,8 +367,9 @@ The copy obeys the following rules:
360 360
   will be considered a directory and the contents of `<src>` will be written
361 361
   at `<dest>/base(<src>)`.
362 362
 
363
-- If multiple `<src>` resources are specified then `<dest>` must be a
364
-  directory, and it must end with a slash `/`.
363
+- If multiple `<src>` resources are specified, either directly or due to the
364
+  use of a wildcard, then `<dest>` must be a directory, and it must end with 
365
+  a slash `/`.
365 366
 
366 367
 - If `<dest>` does not end with a trailing slash, it will be considered a
367 368
   regular file and the contents of `<src>` will be written at `<dest>`.
... ...
@@ -377,11 +385,18 @@ The `COPY` instruction copies new files,directories or remote file URLs to
377 377
 the filesystem of the container  from `<src>` and add them to the at 
378 378
 path `<dest>`. 
379 379
 
380
-Multiple <src> resource may be specified but if they are files or 
380
+Multiple `<src>` resource may be specified but if they are files or 
381 381
 directories then they must be relative to the source directory that is being 
382 382
 built (the context of the build).
383 383
 
384
-`<dest>` is the absolute path to which the source will be copied inside the
384
+Each `<src>` may contain wildcards and matching will be done using Go's
385
+[filepath.Match](http://golang.org/pkg/path/filepath#Match) rules.
386
+For most command line uses this should act as expected, for example:
387
+
388
+    COPY hom* /mydir/        # adds all files starting with "hom"
389
+    COPY hom?.txt /mydir/    # ? is replaced with any single character
390
+
391
+The `<dest>` is the absolute path to which the source will be copied inside the
385 392
 destination container.
386 393
 
387 394
 All new files and directories are created with a UID and GID of 0.
... ...
@@ -405,8 +420,9 @@ The copy obeys the following rules:
405 405
   will be considered a directory and the contents of `<src>` will be written
406 406
   at `<dest>/base(<src>)`.
407 407
 
408
-- If multiple `<src>` resources are specified then `<dest>` must be a
409
-  directory, and it must end with a slash `/`.
408
+- If multiple `<src>` resources are specified, either directly or due to the
409
+  use of a wildcard, then `<dest>` must be a directory, and it must end with 
410
+  a slash `/`.
410 411
 
411 412
 - If `<dest>` does not end with a trailing slash, it will be considered a
412 413
   regular file and the contents of `<src>` will be written at `<dest>`.
... ...
@@ -174,6 +174,29 @@ func TestBuildAddMultipleFilesToFile(t *testing.T) {
174 174
 	logDone("build - multiple add files to file")
175 175
 }
176 176
 
177
+func TestBuildAddMultipleFilesToFileWild(t *testing.T) {
178
+	name := "testaddmultiplefilestofilewild"
179
+	defer deleteImages(name)
180
+	ctx, err := fakeContext(`FROM scratch
181
+	ADD file*.txt test
182
+        `,
183
+		map[string]string{
184
+			"file1.txt": "test1",
185
+			"file2.txt": "test1",
186
+		})
187
+	defer ctx.Close()
188
+	if err != nil {
189
+		t.Fatal(err)
190
+	}
191
+
192
+	expected := "When using ADD with more than one source file, the destination must be a directory and end with a /"
193
+	if _, err := buildImageFromContext(name, ctx, true); err == nil || !strings.Contains(err.Error(), expected) {
194
+		t.Fatalf("Wrong error: (should contain \"%s\") got:\n%v", expected, err)
195
+	}
196
+
197
+	logDone("build - multiple add files to file wild")
198
+}
199
+
177 200
 func TestBuildCopyMultipleFilesToFile(t *testing.T) {
178 201
 	name := "testcopymultiplefilestofile"
179 202
 	defer deleteImages(name)
... ...
@@ -197,6 +220,136 @@ func TestBuildCopyMultipleFilesToFile(t *testing.T) {
197 197
 	logDone("build - multiple copy files to file")
198 198
 }
199 199
 
200
+func TestBuildCopyWildcard(t *testing.T) {
201
+	name := "testcopywildcard"
202
+	defer deleteImages(name)
203
+	server, err := fakeStorage(map[string]string{
204
+		"robots.txt": "hello",
205
+		"index.html": "world",
206
+	})
207
+	if err != nil {
208
+		t.Fatal(err)
209
+	}
210
+	defer server.Close()
211
+	ctx, err := fakeContext(fmt.Sprintf(`FROM busybox
212
+	COPY file*.txt /tmp/
213
+	RUN ls /tmp/file1.txt /tmp/file2.txt
214
+	RUN mkdir /tmp1
215
+	COPY dir* /tmp1/
216
+	RUN ls /tmp1/dirt /tmp1/nested_file /tmp1/nested_dir/nest_nest_file
217
+	RUN mkdir /tmp2
218
+        ADD dir/*dir %s/robots.txt /tmp2/
219
+	RUN ls /tmp2/nest_nest_file /tmp2/robots.txt
220
+	`, server.URL),
221
+		map[string]string{
222
+			"file1.txt":                     "test1",
223
+			"file2.txt":                     "test2",
224
+			"dir/nested_file":               "nested file",
225
+			"dir/nested_dir/nest_nest_file": "2 times nested",
226
+			"dirt": "dirty",
227
+		})
228
+	defer ctx.Close()
229
+	if err != nil {
230
+		t.Fatal(err)
231
+	}
232
+
233
+	id1, err := buildImageFromContext(name, ctx, true)
234
+	if err != nil {
235
+		t.Fatal(err)
236
+	}
237
+
238
+	// Now make sure we use a cache the 2nd time
239
+	id2, err := buildImageFromContext(name, ctx, true)
240
+	if err != nil {
241
+		t.Fatal(err)
242
+	}
243
+
244
+	if id1 != id2 {
245
+		t.Fatal(fmt.Errorf("Didn't use the cache"))
246
+	}
247
+
248
+	logDone("build - copy wild card")
249
+}
250
+
251
+func TestBuildCopyWildcardNoFind(t *testing.T) {
252
+	name := "testcopywildcardnofind"
253
+	defer deleteImages(name)
254
+	ctx, err := fakeContext(`FROM busybox
255
+	COPY file*.txt /tmp/
256
+	`, nil)
257
+	defer ctx.Close()
258
+	if err != nil {
259
+		t.Fatal(err)
260
+	}
261
+
262
+	_, err = buildImageFromContext(name, ctx, true)
263
+	if err == nil {
264
+		t.Fatal(fmt.Errorf("Should have failed to find a file"))
265
+	}
266
+	if !strings.Contains(err.Error(), "No source files were specified") {
267
+		t.Fatalf("Wrong error %v, must be about no source files", err)
268
+	}
269
+
270
+	logDone("build - copy wild card no find")
271
+}
272
+
273
+func TestBuildCopyWildcardCache(t *testing.T) {
274
+	name := "testcopywildcardcache"
275
+	defer deleteImages(name)
276
+	server, err := fakeStorage(map[string]string{
277
+		"robots.txt": "hello",
278
+		"index.html": "world",
279
+	})
280
+	if err != nil {
281
+		t.Fatal(err)
282
+	}
283
+	defer server.Close()
284
+	ctx, err := fakeContext(`FROM busybox
285
+	COPY file1.txt /tmp/
286
+	`,
287
+		map[string]string{
288
+			"file1.txt": "test1",
289
+		})
290
+	defer ctx.Close()
291
+	if err != nil {
292
+		t.Fatal(err)
293
+	}
294
+
295
+	if err != nil {
296
+		t.Fatal(err)
297
+	}
298
+	id1, err := buildImageFromContext(name, ctx, true)
299
+	if err != nil {
300
+		t.Fatal(err)
301
+	}
302
+
303
+	// Now make sure we use a cache the 2nd time even with wild card
304
+	ctx2, err := fakeContext(`FROM busybox
305
+	COPY file*.txt /tmp/
306
+	`,
307
+		map[string]string{
308
+			"file1.txt": "test1",
309
+		})
310
+	defer ctx2.Close()
311
+	if err != nil {
312
+		t.Fatal(err)
313
+	}
314
+
315
+	if err != nil {
316
+		t.Fatal(err)
317
+	}
318
+	id2, err := buildImageFromContext(name, ctx2, true)
319
+	if err != nil {
320
+		t.Fatal(err)
321
+	}
322
+
323
+	if id1 != id2 {
324
+		t.Fatal(fmt.Errorf("Didn't use the cache"))
325
+	}
326
+
327
+	logDone("build - copy wild card cache")
328
+}
329
+
200 330
 func TestBuildAddSingleFileToNonExistDir(t *testing.T) {
201 331
 	name := "testaddsinglefiletononexistdir"
202 332
 	defer deleteImages(name)