Browse code

Merge pull request #17090 from duglin/dockerignoreWildcards

Support multi-dir wildcards in .dockerignore

Alexander Morozov authored on 2015/11/14 07:29:34
Showing 4 changed files
... ...
@@ -232,6 +232,11 @@ eliminates `.` and `..` elements using Go's
232 232
 [filepath.Clean](http://golang.org/pkg/path/filepath/#Clean).  Lines
233 233
 that are blank after preprocessing are ignored.
234 234
 
235
+Beyond Go's filepath.Match rules, Docker also supports a special
236
+wildcard string `**` that matches any number of directories (including
237
+zero). For example, `**/*.go` will exclude all files that end with `.go`
238
+that are found in all directories, including the root of the build context.
239
+
235 240
 Lines starting with `!` (exclamation mark) can be used to make exceptions
236 241
 to exclusions.  The following is an example `.dockerignore` file that
237 242
 uses this mechanism:
... ...
@@ -3470,12 +3470,18 @@ func (s *DockerSuite) TestBuildDockerignore(c *check.C) {
3470 3470
 		RUN [[ ! -e /bla/README.md ]]
3471 3471
 		RUN [[ ! -e /bla/dir/foo ]]
3472 3472
 		RUN [[ ! -e /bla/foo ]]
3473
-		RUN [[ ! -e /bla/.git ]]`
3473
+		RUN [[ ! -e /bla/.git ]]
3474
+		RUN [[ ! -e v.cc ]]
3475
+		RUN [[ ! -e src/v.cc ]]
3476
+		RUN [[ ! -e src/_vendor/v.cc ]]`
3474 3477
 	ctx, err := fakeContext(dockerfile, map[string]string{
3475 3478
 		"Makefile":         "all:",
3476 3479
 		".git/HEAD":        "ref: foo",
3477 3480
 		"src/x.go":         "package main",
3478 3481
 		"src/_vendor/v.go": "package main",
3482
+		"src/_vendor/v.cc": "package main",
3483
+		"src/v.cc":         "package main",
3484
+		"v.cc":             "package main",
3479 3485
 		"dir/foo":          "",
3480 3486
 		".gitignore":       "",
3481 3487
 		"README.md":        "readme",
... ...
@@ -3485,6 +3491,7 @@ pkg
3485 3485
 .gitignore
3486 3486
 src/_vendor
3487 3487
 *.md
3488
+**/*.cc
3488 3489
 dir`,
3489 3490
 	})
3490 3491
 	if err != nil {
... ...
@@ -3534,7 +3541,8 @@ func (s *DockerSuite) TestBuildDockerignoreExceptions(c *check.C) {
3534 3534
 		RUN [[ -f /bla/dir/e ]]
3535 3535
 		RUN [[ -f /bla/dir/e-dir/foo ]]
3536 3536
 		RUN [[ ! -e /bla/foo ]]
3537
-		RUN [[ ! -e /bla/.git ]]`
3537
+		RUN [[ ! -e /bla/.git ]]
3538
+		RUN [[ -e /bla/dir/a.cc ]]`
3538 3539
 	ctx, err := fakeContext(dockerfile, map[string]string{
3539 3540
 		"Makefile":         "all:",
3540 3541
 		".git/HEAD":        "ref: foo",
... ...
@@ -3548,6 +3556,7 @@ func (s *DockerSuite) TestBuildDockerignoreExceptions(c *check.C) {
3548 3548
 		"dir/e-dir/foo":    "",
3549 3549
 		".gitignore":       "",
3550 3550
 		"README.md":        "readme",
3551
+		"dir/a.cc":         "hello",
3551 3552
 		".dockerignore": `
3552 3553
 .git
3553 3554
 pkg
... ...
@@ -3556,7 +3565,9 @@ src/_vendor
3556 3556
 *.md
3557 3557
 dir
3558 3558
 !dir/e*
3559
-!dir/dir/foo`,
3559
+!dir/dir/foo
3560
+**/*.cc
3561
+!**/*.cc`,
3560 3562
 	})
3561 3563
 	if err != nil {
3562 3564
 		c.Fatal(err)
... ...
@@ -3739,7 +3750,7 @@ func (s *DockerSuite) TestBuildDockerignoringWholeDir(c *check.C) {
3739 3739
 
3740 3740
 func (s *DockerSuite) TestBuildDockerignoringBadExclusion(c *check.C) {
3741 3741
 	testRequires(c, DaemonIsLinux)
3742
-	name := "testbuilddockerignorewholedir"
3742
+	name := "testbuilddockerignorebadexclusion"
3743 3743
 	dockerfile := `
3744 3744
         FROM busybox
3745 3745
 		COPY . /
... ...
@@ -3762,6 +3773,112 @@ func (s *DockerSuite) TestBuildDockerignoringBadExclusion(c *check.C) {
3762 3762
 	}
3763 3763
 }
3764 3764
 
3765
+func (s *DockerSuite) TestBuildDockerignoringWildTopDir(c *check.C) {
3766
+	testRequires(c, DaemonIsLinux)
3767
+
3768
+	dockerfile := `
3769
+        FROM busybox
3770
+		COPY . /
3771
+		RUN [[ ! -e /.dockerignore ]]
3772
+		RUN [[ ! -e /Dockerfile ]]
3773
+		RUN [[ ! -e /file1 ]]
3774
+		RUN [[ ! -e /dir ]]`
3775
+
3776
+	ctx, err := fakeContext(dockerfile, map[string]string{
3777
+		"Dockerfile": "FROM scratch",
3778
+		"file1":      "",
3779
+		"dir/dfile1": "",
3780
+	})
3781
+	c.Assert(err, check.IsNil)
3782
+	defer ctx.Close()
3783
+
3784
+	// All of these should result in ignoring all files
3785
+	for _, variant := range []string{"**", "**/", "**/**", "*"} {
3786
+		ctx.Add(".dockerignore", variant)
3787
+		_, err = buildImageFromContext("noname", ctx, true)
3788
+		c.Assert(err, check.IsNil, check.Commentf("variant: %s", variant))
3789
+	}
3790
+}
3791
+
3792
+func (s *DockerSuite) TestBuildDockerignoringWildDirs(c *check.C) {
3793
+	testRequires(c, DaemonIsLinux)
3794
+
3795
+	dockerfile := `
3796
+        FROM busybox
3797
+		COPY . /
3798
+		RUN [[ -e /.dockerignore ]]
3799
+		RUN [[ -e /Dockerfile ]]
3800
+
3801
+		RUN [[ ! -e /file0 ]]
3802
+		RUN [[ ! -e /dir1/file0 ]]
3803
+		RUN [[ ! -e /dir2/file0 ]]
3804
+
3805
+		RUN [[ ! -e /file1 ]]
3806
+		RUN [[ ! -e /dir1/file1 ]]
3807
+		RUN [[ ! -e /dir1/dir2/file1 ]]
3808
+
3809
+		RUN [[ ! -e /dir1/file2 ]]
3810
+		RUN [[   -e /dir1/dir2/file2 ]]
3811
+
3812
+		RUN [[ ! -e /dir1/dir2/file4 ]]
3813
+		RUN [[ ! -e /dir1/dir2/file5 ]]
3814
+		RUN [[ ! -e /dir1/dir2/file6 ]]
3815
+		RUN [[ ! -e /dir1/dir3/file7 ]]
3816
+		RUN [[ ! -e /dir1/dir3/file8 ]]
3817
+		RUN [[   -e /dir1/dir3 ]]
3818
+		RUN [[   -e /dir1/dir4 ]]
3819
+
3820
+		RUN [[ ! -e 'dir1/dir5/fileAA' ]]
3821
+		RUN [[   -e 'dir1/dir5/fileAB' ]]
3822
+		RUN [[   -e 'dir1/dir5/fileB' ]]   # "." in pattern means nothing
3823
+
3824
+		RUN echo all done!`
3825
+
3826
+	ctx, err := fakeContext(dockerfile, map[string]string{
3827
+		"Dockerfile":      "FROM scratch",
3828
+		"file0":           "",
3829
+		"dir1/file0":      "",
3830
+		"dir1/dir2/file0": "",
3831
+
3832
+		"file1":           "",
3833
+		"dir1/file1":      "",
3834
+		"dir1/dir2/file1": "",
3835
+
3836
+		"dir1/file2":      "",
3837
+		"dir1/dir2/file2": "", // remains
3838
+
3839
+		"dir1/dir2/file4": "",
3840
+		"dir1/dir2/file5": "",
3841
+		"dir1/dir2/file6": "",
3842
+		"dir1/dir3/file7": "",
3843
+		"dir1/dir3/file8": "",
3844
+		"dir1/dir4/file9": "",
3845
+
3846
+		"dir1/dir5/fileAA": "",
3847
+		"dir1/dir5/fileAB": "",
3848
+		"dir1/dir5/fileB":  "",
3849
+
3850
+		".dockerignore": `
3851
+**/file0
3852
+**/*file1
3853
+**/dir1/file2
3854
+dir1/**/file4
3855
+**/dir2/file5
3856
+**/dir1/dir2/file6
3857
+dir1/dir3/**
3858
+**/dir4/**
3859
+**/file?A
3860
+**/file\?B
3861
+**/dir5/file.
3862
+`,
3863
+	})
3864
+	c.Assert(err, check.IsNil)
3865
+	defer ctx.Close()
3866
+
3867
+	_, err = buildImageFromContext("noname", ctx, true)
3868
+	c.Assert(err, check.IsNil)
3869
+}
3870
+
3765 3871
 func (s *DockerSuite) TestBuildLineBreak(c *check.C) {
3766 3872
 	testRequires(c, DaemonIsLinux)
3767 3873
 	name := "testbuildlinebreak"
... ...
@@ -6,7 +6,9 @@ import (
6 6
 	"io"
7 7
 	"os"
8 8
 	"path/filepath"
9
+	"regexp"
9 10
 	"strings"
11
+	"text/scanner"
10 12
 
11 13
 	"github.com/Sirupsen/logrus"
12 14
 )
... ...
@@ -92,15 +94,15 @@ func OptimizedMatches(file string, patterns []string, patDirs [][]string) (bool,
92 92
 			pattern = pattern[1:]
93 93
 		}
94 94
 
95
-		match, err := filepath.Match(pattern, file)
95
+		match, err := regexpMatch(pattern, file)
96 96
 		if err != nil {
97
-			return false, err
97
+			return false, fmt.Errorf("Error in pattern (%s): %s", pattern, err)
98 98
 		}
99 99
 
100 100
 		if !match && parentPath != "." {
101 101
 			// Check to see if the pattern matches one of our parent dirs.
102 102
 			if len(patDirs[i]) <= len(parentPathDirs) {
103
-				match, _ = filepath.Match(strings.Join(patDirs[i], "/"),
103
+				match, _ = regexpMatch(strings.Join(patDirs[i], "/"),
104 104
 					strings.Join(parentPathDirs[:len(patDirs[i])], "/"))
105 105
 			}
106 106
 		}
... ...
@@ -117,6 +119,99 @@ func OptimizedMatches(file string, patterns []string, patDirs [][]string) (bool,
117 117
 	return matched, nil
118 118
 }
119 119
 
120
+// regexpMatch tries to match the logic of filepath.Match but
121
+// does so using regexp logic. We do this so that we can expand the
122
+// wildcard set to include other things, like "**" to mean any number
123
+// of directories.  This means that we should be backwards compatible
124
+// with filepath.Match(). We'll end up supporting more stuff, due to
125
+// the fact that we're using regexp, but that's ok - it does no harm.
126
+func regexpMatch(pattern, path string) (bool, error) {
127
+	regStr := "^"
128
+
129
+	// Do some syntax checking on the pattern.
130
+	// filepath's Match() has some really weird rules that are inconsistent
131
+	// so instead of trying to dup their logic, just call Match() for its
132
+	// error state and if there is an error in the pattern return it.
133
+	// If this becomes an issue we can remove this since its really only
134
+	// needed in the error (syntax) case - which isn't really critical.
135
+	if _, err := filepath.Match(pattern, path); err != nil {
136
+		return false, err
137
+	}
138
+
139
+	// Go through the pattern and convert it to a regexp.
140
+	// We use a scanner so we can support utf-8 chars.
141
+	var scan scanner.Scanner
142
+	scan.Init(strings.NewReader(pattern))
143
+
144
+	sl := string(os.PathSeparator)
145
+	escSL := sl
146
+	if sl == `\` {
147
+		escSL += `\`
148
+	}
149
+
150
+	for scan.Peek() != scanner.EOF {
151
+		ch := scan.Next()
152
+
153
+		if ch == '*' {
154
+			if scan.Peek() == '*' {
155
+				// is some flavor of "**"
156
+				scan.Next()
157
+
158
+				if scan.Peek() == scanner.EOF {
159
+					// is "**EOF" - to align with .gitignore just accept all
160
+					regStr += ".*"
161
+				} else {
162
+					// is "**"
163
+					regStr += "((.*" + escSL + ")|([^" + escSL + "]*))"
164
+				}
165
+
166
+				// Treat **/ as ** so eat the "/"
167
+				if string(scan.Peek()) == sl {
168
+					scan.Next()
169
+				}
170
+			} else {
171
+				// is "*" so map it to anything but "/"
172
+				regStr += "[^" + escSL + "]*"
173
+			}
174
+		} else if ch == '?' {
175
+			// "?" is any char except "/"
176
+			regStr += "[^" + escSL + "]"
177
+		} else if strings.Index(".$", string(ch)) != -1 {
178
+			// Escape some regexp special chars that have no meaning
179
+			// in golang's filepath.Match
180
+			regStr += `\` + string(ch)
181
+		} else if ch == '\\' {
182
+			// escape next char. Note that a trailing \ in the pattern
183
+			// will be left alone (but need to escape it)
184
+			if sl == `\` {
185
+				// On windows map "\" to "\\", meaning an escaped backslash,
186
+				// and then just continue because filepath.Match on
187
+				// Windows doesn't allow escaping at all
188
+				regStr += escSL
189
+				continue
190
+			}
191
+			if scan.Peek() != scanner.EOF {
192
+				regStr += `\` + string(scan.Next())
193
+			} else {
194
+				regStr += `\`
195
+			}
196
+		} else {
197
+			regStr += string(ch)
198
+		}
199
+	}
200
+
201
+	regStr += "$"
202
+
203
+	res, err := regexp.MatchString(regStr, path)
204
+
205
+	// Map regexp's error to filepath's so no one knows we're not using filepath
206
+	if err != nil {
207
+		err = filepath.ErrBadPattern
208
+	}
209
+
210
+	return res, err
211
+}
212
+
120 213
 // CopyFile copies from src to dst until either EOF is reached
121 214
 // on src or an error occurs. It verifies src exists and remove
122 215
 // the dst if it exists.
... ...
@@ -5,6 +5,8 @@ import (
5 5
 	"os"
6 6
 	"path"
7 7
 	"path/filepath"
8
+	"runtime"
9
+	"strings"
8 10
 	"testing"
9 11
 )
10 12
 
... ...
@@ -297,6 +299,84 @@ func TestMatchesWithMalformedPatterns(t *testing.T) {
297 297
 	}
298 298
 }
299 299
 
300
+// Test lots of variants of patterns & strings
301
+func TestMatches(t *testing.T) {
302
+	tests := []struct {
303
+		pattern string
304
+		text    string
305
+		pass    bool
306
+	}{
307
+		{"**", "file", true},
308
+		{"**", "file/", true},
309
+		{"**/", "file", true}, // weird one
310
+		{"**/", "file/", true},
311
+		{"**", "/", true},
312
+		{"**/", "/", true},
313
+		{"**", "dir/file", true},
314
+		{"**/", "dir/file", false},
315
+		{"**", "dir/file/", true},
316
+		{"**/", "dir/file/", true},
317
+		{"**/**", "dir/file", true},
318
+		{"**/**", "dir/file/", true},
319
+		{"dir/**", "dir/file", true},
320
+		{"dir/**", "dir/file/", true},
321
+		{"dir/**", "dir/dir2/file", true},
322
+		{"dir/**", "dir/dir2/file/", true},
323
+		{"**/dir2/*", "dir/dir2/file", true},
324
+		{"**/dir2/*", "dir/dir2/file/", false},
325
+		{"**/dir2/**", "dir/dir2/dir3/file", true},
326
+		{"**/dir2/**", "dir/dir2/dir3/file/", true},
327
+		{"**file", "file", true},
328
+		{"**file", "dir/file", true},
329
+		{"**/file", "dir/file", true},
330
+		{"**file", "dir/dir/file", true},
331
+		{"**/file", "dir/dir/file", true},
332
+		{"**/file*", "dir/dir/file", true},
333
+		{"**/file*", "dir/dir/file.txt", true},
334
+		{"**/file*txt", "dir/dir/file.txt", true},
335
+		{"**/file*.txt", "dir/dir/file.txt", true},
336
+		{"**/file*.txt*", "dir/dir/file.txt", true},
337
+		{"**/**/*.txt", "dir/dir/file.txt", true},
338
+		{"**/**/*.txt2", "dir/dir/file.txt", false},
339
+		{"**/*.txt", "file.txt", true},
340
+		{"**/**/*.txt", "file.txt", true},
341
+		{"a**/*.txt", "a/file.txt", true},
342
+		{"a**/*.txt", "a/dir/file.txt", true},
343
+		{"a**/*.txt", "a/dir/dir/file.txt", true},
344
+		{"a/*.txt", "a/dir/file.txt", false},
345
+		{"a/*.txt", "a/file.txt", true},
346
+		{"a/*.txt**", "a/file.txt", true},
347
+		{"a[b-d]e", "ae", false},
348
+		{"a[b-d]e", "ace", true},
349
+		{"a[b-d]e", "aae", false},
350
+		{"a[^b-d]e", "aze", true},
351
+		{".*", ".foo", true},
352
+		{".*", "foo", false},
353
+		{"abc.def", "abcdef", false},
354
+		{"abc.def", "abc.def", true},
355
+		{"abc.def", "abcZdef", false},
356
+		{"abc?def", "abcZdef", true},
357
+		{"abc?def", "abcdef", false},
358
+		{"a\\*b", "a*b", true},
359
+		{"a\\", "a", false},
360
+		{"a\\", "a\\", false},
361
+		{"a\\\\", "a\\", true},
362
+		{"**/foo/bar", "foo/bar", true},
363
+		{"**/foo/bar", "dir/foo/bar", true},
364
+		{"**/foo/bar", "dir/dir2/foo/bar", true},
365
+		{"abc/**", "abc", false},
366
+		{"abc/**", "abc/def", true},
367
+		{"abc/**", "abc/def/ghi", true},
368
+	}
369
+
370
+	for _, test := range tests {
371
+		res, _ := regexpMatch(test.pattern, test.text)
372
+		if res != test.pass {
373
+			t.Fatalf("Failed: %v - res:%v", test, res)
374
+		}
375
+	}
376
+}
377
+
300 378
 // An empty string should return true from Empty.
301 379
 func TestEmpty(t *testing.T) {
302 380
 	empty := empty("")
... ...
@@ -400,3 +480,94 @@ func TestCreateIfNotExistsFile(t *testing.T) {
400 400
 		t.Fatalf("Should have been a file, seems it's not")
401 401
 	}
402 402
 }
403
+
404
+// These matchTests are stolen from go's filepath Match tests.
405
+type matchTest struct {
406
+	pattern, s string
407
+	match      bool
408
+	err        error
409
+}
410
+
411
+var matchTests = []matchTest{
412
+	{"abc", "abc", true, nil},
413
+	{"*", "abc", true, nil},
414
+	{"*c", "abc", true, nil},
415
+	{"a*", "a", true, nil},
416
+	{"a*", "abc", true, nil},
417
+	{"a*", "ab/c", false, nil},
418
+	{"a*/b", "abc/b", true, nil},
419
+	{"a*/b", "a/c/b", false, nil},
420
+	{"a*b*c*d*e*/f", "axbxcxdxe/f", true, nil},
421
+	{"a*b*c*d*e*/f", "axbxcxdxexxx/f", true, nil},
422
+	{"a*b*c*d*e*/f", "axbxcxdxe/xxx/f", false, nil},
423
+	{"a*b*c*d*e*/f", "axbxcxdxexxx/fff", false, nil},
424
+	{"a*b?c*x", "abxbbxdbxebxczzx", true, nil},
425
+	{"a*b?c*x", "abxbbxdbxebxczzy", false, nil},
426
+	{"ab[c]", "abc", true, nil},
427
+	{"ab[b-d]", "abc", true, nil},
428
+	{"ab[e-g]", "abc", false, nil},
429
+	{"ab[^c]", "abc", false, nil},
430
+	{"ab[^b-d]", "abc", false, nil},
431
+	{"ab[^e-g]", "abc", true, nil},
432
+	{"a\\*b", "a*b", true, nil},
433
+	{"a\\*b", "ab", false, nil},
434
+	{"a?b", "a☺b", true, nil},
435
+	{"a[^a]b", "a☺b", true, nil},
436
+	{"a???b", "a☺b", false, nil},
437
+	{"a[^a][^a][^a]b", "a☺b", false, nil},
438
+	{"[a-ζ]*", "α", true, nil},
439
+	{"*[a-ζ]", "A", false, nil},
440
+	{"a?b", "a/b", false, nil},
441
+	{"a*b", "a/b", false, nil},
442
+	{"[\\]a]", "]", true, nil},
443
+	{"[\\-]", "-", true, nil},
444
+	{"[x\\-]", "x", true, nil},
445
+	{"[x\\-]", "-", true, nil},
446
+	{"[x\\-]", "z", false, nil},
447
+	{"[\\-x]", "x", true, nil},
448
+	{"[\\-x]", "-", true, nil},
449
+	{"[\\-x]", "a", false, nil},
450
+	{"[]a]", "]", false, filepath.ErrBadPattern},
451
+	{"[-]", "-", false, filepath.ErrBadPattern},
452
+	{"[x-]", "x", false, filepath.ErrBadPattern},
453
+	{"[x-]", "-", false, filepath.ErrBadPattern},
454
+	{"[x-]", "z", false, filepath.ErrBadPattern},
455
+	{"[-x]", "x", false, filepath.ErrBadPattern},
456
+	{"[-x]", "-", false, filepath.ErrBadPattern},
457
+	{"[-x]", "a", false, filepath.ErrBadPattern},
458
+	{"\\", "a", false, filepath.ErrBadPattern},
459
+	{"[a-b-c]", "a", false, filepath.ErrBadPattern},
460
+	{"[", "a", false, filepath.ErrBadPattern},
461
+	{"[^", "a", false, filepath.ErrBadPattern},
462
+	{"[^bc", "a", false, filepath.ErrBadPattern},
463
+	{"a[", "a", false, filepath.ErrBadPattern}, // was nil but IMO its wrong
464
+	{"a[", "ab", false, filepath.ErrBadPattern},
465
+	{"*x", "xxx", true, nil},
466
+}
467
+
468
+func errp(e error) string {
469
+	if e == nil {
470
+		return "<nil>"
471
+	}
472
+	return e.Error()
473
+}
474
+
475
+// TestMatch test's our version of filepath.Match, called regexpMatch.
476
+func TestMatch(t *testing.T) {
477
+	for _, tt := range matchTests {
478
+		pattern := tt.pattern
479
+		s := tt.s
480
+		if runtime.GOOS == "windows" {
481
+			if strings.Index(pattern, "\\") >= 0 {
482
+				// no escape allowed on windows.
483
+				continue
484
+			}
485
+			pattern = filepath.Clean(pattern)
486
+			s = filepath.Clean(s)
487
+		}
488
+		ok, err := regexpMatch(pattern, s)
489
+		if ok != tt.match || err != tt.err {
490
+			t.Fatalf("Match(%#q, %#q) = %v, %q want %v, %q", pattern, s, ok, errp(err), tt.match, errp(tt.err))
491
+		}
492
+	}
493
+}