Browse code

Keep parser.Directive internal to parser

Signed-off-by: Daniel Nephin <dnephin@docker.com>

Daniel Nephin authored on 2017/04/13 02:47:19
Showing 11 changed files
... ...
@@ -62,7 +62,7 @@ type Builder struct {
62 62
 	disableCommit bool
63 63
 	cacheBusted   bool
64 64
 	buildArgs     *buildArgs
65
-	directive     *parser.Directive
65
+	escapeToken   rune
66 66
 
67 67
 	imageCache builder.ImageCache
68 68
 	from       builder.Image
... ...
@@ -122,7 +122,7 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
122 122
 		runConfig:     new(container.Config),
123 123
 		tmpContainers: map[string]struct{}{},
124 124
 		buildArgs:     newBuildArgs(config.BuildArgs),
125
-		directive:     parser.NewDefaultDirective(),
125
+		escapeToken:   parser.DefaultEscapeToken,
126 126
 	}
127 127
 	b.imageContexts = &imageContexts{b: b}
128 128
 	return b, nil
... ...
@@ -190,9 +190,9 @@ func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (stri
190 190
 		return "", err
191 191
 	}
192 192
 
193
-	addNodesForLabelOption(dockerfile, b.options.Labels)
193
+	addNodesForLabelOption(dockerfile.AST, b.options.Labels)
194 194
 
195
-	if err := checkDispatchDockerfile(dockerfile); err != nil {
195
+	if err := checkDispatchDockerfile(dockerfile.AST); err != nil {
196 196
 		return "", err
197 197
 	}
198 198
 
... ...
@@ -220,10 +220,13 @@ func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (stri
220 220
 	return b.image, nil
221 221
 }
222 222
 
223
-func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Node) (string, error) {
224
-	total := len(dockerfile.Children)
223
+func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) (string, error) {
224
+	// TODO: pass this to dispatchRequest instead
225
+	b.escapeToken = dockerfile.EscapeToken
226
+
227
+	total := len(dockerfile.AST.Children)
225 228
 	var shortImgID string
226
-	for i, n := range dockerfile.Children {
229
+	for i, n := range dockerfile.AST.Children {
227 230
 		select {
228 231
 		case <-b.clientCtx.Done():
229 232
 			logrus.Debug("Builder: build cancelled!")
... ...
@@ -320,13 +323,13 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
320 320
 		return nil, err
321 321
 	}
322 322
 
323
-	ast, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")), b.directive)
323
+	result, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")))
324 324
 	if err != nil {
325 325
 		return nil, err
326 326
 	}
327 327
 
328 328
 	// ensure that the commands are valid
329
-	for _, n := range ast.Children {
329
+	for _, n := range result.AST.Children {
330 330
 		if !validCommitCommands[n.Value] {
331 331
 			return nil, fmt.Errorf("%s is not a valid change command", n.Value)
332 332
 		}
... ...
@@ -337,11 +340,11 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
337 337
 	b.Stderr = ioutil.Discard
338 338
 	b.disableCommit = true
339 339
 
340
-	if err := checkDispatchDockerfile(ast); err != nil {
340
+	if err := checkDispatchDockerfile(result.AST); err != nil {
341 341
 		return nil, err
342 342
 	}
343 343
 
344
-	if err := dispatchFromDockerfile(b, ast); err != nil {
344
+	if err := dispatchFromDockerfile(b, result); err != nil {
345 345
 		return nil, err
346 346
 	}
347 347
 	return b.runConfig, nil
... ...
@@ -356,8 +359,12 @@ func checkDispatchDockerfile(dockerfile *parser.Node) error {
356 356
 	return nil
357 357
 }
358 358
 
359
-func dispatchFromDockerfile(b *Builder, ast *parser.Node) error {
359
+func dispatchFromDockerfile(b *Builder, result *parser.Result) error {
360
+	// TODO: pass this to dispatchRequest instead
361
+	b.escapeToken = result.EscapeToken
362
+	ast := result.AST
360 363
 	total := len(ast.Children)
364
+
361 365
 	for i, n := range ast.Children {
362 366
 		if err := b.dispatch(i, total, n); err != nil {
363 367
 			return err
... ...
@@ -10,8 +10,7 @@ import (
10 10
 
11 11
 func TestAddNodesForLabelOption(t *testing.T) {
12 12
 	dockerfile := "FROM scratch"
13
-	d := parser.NewDefaultDirective()
14
-	nodes, err := parser.Parse(strings.NewReader(dockerfile), d)
13
+	result, err := parser.Parse(strings.NewReader(dockerfile))
15 14
 	assert.NilError(t, err)
16 15
 
17 16
 	labels := map[string]string{
... ...
@@ -21,6 +20,7 @@ func TestAddNodesForLabelOption(t *testing.T) {
21 21
 		"org.b": "cli-b",
22 22
 		"org.a": "cli-a",
23 23
 	}
24
+	nodes := result.AST
24 25
 	addNodesForLabelOption(nodes, labels)
25 26
 
26 27
 	expected := []string{
... ...
@@ -200,7 +200,7 @@ func from(b *Builder, args []string, attributes map[string]bool, original string
200 200
 		substituionArgs = append(substituionArgs, key+"="+value)
201 201
 	}
202 202
 
203
-	name, err := ProcessWord(args[0], substituionArgs, b.directive.EscapeToken())
203
+	name, err := ProcessWord(args[0], substituionArgs, b.escapeToken)
204 204
 	if err != nil {
205 205
 		return err
206 206
 	}
... ...
@@ -10,7 +10,6 @@ import (
10 10
 	"github.com/docker/docker/api/types/container"
11 11
 	"github.com/docker/docker/api/types/strslice"
12 12
 	"github.com/docker/docker/builder"
13
-	"github.com/docker/docker/builder/dockerfile/parser"
14 13
 	"github.com/docker/docker/pkg/testutil/assert"
15 14
 	"github.com/docker/go-connections/nat"
16 15
 )
... ...
@@ -205,7 +204,6 @@ func newBuilderWithMockBackend() *Builder {
205 205
 		options:   &types.ImageBuildOptions{},
206 206
 		docker:    &MockBackend{},
207 207
 		buildArgs: newBuildArgs(make(map[string]*string)),
208
-		directive: parser.NewDefaultDirective(),
209 208
 	}
210 209
 	b.imageContexts = &imageContexts{b: b}
211 210
 	return b
... ...
@@ -180,7 +180,7 @@ func (b *Builder) evaluateEnv(cmd string, str string, envs []string) ([]string,
180 180
 			return []string{word}, err
181 181
 		}
182 182
 	}
183
-	return processFunc(str, envs, b.directive.EscapeToken())
183
+	return processFunc(str, envs, b.escapeToken)
184 184
 }
185 185
 
186 186
 // buildArgsWithoutConfigEnv returns a list of key=value pairs for all the build
... ...
@@ -171,7 +171,7 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) {
171 171
 	}()
172 172
 
173 173
 	r := strings.NewReader(testCase.dockerfile)
174
-	n, err := parser.Parse(r, parser.NewDefaultDirective())
174
+	result, err := parser.Parse(r)
175 175
 
176 176
 	if err != nil {
177 177
 		t.Fatalf("Error when parsing Dockerfile: %s", err)
... ...
@@ -188,9 +188,9 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) {
188 188
 		Stdout:    ioutil.Discard,
189 189
 		context:   context,
190 190
 		buildArgs: newBuildArgs(options.BuildArgs),
191
-		directive: parser.NewDefaultDirective(),
192 191
 	}
193 192
 
193
+	n := result.AST
194 194
 	err = b.dispatch(0, len(n.Children), n.Children[0])
195 195
 
196 196
 	if err == nil {
... ...
@@ -462,12 +462,12 @@ func (b *Builder) processImageFrom(img builder.Image) error {
462 462
 
463 463
 	// parse the ONBUILD triggers by invoking the parser
464 464
 	for _, step := range onBuildTriggers {
465
-		ast, err := parser.Parse(strings.NewReader(step), b.directive)
465
+		result, err := parser.Parse(strings.NewReader(step))
466 466
 		if err != nil {
467 467
 			return err
468 468
 		}
469 469
 
470
-		for _, n := range ast.Children {
470
+		for _, n := range result.AST.Children {
471 471
 			if err := checkDispatch(n); err != nil {
472 472
 				return err
473 473
 			}
... ...
@@ -481,7 +481,7 @@ func (b *Builder) processImageFrom(img builder.Image) error {
481 481
 			}
482 482
 		}
483 483
 
484
-		if err := dispatchFromDockerfile(b, ast); err != nil {
484
+		if err := dispatchFromDockerfile(b, result); err != nil {
485 485
 			return err
486 486
 		}
487 487
 	}
... ...
@@ -650,7 +650,7 @@ func (b *Builder) clearTmp() {
650 650
 }
651 651
 
652 652
 // readAndParseDockerfile reads a Dockerfile from the current context.
653
-func (b *Builder) readAndParseDockerfile() (*parser.Node, error) {
653
+func (b *Builder) readAndParseDockerfile() (*parser.Result, error) {
654 654
 	// If no -f was specified then look for 'Dockerfile'. If we can't find
655 655
 	// that then look for 'dockerfile'.  If neither are found then default
656 656
 	// back to 'Dockerfile' and use that in the error message.
... ...
@@ -664,9 +664,9 @@ func (b *Builder) readAndParseDockerfile() (*parser.Node, error) {
664 664
 		}
665 665
 	}
666 666
 
667
-	nodes, err := b.parseDockerfile()
667
+	result, err := b.parseDockerfile()
668 668
 	if err != nil {
669
-		return nodes, err
669
+		return nil, err
670 670
 	}
671 671
 
672 672
 	// After the Dockerfile has been parsed, we need to check the .dockerignore
... ...
@@ -680,10 +680,10 @@ func (b *Builder) readAndParseDockerfile() (*parser.Node, error) {
680 680
 	if dockerIgnore, ok := b.context.(builder.DockerIgnoreContext); ok {
681 681
 		dockerIgnore.Process([]string{b.options.Dockerfile})
682 682
 	}
683
-	return nodes, nil
683
+	return result, nil
684 684
 }
685 685
 
686
-func (b *Builder) parseDockerfile() (*parser.Node, error) {
686
+func (b *Builder) parseDockerfile() (*parser.Result, error) {
687 687
 	f, err := b.context.Open(b.options.Dockerfile)
688 688
 	if err != nil {
689 689
 		if os.IsNotExist(err) {
... ...
@@ -702,5 +702,5 @@ func (b *Builder) parseDockerfile() (*parser.Node, error) {
702 702
 			return nil, fmt.Errorf("The Dockerfile (%s) cannot be empty", b.options.Dockerfile)
703 703
 		}
704 704
 	}
705
-	return parser.Parse(f, b.directive)
705
+	return parser.Parse(f)
706 706
 }
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	"os"
6 6
 
7 7
 	"github.com/docker/docker/builder/dockerfile/parser"
8
+	"go/ast"
8 9
 )
9 10
 
10 11
 func main() {
... ...
@@ -23,12 +24,10 @@ func main() {
23 23
 		}
24 24
 		defer f.Close()
25 25
 
26
-		d := parser.NewDefaultDirective()
27
-		ast, err := parser.Parse(f, d)
26
+		result, err := parser.Parse(f)
28 27
 		if err != nil {
29 28
 			panic(err)
30
-		} else {
31
-			fmt.Println(ast.Dump())
32 29
 		}
30
+		fmt.Println(result.AST.Dump())
33 31
 	}
34 32
 }
... ...
@@ -28,10 +28,9 @@ var validJSONArraysOfStrings = map[string][]string{
28 28
 
29 29
 func TestJSONArraysOfStrings(t *testing.T) {
30 30
 	for json, expected := range validJSONArraysOfStrings {
31
-		d := Directive{}
32
-		d.SetEscapeToken(DefaultEscapeToken)
31
+		d := NewDefaultDirective()
33 32
 
34
-		if node, _, err := parseJSON(json, &d); err != nil {
33
+		if node, _, err := parseJSON(json, d); err != nil {
35 34
 			t.Fatalf("%q should be a valid JSON array of strings, but wasn't! (err: %q)", json, err)
36 35
 		} else {
37 36
 			i := 0
... ...
@@ -51,10 +50,9 @@ func TestJSONArraysOfStrings(t *testing.T) {
51 51
 		}
52 52
 	}
53 53
 	for _, json := range invalidJSONArraysOfStrings {
54
-		d := Directive{}
55
-		d.SetEscapeToken(DefaultEscapeToken)
54
+		d := NewDefaultDirective()
56 55
 
57
-		if _, _, err := parseJSON(json, &d); err != errDockerfileNotStringArray {
56
+		if _, _, err := parseJSON(json, d); err != errDockerfileNotStringArray {
58 57
 			t.Fatalf("%q should be an invalid JSON array of strings, but wasn't!", json)
59 58
 		}
60 59
 	}
... ...
@@ -34,7 +34,7 @@ type Node struct {
34 34
 	Original   string          // original line used before parsing
35 35
 	Flags      []string        // only top Node should have this set
36 36
 	StartLine  int             // the line in the original dockerfile where the node begins
37
-	EndLine    int             // the line in the original dockerfile where the node ends
37
+	endLine    int             // the line in the original dockerfile where the node ends
38 38
 }
39 39
 
40 40
 // Dump dumps the AST defined by `node` as a list of sexps.
... ...
@@ -70,7 +70,7 @@ var (
70 70
 )
71 71
 
72 72
 // DefaultEscapeToken is the default escape token
73
-const DefaultEscapeToken = "\\"
73
+const DefaultEscapeToken = '\\'
74 74
 
75 75
 // Directive is the structure used during a build run to hold the state of
76 76
 // parsing directives.
... ...
@@ -81,8 +81,8 @@ type Directive struct {
81 81
 	escapeSeen            bool           // Whether the escape directive has been seen
82 82
 }
83 83
 
84
-// SetEscapeToken sets the default token for escaping characters in a Dockerfile.
85
-func (d *Directive) SetEscapeToken(s string) error {
84
+// setEscapeToken sets the default token for escaping characters in a Dockerfile.
85
+func (d *Directive) setEscapeToken(s string) error {
86 86
 	if s != "`" && s != "\\" {
87 87
 		return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s)
88 88
 	}
... ...
@@ -91,18 +91,13 @@ func (d *Directive) SetEscapeToken(s string) error {
91 91
 	return nil
92 92
 }
93 93
 
94
-// EscapeToken returns the escape token
95
-func (d *Directive) EscapeToken() rune {
96
-	return d.escapeToken
97
-}
98
-
99 94
 // NewDefaultDirective returns a new Directive with the default escapeToken token
100 95
 func NewDefaultDirective() *Directive {
101 96
 	directive := Directive{
102 97
 		escapeSeen:           false,
103 98
 		lookingForDirectives: true,
104 99
 	}
105
-	directive.SetEscapeToken(DefaultEscapeToken)
100
+	directive.setEscapeToken(string(DefaultEscapeToken))
106 101
 	return &directive
107 102
 }
108 103
 
... ...
@@ -200,7 +195,7 @@ func handleParserDirective(line string, d *Directive) (bool, error) {
200 200
 	}
201 201
 	for i, n := range tokenEscapeCommand.SubexpNames() {
202 202
 		if n == "escapechar" {
203
-			if err := d.SetEscapeToken(tecMatch[i]); err != nil {
203
+			if err := d.setEscapeToken(tecMatch[i]); err != nil {
204 204
 				return false, err
205 205
 			}
206 206
 			return true, nil
... ...
@@ -209,9 +204,16 @@ func handleParserDirective(line string, d *Directive) (bool, error) {
209 209
 	return false, nil
210 210
 }
211 211
 
212
-// Parse is the main parse routine.
213
-// It handles an io.ReadWriteCloser and returns the root of the AST.
214
-func Parse(rwc io.Reader, d *Directive) (*Node, error) {
212
+// Result is the result of parsing a Dockerfile
213
+type Result struct {
214
+	AST         *Node
215
+	EscapeToken rune
216
+}
217
+
218
+// Parse reads lines from a Reader, parses the lines into an AST and returns
219
+// the AST and escape token
220
+func Parse(rwc io.Reader) (*Result, error) {
221
+	d := NewDefaultDirective()
215 222
 	currentLine := 0
216 223
 	root := &Node{}
217 224
 	root.StartLine = -1
... ...
@@ -267,18 +269,18 @@ func Parse(rwc io.Reader, d *Directive) (*Node, error) {
267 267
 		if child != nil {
268 268
 			// Update the line information for the current child.
269 269
 			child.StartLine = startLine
270
-			child.EndLine = currentLine
270
+			child.endLine = currentLine
271 271
 			// Update the line information for the root. The starting line of the root is always the
272 272
 			// starting line of the first child and the ending line is the ending line of the last child.
273 273
 			if root.StartLine < 0 {
274 274
 				root.StartLine = currentLine
275 275
 			}
276
-			root.EndLine = currentLine
276
+			root.endLine = currentLine
277 277
 			root.Children = append(root.Children, child)
278 278
 		}
279 279
 	}
280 280
 
281
-	return root, nil
281
+	return &Result{AST: root, EscapeToken: d.escapeToken}, nil
282 282
 }
283 283
 
284 284
 // covers comments and empty lines. Lines should be trimmed before passing to
... ...
@@ -8,6 +8,8 @@ import (
8 8
 	"path/filepath"
9 9
 	"runtime"
10 10
 	"testing"
11
+
12
+	"github.com/docker/docker/pkg/testutil/assert"
11 13
 )
12 14
 
13 15
 const testDir = "testfiles"
... ...
@@ -16,17 +18,11 @@ const testFileLineInfo = "testfile-line/Dockerfile"
16 16
 
17 17
 func getDirs(t *testing.T, dir string) []string {
18 18
 	f, err := os.Open(dir)
19
-	if err != nil {
20
-		t.Fatal(err)
21
-	}
22
-
19
+	assert.NilError(t, err)
23 20
 	defer f.Close()
24 21
 
25 22
 	dirs, err := f.Readdirnames(0)
26
-	if err != nil {
27
-		t.Fatal(err)
28
-	}
29
-
23
+	assert.NilError(t, err)
30 24
 	return dirs
31 25
 }
32 26
 
... ...
@@ -35,15 +31,11 @@ func TestTestNegative(t *testing.T) {
35 35
 		dockerfile := filepath.Join(negativeTestDir, dir, "Dockerfile")
36 36
 
37 37
 		df, err := os.Open(dockerfile)
38
-		if err != nil {
39
-			t.Fatalf("Dockerfile missing for %s: %v", dir, err)
40
-		}
38
+		assert.NilError(t, err)
41 39
 		defer df.Close()
42 40
 
43
-		_, err = Parse(df, NewDefaultDirective())
44
-		if err == nil {
45
-			t.Fatalf("No error parsing broken dockerfile for %s", dir)
46
-		}
41
+		_, err = Parse(df)
42
+		assert.Error(t, err, "")
47 43
 	}
48 44
 }
49 45
 
... ...
@@ -53,31 +45,21 @@ func TestTestData(t *testing.T) {
53 53
 		resultfile := filepath.Join(testDir, dir, "result")
54 54
 
55 55
 		df, err := os.Open(dockerfile)
56
-		if err != nil {
57
-			t.Fatalf("Dockerfile missing for %s: %v", dir, err)
58
-		}
56
+		assert.NilError(t, err)
59 57
 		defer df.Close()
60 58
 
61
-		ast, err := Parse(df, NewDefaultDirective())
62
-		if err != nil {
63
-			t.Fatalf("Error parsing %s's dockerfile: %v", dir, err)
64
-		}
59
+		result, err := Parse(df)
60
+		assert.NilError(t, err)
65 61
 
66 62
 		content, err := ioutil.ReadFile(resultfile)
67
-		if err != nil {
68
-			t.Fatalf("Error reading %s's result file: %v", dir, err)
69
-		}
63
+		assert.NilError(t, err)
70 64
 
71 65
 		if runtime.GOOS == "windows" {
72 66
 			// CRLF --> CR to match Unix behavior
73 67
 			content = bytes.Replace(content, []byte{'\x0d', '\x0a'}, []byte{'\x0a'}, -1)
74 68
 		}
75 69
 
76
-		if ast.Dump()+"\n" != string(content) {
77
-			fmt.Fprintln(os.Stderr, "Result:\n"+ast.Dump())
78
-			fmt.Fprintln(os.Stderr, "Expected:\n"+string(content))
79
-			t.Fatalf("%s: AST dump of dockerfile does not match result", dir)
80
-		}
70
+		assert.Equal(t, result.AST.Dump()+"\n", string(content))
81 71
 	}
82 72
 }
83 73
 
... ...
@@ -119,46 +101,33 @@ func TestParseWords(t *testing.T) {
119 119
 
120 120
 	for _, test := range tests {
121 121
 		words := parseWords(test["input"][0], NewDefaultDirective())
122
-		if len(words) != len(test["expect"]) {
123
-			t.Fatalf("length check failed. input: %v, expect: %q, output: %q", test["input"][0], test["expect"], words)
124
-		}
125
-		for i, word := range words {
126
-			if word != test["expect"][i] {
127
-				t.Fatalf("word check failed for word: %q. input: %q, expect: %q, output: %q", word, test["input"][0], test["expect"], words)
128
-			}
129
-		}
122
+		assert.DeepEqual(t, words, test["expect"])
130 123
 	}
131 124
 }
132 125
 
133 126
 func TestLineInformation(t *testing.T) {
134 127
 	df, err := os.Open(testFileLineInfo)
135
-	if err != nil {
136
-		t.Fatalf("Dockerfile missing for %s: %v", testFileLineInfo, err)
137
-	}
128
+	assert.NilError(t, err)
138 129
 	defer df.Close()
139 130
 
140
-	ast, err := Parse(df, NewDefaultDirective())
141
-	if err != nil {
142
-		t.Fatalf("Error parsing dockerfile %s: %v", testFileLineInfo, err)
143
-	}
131
+	result, err := Parse(df)
132
+	assert.NilError(t, err)
144 133
 
145
-	if ast.StartLine != 5 || ast.EndLine != 31 {
146
-		fmt.Fprintf(os.Stderr, "Wrong root line information: expected(%d-%d), actual(%d-%d)\n", 5, 31, ast.StartLine, ast.EndLine)
134
+	ast := result.AST
135
+	if ast.StartLine != 5 || ast.endLine != 31 {
136
+		fmt.Fprintf(os.Stderr, "Wrong root line information: expected(%d-%d), actual(%d-%d)\n", 5, 31, ast.StartLine, ast.endLine)
147 137
 		t.Fatal("Root line information doesn't match result.")
148 138
 	}
149
-	if len(ast.Children) != 3 {
150
-		fmt.Fprintf(os.Stderr, "Wrong number of child: expected(%d), actual(%d)\n", 3, len(ast.Children))
151
-		t.Fatalf("Root line information doesn't match result for %s", testFileLineInfo)
152
-	}
139
+	assert.Equal(t, len(ast.Children), 3)
153 140
 	expected := [][]int{
154 141
 		{5, 5},
155 142
 		{11, 12},
156 143
 		{17, 31},
157 144
 	}
158 145
 	for i, child := range ast.Children {
159
-		if child.StartLine != expected[i][0] || child.EndLine != expected[i][1] {
146
+		if child.StartLine != expected[i][0] || child.endLine != expected[i][1] {
160 147
 			t.Logf("Wrong line information for child %d: expected(%d-%d), actual(%d-%d)\n",
161
-				i, expected[i][0], expected[i][1], child.StartLine, child.EndLine)
148
+				i, expected[i][0], expected[i][1], child.StartLine, child.endLine)
162 149
 			t.Fatal("Root line information doesn't match result.")
163 150
 		}
164 151
 	}