Browse code

Fix processing of unset build-args during build

This reverts 26103. 26103 was trying to make it so that if someone did:
docker build --build-arg FOO .
and FOO wasn't set as an env var then it would pick-up FOO from the
Dockerfile's ARG cmd. However, it went too far and removed the ability
to specify a build arg w/o any value. Meaning it required the --build-arg
param to always be in the form "name=value", and not just "name".

This PR does the right fix - it allows just "name" and it'll grab the value
from the env vars if set. If "name" isn't set in the env then it still needs
to send "name" to the server so that a warning can be printed about an
unused --build-arg. And this is why buildArgs in the options is now a
*string instead of just a string - 'nil' == mentioned but no value.

Closes #29084

Signed-off-by: Doug Davis <dug@us.ibm.com>

Doug Davis authored on 2016/12/03 22:46:04
Showing 12 changed files
... ...
@@ -84,14 +84,28 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
84 84
 		options.Ulimits = buildUlimits
85 85
 	}
86 86
 
87
-	var buildArgs = map[string]string{}
87
+	var buildArgs = map[string]*string{}
88 88
 	buildArgsJSON := r.FormValue("buildargs")
89
+
90
+	// Note that there are two ways a --build-arg might appear in the
91
+	// json of the query param:
92
+	//     "foo":"bar"
93
+	// and "foo":nil
94
+	// The first is the normal case, ie. --build-arg foo=bar
95
+	// or  --build-arg foo
96
+	// where foo's value was picked up from an env var.
97
+	// The second ("foo":nil) is where they put --build-arg foo
98
+	// but "foo" isn't set as an env var. In that case we can't just drop
99
+	// the fact they mentioned it, we need to pass that along to the builder
100
+	// so that it can print a warning about "foo" being unused if there is
101
+	// no "ARG foo" in the Dockerfile.
89 102
 	if buildArgsJSON != "" {
90 103
 		if err := json.Unmarshal([]byte(buildArgsJSON), &buildArgs); err != nil {
91 104
 			return nil, err
92 105
 		}
93 106
 		options.BuildArgs = buildArgs
94 107
 	}
108
+
95 109
 	var labels = map[string]string{}
96 110
 	labelsJSON := r.FormValue("labels")
97 111
 	if labelsJSON != "" {
... ...
@@ -160,10 +160,13 @@ type ImageBuildOptions struct {
160 160
 	ShmSize        int64
161 161
 	Dockerfile     string
162 162
 	Ulimits        []*units.Ulimit
163
-	BuildArgs      map[string]string
164
-	AuthConfigs    map[string]AuthConfig
165
-	Context        io.Reader
166
-	Labels         map[string]string
163
+	// See the parsing of buildArgs in api/server/router/build/build_routes.go
164
+	// for an explaination of why BuildArgs needs to use *string instead of
165
+	// just a string
166
+	BuildArgs   map[string]*string
167
+	AuthConfigs map[string]AuthConfig
168
+	Context     io.Reader
169
+	Labels      map[string]string
167 170
 	// squash the resulting image's layers to the parent
168 171
 	// preserves the original image and creates a new one from the parent with all
169 172
 	// the changes applied to a single layer
... ...
@@ -125,7 +125,7 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
125 125
 		config = new(types.ImageBuildOptions)
126 126
 	}
127 127
 	if config.BuildArgs == nil {
128
-		config.BuildArgs = make(map[string]string)
128
+		config.BuildArgs = make(map[string]*string)
129 129
 	}
130 130
 	ctx, cancel := context.WithCancel(clientCtx)
131 131
 	b = &Builder{
... ...
@@ -384,8 +384,8 @@ func run(b *Builder, args []string, attributes map[string]bool, original string)
384 384
 			// the entire file (see 'leftoverArgs' processing in evaluator.go )
385 385
 			continue
386 386
 		}
387
-		if _, ok := configEnv[key]; !ok {
388
-			cmdBuildEnv = append(cmdBuildEnv, fmt.Sprintf("%s=%s", key, val))
387
+		if _, ok := configEnv[key]; !ok && val != nil {
388
+			cmdBuildEnv = append(cmdBuildEnv, fmt.Sprintf("%s=%s", key, *val))
389 389
 		}
390 390
 	}
391 391
 
... ...
@@ -728,7 +728,7 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string)
728 728
 
729 729
 	var (
730 730
 		name       string
731
-		value      string
731
+		newValue   string
732 732
 		hasDefault bool
733 733
 	)
734 734
 
... ...
@@ -745,7 +745,7 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string)
745 745
 		}
746 746
 
747 747
 		name = parts[0]
748
-		value = parts[1]
748
+		newValue = parts[1]
749 749
 		hasDefault = true
750 750
 	} else {
751 751
 		name = arg
... ...
@@ -756,9 +756,12 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string)
756 756
 
757 757
 	// If there is a default value associated with this arg then add it to the
758 758
 	// b.buildArgs if one is not already passed to the builder. The args passed
759
-	// to builder override the default value of 'arg'.
760
-	if _, ok := b.options.BuildArgs[name]; !ok && hasDefault {
761
-		b.options.BuildArgs[name] = value
759
+	// to builder override the default value of 'arg'. Note that a 'nil' for
760
+	// a value means that the user specified "--build-arg FOO" and "FOO" wasn't
761
+	// defined as an env var - and in that case we DO want to use the default
762
+	// value specified in the ARG cmd.
763
+	if baValue, ok := b.options.BuildArgs[name]; (!ok || baValue == nil) && hasDefault {
764
+		b.options.BuildArgs[name] = &newValue
762 765
 	}
763 766
 
764 767
 	return b.commit("", b.runConfig.Cmd, fmt.Sprintf("ARG %s", arg))
... ...
@@ -460,7 +460,7 @@ func TestStopSignal(t *testing.T) {
460 460
 }
461 461
 
462 462
 func TestArg(t *testing.T) {
463
-	buildOptions := &types.ImageBuildOptions{BuildArgs: make(map[string]string)}
463
+	buildOptions := &types.ImageBuildOptions{BuildArgs: make(map[string]*string)}
464 464
 
465 465
 	b := &Builder{flags: &BFlags{}, runConfig: &container.Config{}, disableCommit: true, allowedBuildArgs: make(map[string]bool), options: buildOptions}
466 466
 
... ...
@@ -488,7 +488,7 @@ func TestArg(t *testing.T) {
488 488
 		t.Fatalf("%s argument should be a build arg", argName)
489 489
 	}
490 490
 
491
-	if val != "bar" {
491
+	if *val != "bar" {
492 492
 		t.Fatalf("%s argument should have default value 'bar', got %s", argName, val)
493 493
 	}
494 494
 }
... ...
@@ -158,7 +158,7 @@ func (b *Builder) dispatch(stepN int, stepTotal int, ast *parser.Node) error {
158 158
 			// the entire file (see 'leftoverArgs' processing in evaluator.go )
159 159
 			continue
160 160
 		}
161
-		envs = append(envs, fmt.Sprintf("%s=%s", key, val))
161
+		envs = append(envs, fmt.Sprintf("%s=%s", key, *val))
162 162
 	}
163 163
 	for ast.Next != nil {
164 164
 		ast = ast.Next
... ...
@@ -67,7 +67,7 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command {
67 67
 	ulimits := make(map[string]*units.Ulimit)
68 68
 	options := buildOptions{
69 69
 		tags:      opts.NewListOpts(validateTag),
70
-		buildArgs: opts.NewListOpts(runconfigopts.ValidateArg),
70
+		buildArgs: opts.NewListOpts(runconfigopts.ValidateEnv),
71 71
 		ulimits:   runconfigopts.NewUlimitOpt(&ulimits),
72 72
 		labels:    opts.NewListOpts(runconfigopts.ValidateEnv),
73 73
 	}
... ...
@@ -300,7 +300,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
300 300
 		Dockerfile:     relDockerfile,
301 301
 		ShmSize:        shmSize,
302 302
 		Ulimits:        options.ulimits.GetList(),
303
-		BuildArgs:      runconfigopts.ConvertKVStringsToMap(options.buildArgs.GetAll()),
303
+		BuildArgs:      runconfigopts.ConvertKVStringsToMapWithNil(options.buildArgs.GetAll()),
304 304
 		AuthConfigs:    authConfigs,
305 305
 		Labels:         runconfigopts.ConvertKVStringsToMap(options.labels.GetAll()),
306 306
 		CacheFrom:      options.cacheFrom,
... ...
@@ -27,6 +27,8 @@ func TestImageBuildError(t *testing.T) {
27 27
 }
28 28
 
29 29
 func TestImageBuild(t *testing.T) {
30
+	v1 := "value1"
31
+	v2 := "value2"
30 32
 	emptyRegistryConfig := "bnVsbA=="
31 33
 	buildCases := []struct {
32 34
 		buildOptions           types.ImageBuildOptions
... ...
@@ -105,13 +107,14 @@ func TestImageBuild(t *testing.T) {
105 105
 		},
106 106
 		{
107 107
 			buildOptions: types.ImageBuildOptions{
108
-				BuildArgs: map[string]string{
109
-					"ARG1": "value1",
110
-					"ARG2": "value2",
108
+				BuildArgs: map[string]*string{
109
+					"ARG1": &v1,
110
+					"ARG2": &v2,
111
+					"ARG3": nil,
111 112
 				},
112 113
 			},
113 114
 			expectedQueryParams: map[string]string{
114
-				"buildargs": `{"ARG1":"value1","ARG2":"value2"}`,
115
+				"buildargs": `{"ARG1":"value1","ARG2":"value2","ARG3":null}`,
115 116
 				"rm":        "0",
116 117
 			},
117 118
 			expectedTags:           []string{},
... ...
@@ -6070,6 +6070,71 @@ func (s *DockerSuite) TestBuildBuildTimeArgUnconsumedArg(c *check.C) {
6070 6070
 
6071 6071
 }
6072 6072
 
6073
+func (s *DockerSuite) TestBuildBuildTimeArgEnv(c *check.C) {
6074
+	testRequires(c, DaemonIsLinux) // Windows does not support ARG
6075
+	args := []string{
6076
+		"build",
6077
+		"--build-arg", fmt.Sprintf("FOO1=fromcmd"),
6078
+		"--build-arg", fmt.Sprintf("FOO2="),
6079
+		"--build-arg", fmt.Sprintf("FOO3"), // set in env
6080
+		"--build-arg", fmt.Sprintf("FOO4"), // not set in env
6081
+		"--build-arg", fmt.Sprintf("FOO5=fromcmd"),
6082
+		// FOO6 is not set at all
6083
+		"--build-arg", fmt.Sprintf("FOO7=fromcmd"), // should produce a warning
6084
+		"--build-arg", fmt.Sprintf("FOO8="), // should produce a warning
6085
+		"--build-arg", fmt.Sprintf("FOO9"), // should produce a warning
6086
+		".",
6087
+	}
6088
+
6089
+	dockerfile := fmt.Sprintf(`FROM busybox
6090
+		ARG FOO1=fromfile
6091
+		ARG FOO2=fromfile
6092
+		ARG FOO3=fromfile
6093
+		ARG FOO4=fromfile
6094
+		ARG FOO5
6095
+		ARG FOO6
6096
+		RUN env
6097
+		RUN [ "$FOO1" == "fromcmd" ]
6098
+		RUN [ "$FOO2" == "" ]
6099
+		RUN [ "$FOO3" == "fromenv" ]
6100
+		RUN [ "$FOO4" == "fromfile" ]
6101
+		RUN [ "$FOO5" == "fromcmd" ]
6102
+		# The following should not exist at all in the env
6103
+		RUN [ "$(env | grep FOO6)" == "" ]
6104
+		RUN [ "$(env | grep FOO7)" == "" ]
6105
+		RUN [ "$(env | grep FOO8)" == "" ]
6106
+		RUN [ "$(env | grep FOO9)" == "" ]
6107
+	    `)
6108
+
6109
+	ctx, err := fakeContext(dockerfile, nil)
6110
+	c.Assert(err, check.IsNil)
6111
+	defer ctx.Close()
6112
+
6113
+	cmd := exec.Command(dockerBinary, args...)
6114
+	cmd.Dir = ctx.Dir
6115
+	cmd.Env = append(os.Environ(),
6116
+		"FOO1=fromenv",
6117
+		"FOO2=fromenv",
6118
+		"FOO3=fromenv")
6119
+	out, _, err := runCommandWithOutput(cmd)
6120
+	if err != nil {
6121
+		c.Fatal(err, out)
6122
+	}
6123
+
6124
+	// Now check to make sure we got a warning msg about unused build-args
6125
+	i := strings.Index(out, "[Warning]")
6126
+	if i < 0 {
6127
+		c.Fatalf("Missing the build-arg warning in %q", out)
6128
+	}
6129
+
6130
+	out = out[i:] // "out" should contain just the warning message now
6131
+
6132
+	// These were specified on a --build-arg but no ARG was in the Dockerfile
6133
+	c.Assert(out, checker.Contains, "FOO7")
6134
+	c.Assert(out, checker.Contains, "FOO8")
6135
+	c.Assert(out, checker.Contains, "FOO9")
6136
+}
6137
+
6073 6138
 func (s *DockerSuite) TestBuildBuildTimeArgQuotedValVariants(c *check.C) {
6074 6139
 	imgName := "bldargtest"
6075 6140
 	envKey := "foo"
... ...
@@ -59,21 +59,6 @@ func doesEnvExist(name string) bool {
59 59
 	return false
60 60
 }
61 61
 
62
-// ValidateArg validates a build-arg variable and returns it.
63
-// Build-arg is in the form of <varname>=<value> where <varname> is required.
64
-func ValidateArg(val string) (string, error) {
65
-	arr := strings.Split(val, "=")
66
-	if len(arr) > 1 && isNotEmpty(arr[0]) {
67
-		return val, nil
68
-	}
69
-
70
-	return "", fmt.Errorf("bad format for build-arg: %s", val)
71
-}
72
-
73
-func isNotEmpty(val string) bool {
74
-	return len(val) > 0
75
-}
76
-
77 62
 // ValidateExtraHost validates that the specified string is a valid extrahost and returns it.
78 63
 // ExtraHost is in the form of name:ip where the ip has to be a valid ip (IPv4 or IPv6).
79 64
 func ValidateExtraHost(val string) (string, error) {
... ...
@@ -66,42 +66,6 @@ func TestValidateEnv(t *testing.T) {
66 66
 	}
67 67
 }
68 68
 
69
-func TestValidateArg(t *testing.T) {
70
-	valids := map[string]string{
71
-		"_=a":                "_=a",
72
-		"var1=value1":        "var1=value1",
73
-		"_var1=value1":       "_var1=value1",
74
-		"var2=value2=value3": "var2=value2=value3",
75
-		"var3=abc!qwe":       "var3=abc!qwe",
76
-		"var_4=value 4":      "var_4=value 4",
77
-		"var_5=":             "var_5=",
78
-	}
79
-	for value, expected := range valids {
80
-		actual, err := ValidateArg(value)
81
-		if err != nil {
82
-			t.Fatal(err)
83
-		}
84
-		if actual != expected {
85
-			t.Fatalf("Expected [%v], got [%v]", expected, actual)
86
-		}
87
-	}
88
-
89
-	invalid := map[string]string{
90
-		"foo":  "bad format",
91
-		"=foo": "bad format",
92
-		"cc c": "bad format",
93
-	}
94
-	for value, expectedError := range invalid {
95
-		if _, err := ValidateArg(value); err == nil {
96
-			t.Fatalf("ValidateArg(`%s`) should have failed validation", value)
97
-		} else {
98
-			if !strings.Contains(err.Error(), expectedError) {
99
-				t.Fatalf("ValidateArg(`%s`) error should contain %q", value, expectedError)
100
-			}
101
-		}
102
-	}
103
-}
104
-
105 69
 func TestValidateExtraHosts(t *testing.T) {
106 70
 	valid := []string{
107 71
 		`myhost:192.168.0.1`,
... ...
@@ -705,6 +705,25 @@ func ConvertKVStringsToMap(values []string) map[string]string {
705 705
 	return result
706 706
 }
707 707
 
708
+// ConvertKVStringsToMapWithNil converts ["key=value"] to {"key":"value"}
709
+// but set unset keys to nil - meaning the ones with no "=" in them.
710
+// We use this in cases where we need to distinguish between
711
+//   FOO=  and FOO
712
+// where the latter case just means FOO was mentioned but not given a value
713
+func ConvertKVStringsToMapWithNil(values []string) map[string]*string {
714
+	result := make(map[string]*string, len(values))
715
+	for _, value := range values {
716
+		kv := strings.SplitN(value, "=", 2)
717
+		if len(kv) == 1 {
718
+			result[kv[0]] = nil
719
+		} else {
720
+			result[kv[0]] = &kv[1]
721
+		}
722
+	}
723
+
724
+	return result
725
+}
726
+
708 727
 func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) {
709 728
 	loggingOptsMap := ConvertKVStringsToMap(loggingOpts)
710 729
 	if loggingDriver == "none" && len(loggingOpts) > 0 {