Browse code

compose: fix environment interpolation from the client

For an environment variable defined in the yaml without value,
the value needs to be propagated from the client, as in Docker Compose.

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>

Akihiro Suda authored on 2017/02/07 18:44:47
Showing 4 changed files
... ...
@@ -115,6 +115,16 @@ func getConfigDetails(opts deployOptions) (composetypes.ConfigDetails, error) {
115 115
 	}
116 116
 	// TODO: support multiple files
117 117
 	details.ConfigFiles = []composetypes.ConfigFile{*configFile}
118
+	env := os.Environ()
119
+	details.Environment = make(map[string]string, len(env))
120
+	for _, s := range env {
121
+		// if value is empty, s is like "K=", not "K".
122
+		if !strings.Contains(s, "=") {
123
+			return details, fmt.Errorf("unexpected environment %q", s)
124
+		}
125
+		kv := strings.SplitN(s, "=", 2)
126
+		details.Environment[kv[0]] = kv[1]
127
+	}
118 128
 	return details, nil
119 129
 }
120 130
 
... ...
@@ -1 +1,4 @@
1 1
 BAR=2
2
+
3
+# overridden in configDetails.Environment
4
+QUX=1
... ...
@@ -2,15 +2,16 @@ package loader
2 2
 
3 3
 import (
4 4
 	"fmt"
5
-	"os"
6 5
 	"path"
7 6
 	"reflect"
8 7
 	"regexp"
9 8
 	"sort"
10 9
 	"strings"
11 10
 
11
+	"github.com/Sirupsen/logrus"
12 12
 	"github.com/docker/docker/cli/compose/interpolation"
13 13
 	"github.com/docker/docker/cli/compose/schema"
14
+	"github.com/docker/docker/cli/compose/template"
14 15
 	"github.com/docker/docker/cli/compose/types"
15 16
 	"github.com/docker/docker/opts"
16 17
 	runconfigopts "github.com/docker/docker/runconfig/opts"
... ...
@@ -69,13 +70,17 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
69 69
 	}
70 70
 
71 71
 	cfg := types.Config{}
72
+	lookupEnv := func(k string) (string, bool) {
73
+		v, ok := configDetails.Environment[k]
74
+		return v, ok
75
+	}
72 76
 	if services, ok := configDict["services"]; ok {
73
-		servicesConfig, err := interpolation.Interpolate(services.(types.Dict), "service", os.LookupEnv)
77
+		servicesConfig, err := interpolation.Interpolate(services.(types.Dict), "service", lookupEnv)
74 78
 		if err != nil {
75 79
 			return nil, err
76 80
 		}
77 81
 
78
-		servicesList, err := LoadServices(servicesConfig, configDetails.WorkingDir)
82
+		servicesList, err := LoadServices(servicesConfig, configDetails.WorkingDir, lookupEnv)
79 83
 		if err != nil {
80 84
 			return nil, err
81 85
 		}
... ...
@@ -84,7 +89,7 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
84 84
 	}
85 85
 
86 86
 	if networks, ok := configDict["networks"]; ok {
87
-		networksConfig, err := interpolation.Interpolate(networks.(types.Dict), "network", os.LookupEnv)
87
+		networksConfig, err := interpolation.Interpolate(networks.(types.Dict), "network", lookupEnv)
88 88
 		if err != nil {
89 89
 			return nil, err
90 90
 		}
... ...
@@ -98,7 +103,7 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
98 98
 	}
99 99
 
100 100
 	if volumes, ok := configDict["volumes"]; ok {
101
-		volumesConfig, err := interpolation.Interpolate(volumes.(types.Dict), "volume", os.LookupEnv)
101
+		volumesConfig, err := interpolation.Interpolate(volumes.(types.Dict), "volume", lookupEnv)
102 102
 		if err != nil {
103 103
 			return nil, err
104 104
 		}
... ...
@@ -112,7 +117,7 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
112 112
 	}
113 113
 
114 114
 	if secrets, ok := configDict["secrets"]; ok {
115
-		secretsConfig, err := interpolation.Interpolate(secrets.(types.Dict), "secret", os.LookupEnv)
115
+		secretsConfig, err := interpolation.Interpolate(secrets.(types.Dict), "secret", lookupEnv)
116 116
 		if err != nil {
117 117
 			return nil, err
118 118
 		}
... ...
@@ -308,11 +313,11 @@ func formatInvalidKeyError(keyPrefix string, key interface{}) error {
308 308
 
309 309
 // LoadServices produces a ServiceConfig map from a compose file Dict
310 310
 // the servicesDict is not validated if directly used. Use Load() to enable validation
311
-func LoadServices(servicesDict types.Dict, workingDir string) ([]types.ServiceConfig, error) {
311
+func LoadServices(servicesDict types.Dict, workingDir string, lookupEnv template.Mapping) ([]types.ServiceConfig, error) {
312 312
 	var services []types.ServiceConfig
313 313
 
314 314
 	for name, serviceDef := range servicesDict {
315
-		serviceConfig, err := LoadService(name, serviceDef.(types.Dict), workingDir)
315
+		serviceConfig, err := loadService(name, serviceDef.(types.Dict), workingDir, lookupEnv)
316 316
 		if err != nil {
317 317
 			return nil, err
318 318
 		}
... ...
@@ -324,22 +329,39 @@ func LoadServices(servicesDict types.Dict, workingDir string) ([]types.ServiceCo
324 324
 
325 325
 // LoadService produces a single ServiceConfig from a compose file Dict
326 326
 // the serviceDict is not validated if directly used. Use Load() to enable validation
327
-func LoadService(name string, serviceDict types.Dict, workingDir string) (*types.ServiceConfig, error) {
327
+func LoadService(name string, serviceDict types.Dict, workingDir string, lookupEnv template.Mapping) (*types.ServiceConfig, error) {
328 328
 	serviceConfig := &types.ServiceConfig{}
329 329
 	if err := transform(serviceDict, serviceConfig); err != nil {
330 330
 		return nil, err
331 331
 	}
332 332
 	serviceConfig.Name = name
333 333
 
334
-	if err := resolveEnvironment(serviceConfig, workingDir); err != nil {
334
+	if err := resolveEnvironment(serviceConfig, workingDir, lookupEnv); err != nil {
335 335
 		return nil, err
336 336
 	}
337 337
 
338
-	resolveVolumePaths(serviceConfig.Volumes, workingDir)
338
+	resolveVolumePaths(serviceConfig.Volumes, workingDir, lookupEnv)
339 339
 	return serviceConfig, nil
340 340
 }
341 341
 
342
-func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string) error {
342
+func updateEnvironment(environment map[string]string, vars map[string]string, lookupEnv template.Mapping) map[string]string {
343
+	result := make(map[string]string, len(environment))
344
+	for k, v := range environment {
345
+		result[k]=v
346
+	}
347
+	for k, v := range vars {
348
+		interpolatedV, ok := lookupEnv(k)
349
+		if ok {
350
+			// lookupEnv is prioritized over vars
351
+			result[k] = interpolatedV
352
+		} else {
353
+			result[k] = v
354
+		}
355
+	}
356
+	return result
357
+}
358
+
359
+func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string, lookupEnv template.Mapping) error {
343 360
 	environment := make(map[string]string)
344 361
 
345 362
 	if len(serviceConfig.EnvFile) > 0 {
... ...
@@ -353,36 +375,35 @@ func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string) e
353 353
 			}
354 354
 			envVars = append(envVars, fileVars...)
355 355
 		}
356
-
357
-		for k, v := range runconfigopts.ConvertKVStringsToMap(envVars) {
358
-			environment[k] = v
359
-		}
360
-	}
361
-
362
-	for k, v := range serviceConfig.Environment {
363
-		environment[k] = v
356
+		environment = updateEnvironment(environment,
357
+			runconfigopts.ConvertKVStringsToMap(envVars), lookupEnv)
364 358
 	}
365 359
 
366
-	serviceConfig.Environment = environment
367
-
360
+	serviceConfig.Environment = updateEnvironment(environment,
361
+		serviceConfig.Environment, lookupEnv)
368 362
 	return nil
369 363
 }
370 364
 
371
-func resolveVolumePaths(volumes []types.ServiceVolumeConfig, workingDir string) {
365
+func resolveVolumePaths(volumes []types.ServiceVolumeConfig, workingDir string, lookupEnv template.Mapping) {
372 366
 	for i, volume := range volumes {
373 367
 		if volume.Type != "bind" {
374 368
 			continue
375 369
 		}
376 370
 
377
-		volume.Source = absPath(workingDir, expandUser(volume.Source))
371
+		volume.Source = absPath(workingDir, expandUser(volume.Source, lookupEnv))
378 372
 		volumes[i] = volume
379 373
 	}
380 374
 }
381 375
 
382 376
 // TODO: make this more robust
383
-func expandUser(path string) string {
377
+func expandUser(path string, lookupEnv template.Mapping) string {
384 378
 	if strings.HasPrefix(path, "~") {
385
-		return strings.Replace(path, "~", os.Getenv("HOME"), 1)
379
+		home, ok := lookupEnv("HOME")
380
+		if !ok {
381
+			logrus.Warn("cannot expand '~', because the environment lacks HOME")
382
+			return path
383
+		}
384
+		return strings.Replace(path, "~", home, 1)
386 385
 	}
387 386
 	return path
388 387
 }
... ...
@@ -12,7 +12,7 @@ import (
12 12
 	"github.com/stretchr/testify/assert"
13 13
 )
14 14
 
15
-func buildConfigDetails(source types.Dict) types.ConfigDetails {
15
+func buildConfigDetails(source types.Dict, env map[string]string) types.ConfigDetails {
16 16
 	workingDir, err := os.Getwd()
17 17
 	if err != nil {
18 18
 		panic(err)
... ...
@@ -23,7 +23,7 @@ func buildConfigDetails(source types.Dict) types.ConfigDetails {
23 23
 		ConfigFiles: []types.ConfigFile{
24 24
 			{Filename: "filename.yml", Config: source},
25 25
 		},
26
-		Environment: nil,
26
+		Environment: env,
27 27
 	}
28 28
 }
29 29
 
... ...
@@ -154,7 +154,7 @@ func TestParseYAML(t *testing.T) {
154 154
 }
155 155
 
156 156
 func TestLoad(t *testing.T) {
157
-	actual, err := Load(buildConfigDetails(sampleDict))
157
+	actual, err := Load(buildConfigDetails(sampleDict, nil))
158 158
 	if !assert.NoError(t, err) {
159 159
 		return
160 160
 	}
... ...
@@ -173,7 +173,7 @@ services:
173 173
 secrets:
174 174
   super:
175 175
     external: true
176
-`)
176
+`, nil)
177 177
 	if !assert.NoError(t, err) {
178 178
 		return
179 179
 	}
... ...
@@ -182,7 +182,7 @@ secrets:
182 182
 }
183 183
 
184 184
 func TestParseAndLoad(t *testing.T) {
185
-	actual, err := loadYAML(sampleYAML)
185
+	actual, err := loadYAML(sampleYAML, nil)
186 186
 	if !assert.NoError(t, err) {
187 187
 		return
188 188
 	}
... ...
@@ -192,15 +192,15 @@ func TestParseAndLoad(t *testing.T) {
192 192
 }
193 193
 
194 194
 func TestInvalidTopLevelObjectType(t *testing.T) {
195
-	_, err := loadYAML("1")
195
+	_, err := loadYAML("1", nil)
196 196
 	assert.Error(t, err)
197 197
 	assert.Contains(t, err.Error(), "Top-level object must be a mapping")
198 198
 
199
-	_, err = loadYAML("\"hello\"")
199
+	_, err = loadYAML("\"hello\"", nil)
200 200
 	assert.Error(t, err)
201 201
 	assert.Contains(t, err.Error(), "Top-level object must be a mapping")
202 202
 
203
-	_, err = loadYAML("[\"hello\"]")
203
+	_, err = loadYAML("[\"hello\"]", nil)
204 204
 	assert.Error(t, err)
205 205
 	assert.Contains(t, err.Error(), "Top-level object must be a mapping")
206 206
 }
... ...
@@ -211,7 +211,7 @@ version: "3"
211 211
 123:
212 212
   foo:
213 213
     image: busybox
214
-`)
214
+`, nil)
215 215
 	assert.Error(t, err)
216 216
 	assert.Contains(t, err.Error(), "Non-string key at top level: 123")
217 217
 
... ...
@@ -222,7 +222,7 @@ services:
222 222
     image: busybox
223 223
   123:
224 224
     image: busybox
225
-`)
225
+`, nil)
226 226
 	assert.Error(t, err)
227 227
 	assert.Contains(t, err.Error(), "Non-string key in services: 123")
228 228
 
... ...
@@ -236,7 +236,7 @@ networks:
236 236
     ipam:
237 237
       config:
238 238
         - 123: oh dear
239
-`)
239
+`, nil)
240 240
 	assert.Error(t, err)
241 241
 	assert.Contains(t, err.Error(), "Non-string key in networks.default.ipam.config[0]: 123")
242 242
 
... ...
@@ -247,7 +247,7 @@ services:
247 247
     image: busybox
248 248
     environment:
249 249
       1: FOO
250
-`)
250
+`, nil)
251 251
 	assert.Error(t, err)
252 252
 	assert.Contains(t, err.Error(), "Non-string key in services.dict-env.environment: 1")
253 253
 }
... ...
@@ -258,7 +258,7 @@ version: "3"
258 258
 services:
259 259
   foo:
260 260
     image: busybox
261
-`)
261
+`, nil)
262 262
 	assert.NoError(t, err)
263 263
 
264 264
 	_, err = loadYAML(`
... ...
@@ -266,7 +266,7 @@ version: "3.0"
266 266
 services:
267 267
   foo:
268 268
     image: busybox
269
-`)
269
+`, nil)
270 270
 	assert.NoError(t, err)
271 271
 }
272 272
 
... ...
@@ -276,7 +276,7 @@ version: "2"
276 276
 services:
277 277
   foo:
278 278
     image: busybox
279
-`)
279
+`, nil)
280 280
 	assert.Error(t, err)
281 281
 	assert.Contains(t, err.Error(), "version")
282 282
 
... ...
@@ -285,7 +285,7 @@ version: "2.0"
285 285
 services:
286 286
   foo:
287 287
     image: busybox
288
-`)
288
+`, nil)
289 289
 	assert.Error(t, err)
290 290
 	assert.Contains(t, err.Error(), "version")
291 291
 }
... ...
@@ -296,7 +296,7 @@ version: 3
296 296
 services:
297 297
   foo:
298 298
     image: busybox
299
-`)
299
+`, nil)
300 300
 	assert.Error(t, err)
301 301
 	assert.Contains(t, err.Error(), "version must be a string")
302 302
 }
... ...
@@ -305,7 +305,7 @@ func TestV1Unsupported(t *testing.T) {
305 305
 	_, err := loadYAML(`
306 306
 foo:
307 307
   image: busybox
308
-`)
308
+`, nil)
309 309
 	assert.Error(t, err)
310 310
 }
311 311
 
... ...
@@ -315,7 +315,7 @@ version: "3"
315 315
 services:
316 316
   - foo:
317 317
       image: busybox
318
-`)
318
+`, nil)
319 319
 	assert.Error(t, err)
320 320
 	assert.Contains(t, err.Error(), "services must be a mapping")
321 321
 
... ...
@@ -323,7 +323,7 @@ services:
323 323
 version: "3"
324 324
 services:
325 325
   foo: busybox
326
-`)
326
+`, nil)
327 327
 	assert.Error(t, err)
328 328
 	assert.Contains(t, err.Error(), "services.foo must be a mapping")
329 329
 
... ...
@@ -332,7 +332,7 @@ version: "3"
332 332
 networks:
333 333
   - default:
334 334
       driver: bridge
335
-`)
335
+`, nil)
336 336
 	assert.Error(t, err)
337 337
 	assert.Contains(t, err.Error(), "networks must be a mapping")
338 338
 
... ...
@@ -340,7 +340,7 @@ networks:
340 340
 version: "3"
341 341
 networks:
342 342
   default: bridge
343
-`)
343
+`, nil)
344 344
 	assert.Error(t, err)
345 345
 	assert.Contains(t, err.Error(), "networks.default must be a mapping")
346 346
 
... ...
@@ -349,7 +349,7 @@ version: "3"
349 349
 volumes:
350 350
   - data:
351 351
       driver: local
352
-`)
352
+`, nil)
353 353
 	assert.Error(t, err)
354 354
 	assert.Contains(t, err.Error(), "volumes must be a mapping")
355 355
 
... ...
@@ -357,7 +357,7 @@ volumes:
357 357
 version: "3"
358 358
 volumes:
359 359
   data: local
360
-`)
360
+`, nil)
361 361
 	assert.Error(t, err)
362 362
 	assert.Contains(t, err.Error(), "volumes.data must be a mapping")
363 363
 }
... ...
@@ -368,7 +368,7 @@ version: "3"
368 368
 services:
369 369
   foo:
370 370
     image: ["busybox", "latest"]
371
-`)
371
+`, nil)
372 372
 	assert.Error(t, err)
373 373
 	assert.Contains(t, err.Error(), "services.foo.image must be a string")
374 374
 }
... ...
@@ -383,6 +383,7 @@ services:
383 383
       FOO: "1"
384 384
       BAR: 2
385 385
       BAZ: 2.5
386
+      QUX:
386 387
       QUUX:
387 388
   list-env:
388 389
     image: busybox
... ...
@@ -390,14 +391,16 @@ services:
390 390
       - FOO=1
391 391
       - BAR=2
392 392
       - BAZ=2.5
393
+      - QUX
393 394
       - QUUX=
394
-`)
395
+`, map[string]string{"QUX": "qux"})
395 396
 	assert.NoError(t, err)
396 397
 
397 398
 	expected := types.MappingWithEquals{
398 399
 		"FOO":  "1",
399 400
 		"BAR":  "2",
400 401
 		"BAZ":  "2.5",
402
+		"QUX":  "qux",
401 403
 		"QUUX": "",
402 404
 	}
403 405
 
... ...
@@ -416,7 +419,7 @@ services:
416 416
     image: busybox
417 417
     environment:
418 418
       FOO: ["1"]
419
-`)
419
+`, nil)
420 420
 	assert.Error(t, err)
421 421
 	assert.Contains(t, err.Error(), "services.dict-env.environment.FOO must be a string, number or null")
422 422
 }
... ...
@@ -428,12 +431,13 @@ services:
428 428
   dict-env:
429 429
     image: busybox
430 430
     environment: "FOO=1"
431
-`)
431
+`, nil)
432 432
 	assert.Error(t, err)
433 433
 	assert.Contains(t, err.Error(), "services.dict-env.environment must be a mapping")
434 434
 }
435 435
 
436 436
 func TestEnvironmentInterpolation(t *testing.T) {
437
+	home := "/home/foo"
437 438
 	config, err := loadYAML(`
438 439
 version: "3"
439 440
 services:
... ...
@@ -450,12 +454,13 @@ networks:
450 450
 volumes:
451 451
   test:
452 452
     driver: $HOME
453
-`)
453
+`, map[string]string{
454
+		"HOME": home,
455
+		"FOO":  "foo",
456
+	})
454 457
 
455 458
 	assert.NoError(t, err)
456 459
 
457
-	home := os.Getenv("HOME")
458
-
459 460
 	expectedLabels := types.MappingWithEquals{
460 461
 		"home1":       home,
461 462
 		"home2":       home,
... ...
@@ -483,7 +488,7 @@ services:
483 483
 `))
484 484
 	assert.NoError(t, err)
485 485
 
486
-	configDetails := buildConfigDetails(dict)
486
+	configDetails := buildConfigDetails(dict, nil)
487 487
 
488 488
 	_, err = Load(configDetails)
489 489
 	assert.NoError(t, err)
... ...
@@ -506,7 +511,7 @@ services:
506 506
 `))
507 507
 	assert.NoError(t, err)
508 508
 
509
-	configDetails := buildConfigDetails(dict)
509
+	configDetails := buildConfigDetails(dict, nil)
510 510
 
511 511
 	_, err = Load(configDetails)
512 512
 	assert.NoError(t, err)
... ...
@@ -529,7 +534,7 @@ services:
529 529
   bar:
530 530
     extends:
531 531
       service: foo
532
-`)
532
+`, nil)
533 533
 
534 534
 	assert.Error(t, err)
535 535
 	assert.IsType(t, &ForbiddenPropertiesError{}, err)
... ...
@@ -601,7 +606,8 @@ func TestFullExample(t *testing.T) {
601 601
 	bytes, err := ioutil.ReadFile("full-example.yml")
602 602
 	assert.NoError(t, err)
603 603
 
604
-	config, err := loadYAML(string(bytes))
604
+	homeDir := "/home/foo"
605
+	config, err := loadYAML(string(bytes), map[string]string{"HOME": homeDir, "QUX": "2"})
605 606
 	if !assert.NoError(t, err) {
606 607
 		return
607 608
 	}
... ...
@@ -609,7 +615,6 @@ func TestFullExample(t *testing.T) {
609 609
 	workingDir, err := os.Getwd()
610 610
 	assert.NoError(t, err)
611 611
 
612
-	homeDir := os.Getenv("HOME")
613 612
 	stopGracePeriod := time.Duration(20 * time.Second)
614 613
 
615 614
 	expectedServiceConfig := types.ServiceConfig{
... ...
@@ -664,6 +669,7 @@ func TestFullExample(t *testing.T) {
664 664
 			"FOO":            "1",
665 665
 			"BAR":            "2",
666 666
 			"BAZ":            "3",
667
+			"QUX":            "2",
667 668
 		},
668 669
 		EnvFile: []string{
669 670
 			"./example1.env",
... ...
@@ -955,13 +961,13 @@ func TestFullExample(t *testing.T) {
955 955
 	assert.Equal(t, expectedVolumeConfig, config.Volumes)
956 956
 }
957 957
 
958
-func loadYAML(yaml string) (*types.Config, error) {
958
+func loadYAML(yaml string, env map[string]string) (*types.Config, error) {
959 959
 	dict, err := ParseYAML([]byte(yaml))
960 960
 	if err != nil {
961 961
 		return nil, err
962 962
 	}
963 963
 
964
-	return Load(buildConfigDetails(dict))
964
+	return Load(buildConfigDetails(dict, env))
965 965
 }
966 966
 
967 967
 func serviceSort(services []types.ServiceConfig) []types.ServiceConfig {