Browse code

Fix copy when used with scratch and images with empty RootFS

Commit the rwLayer to get the correct DiffID
Refacator copy in thebuilder
move more code into exportImage
cleanup some windows tests
Release the newly commited layer.
Set the imageID on the buildStage after exporting a new image.
Move archiver to BuildManager.
Have ReleaseableLayer.Commit return a layer
and store the Image from exportImage in the local imageSources cache
Remove NewChild from image interface.

Signed-off-by: Daniel Nephin <dnephin@docker.com>

Daniel Nephin authored on 2017/05/26 06:03:29
Showing 20 changed files
... ...
@@ -8,6 +8,7 @@ import (
8 8
 	"github.com/docker/docker/builder"
9 9
 	"github.com/docker/docker/builder/dockerfile"
10 10
 	"github.com/docker/docker/image"
11
+	"github.com/docker/docker/pkg/idtools"
11 12
 	"github.com/docker/docker/pkg/stringid"
12 13
 	"github.com/pkg/errors"
13 14
 	"golang.org/x/net/context"
... ...
@@ -26,8 +27,8 @@ type Backend struct {
26 26
 }
27 27
 
28 28
 // NewBackend creates a new build backend from components
29
-func NewBackend(components ImageComponent, builderBackend builder.Backend) *Backend {
30
-	manager := dockerfile.NewBuildManager(builderBackend)
29
+func NewBackend(components ImageComponent, builderBackend builder.Backend, idMappings *idtools.IDMappings) *Backend {
30
+	manager := dockerfile.NewBuildManager(builderBackend, idMappings)
31 31
 	return &Backend{imageComponent: components, manager: manager}
32 32
 }
33 33
 
... ...
@@ -11,9 +11,7 @@ import (
11 11
 	"github.com/docker/docker/api/types/backend"
12 12
 	"github.com/docker/docker/api/types/container"
13 13
 	containerpkg "github.com/docker/docker/container"
14
-	"github.com/docker/docker/image"
15 14
 	"github.com/docker/docker/layer"
16
-	"github.com/docker/docker/pkg/idtools"
17 15
 	"golang.org/x/net/context"
18 16
 )
19 17
 
... ...
@@ -45,9 +43,7 @@ type Backend interface {
45 45
 	// ContainerCreateWorkdir creates the workdir
46 46
 	ContainerCreateWorkdir(containerID string) error
47 47
 
48
-	CreateImage(config []byte, parent string) (string, error)
49
-
50
-	IDMappings() *idtools.IDMappings
48
+	CreateImage(config []byte, parent string) (Image, error)
51 49
 
52 50
 	ImageCacheBuilder
53 51
 }
... ...
@@ -98,12 +94,12 @@ type Image interface {
98 98
 	ImageID() string
99 99
 	RunConfig() *container.Config
100 100
 	MarshalJSON() ([]byte, error)
101
-	NewChild(child image.ChildConfig) *image.Image
102 101
 }
103 102
 
104 103
 // ReleaseableLayer is an image layer that can be mounted and released
105 104
 type ReleaseableLayer interface {
106 105
 	Release() error
107 106
 	Mount() (string, error)
107
+	Commit() (ReleaseableLayer, error)
108 108
 	DiffID() layer.DiffID
109 109
 }
... ...
@@ -17,6 +17,7 @@ import (
17 17
 	"github.com/docker/docker/builder/remotecontext"
18 18
 	"github.com/docker/docker/pkg/archive"
19 19
 	"github.com/docker/docker/pkg/chrootarchive"
20
+	"github.com/docker/docker/pkg/idtools"
20 21
 	"github.com/docker/docker/pkg/streamformatter"
21 22
 	"github.com/docker/docker/pkg/stringid"
22 23
 	"github.com/pkg/errors"
... ...
@@ -39,15 +40,17 @@ var validCommitCommands = map[string]bool{
39 39
 
40 40
 // BuildManager is shared across all Builder objects
41 41
 type BuildManager struct {
42
+	archiver  *archive.Archiver
42 43
 	backend   builder.Backend
43 44
 	pathCache pathCache // TODO: make this persistent
44 45
 }
45 46
 
46 47
 // NewBuildManager creates a BuildManager
47
-func NewBuildManager(b builder.Backend) *BuildManager {
48
+func NewBuildManager(b builder.Backend, idMappings *idtools.IDMappings) *BuildManager {
48 49
 	return &BuildManager{
49 50
 		backend:   b,
50 51
 		pathCache: &syncmap.Map{},
52
+		archiver:  chrootarchive.NewArchiver(idMappings),
51 53
 	}
52 54
 }
53 55
 
... ...
@@ -75,6 +78,7 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
75 75
 		ProgressWriter: config.ProgressWriter,
76 76
 		Backend:        bm.backend,
77 77
 		PathCache:      bm.pathCache,
78
+		Archiver:       bm.archiver,
78 79
 	}
79 80
 	return newBuilder(ctx, builderOptions).build(source, dockerfile)
80 81
 }
... ...
@@ -85,6 +89,7 @@ type builderOptions struct {
85 85
 	Backend        builder.Backend
86 86
 	ProgressWriter backend.ProgressWriter
87 87
 	PathCache      pathCache
88
+	Archiver       *archive.Archiver
88 89
 }
89 90
 
90 91
 // Builder is a Dockerfile builder
... ...
@@ -124,7 +129,7 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
124 124
 		Aux:              options.ProgressWriter.AuxFormatter,
125 125
 		Output:           options.ProgressWriter.Output,
126 126
 		docker:           options.Backend,
127
-		archiver:         chrootarchive.NewArchiver(options.Backend.IDMappings()),
127
+		archiver:         options.Archiver,
128 128
 		buildArgs:        newBuildArgs(config.BuildArgs),
129 129
 		buildStages:      newBuildStages(),
130 130
 		imageSources:     newImageSources(clientCtx, options),
... ...
@@ -368,7 +368,7 @@ type copyFileOptions struct {
368 368
 	archiver   *archive.Archiver
369 369
 }
370 370
 
371
-func copyFile(dest copyInfo, source copyInfo, options copyFileOptions) error {
371
+func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) error {
372 372
 	srcPath, err := source.fullPath()
373 373
 	if err != nil {
374 374
 		return err
... ...
@@ -379,36 +379,63 @@ func copyFile(dest copyInfo, source copyInfo, options copyFileOptions) error {
379 379
 	}
380 380
 
381 381
 	archiver := options.archiver
382
-	rootIDs := archiver.IDMappings.RootPair()
383 382
 
384 383
 	src, err := os.Stat(srcPath)
385 384
 	if err != nil {
386 385
 		return errors.Wrapf(err, "source path not found")
387 386
 	}
388 387
 	if src.IsDir() {
389
-		if err := archiver.CopyWithTar(srcPath, destPath); err != nil {
390
-			return err
391
-		}
392
-		return fixPermissions(srcPath, destPath, rootIDs)
388
+		return copyDirectory(archiver, srcPath, destPath)
393 389
 	}
394
-
395 390
 	if options.decompress && archive.IsArchivePath(srcPath) {
396
-		// To support the untar feature we need to clean up the path a little bit
397
-		// because tar is not very forgiving
398
-		tarDest := dest.path
399
-		// TODO: could this be just TrimSuffix()?
400
-		if strings.HasSuffix(tarDest, string(os.PathSeparator)) {
401
-			tarDest = filepath.Dir(dest.path)
402
-		}
403
-		return archiver.UntarPath(srcPath, tarDest)
391
+		return archiver.UntarPath(srcPath, destPath)
404 392
 	}
405 393
 
406
-	if err := idtools.MkdirAllAndChownNew(filepath.Dir(destPath), 0755, rootIDs); err != nil {
394
+	destExistsAsDir, err := isExistingDirectory(destPath)
395
+	if err != nil {
407 396
 		return err
408 397
 	}
409
-	if err := archiver.CopyFileWithTar(srcPath, destPath); err != nil {
410
-		return err
398
+	// dest.path must be used because destPath has already been cleaned of any
399
+	// trailing slash
400
+	if endsInSlash(dest.path) || destExistsAsDir {
401
+		// source.path must be used to get the correct filename when the source
402
+		// is a symlink
403
+		destPath = filepath.Join(destPath, filepath.Base(source.path))
404
+	}
405
+	return copyFile(archiver, srcPath, destPath)
406
+}
407
+
408
+func copyDirectory(archiver *archive.Archiver, source, dest string) error {
409
+	if err := archiver.CopyWithTar(source, dest); err != nil {
410
+		return errors.Wrapf(err, "failed to copy directory")
411
+	}
412
+	return fixPermissions(source, dest, archiver.IDMappings.RootPair())
413
+}
414
+
415
+func copyFile(archiver *archive.Archiver, source, dest string) error {
416
+	rootIDs := archiver.IDMappings.RootPair()
417
+
418
+	if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest), 0755, rootIDs); err != nil {
419
+		return errors.Wrapf(err, "failed to create new directory")
420
+	}
421
+	if err := archiver.CopyFileWithTar(source, dest); err != nil {
422
+		return errors.Wrapf(err, "failed to copy file")
423
+	}
424
+	return fixPermissions(source, dest, rootIDs)
425
+}
426
+
427
+func endsInSlash(path string) bool {
428
+	return strings.HasSuffix(path, string(os.PathSeparator))
429
+}
430
+
431
+// isExistingDirectory returns true if the path exists and is a directory
432
+func isExistingDirectory(path string) (bool, error) {
433
+	destStat, err := os.Stat(path)
434
+	switch {
435
+	case os.IsNotExist(err):
436
+		return false, nil
437
+	case err != nil:
438
+		return false, err
411 439
 	}
412
-	// TODO: do I have to change destPath to the filename?
413
-	return fixPermissions(srcPath, destPath, rootIDs)
440
+	return destStat.IsDir(), nil
414 441
 }
415 442
new file mode 100644
... ...
@@ -0,0 +1,45 @@
0
+package dockerfile
1
+
2
+import (
3
+	"testing"
4
+
5
+	"github.com/docker/docker/pkg/testutil/tempfile"
6
+	"github.com/stretchr/testify/assert"
7
+)
8
+
9
+func TestIsExistingDirectory(t *testing.T) {
10
+	tmpfile := tempfile.NewTempFile(t, "file-exists-test", "something")
11
+	defer tmpfile.Remove()
12
+	tmpdir := tempfile.NewTempDir(t, "dir-exists-test")
13
+	defer tmpdir.Remove()
14
+
15
+	var testcases = []struct {
16
+		doc      string
17
+		path     string
18
+		expected bool
19
+	}{
20
+		{
21
+			doc:      "directory exists",
22
+			path:     tmpdir.Path,
23
+			expected: true,
24
+		},
25
+		{
26
+			doc:      "path doesn't exist",
27
+			path:     "/bogus/path/does/not/exist",
28
+			expected: false,
29
+		},
30
+		{
31
+			doc:      "file exists",
32
+			path:     tmpfile.Name(),
33
+			expected: false,
34
+		},
35
+	}
36
+
37
+	for _, testcase := range testcases {
38
+		result, err := isExistingDirectory(testcase.path)
39
+		if !assert.NoError(t, err) {
40
+			continue
41
+		}
42
+		assert.Equal(t, testcase.expected, result, testcase.doc)
43
+	}
44
+}
... ...
@@ -1,3 +1,5 @@
1
+// +build !windows
2
+
1 3
 package dockerfile
2 4
 
3 5
 import (
... ...
@@ -7,20 +9,8 @@ import (
7 7
 	"github.com/docker/docker/pkg/idtools"
8 8
 )
9 9
 
10
-func pathExists(path string) (bool, error) {
11
-	_, err := os.Stat(path)
12
-	switch {
13
-	case err == nil:
14
-		return true, nil
15
-	case os.IsNotExist(err):
16
-		return false, nil
17
-	}
18
-	return false, err
19
-}
20
-
21
-// TODO: review this
22 10
 func fixPermissions(source, destination string, rootIDs idtools.IDPair) error {
23
-	doChownDestination, err := chownDestinationRoot(destination)
11
+	skipChownRoot, err := isExistingDirectory(destination)
24 12
 	if err != nil {
25 13
 		return err
26 14
 	}
... ...
@@ -30,7 +20,7 @@ func fixPermissions(source, destination string, rootIDs idtools.IDPair) error {
30 30
 	return filepath.Walk(source, func(fullpath string, info os.FileInfo, err error) error {
31 31
 		// Do not alter the walk root iff. it existed before, as it doesn't fall under
32 32
 		// the domain of "things we should chown".
33
-		if !doChownDestination && (source == fullpath) {
33
+		if skipChownRoot && source == fullpath {
34 34
 			return nil
35 35
 		}
36 36
 
... ...
@@ -44,21 +34,3 @@ func fixPermissions(source, destination string, rootIDs idtools.IDPair) error {
44 44
 		return os.Lchown(fullpath, rootIDs.UID, rootIDs.GID)
45 45
 	})
46 46
 }
47
-
48
-// If the destination didn't already exist, or the destination isn't a
49
-// directory, then we should Lchown the destination. Otherwise, we shouldn't
50
-// Lchown the destination.
51
-func chownDestinationRoot(destination string) (bool, error) {
52
-	destExists, err := pathExists(destination)
53
-	if err != nil {
54
-		return false, err
55
-	}
56
-	destStat, err := os.Stat(destination)
57
-	if err != nil {
58
-		// This should *never* be reached, because the destination must've already
59
-		// been created while untar-ing the context.
60
-		return false, err
61
-	}
62
-
63
-	return !destExists || !destStat.IsDir(), nil
64
-}
... ...
@@ -8,7 +8,7 @@ import (
8 8
 	"github.com/docker/docker/api/types/backend"
9 9
 	"github.com/docker/docker/builder"
10 10
 	"github.com/docker/docker/builder/remotecontext"
11
-	"github.com/docker/docker/layer"
11
+	dockerimage "github.com/docker/docker/image"
12 12
 	"github.com/pkg/errors"
13 13
 	"golang.org/x/net/context"
14 14
 )
... ...
@@ -92,6 +92,7 @@ type getAndMountFunc func(string) (builder.Image, builder.ReleaseableLayer, erro
92 92
 // all images so they can be unmounted at the end of the build.
93 93
 type imageSources struct {
94 94
 	byImageID map[string]*imageMount
95
+	withoutID []*imageMount
95 96
 	getImage  getAndMountFunc
96 97
 	cache     pathCache // TODO: remove
97 98
 }
... ...
@@ -121,7 +122,7 @@ func (m *imageSources) Get(idOrRef string) (*imageMount, error) {
121 121
 		return nil, err
122 122
 	}
123 123
 	im := newImageMount(image, layer)
124
-	m.byImageID[image.ImageID()] = im
124
+	m.Add(im)
125 125
 	return im, nil
126 126
 }
127 127
 
... ...
@@ -132,9 +133,25 @@ func (m *imageSources) Unmount() (retErr error) {
132 132
 			retErr = err
133 133
 		}
134 134
 	}
135
+	for _, im := range m.withoutID {
136
+		if err := im.unmount(); err != nil {
137
+			logrus.Error(err)
138
+			retErr = err
139
+		}
140
+	}
135 141
 	return
136 142
 }
137 143
 
144
+func (m *imageSources) Add(im *imageMount) {
145
+	switch im.image {
146
+	case nil:
147
+		im.image = &dockerimage.Image{}
148
+		m.withoutID = append(m.withoutID, im)
149
+	default:
150
+		m.byImageID[im.image.ImageID()] = im
151
+	}
152
+}
153
+
138 154
 // imageMount is a reference to an image that can be used as a builder.Source
139 155
 type imageMount struct {
140 156
 	image  builder.Image
... ...
@@ -172,6 +189,7 @@ func (im *imageMount) unmount() error {
172 172
 	if err := im.layer.Release(); err != nil {
173 173
 		return errors.Wrapf(err, "failed to unmount previous build image %s", im.image.ImageID())
174 174
 	}
175
+	im.layer = nil
175 176
 	return nil
176 177
 }
177 178
 
... ...
@@ -179,10 +197,10 @@ func (im *imageMount) Image() builder.Image {
179 179
 	return im.image
180 180
 }
181 181
 
182
-func (im *imageMount) ImageID() string {
183
-	return im.image.ImageID()
182
+func (im *imageMount) Layer() builder.ReleaseableLayer {
183
+	return im.layer
184 184
 }
185 185
 
186
-func (im *imageMount) DiffID() layer.DiffID {
187
-	return im.layer.DiffID()
186
+func (im *imageMount) ImageID() string {
187
+	return im.image.ImageID()
188 188
 }
... ...
@@ -12,7 +12,6 @@ import (
12 12
 	"github.com/docker/docker/api/types"
13 13
 	"github.com/docker/docker/api/types/backend"
14 14
 	"github.com/docker/docker/api/types/container"
15
-	"github.com/docker/docker/builder"
16 15
 	"github.com/docker/docker/image"
17 16
 	"github.com/docker/docker/pkg/stringid"
18 17
 	"github.com/pkg/errors"
... ...
@@ -65,14 +64,44 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta
65 65
 	return nil
66 66
 }
67 67
 
68
-func (b *Builder) exportImage(state *dispatchState, image builder.Image) error {
69
-	config, err := image.MarshalJSON()
68
+func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
69
+	newLayer, err := imageMount.Layer().Commit()
70
+	if err != nil {
71
+		return err
72
+	}
73
+
74
+	// add an image mount without an image so the layer is properly unmounted
75
+	// if there is an error before we can add the full mount with image
76
+	b.imageSources.Add(newImageMount(nil, newLayer))
77
+
78
+	parentImage, ok := imageMount.Image().(*image.Image)
79
+	if !ok {
80
+		return errors.Errorf("unexpected image type")
81
+	}
82
+
83
+	newImage := image.NewChildImage(parentImage, image.ChildConfig{
84
+		Author:          state.maintainer,
85
+		ContainerConfig: runConfig,
86
+		DiffID:          newLayer.DiffID(),
87
+		Config:          copyRunConfig(state.runConfig),
88
+	})
89
+
90
+	// TODO: it seems strange to marshal this here instead of just passing in the
91
+	// image struct
92
+	config, err := newImage.MarshalJSON()
70 93
 	if err != nil {
71 94
 		return errors.Wrap(err, "failed to encode image config")
72 95
 	}
73 96
 
74
-	state.imageID, err = b.docker.CreateImage(config, state.imageID)
75
-	return err
97
+	exportedImage, err := b.docker.CreateImage(config, state.imageID)
98
+	if err != nil {
99
+		return errors.Wrapf(err, "failed to export image")
100
+	}
101
+
102
+	state.imageID = exportedImage.ImageID()
103
+	b.imageSources.Add(newImageMount(exportedImage, newLayer))
104
+	b.buildStages.update(state.imageID)
105
+	return nil
76 106
 }
77 107
 
78 108
 func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error {
... ...
@@ -82,46 +111,46 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error
82 82
 	runConfigWithCommentCmd := copyRunConfig(
83 83
 		state.runConfig,
84 84
 		withCmdCommentString(fmt.Sprintf("%s %s in %s ", inst.cmdName, srcHash, inst.dest)))
85
-	containerID, err := b.probeAndCreate(state, runConfigWithCommentCmd)
86
-	if err != nil || containerID == "" {
87
-		return err
88
-	}
89
-
90
-	// Twiddle the destination when it's a relative path - meaning, make it
91
-	// relative to the WORKINGDIR
92
-	dest, err := normaliseDest(inst.cmdName, state.runConfig.WorkingDir, inst.dest)
93
-	if err != nil {
85
+	hit, err := b.probeCache(state, runConfigWithCommentCmd)
86
+	if err != nil || hit {
94 87
 		return err
95 88
 	}
96 89
 
97 90
 	imageMount, err := b.imageSources.Get(state.imageID)
98 91
 	if err != nil {
99
-		return err
92
+		return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
100 93
 	}
101
-	destSource, err := imageMount.Source()
94
+	destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount)
102 95
 	if err != nil {
103
-		return errors.Wrapf(err, "failed to mount copy source")
96
+		return err
104 97
 	}
105 98
 
106
-	destInfo := newCopyInfoFromSource(destSource, dest, "")
107 99
 	opts := copyFileOptions{
108 100
 		decompress: inst.allowLocalDecompression,
109 101
 		archiver:   b.archiver,
110 102
 	}
111 103
 	for _, info := range inst.infos {
112
-		if err := copyFile(destInfo, info, opts); err != nil {
113
-			return err
104
+		if err := performCopyForInfo(destInfo, info, opts); err != nil {
105
+			return errors.Wrapf(err, "failed to copy files")
114 106
 		}
115 107
 	}
108
+	return b.exportImage(state, imageMount, runConfigWithCommentCmd)
109
+}
116 110
 
117
-	newImage := imageMount.Image().NewChild(image.ChildConfig{
118
-		Author:          state.maintainer,
119
-		DiffID:          imageMount.DiffID(),
120
-		ContainerConfig: runConfigWithCommentCmd,
121
-		// TODO: ContainerID?
122
-		// TODO: Config?
123
-	})
124
-	return b.exportImage(state, newImage)
111
+func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount) (copyInfo, error) {
112
+	// Twiddle the destination when it's a relative path - meaning, make it
113
+	// relative to the WORKINGDIR
114
+	dest, err := normaliseDest(workingDir, inst.dest)
115
+	if err != nil {
116
+		return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName)
117
+	}
118
+
119
+	destMount, err := imageMount.Source()
120
+	if err != nil {
121
+		return copyInfo{}, errors.Wrapf(err, "failed to mount copy source")
122
+	}
123
+
124
+	return newCopyInfoFromSource(destMount, dest, ""), nil
125 125
 }
126 126
 
127 127
 // For backwards compat, if there's just one info then use it as the
... ...
@@ -12,7 +12,7 @@ import (
12 12
 
13 13
 // normaliseDest normalises the destination of a COPY/ADD command in a
14 14
 // platform semantically consistent way.
15
-func normaliseDest(cmdName, workingDir, requested string) (string, error) {
15
+func normaliseDest(workingDir, requested string) (string, error) {
16 16
 	dest := filepath.FromSlash(requested)
17 17
 	endsInSlash := strings.HasSuffix(requested, string(os.PathSeparator))
18 18
 	if !system.IsAbs(requested) {
... ...
@@ -12,7 +12,7 @@ import (
12 12
 
13 13
 // normaliseDest normalises the destination of a COPY/ADD command in a
14 14
 // platform semantically consistent way.
15
-func normaliseDest(cmdName, workingDir, requested string) (string, error) {
15
+func normaliseDest(workingDir, requested string) (string, error) {
16 16
 	dest := filepath.FromSlash(requested)
17 17
 	endsInSlash := strings.HasSuffix(dest, string(os.PathSeparator))
18 18
 
... ...
@@ -32,7 +32,7 @@ func normaliseDest(cmdName, workingDir, requested string) (string, error) {
32 32
 	// we only want to validate where the DriveColon part has been supplied.
33 33
 	if filepath.IsAbs(dest) {
34 34
 		if strings.ToUpper(string(dest[0])) != "C" {
35
-			return "", fmt.Errorf("Windows does not support %s with a destinations not on the system drive (C:)", cmdName)
35
+			return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
36 36
 		}
37 37
 		dest = dest[2:] // Strip the drive letter
38 38
 	}
... ...
@@ -44,7 +44,7 @@ func normaliseDest(cmdName, workingDir, requested string) (string, error) {
44 44
 		}
45 45
 		if !system.IsAbs(dest) {
46 46
 			if string(workingDir[0]) != "C" {
47
-				return "", fmt.Errorf("Windows does not support %s with relative paths when WORKDIR is not the system drive", cmdName)
47
+				return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
48 48
 			}
49 49
 			dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
50 50
 			// Make sure we preserve any trailing slash
... ...
@@ -2,16 +2,22 @@
2 2
 
3 3
 package dockerfile
4 4
 
5
-import "testing"
5
+import (
6
+	"fmt"
7
+	"testing"
8
+
9
+	"github.com/docker/docker/pkg/testutil"
10
+	"github.com/stretchr/testify/assert"
11
+)
6 12
 
7 13
 func TestNormaliseDest(t *testing.T) {
8 14
 	tests := []struct{ current, requested, expected, etext string }{
9
-		{``, `D:\`, ``, `Windows does not support TEST with a destinations not on the system drive (C:)`},
10
-		{``, `e:/`, ``, `Windows does not support TEST with a destinations not on the system drive (C:)`},
15
+		{``, `D:\`, ``, `Windows does not support destinations not on the system drive (C:)`},
16
+		{``, `e:/`, ``, `Windows does not support destinations not on the system drive (C:)`},
11 17
 		{`invalid`, `./c1`, ``, `Current WorkingDir invalid is not platform consistent`},
12 18
 		{`C:`, ``, ``, `Current WorkingDir C: is not platform consistent`},
13 19
 		{`C`, ``, ``, `Current WorkingDir C is not platform consistent`},
14
-		{`D:\`, `.`, ``, "Windows does not support TEST with relative paths when WORKDIR is not the system drive"},
20
+		{`D:\`, `.`, ``, "Windows does not support relative paths when WORKDIR is not the system drive"},
15 21
 		{``, `D`, `D`, ``},
16 22
 		{``, `./a1`, `.\a1`, ``},
17 23
 		{``, `.\b1`, `.\b1`, ``},
... ...
@@ -32,20 +38,16 @@ func TestNormaliseDest(t *testing.T) {
32 32
 		{`C:\wdm`, `foo/bar/`, `\wdm\foo\bar\`, ``},
33 33
 		{`C:\wdn`, `foo\bar/`, `\wdn\foo\bar\`, ``},
34 34
 	}
35
-	for _, i := range tests {
36
-		got, err := normaliseDest("TEST", i.current, i.requested)
37
-		if err != nil && i.etext == "" {
38
-			t.Fatalf("TestNormaliseDest Got unexpected error %q for %s %s. ", err.Error(), i.current, i.requested)
39
-		}
40
-		if i.etext != "" && ((err == nil) || (err != nil && err.Error() != i.etext)) {
41
-			if err == nil {
42
-				t.Fatalf("TestNormaliseDest Expected an error for %s %s but didn't get one", i.current, i.requested)
43
-			} else {
44
-				t.Fatalf("TestNormaliseDest Wrong error text for %s %s - %s", i.current, i.requested, err.Error())
35
+	for _, testcase := range tests {
36
+		msg := fmt.Sprintf("Input: %s, %s", testcase.current, testcase.requested)
37
+		actual, err := normaliseDest(testcase.current, testcase.requested)
38
+		if testcase.etext == "" {
39
+			if !assert.NoError(t, err, msg) {
40
+				continue
45 41
 			}
46
-		}
47
-		if i.etext == "" && got != i.expected {
48
-			t.Fatalf("TestNormaliseDest Expected %q for %q and %q. Got %q", i.expected, i.current, i.requested, got)
42
+			assert.Equal(t, testcase.expected, actual, msg)
43
+		} else {
44
+			testutil.ErrorContains(t, err, testcase.etext)
49 45
 		}
50 46
 	}
51 47
 }
... ...
@@ -9,9 +9,7 @@ import (
9 9
 	"github.com/docker/docker/api/types/container"
10 10
 	"github.com/docker/docker/builder"
11 11
 	containerpkg "github.com/docker/docker/container"
12
-	"github.com/docker/docker/image"
13 12
 	"github.com/docker/docker/layer"
14
-	"github.com/docker/docker/pkg/idtools"
15 13
 	"golang.org/x/net/context"
16 14
 )
17 15
 
... ...
@@ -80,12 +78,8 @@ func (m *MockBackend) MakeImageCache(cacheFrom []string) builder.ImageCache {
80 80
 	return nil
81 81
 }
82 82
 
83
-func (m *MockBackend) CreateImage(config []byte, parent string) (string, error) {
84
-	return "c411d1d", nil
85
-}
86
-
87
-func (m *MockBackend) IDMappings() *idtools.IDMappings {
88
-	return &idtools.IDMappings{}
83
+func (m *MockBackend) CreateImage(config []byte, parent string) (builder.Image, error) {
84
+	return nil, nil
89 85
 }
90 86
 
91 87
 type mockImage struct {
... ...
@@ -106,10 +100,6 @@ func (i *mockImage) MarshalJSON() ([]byte, error) {
106 106
 	return json.Marshal(rawImage(*i))
107 107
 }
108 108
 
109
-func (i *mockImage) NewChild(child image.ChildConfig) *image.Image {
110
-	return nil
111
-}
112
-
113 109
 type mockImageCache struct {
114 110
 	getCacheFunc func(parentID string, cfg *container.Config) (string, error)
115 111
 }
... ...
@@ -131,6 +121,10 @@ func (l *mockLayer) Mount() (string, error) {
131 131
 	return "mountPath", nil
132 132
 }
133 133
 
134
+func (l *mockLayer) Commit() (builder.ReleaseableLayer, error) {
135
+	return nil, nil
136
+}
137
+
134 138
 func (l *mockLayer) DiffID() layer.DiffID {
135
-	return layer.DiffID("abcdef12345")
139
+	return layer.DiffID("abcdef")
136 140
 }
... ...
@@ -449,7 +449,7 @@ func initRouter(s *apiserver.Server, d *daemon.Daemon, c *cluster.Cluster) {
449 449
 		image.NewRouter(d, decoder),
450 450
 		systemrouter.NewRouter(d, c),
451 451
 		volume.NewRouter(d),
452
-		build.NewRouter(buildbackend.NewBackend(d, d), d),
452
+		build.NewRouter(buildbackend.NewBackend(d, d, d.IDMappings()), d),
453 453
 		swarmrouter.NewRouter(c),
454 454
 		pluginrouter.NewRouter(d.PluginManager()),
455 455
 		distributionrouter.NewRouter(d),
... ...
@@ -359,4 +359,4 @@ func (daemon *Daemon) containerCopy(container *container.Container, resource str
359 359
 	})
360 360
 	daemon.LogContainerEvent(container, "copy")
361 361
 	return reader, nil
362
-}
363 362
\ No newline at end of file
363
+}
... ...
@@ -1,6 +1,8 @@
1 1
 package daemon
2 2
 
3 3
 import (
4
+	"io"
5
+
4 6
 	"github.com/Sirupsen/logrus"
5 7
 	"github.com/docker/distribution/reference"
6 8
 	"github.com/docker/docker/api/types"
... ...
@@ -13,7 +15,6 @@ import (
13 13
 	"github.com/docker/docker/registry"
14 14
 	"github.com/pkg/errors"
15 15
 	"golang.org/x/net/context"
16
-	"io"
17 16
 )
18 17
 
19 18
 type releaseableLayer struct {
... ...
@@ -23,12 +24,14 @@ type releaseableLayer struct {
23 23
 }
24 24
 
25 25
 func (rl *releaseableLayer) Mount() (string, error) {
26
-	if rl.roLayer == nil {
27
-		return "", errors.New("can not mount an image with no root FS")
28
-	}
29 26
 	var err error
27
+	var chainID layer.ChainID
28
+	if rl.roLayer != nil {
29
+		chainID = rl.roLayer.ChainID()
30
+	}
31
+
30 32
 	mountID := stringid.GenerateRandomID()
31
-	rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, rl.roLayer.ChainID(), nil)
33
+	rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, nil)
32 34
 	if err != nil {
33 35
 		return "", errors.Wrap(err, "failed to create rwlayer")
34 36
 	}
... ...
@@ -36,15 +39,41 @@ func (rl *releaseableLayer) Mount() (string, error) {
36 36
 	return rl.rwLayer.Mount("")
37 37
 }
38 38
 
39
-func (rl *releaseableLayer) Release() error {
40
-	rl.releaseRWLayer()
41
-	return rl.releaseROLayer()
39
+func (rl *releaseableLayer) Commit() (builder.ReleaseableLayer, error) {
40
+	var chainID layer.ChainID
41
+	if rl.roLayer != nil {
42
+		chainID = rl.roLayer.ChainID()
43
+	}
44
+
45
+	stream, err := rl.rwLayer.TarStream()
46
+	if err != nil {
47
+		return nil, err
48
+	}
49
+
50
+	newLayer, err := rl.layerStore.Register(stream, chainID)
51
+	if err != nil {
52
+		return nil, err
53
+	}
54
+
55
+	if layer.IsEmpty(newLayer.DiffID()) {
56
+		_, err := rl.layerStore.Release(newLayer)
57
+		return &releaseableLayer{layerStore: rl.layerStore}, err
58
+	}
59
+	return &releaseableLayer{layerStore: rl.layerStore, roLayer: newLayer}, nil
42 60
 }
43 61
 
44 62
 func (rl *releaseableLayer) DiffID() layer.DiffID {
63
+	if rl.roLayer == nil {
64
+		return layer.DigestSHA256EmptyTar
65
+	}
45 66
 	return rl.roLayer.DiffID()
46 67
 }
47 68
 
69
+func (rl *releaseableLayer) Release() error {
70
+	rl.releaseRWLayer()
71
+	return rl.releaseROLayer()
72
+}
73
+
48 74
 func (rl *releaseableLayer) releaseRWLayer() error {
49 75
 	if rl.rwLayer == nil {
50 76
 		return nil
... ...
@@ -67,8 +96,8 @@ func (rl *releaseableLayer) releaseROLayer() error {
67 67
 }
68 68
 
69 69
 func newReleasableLayerForImage(img *image.Image, layerStore layer.Store) (builder.ReleaseableLayer, error) {
70
-	if img.RootFS.ChainID() == "" {
71
-		return nil, nil
70
+	if img == nil || img.RootFS.ChainID() == "" {
71
+		return &releaseableLayer{layerStore: layerStore}, nil
72 72
 	}
73 73
 	// Hold a reference to the image layer so that it can't be removed before
74 74
 	// it is released
... ...
@@ -109,6 +138,11 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi
109 109
 // Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent
110 110
 // leaking of layers.
111 111
 func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) {
112
+	if refOrID == "" {
113
+		layer, err := newReleasableLayerForImage(nil, daemon.layerStore)
114
+		return nil, layer, err
115
+	}
116
+
112 117
 	if !opts.ForcePull {
113 118
 		image, _ := daemon.GetImage(refOrID)
114 119
 		// TODO: shouldn't we error out if error is different from "not found" ?
... ...
@@ -129,19 +163,19 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st
129 129
 // CreateImage creates a new image by adding a config and ID to the image store.
130 130
 // This is similar to LoadImage() except that it receives JSON encoded bytes of
131 131
 // an image instead of a tar archive.
132
-func (daemon *Daemon) CreateImage(config []byte, parent string) (string, error) {
132
+func (daemon *Daemon) CreateImage(config []byte, parent string) (builder.Image, error) {
133 133
 	id, err := daemon.imageStore.Create(config)
134 134
 	if err != nil {
135
-		return "", err
135
+		return nil, errors.Wrapf(err, "failed to create image")
136 136
 	}
137 137
 
138 138
 	if parent != "" {
139 139
 		if err := daemon.imageStore.SetParent(id, image.ID(parent)); err != nil {
140
-			return "", err
140
+			return nil, errors.Wrapf(err, "failed to set parent %s", parent)
141 141
 		}
142 142
 	}
143
-	// TODO: do we need any daemon.LogContainerEventWithAttributes?
144
-	return id.String(), nil
143
+
144
+	return daemon.imageStore.Get(id)
145 145
 }
146 146
 
147 147
 // IDMappings returns uid/gid mappings for the builder
... ...
@@ -188,7 +188,7 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
188 188
 		Config:          newConfig,
189 189
 		DiffID:          l.DiffID(),
190 190
 	}
191
-	config, err := json.Marshal(parent.NewChild(cc))
191
+	config, err := json.Marshal(image.NewChildImage(parent, cc))
192 192
 	if err != nil {
193 193
 		return "", err
194 194
 	}
... ...
@@ -4,14 +4,14 @@ import (
4 4
 	"encoding/json"
5 5
 	"errors"
6 6
 	"io"
7
+	"runtime"
8
+	"strings"
7 9
 	"time"
8 10
 
9 11
 	"github.com/docker/docker/api/types/container"
10 12
 	"github.com/docker/docker/dockerversion"
11 13
 	"github.com/docker/docker/layer"
12 14
 	"github.com/opencontainers/go-digest"
13
-	"runtime"
14
-	"strings"
15 15
 )
16 16
 
17 17
 // ID is the content-addressable ID of an image.
... ...
@@ -125,10 +125,13 @@ type ChildConfig struct {
125 125
 	Config          *container.Config
126 126
 }
127 127
 
128
-// NewChild creates a new Image as a child of this image.
129
-func (img *Image) NewChild(child ChildConfig) *Image {
128
+// NewChildImage creates a new Image as a child of this image.
129
+func NewChildImage(img *Image, child ChildConfig) *Image {
130 130
 	isEmptyLayer := layer.IsEmpty(child.DiffID)
131 131
 	rootFS := img.RootFS
132
+	if rootFS == nil {
133
+		rootFS = NewRootFS()
134
+	}
132 135
 	if !isEmptyLayer {
133 136
 		rootFS.Append(child.DiffID)
134 137
 	}
... ...
@@ -2,7 +2,6 @@ package image
2 2
 
3 3
 import (
4 4
 	"encoding/json"
5
-	"errors"
6 5
 	"fmt"
7 6
 	"sync"
8 7
 
... ...
@@ -10,6 +9,7 @@ import (
10 10
 	"github.com/docker/distribution/digestset"
11 11
 	"github.com/docker/docker/layer"
12 12
 	"github.com/opencontainers/go-digest"
13
+	"github.com/pkg/errors"
13 14
 )
14 15
 
15 16
 // Store is an interface for creating and accessing images
... ...
@@ -147,7 +147,7 @@ func (is *store) Create(config []byte) (ID, error) {
147 147
 	if layerID != "" {
148 148
 		l, err = is.ls.Get(layerID)
149 149
 		if err != nil {
150
-			return "", err
150
+			return "", errors.Wrapf(err, "failed to get layer %s", layerID)
151 151
 		}
152 152
 	}
153 153
 
... ...
@@ -5870,15 +5870,17 @@ func (s *DockerSuite) TestBuildCopyFromPreviousRootFS(c *check.C) {
5870 5870
 	dockerfile := `
5871 5871
 		FROM busybox AS first
5872 5872
 		COPY foo bar
5873
+
5873 5874
 		FROM busybox
5874
-    %s
5875
-    COPY baz baz
5876
-    RUN echo mno > baz/cc
5875
+		%s
5876
+		COPY baz baz
5877
+		RUN echo mno > baz/cc
5878
+
5877 5879
 		FROM busybox
5878
-    COPY bar /
5879
-    COPY --from=1 baz sub/
5880
-    COPY --from=0 bar baz
5881
-    COPY --from=first bar bay`
5880
+		COPY bar /
5881
+		COPY --from=1 baz sub/
5882
+		COPY --from=0 bar baz
5883
+		COPY --from=first bar bay`
5882 5884
 
5883 5885
 	ctx := fakecontext.New(c, "",
5884 5886
 		fakecontext.WithDockerfile(fmt.Sprintf(dockerfile, "")),
... ...
@@ -5892,32 +5894,25 @@ func (s *DockerSuite) TestBuildCopyFromPreviousRootFS(c *check.C) {
5892 5892
 
5893 5893
 	cli.BuildCmd(c, "build1", build.WithExternalBuildContext(ctx))
5894 5894
 
5895
-	out := cli.DockerCmd(c, "run", "build1", "cat", "bar").Combined()
5896
-	c.Assert(strings.TrimSpace(out), check.Equals, "def")
5897
-	out = cli.DockerCmd(c, "run", "build1", "cat", "sub/aa").Combined()
5898
-	c.Assert(strings.TrimSpace(out), check.Equals, "ghi")
5899
-	out = cli.DockerCmd(c, "run", "build1", "cat", "sub/cc").Combined()
5900
-	c.Assert(strings.TrimSpace(out), check.Equals, "mno")
5901
-	out = cli.DockerCmd(c, "run", "build1", "cat", "baz").Combined()
5902
-	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5903
-	out = cli.DockerCmd(c, "run", "build1", "cat", "bay").Combined()
5904
-	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5895
+	cli.DockerCmd(c, "run", "build1", "cat", "bar").Assert(c, icmd.Expected{Out: "def"})
5896
+	cli.DockerCmd(c, "run", "build1", "cat", "sub/aa").Assert(c, icmd.Expected{Out: "ghi"})
5897
+	cli.DockerCmd(c, "run", "build1", "cat", "sub/cc").Assert(c, icmd.Expected{Out: "mno"})
5898
+	cli.DockerCmd(c, "run", "build1", "cat", "baz").Assert(c, icmd.Expected{Out: "abc"})
5899
+	cli.DockerCmd(c, "run", "build1", "cat", "bay").Assert(c, icmd.Expected{Out: "abc"})
5905 5900
 
5906 5901
 	result := cli.BuildCmd(c, "build2", build.WithExternalBuildContext(ctx))
5907 5902
 
5908 5903
 	// all commands should be cached
5909 5904
 	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 7)
5905
+	c.Assert(getIDByName(c, "build1"), checker.Equals, getIDByName(c, "build2"))
5910 5906
 
5911 5907
 	err := ioutil.WriteFile(filepath.Join(ctx.Dir, "Dockerfile"), []byte(fmt.Sprintf(dockerfile, "COPY baz/aa foo")), 0644)
5912 5908
 	c.Assert(err, checker.IsNil)
5913 5909
 
5914 5910
 	// changing file in parent block should not affect last block
5915 5911
 	result = cli.BuildCmd(c, "build3", build.WithExternalBuildContext(ctx))
5916
-
5917 5912
 	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 5)
5918 5913
 
5919
-	c.Assert(getIDByName(c, "build1"), checker.Equals, getIDByName(c, "build2"))
5920
-
5921 5914
 	err = ioutil.WriteFile(filepath.Join(ctx.Dir, "foo"), []byte("pqr"), 0644)
5922 5915
 	c.Assert(err, checker.IsNil)
5923 5916
 
... ...
@@ -5925,10 +5920,8 @@ func (s *DockerSuite) TestBuildCopyFromPreviousRootFS(c *check.C) {
5925 5925
 	result = cli.BuildCmd(c, "build4", build.WithExternalBuildContext(ctx))
5926 5926
 	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 5)
5927 5927
 
5928
-	out = cli.DockerCmd(c, "run", "build4", "cat", "bay").Combined()
5929
-	c.Assert(strings.TrimSpace(out), check.Equals, "pqr")
5930
-	out = cli.DockerCmd(c, "run", "build4", "cat", "baz").Combined()
5931
-	c.Assert(strings.TrimSpace(out), check.Equals, "pqr")
5928
+	cli.DockerCmd(c, "run", "build4", "cat", "bay").Assert(c, icmd.Expected{Out: "pqr"})
5929
+	cli.DockerCmd(c, "run", "build4", "cat", "baz").Assert(c, icmd.Expected{Out: "pqr"})
5932 5930
 }
5933 5931
 
5934 5932
 func (s *DockerSuite) TestBuildCopyFromPreviousRootFSErrors(c *check.C) {
... ...
@@ -34,3 +34,23 @@ func (f *TempFile) Name() string {
34 34
 func (f *TempFile) Remove() {
35 35
 	os.Remove(f.Name())
36 36
 }
37
+
38
+// TempDir is a temporary directory that can be used with unit tests. TempDir
39
+// reduces the boilerplate setup required in each test case by handling
40
+// setup errors.
41
+type TempDir struct {
42
+	Path string
43
+}
44
+
45
+// NewTempDir returns a new temp file with contents
46
+func NewTempDir(t require.TestingT, prefix string) *TempDir {
47
+	path, err := ioutil.TempDir("", prefix+"-")
48
+	require.NoError(t, err)
49
+
50
+	return &TempDir{Path: path}
51
+}
52
+
53
+// Remove removes the file
54
+func (f *TempDir) Remove() {
55
+	os.Remove(f.Path)
56
+}