Browse code

Add support for COPY from previous rootfs

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2017/03/16 07:28:06
Showing 14 changed files
... ...
@@ -146,6 +146,9 @@ type Backend interface {
146 146
 
147 147
 	// SquashImage squashes the fs layers from the provided image down to the specified `to` image
148 148
 	SquashImage(from string, to string) (string, error)
149
+
150
+	// MountImage returns mounted path with rootfs of an image.
151
+	MountImage(name string) (string, func() error, error)
149 152
 }
150 153
 
151 154
 // Image represents a Docker image used by the builder.
... ...
@@ -69,7 +69,8 @@ type Builder struct {
69 69
 	runConfig        *container.Config // runconfig for cmd, run, entrypoint etc.
70 70
 	flags            *BFlags
71 71
 	tmpContainers    map[string]struct{}
72
-	image            string // imageID
72
+	image            string         // imageID
73
+	imageContexts    *imageContexts // helper for storing contexts from builds
73 74
 	noBaseImage      bool
74 75
 	maintainer       string
75 76
 	cmdSet           bool
... ...
@@ -88,12 +89,13 @@ type Builder struct {
88 88
 
89 89
 // BuildManager implements builder.Backend and is shared across all Builder objects.
90 90
 type BuildManager struct {
91
-	backend builder.Backend
91
+	backend   builder.Backend
92
+	pathCache *pathCache // TODO: make this persistent
92 93
 }
93 94
 
94 95
 // NewBuildManager creates a BuildManager.
95 96
 func NewBuildManager(b builder.Backend) (bm *BuildManager) {
96
-	return &BuildManager{backend: b}
97
+	return &BuildManager{backend: b, pathCache: &pathCache{}}
97 98
 }
98 99
 
99 100
 // BuildFromContext builds a new image from a given context.
... ...
@@ -118,6 +120,7 @@ func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser,
118 118
 	if err != nil {
119 119
 		return "", err
120 120
 	}
121
+	b.imageContexts.cache = bm.pathCache
121 122
 	return b.build(pg.StdoutFormatter, pg.StderrFormatter, pg.Output)
122 123
 }
123 124
 
... ...
@@ -147,6 +150,7 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
147 147
 			LookingForDirectives: true,
148 148
 		},
149 149
 	}
150
+	b.imageContexts = &imageContexts{b: b}
150 151
 
151 152
 	parser.SetEscapeToken(parser.DefaultEscapeToken, &b.directive) // Assume the default token for escape
152 153
 
... ...
@@ -239,6 +243,8 @@ func (b *Builder) processLabels() error {
239 239
 // * Print a happy message and return the image ID.
240 240
 //
241 241
 func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (string, error) {
242
+	defer b.imageContexts.unmount()
243
+
242 244
 	b.Stdout = stdout
243 245
 	b.Stderr = stderr
244 246
 	b.Output = out
... ...
@@ -8,7 +8,6 @@ package dockerfile
8 8
 // package.
9 9
 
10 10
 import (
11
-	"errors"
12 11
 	"fmt"
13 12
 	"regexp"
14 13
 	"runtime"
... ...
@@ -25,6 +24,7 @@ import (
25 25
 	"github.com/docker/docker/builder"
26 26
 	"github.com/docker/docker/pkg/signal"
27 27
 	"github.com/docker/go-connections/nat"
28
+	"github.com/pkg/errors"
28 29
 )
29 30
 
30 31
 // ENV foo bar
... ...
@@ -169,7 +169,7 @@ func add(b *Builder, args []string, attributes map[string]bool, original string)
169 169
 		return err
170 170
 	}
171 171
 
172
-	return b.runContextCommand(args, true, true, "ADD")
172
+	return b.runContextCommand(args, true, true, "ADD", nil)
173 173
 }
174 174
 
175 175
 // COPY foo /path
... ...
@@ -181,11 +181,26 @@ func dispatchCopy(b *Builder, args []string, attributes map[string]bool, origina
181 181
 		return errAtLeastTwoArguments("COPY")
182 182
 	}
183 183
 
184
+	flFrom := b.flags.AddString("from", "")
185
+
184 186
 	if err := b.flags.Parse(); err != nil {
185 187
 		return err
186 188
 	}
187 189
 
188
-	return b.runContextCommand(args, false, false, "COPY")
190
+	var contextID *int
191
+	if flFrom.IsUsed() {
192
+		var err error
193
+		context, err := strconv.Atoi(flFrom.Value)
194
+		if err != nil {
195
+			return errors.Wrap(err, "from expects an integer value corresponding to the context number")
196
+		}
197
+		if err := b.imageContexts.validate(context); err != nil {
198
+			return err
199
+		}
200
+		contextID = &context
201
+	}
202
+
203
+	return b.runContextCommand(args, false, false, "COPY", contextID)
189 204
 }
190 205
 
191 206
 // FROM imagename
... ...
@@ -206,6 +221,7 @@ func from(b *Builder, args []string, attributes map[string]bool, original string
206 206
 	var image builder.Image
207 207
 
208 208
 	b.resetImageCache()
209
+	b.imageContexts.new()
209 210
 
210 211
 	// Windows cannot support a container with no base image.
211 212
 	if name == api.NoBaseImageSpecifier {
... ...
@@ -227,6 +243,7 @@ func from(b *Builder, args []string, attributes map[string]bool, original string
227 227
 				return err
228 228
 			}
229 229
 		}
230
+		b.imageContexts.update(image.ImageID())
230 231
 	}
231 232
 	b.from = image
232 233
 
... ...
@@ -191,6 +191,7 @@ func TestLabel(t *testing.T) {
191 191
 
192 192
 func TestFrom(t *testing.T) {
193 193
 	b := &Builder{flags: &BFlags{}, runConfig: &container.Config{}, disableCommit: true}
194
+	b.imageContexts = &imageContexts{b: b}
194 195
 
195 196
 	err := from(b, []string{"scratch"}, nil, "")
196 197
 
197 198
new file mode 100644
... ...
@@ -0,0 +1,121 @@
0
+package dockerfile
1
+
2
+import (
3
+	"sync"
4
+
5
+	"github.com/Sirupsen/logrus"
6
+	"github.com/docker/docker/builder"
7
+	"github.com/docker/docker/builder/remotecontext"
8
+	"github.com/pkg/errors"
9
+)
10
+
11
+// imageContexts is a helper for stacking up built image rootfs and reusing
12
+// them as contexts
13
+type imageContexts struct {
14
+	b     *Builder
15
+	list  []*imageMount
16
+	cache *pathCache
17
+}
18
+
19
+type imageMount struct {
20
+	id      string
21
+	ctx     builder.Context
22
+	release func() error
23
+}
24
+
25
+func (ic *imageContexts) new() {
26
+	ic.list = append(ic.list, &imageMount{})
27
+}
28
+
29
+func (ic *imageContexts) update(imageID string) {
30
+	ic.list[len(ic.list)-1].id = imageID
31
+}
32
+
33
+func (ic *imageContexts) validate(i int) error {
34
+	if i < 0 || i >= len(ic.list)-1 {
35
+		var extraMsg string
36
+		if i == len(ic.list)-1 {
37
+			extraMsg = " refers current build block"
38
+		}
39
+		return errors.Errorf("invalid from flag value %d%s", i, extraMsg)
40
+	}
41
+	return nil
42
+}
43
+
44
+func (ic *imageContexts) context(i int) (builder.Context, error) {
45
+	if err := ic.validate(i); err != nil {
46
+		return nil, err
47
+	}
48
+	im := ic.list[i]
49
+	if im.ctx == nil {
50
+		if im.id == "" {
51
+			return nil, errors.Errorf("could not copy from empty context")
52
+		}
53
+		p, release, err := ic.b.docker.MountImage(im.id)
54
+		if err != nil {
55
+			return nil, errors.Wrapf(err, "failed to mount %s", im.id)
56
+		}
57
+		ctx, err := remotecontext.NewLazyContext(p)
58
+		if err != nil {
59
+			return nil, errors.Wrapf(err, "failed to create lazycontext for %s", p)
60
+		}
61
+		logrus.Debugf("mounted image: %s %s", im.id, p)
62
+		im.release = release
63
+		im.ctx = ctx
64
+	}
65
+	return im.ctx, nil
66
+}
67
+
68
+func (ic *imageContexts) unmount() (retErr error) {
69
+	for _, im := range ic.list {
70
+		if im.release != nil {
71
+			if err := im.release(); err != nil {
72
+				logrus.Error(errors.Wrapf(err, "failed to unmount previous build image"))
73
+				retErr = err
74
+			}
75
+		}
76
+	}
77
+	return
78
+}
79
+
80
+func (ic *imageContexts) getCache(i int, path string) (interface{}, bool) {
81
+	if ic.cache != nil {
82
+		im := ic.list[i]
83
+		if im.id == "" {
84
+			return nil, false
85
+		}
86
+		return ic.cache.get(im.id + path)
87
+	}
88
+	return nil, false
89
+}
90
+
91
+func (ic *imageContexts) setCache(i int, path string, v interface{}) {
92
+	if ic.cache != nil {
93
+		ic.cache.set(ic.list[i].id+path, v)
94
+	}
95
+}
96
+
97
+type pathCache struct {
98
+	mu    sync.Mutex
99
+	items map[string]interface{}
100
+}
101
+
102
+func (c *pathCache) set(k string, v interface{}) {
103
+	c.mu.Lock()
104
+	if c.items == nil {
105
+		c.items = make(map[string]interface{})
106
+	}
107
+	c.items[k] = v
108
+	c.mu.Unlock()
109
+}
110
+
111
+func (c *pathCache) get(k string) (interface{}, bool) {
112
+	c.mu.Lock()
113
+	if c.items == nil {
114
+		c.mu.Unlock()
115
+		return nil, false
116
+	}
117
+	v, ok := c.items[k]
118
+	c.mu.Unlock()
119
+	return v, ok
120
+}
... ...
@@ -6,7 +6,6 @@ package dockerfile
6 6
 import (
7 7
 	"crypto/sha256"
8 8
 	"encoding/hex"
9
-	"errors"
10 9
 	"fmt"
11 10
 	"io"
12 11
 	"io/ioutil"
... ...
@@ -14,6 +13,8 @@ import (
14 14
 	"net/url"
15 15
 	"os"
16 16
 	"path/filepath"
17
+	"regexp"
18
+	"runtime"
17 19
 	"sort"
18 20
 	"strings"
19 21
 	"time"
... ...
@@ -36,6 +37,7 @@ import (
36 36
 	"github.com/docker/docker/pkg/tarsum"
37 37
 	"github.com/docker/docker/pkg/urlutil"
38 38
 	"github.com/docker/docker/runconfig/opts"
39
+	"github.com/pkg/errors"
39 40
 )
40 41
 
41 42
 func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) error {
... ...
@@ -83,6 +85,7 @@ func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) e
83 83
 	}
84 84
 
85 85
 	b.image = imageID
86
+	b.imageContexts.update(imageID)
86 87
 	return nil
87 88
 }
88 89
 
... ...
@@ -91,11 +94,7 @@ type copyInfo struct {
91 91
 	decompress bool
92 92
 }
93 93
 
94
-func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalDecompression bool, cmdName string) error {
95
-	if b.context == nil {
96
-		return fmt.Errorf("No context given. Impossible to use %s", cmdName)
97
-	}
98
-
94
+func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalDecompression bool, cmdName string, contextID *int) error {
99 95
 	if len(args) < 2 {
100 96
 		return fmt.Errorf("Invalid %s format - at least two arguments required", cmdName)
101 97
 	}
... ...
@@ -129,7 +128,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD
129 129
 			continue
130 130
 		}
131 131
 		// not a URL
132
-		subInfos, err := b.calcCopyInfo(cmdName, orig, allowLocalDecompression, true)
132
+		subInfos, err := b.calcCopyInfo(cmdName, orig, allowLocalDecompression, true, contextID)
133 133
 		if err != nil {
134 134
 			return err
135 135
 		}
... ...
@@ -299,20 +298,41 @@ func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) {
299 299
 	return &builder.HashedFileInfo{FileInfo: builder.PathFileInfo{FileInfo: tmpFileSt, FilePath: tmpFileName}, FileHash: hash}, nil
300 300
 }
301 301
 
302
-func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression, allowWildcards bool) ([]copyInfo, error) {
302
+func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression, allowWildcards bool, contextID *int) ([]copyInfo, error) {
303 303
 
304 304
 	// Work in daemon-specific OS filepath semantics
305 305
 	origPath = filepath.FromSlash(origPath)
306 306
 
307
+	// validate windows paths from other images
308
+	if contextID != nil && runtime.GOOS == "windows" {
309
+		forbid := regexp.MustCompile("(?i)^" + string(os.PathSeparator) + "?(windows(" + string(os.PathSeparator) + ".+)?)?$")
310
+		if p := filepath.Clean(origPath); p == "." || forbid.MatchString(p) {
311
+			return nil, errors.Errorf("copy from %s is not allowed on windows", origPath)
312
+		}
313
+	}
314
+
307 315
 	if origPath != "" && origPath[0] == os.PathSeparator && len(origPath) > 1 {
308 316
 		origPath = origPath[1:]
309 317
 	}
310 318
 	origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator))
311 319
 
320
+	context := b.context
321
+	var err error
322
+	if contextID != nil {
323
+		context, err = b.imageContexts.context(*contextID)
324
+		if err != nil {
325
+			return nil, err
326
+		}
327
+	}
328
+
329
+	if context == nil {
330
+		return nil, errors.Errorf("No context given. Impossible to use %s", cmdName)
331
+	}
332
+
312 333
 	// Deal with wildcards
313 334
 	if allowWildcards && containsWildcards(origPath) {
314 335
 		var copyInfos []copyInfo
315
-		if err := b.context.Walk("", func(path string, info builder.FileInfo, err error) error {
336
+		if err := context.Walk("", func(path string, info builder.FileInfo, err error) error {
316 337
 			if err != nil {
317 338
 				return err
318 339
 			}
... ...
@@ -326,7 +346,7 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
326 326
 
327 327
 			// Note we set allowWildcards to false in case the name has
328 328
 			// a * in it
329
-			subInfos, err := b.calcCopyInfo(cmdName, path, allowLocalDecompression, false)
329
+			subInfos, err := b.calcCopyInfo(cmdName, path, allowLocalDecompression, false, contextID)
330 330
 			if err != nil {
331 331
 				return err
332 332
 			}
... ...
@@ -339,8 +359,7 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
339 339
 	}
340 340
 
341 341
 	// Must be a dir or a file
342
-
343
-	statPath, fi, err := b.context.Stat(origPath)
342
+	statPath, fi, err := context.Stat(origPath)
344 343
 	if err != nil {
345 344
 		return nil, err
346 345
 	}
... ...
@@ -351,6 +370,13 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
351 351
 	if !handleHash {
352 352
 		return copyInfos, nil
353 353
 	}
354
+	if contextID != nil {
355
+		// fast-cache based on imageID
356
+		if h, ok := b.imageContexts.getCache(*contextID, origPath); ok {
357
+			hfi.SetHash(h.(string))
358
+			return copyInfos, nil
359
+		}
360
+	}
354 361
 
355 362
 	// Deal with the single file case
356 363
 	if !fi.IsDir() {
... ...
@@ -359,7 +385,7 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
359 359
 	}
360 360
 	// Must be a dir
361 361
 	var subfiles []string
362
-	err = b.context.Walk(statPath, func(path string, info builder.FileInfo, err error) error {
362
+	err = context.Walk(statPath, func(path string, info builder.FileInfo, err error) error {
363 363
 		if err != nil {
364 364
 			return err
365 365
 		}
... ...
@@ -375,6 +401,9 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
375 375
 	hasher := sha256.New()
376 376
 	hasher.Write([]byte(strings.Join(subfiles, ",")))
377 377
 	hfi.SetHash("dir:" + hex.EncodeToString(hasher.Sum(nil)))
378
+	if contextID != nil {
379
+		b.imageContexts.setCache(*contextID, origPath, hfi.Hash())
380
+	}
378 381
 
379 382
 	return copyInfos, nil
380 383
 }
... ...
@@ -468,6 +497,7 @@ func (b *Builder) probeCache() (bool, error) {
468 468
 	fmt.Fprint(b.Stdout, " ---> Using cache\n")
469 469
 	logrus.Debugf("[BUILDER] Use cached version: %s", b.runConfig.Cmd)
470 470
 	b.image = string(cache)
471
+	b.imageContexts.update(b.image)
471 472
 
472 473
 	return true, nil
473 474
 }
474 475
new file mode 100644
... ...
@@ -0,0 +1,34 @@
0
+package remotecontext
1
+
2
+import (
3
+	"archive/tar"
4
+	"crypto/sha256"
5
+	"hash"
6
+	"os"
7
+
8
+	"github.com/docker/docker/pkg/archive"
9
+	"github.com/docker/docker/pkg/tarsum"
10
+)
11
+
12
+// NewFileHash returns new hash that is used for the builder cache keys
13
+func NewFileHash(path, name string, fi os.FileInfo) (hash.Hash, error) {
14
+	hdr, err := archive.FileInfoHeader(path, name, fi)
15
+	if err != nil {
16
+		return nil, err
17
+	}
18
+	tsh := &tarsumHash{hdr: hdr, Hash: sha256.New()}
19
+	tsh.Reset() // initialize header
20
+	return tsh, nil
21
+}
22
+
23
+type tarsumHash struct {
24
+	hash.Hash
25
+	hdr *tar.Header
26
+}
27
+
28
+// Reset resets the Hash to its initial state.
29
+func (tsh *tarsumHash) Reset() {
30
+	// comply with hash.Hash and reset to the state hash had before any writes
31
+	tsh.Hash.Reset()
32
+	tarsum.WriteV1Header(tsh.hdr, tsh.Hash)
33
+}
0 34
new file mode 100644
... ...
@@ -0,0 +1,149 @@
0
+package remotecontext
1
+
2
+import (
3
+	"encoding/hex"
4
+	"io"
5
+	"os"
6
+	"path/filepath"
7
+
8
+	"github.com/docker/docker/builder"
9
+	"github.com/docker/docker/pkg/pools"
10
+	"github.com/docker/docker/pkg/symlink"
11
+	"github.com/pkg/errors"
12
+)
13
+
14
+// NewLazyContext creates a new LazyContext. LazyContext defines a hashed build
15
+// context based on a root directory. Individual files are hashed first time
16
+// they are asked. It is not safe to call methods of LazyContext concurrently.
17
+func NewLazyContext(root string) (builder.Context, error) {
18
+	return &lazyContext{
19
+		root: root,
20
+		sums: make(map[string]string),
21
+	}, nil
22
+}
23
+
24
+type lazyContext struct {
25
+	root string
26
+	sums map[string]string
27
+}
28
+
29
+func (c *lazyContext) Close() error {
30
+	return nil
31
+}
32
+
33
+func (c *lazyContext) Open(path string) (io.ReadCloser, error) {
34
+	cleanPath, fullPath, err := c.normalize(path)
35
+	if err != nil {
36
+		return nil, err
37
+	}
38
+
39
+	r, err := os.Open(fullPath)
40
+	if err != nil {
41
+		return nil, errors.WithStack(convertPathError(err, cleanPath))
42
+	}
43
+	return r, nil
44
+}
45
+
46
+func (c *lazyContext) Stat(path string) (string, builder.FileInfo, error) {
47
+	// TODO: although stat returns builder.FileInfo it builder.Context actually requires Hashed
48
+	cleanPath, fullPath, err := c.normalize(path)
49
+	if err != nil {
50
+		return "", nil, err
51
+	}
52
+
53
+	st, err := os.Lstat(fullPath)
54
+	if err != nil {
55
+		return "", nil, errors.WithStack(convertPathError(err, cleanPath))
56
+	}
57
+
58
+	relPath, err := filepath.Rel(c.root, fullPath)
59
+	if err != nil {
60
+		return "", nil, errors.WithStack(convertPathError(err, cleanPath))
61
+	}
62
+
63
+	sum, ok := c.sums[relPath]
64
+	if !ok {
65
+		sum, err = c.prepareHash(relPath, st)
66
+		if err != nil {
67
+			return "", nil, err
68
+		}
69
+	}
70
+
71
+	fi := &builder.HashedFileInfo{
72
+		builder.PathFileInfo{st, fullPath, filepath.Base(cleanPath)},
73
+		sum,
74
+	}
75
+	return relPath, fi, nil
76
+}
77
+
78
+func (c *lazyContext) Walk(root string, walkFn builder.WalkFunc) error {
79
+	_, fullPath, err := c.normalize(root)
80
+	if err != nil {
81
+		return err
82
+	}
83
+	return filepath.Walk(fullPath, func(fullPath string, fi os.FileInfo, err error) error {
84
+		relPath, err := filepath.Rel(c.root, fullPath)
85
+		if err != nil {
86
+			return errors.WithStack(err)
87
+		}
88
+		if relPath == "." {
89
+			return nil
90
+		}
91
+
92
+		sum, ok := c.sums[relPath]
93
+		if !ok {
94
+			sum, err = c.prepareHash(relPath, fi)
95
+			if err != nil {
96
+				return err
97
+			}
98
+		}
99
+
100
+		hfi := &builder.HashedFileInfo{
101
+			builder.PathFileInfo{FileInfo: fi, FilePath: fullPath},
102
+			sum,
103
+		}
104
+		if err := walkFn(relPath, hfi, nil); err != nil {
105
+			return err
106
+		}
107
+		return nil
108
+	})
109
+}
110
+
111
+func (c *lazyContext) prepareHash(relPath string, fi os.FileInfo) (string, error) {
112
+	p := filepath.Join(c.root, relPath)
113
+	h, err := NewFileHash(p, relPath, fi)
114
+	if err != nil {
115
+		return "", errors.Wrapf(err, "failed to create hash for %s", relPath)
116
+	}
117
+	if fi.Mode().IsRegular() && fi.Size() > 0 {
118
+		f, err := os.Open(p)
119
+		if err != nil {
120
+			return "", errors.Wrapf(err, "failed to open %s", relPath)
121
+		}
122
+		defer f.Close()
123
+		if _, err := pools.Copy(h, f); err != nil {
124
+			return "", errors.Wrapf(err, "failed to copy file data for %s", relPath)
125
+		}
126
+	}
127
+	sum := hex.EncodeToString(h.Sum(nil))
128
+	c.sums[relPath] = sum
129
+	return sum, nil
130
+}
131
+
132
+func (c *lazyContext) normalize(path string) (cleanPath, fullPath string, err error) {
133
+	// todo: combine these helpers with tarsum after they are moved to same package
134
+	cleanPath = filepath.Clean(string(os.PathSeparator) + path)[1:]
135
+	fullPath, err = symlink.FollowSymlinkInScope(filepath.Join(c.root, path), c.root)
136
+	if err != nil {
137
+		return "", "", errors.Wrapf(err, "forbidden path outside the build context: %s (%s)", path, fullPath)
138
+	}
139
+	return
140
+}
141
+
142
+func convertPathError(err error, cleanpath string) error {
143
+	if err, ok := err.(*os.PathError); ok {
144
+		err.Path = cleanpath
145
+		return err
146
+	}
147
+	return err
148
+}
... ...
@@ -1,7 +1,6 @@
1 1
 package daemon
2 2
 
3 3
 import (
4
-	"errors"
5 4
 	"io"
6 5
 	"os"
7 6
 	"path/filepath"
... ...
@@ -10,11 +9,14 @@ import (
10 10
 	"github.com/docker/docker/api/types"
11 11
 	"github.com/docker/docker/builder"
12 12
 	"github.com/docker/docker/container"
13
+	"github.com/docker/docker/layer"
13 14
 	"github.com/docker/docker/pkg/archive"
14 15
 	"github.com/docker/docker/pkg/chrootarchive"
15 16
 	"github.com/docker/docker/pkg/idtools"
16 17
 	"github.com/docker/docker/pkg/ioutils"
18
+	"github.com/docker/docker/pkg/stringid"
17 19
 	"github.com/docker/docker/pkg/system"
20
+	"github.com/pkg/errors"
18 21
 )
19 22
 
20 23
 // ErrExtractPointNotDirectory is used to convey that the operation to extract
... ...
@@ -454,3 +456,34 @@ func (daemon *Daemon) CopyOnBuild(cID string, destPath string, src builder.FileI
454 454
 
455 455
 	return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists)
456 456
 }
457
+
458
+// MountImage returns mounted path with rootfs of an image.
459
+func (daemon *Daemon) MountImage(name string) (string, func() error, error) {
460
+	img, err := daemon.GetImage(name)
461
+	if err != nil {
462
+		return "", nil, errors.Wrapf(err, "no such image: %s", name)
463
+	}
464
+
465
+	mountID := stringid.GenerateRandomID()
466
+	rwLayer, err := daemon.layerStore.CreateRWLayer(mountID, img.RootFS.ChainID(), nil)
467
+	if err != nil {
468
+		return "", nil, errors.Wrap(err, "failed to create rwlayer")
469
+	}
470
+
471
+	mountPath, err := rwLayer.Mount("")
472
+	if err != nil {
473
+		metadata, releaseErr := daemon.layerStore.ReleaseRWLayer(rwLayer)
474
+		if releaseErr != nil {
475
+			err = errors.Wrapf(err, "failed to release rwlayer: %s", releaseErr.Error())
476
+		}
477
+		layer.LogReleaseMetadata(metadata)
478
+		return "", nil, errors.Wrap(err, "failed to mount rwlayer")
479
+	}
480
+
481
+	return mountPath, func() error {
482
+		rwLayer.Unmount()
483
+		metadata, err := daemon.layerStore.ReleaseRWLayer(rwLayer)
484
+		layer.LogReleaseMetadata(metadata)
485
+		return err
486
+	}, nil
487
+}
... ...
@@ -5755,6 +5755,138 @@ func (s *DockerSuite) TestBuildContChar(c *check.C) {
5755 5755
 	c.Assert(result.Combined(), checker.Contains, "Step 2/2 : RUN echo hi \\\\\n")
5756 5756
 }
5757 5757
 
5758
+func (s *DockerSuite) TestBuildCopyFromPreviousRootFS(c *check.C) {
5759
+	dockerfile := `
5760
+		FROM busybox
5761
+		COPY foo bar
5762
+		FROM busybox
5763
+    %s
5764
+    COPY baz baz
5765
+    RUN echo mno > baz/cc
5766
+		FROM busybox
5767
+    COPY bar /
5768
+    COPY --from=1 baz sub/
5769
+    COPY --from=0 bar baz
5770
+    COPY --from=0 bar bay`
5771
+	ctx := fakeContext(c, fmt.Sprintf(dockerfile, ""), map[string]string{
5772
+		"Dockerfile": dockerfile,
5773
+		"foo":        "abc",
5774
+		"bar":        "def",
5775
+		"baz/aa":     "ghi",
5776
+		"baz/bb":     "jkl",
5777
+	})
5778
+	defer ctx.Close()
5779
+
5780
+	result := buildImage("build1", withExternalBuildContext(ctx))
5781
+	result.Assert(c, icmd.Success)
5782
+
5783
+	out, _ := dockerCmd(c, "run", "build1", "cat", "bar")
5784
+	c.Assert(strings.TrimSpace(out), check.Equals, "def")
5785
+	out, _ = dockerCmd(c, "run", "build1", "cat", "sub/aa")
5786
+	c.Assert(strings.TrimSpace(out), check.Equals, "ghi")
5787
+	out, _ = dockerCmd(c, "run", "build1", "cat", "sub/cc")
5788
+	c.Assert(strings.TrimSpace(out), check.Equals, "mno")
5789
+	out, _ = dockerCmd(c, "run", "build1", "cat", "baz")
5790
+	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5791
+	out, _ = dockerCmd(c, "run", "build1", "cat", "bay")
5792
+	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5793
+
5794
+	result = buildImage("build2", withExternalBuildContext(ctx))
5795
+	result.Assert(c, icmd.Success)
5796
+
5797
+	// all commands should be cached
5798
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 7)
5799
+
5800
+	err := ioutil.WriteFile(filepath.Join(ctx.Dir, "Dockerfile"), []byte(fmt.Sprintf(dockerfile, "COPY baz/aa foo")), 0644)
5801
+	c.Assert(err, checker.IsNil)
5802
+
5803
+	// changing file in parent block should not affect last block
5804
+	result = buildImage("build3", withExternalBuildContext(ctx))
5805
+	result.Assert(c, icmd.Success)
5806
+
5807
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 5)
5808
+
5809
+	c.Assert(getIDByName(c, "build1"), checker.Equals, getIDByName(c, "build2"))
5810
+
5811
+	err = ioutil.WriteFile(filepath.Join(ctx.Dir, "foo"), []byte("pqr"), 0644)
5812
+	c.Assert(err, checker.IsNil)
5813
+
5814
+	// changing file in parent block should affect both first and last block
5815
+	result = buildImage("build4", withExternalBuildContext(ctx))
5816
+	result.Assert(c, icmd.Success)
5817
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 5)
5818
+
5819
+	out, _ = dockerCmd(c, "run", "build4", "cat", "bay")
5820
+	c.Assert(strings.TrimSpace(out), check.Equals, "pqr")
5821
+	out, _ = dockerCmd(c, "run", "build4", "cat", "baz")
5822
+	c.Assert(strings.TrimSpace(out), check.Equals, "pqr")
5823
+}
5824
+
5825
+func (s *DockerSuite) TestBuildCopyFromPreviousRootFSErrors(c *check.C) {
5826
+	dockerfile := `
5827
+		FROM busybox
5828
+		COPY --from=foo foo bar`
5829
+
5830
+	ctx := fakeContext(c, dockerfile, map[string]string{
5831
+		"Dockerfile": dockerfile,
5832
+		"foo":        "abc",
5833
+	})
5834
+	defer ctx.Close()
5835
+
5836
+	buildImage("build1", withExternalBuildContext(ctx)).Assert(c, icmd.Expected{
5837
+		ExitCode: 1,
5838
+		Err:      "from expects an integer value corresponding to the context number",
5839
+	})
5840
+
5841
+	dockerfile = `
5842
+		FROM busybox
5843
+		COPY --from=0 foo bar`
5844
+
5845
+	ctx = fakeContext(c, dockerfile, map[string]string{
5846
+		"Dockerfile": dockerfile,
5847
+		"foo":        "abc",
5848
+	})
5849
+	defer ctx.Close()
5850
+
5851
+	buildImage("build1", withExternalBuildContext(ctx)).Assert(c, icmd.Expected{
5852
+		ExitCode: 1,
5853
+		Err:      "invalid from flag value 0 refers current build block",
5854
+	})
5855
+}
5856
+
5857
+func (s *DockerSuite) TestBuildCopyFromPreviousFrom(c *check.C) {
5858
+	dockerfile := `
5859
+		FROM busybox
5860
+		COPY foo bar`
5861
+	ctx := fakeContext(c, dockerfile, map[string]string{
5862
+		"Dockerfile": dockerfile,
5863
+		"foo":        "abc",
5864
+	})
5865
+	defer ctx.Close()
5866
+
5867
+	result := buildImage("build1", withExternalBuildContext(ctx))
5868
+	result.Assert(c, icmd.Success)
5869
+
5870
+	dockerfile = `
5871
+		FROM build1:latest
5872
+    FROM busybox
5873
+		COPY --from=0 bar /
5874
+		COPY foo /`
5875
+	ctx = fakeContext(c, dockerfile, map[string]string{
5876
+		"Dockerfile": dockerfile,
5877
+		"foo":        "def",
5878
+	})
5879
+	defer ctx.Close()
5880
+
5881
+	result = buildImage("build2", withExternalBuildContext(ctx))
5882
+	result.Assert(c, icmd.Success)
5883
+
5884
+	out, _ := dockerCmd(c, "run", "build2", "cat", "bar")
5885
+	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5886
+	out, _ = dockerCmd(c, "run", "build2", "cat", "foo")
5887
+	c.Assert(strings.TrimSpace(out), check.Equals, "def")
5888
+}
5889
+
5758 5890
 // TestBuildOpaqueDirectory tests that a build succeeds which
5759 5891
 // creates opaque directories.
5760 5892
 // See https://github.com/docker/docker/issues/25244
... ...
@@ -240,6 +240,38 @@ func (compression *Compression) Extension() string {
240 240
 	return ""
241 241
 }
242 242
 
243
+// FileInfoHeader creates a populated Header from fi.
244
+// Compared to archive pkg this function fills in more information.
245
+func FileInfoHeader(path, name string, fi os.FileInfo) (*tar.Header, error) {
246
+	var link string
247
+	if fi.Mode()&os.ModeSymlink != 0 {
248
+		var err error
249
+		link, err = os.Readlink(path)
250
+		if err != nil {
251
+			return nil, err
252
+		}
253
+	}
254
+	hdr, err := tar.FileInfoHeader(fi, link)
255
+	if err != nil {
256
+		return nil, err
257
+	}
258
+	hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
259
+	name, err = canonicalTarName(name, fi.IsDir())
260
+	if err != nil {
261
+		return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err)
262
+	}
263
+	hdr.Name = name
264
+	if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil {
265
+		return nil, err
266
+	}
267
+	capability, _ := system.Lgetxattr(path, "security.capability")
268
+	if capability != nil {
269
+		hdr.Xattrs = make(map[string]string)
270
+		hdr.Xattrs["security.capability"] = string(capability)
271
+	}
272
+	return hdr, nil
273
+}
274
+
243 275
 type tarWhiteoutConverter interface {
244 276
 	ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error)
245 277
 	ConvertRead(*tar.Header, string) (bool, error)
... ...
@@ -283,26 +315,7 @@ func (ta *tarAppender) addTarFile(path, name string) error {
283 283
 		return err
284 284
 	}
285 285
 
286
-	link := ""
287
-	if fi.Mode()&os.ModeSymlink != 0 {
288
-		if link, err = os.Readlink(path); err != nil {
289
-			return err
290
-		}
291
-	}
292
-
293
-	hdr, err := tar.FileInfoHeader(fi, link)
294
-	if err != nil {
295
-		return err
296
-	}
297
-	hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
298
-
299
-	name, err = canonicalTarName(name, fi.IsDir())
300
-	if err != nil {
301
-		return fmt.Errorf("tar: cannot canonicalize path: %v", err)
302
-	}
303
-	hdr.Name = name
304
-
305
-	inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys())
286
+	hdr, err := FileInfoHeader(path, name, fi)
306 287
 	if err != nil {
307 288
 		return err
308 289
 	}
... ...
@@ -310,6 +323,10 @@ func (ta *tarAppender) addTarFile(path, name string) error {
310 310
 	// if it's not a directory and has more than 1 link,
311 311
 	// it's hard linked, so set the type flag accordingly
312 312
 	if !fi.IsDir() && hasHardlinks(fi) {
313
+		inode, err := getInodeFromStat(fi.Sys())
314
+		if err != nil {
315
+			return err
316
+		}
313 317
 		// a link should have a name that it links too
314 318
 		// and that linked name should be first in the tar archive
315 319
 		if oldpath, ok := ta.SeenFiles[inode]; ok {
... ...
@@ -321,12 +338,6 @@ func (ta *tarAppender) addTarFile(path, name string) error {
321 321
 		}
322 322
 	}
323 323
 
324
-	capability, _ := system.Lgetxattr(path, "security.capability")
325
-	if capability != nil {
326
-		hdr.Xattrs = make(map[string]string)
327
-		hdr.Xattrs["security.capability"] = string(capability)
328
-	}
329
-
330 324
 	//handle re-mapping container ID mappings back to host ID mappings before
331 325
 	//writing tar headers/files. We skip whiteout files because they were written
332 326
 	//by the kernel and already have proper ownership relative to the host
... ...
@@ -41,7 +41,7 @@ func chmodTarEntry(perm os.FileMode) os.FileMode {
41 41
 	return perm // noop for unix as golang APIs provide perm bits correctly
42 42
 }
43 43
 
44
-func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, stat interface{}) (inode uint64, err error) {
44
+func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (err error) {
45 45
 	s, ok := stat.(*syscall.Stat_t)
46 46
 
47 47
 	if !ok {
... ...
@@ -49,8 +49,6 @@ func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, st
49 49
 		return
50 50
 	}
51 51
 
52
-	inode = uint64(s.Ino)
53
-
54 52
 	// Currently go does not fill in the major/minors
55 53
 	if s.Mode&syscall.S_IFBLK != 0 ||
56 54
 		s.Mode&syscall.S_IFCHR != 0 {
... ...
@@ -61,6 +59,19 @@ func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, st
61 61
 	return
62 62
 }
63 63
 
64
+func getInodeFromStat(stat interface{}) (inode uint64, err error) {
65
+	s, ok := stat.(*syscall.Stat_t)
66
+
67
+	if !ok {
68
+		err = errors.New("cannot convert stat value to syscall.Stat_t")
69
+		return
70
+	}
71
+
72
+	inode = uint64(s.Ino)
73
+
74
+	return
75
+}
76
+
64 77
 func getFileUIDGID(stat interface{}) (int, int, error) {
65 78
 	s, ok := stat.(*syscall.Stat_t)
66 79
 
... ...
@@ -49,8 +49,13 @@ func chmodTarEntry(perm os.FileMode) os.FileMode {
49 49
 	return perm
50 50
 }
51 51
 
52
-func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, stat interface{}) (inode uint64, err error) {
53
-	// do nothing. no notion of Rdev, Inode, Nlink in stat on Windows
52
+func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (err error) {
53
+	// do nothing. no notion of Rdev, Nlink in stat on Windows
54
+	return
55
+}
56
+
57
+func getInodeFromStat(stat interface{}) (inode uint64, err error) {
58
+	// do nothing. no notion of Inode in stat on Windows
54 59
 	return
55 60
 }
56 61
 
... ...
@@ -3,6 +3,7 @@ package tarsum
3 3
 import (
4 4
 	"archive/tar"
5 5
 	"errors"
6
+	"io"
6 7
 	"sort"
7 8
 	"strconv"
8 9
 	"strings"
... ...
@@ -21,6 +22,13 @@ const (
21 21
 	VersionDev
22 22
 )
23 23
 
24
+// WriteV1Header writes a tar header to a writer in V1 tarsum format.
25
+func WriteV1Header(h *tar.Header, w io.Writer) {
26
+	for _, elem := range v1TarHeaderSelect(h) {
27
+		w.Write([]byte(elem[0] + elem[1]))
28
+	}
29
+}
30
+
24 31
 // VersionLabelForChecksum returns the label for the given tarsum
25 32
 // checksum, i.e., everything before the first `+` character in
26 33
 // the string or an empty string if no label separator is found.