Browse code

Merge pull request #31257 from tonistiigi/nested-build

build: add multi-stage build support

Tõnis Tiigi authored on 2017/03/24 11:14:13
Showing 15 changed files
... ...
@@ -146,6 +146,9 @@ type Backend interface {
146 146
 
147 147
 	// SquashImage squashes the fs layers from the provided image down to the specified `to` image
148 148
 	SquashImage(from string, to string) (string, error)
149
+
150
+	// MountImage returns mounted path with rootfs of an image.
151
+	MountImage(name string) (string, func() error, error)
149 152
 }
150 153
 
151 154
 // Image represents a Docker image used by the builder.
... ...
@@ -71,13 +71,15 @@ type Builder struct {
71 71
 	runConfig        *container.Config // runconfig for cmd, run, entrypoint etc.
72 72
 	flags            *BFlags
73 73
 	tmpContainers    map[string]struct{}
74
-	image            string // imageID
74
+	image            string         // imageID
75
+	imageContexts    *imageContexts // helper for storing contexts from builds
75 76
 	noBaseImage      bool
76 77
 	maintainer       string
77 78
 	cmdSet           bool
78 79
 	disableCommit    bool
79 80
 	cacheBusted      bool
80
-	allowedBuildArgs map[string]bool // list of build-time args that are allowed for expansion/substitution and passing to commands in 'run'.
81
+	allowedBuildArgs map[string]*string  // list of build-time args that are allowed for expansion/substitution and passing to commands in 'run'.
82
+	allBuildArgs     map[string]struct{} // list of all build-time args found during parsing of the Dockerfile
81 83
 	directive        parser.Directive
82 84
 
83 85
 	// TODO: remove once docker.Commit can receive a tag
... ...
@@ -89,12 +91,13 @@ type Builder struct {
89 89
 
90 90
 // BuildManager implements builder.Backend and is shared across all Builder objects.
91 91
 type BuildManager struct {
92
-	backend builder.Backend
92
+	backend   builder.Backend
93
+	pathCache *pathCache // TODO: make this persistent
93 94
 }
94 95
 
95 96
 // NewBuildManager creates a BuildManager.
96 97
 func NewBuildManager(b builder.Backend) (bm *BuildManager) {
97
-	return &BuildManager{backend: b}
98
+	return &BuildManager{backend: b, pathCache: &pathCache{}}
98 99
 }
99 100
 
100 101
 // BuildFromContext builds a new image from a given context.
... ...
@@ -119,6 +122,7 @@ func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser,
119 119
 	if err != nil {
120 120
 		return "", err
121 121
 	}
122
+	b.imageContexts.cache = bm.pathCache
122 123
 	return b.build(pg.StdoutFormatter, pg.StderrFormatter, pg.Output)
123 124
 }
124 125
 
... ...
@@ -129,9 +133,6 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
129 129
 	if config == nil {
130 130
 		config = new(types.ImageBuildOptions)
131 131
 	}
132
-	if config.BuildArgs == nil {
133
-		config.BuildArgs = make(map[string]*string)
134
-	}
135 132
 	ctx, cancel := context.WithCancel(clientCtx)
136 133
 	b = &Builder{
137 134
 		clientCtx:        ctx,
... ...
@@ -144,15 +145,14 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
144 144
 		runConfig:        new(container.Config),
145 145
 		tmpContainers:    map[string]struct{}{},
146 146
 		id:               stringid.GenerateNonCryptoID(),
147
-		allowedBuildArgs: make(map[string]bool),
147
+		allowedBuildArgs: make(map[string]*string),
148
+		allBuildArgs:     make(map[string]struct{}),
148 149
 		directive: parser.Directive{
149 150
 			EscapeSeen:           false,
150 151
 			LookingForDirectives: true,
151 152
 		},
152 153
 	}
153
-	if icb, ok := backend.(builder.ImageCacheBuilder); ok {
154
-		b.imageCache = icb.MakeImageCache(config.CacheFrom)
155
-	}
154
+	b.imageContexts = &imageContexts{b: b}
156 155
 
157 156
 	parser.SetEscapeToken(parser.DefaultEscapeToken, &b.directive) // Assume the default token for escape
158 157
 
... ...
@@ -166,6 +166,14 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
166 166
 	return b, nil
167 167
 }
168 168
 
169
+func (b *Builder) resetImageCache() {
170
+	if icb, ok := b.docker.(builder.ImageCacheBuilder); ok {
171
+		b.imageCache = icb.MakeImageCache(b.options.CacheFrom)
172
+	}
173
+	b.noBaseImage = false
174
+	b.cacheBusted = false
175
+}
176
+
169 177
 // sanitizeRepoAndTags parses the raw "t" parameter received from the client
170 178
 // to a slice of repoAndTag.
171 179
 // It also validates each repoName and tag.
... ...
@@ -237,6 +245,8 @@ func (b *Builder) processLabels() error {
237 237
 // * Print a happy message and return the image ID.
238 238
 //
239 239
 func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (string, error) {
240
+	defer b.imageContexts.unmount()
241
+
240 242
 	b.Stdout = stdout
241 243
 	b.Stderr = stderr
242 244
 	b.Output = out
... ...
@@ -322,7 +332,7 @@ func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (stri
322 322
 func (b *Builder) warnOnUnusedBuildArgs() {
323 323
 	leftoverArgs := []string{}
324 324
 	for arg := range b.options.BuildArgs {
325
-		if !b.isBuildArgAllowed(arg) {
325
+		if _, ok := b.allBuildArgs[arg]; !ok {
326 326
 			leftoverArgs = append(leftoverArgs, arg)
327 327
 		}
328 328
 	}
... ...
@@ -8,7 +8,6 @@ package dockerfile
8 8
 // package.
9 9
 
10 10
 import (
11
-	"errors"
12 11
 	"fmt"
13 12
 	"regexp"
14 13
 	"runtime"
... ...
@@ -25,6 +24,7 @@ import (
25 25
 	"github.com/docker/docker/builder"
26 26
 	"github.com/docker/docker/pkg/signal"
27 27
 	"github.com/docker/go-connections/nat"
28
+	"github.com/pkg/errors"
28 29
 )
29 30
 
30 31
 // ENV foo bar
... ...
@@ -169,7 +169,7 @@ func add(b *Builder, args []string, attributes map[string]bool, original string)
169 169
 		return err
170 170
 	}
171 171
 
172
-	return b.runContextCommand(args, true, true, "ADD")
172
+	return b.runContextCommand(args, true, true, "ADD", nil)
173 173
 }
174 174
 
175 175
 // COPY foo /path
... ...
@@ -181,11 +181,26 @@ func dispatchCopy(b *Builder, args []string, attributes map[string]bool, origina
181 181
 		return errAtLeastTwoArguments("COPY")
182 182
 	}
183 183
 
184
+	flFrom := b.flags.AddString("from", "")
185
+
184 186
 	if err := b.flags.Parse(); err != nil {
185 187
 		return err
186 188
 	}
187 189
 
188
-	return b.runContextCommand(args, false, false, "COPY")
190
+	var contextID *int
191
+	if flFrom.IsUsed() {
192
+		var err error
193
+		context, err := strconv.Atoi(flFrom.Value)
194
+		if err != nil {
195
+			return errors.Wrap(err, "from expects an integer value corresponding to the context number")
196
+		}
197
+		if err := b.imageContexts.validate(context); err != nil {
198
+			return err
199
+		}
200
+		contextID = &context
201
+	}
202
+
203
+	return b.runContextCommand(args, false, false, "COPY", contextID)
189 204
 }
190 205
 
191 206
 // FROM imagename
... ...
@@ -205,6 +220,9 @@ func from(b *Builder, args []string, attributes map[string]bool, original string
205 205
 
206 206
 	var image builder.Image
207 207
 
208
+	b.resetImageCache()
209
+	b.imageContexts.new()
210
+
208 211
 	// Windows cannot support a container with no base image.
209 212
 	if name == api.NoBaseImageSpecifier {
210 213
 		if runtime.GOOS == "windows" {
... ...
@@ -225,9 +243,12 @@ func from(b *Builder, args []string, attributes map[string]bool, original string
225 225
 				return err
226 226
 			}
227 227
 		}
228
+		b.imageContexts.update(image.ImageID())
228 229
 	}
229 230
 	b.from = image
230 231
 
232
+	b.allowedBuildArgs = make(map[string]*string)
233
+
231 234
 	return b.processImageFrom(image)
232 235
 }
233 236
 
... ...
@@ -749,17 +770,13 @@ func arg(b *Builder, args []string, attributes map[string]bool, original string)
749 749
 		hasDefault = false
750 750
 	}
751 751
 	// add the arg to allowed list of build-time args from this step on.
752
-	b.allowedBuildArgs[name] = true
752
+	b.allBuildArgs[name] = struct{}{}
753 753
 
754
-	// If there is a default value associated with this arg then add it to the
755
-	// b.buildArgs if one is not already passed to the builder. The args passed
756
-	// to builder override the default value of 'arg'. Note that a 'nil' for
757
-	// a value means that the user specified "--build-arg FOO" and "FOO" wasn't
758
-	// defined as an env var - and in that case we DO want to use the default
759
-	// value specified in the ARG cmd.
760
-	if baValue, ok := b.options.BuildArgs[name]; (!ok || baValue == nil) && hasDefault {
761
-		b.options.BuildArgs[name] = &newValue
754
+	var value *string
755
+	if hasDefault {
756
+		value = &newValue
762 757
 	}
758
+	b.allowedBuildArgs[name] = value
763 759
 
764 760
 	return b.commit("", b.runConfig.Cmd, fmt.Sprintf("ARG %s", arg))
765 761
 }
... ...
@@ -191,6 +191,7 @@ func TestLabel(t *testing.T) {
191 191
 
192 192
 func TestFrom(t *testing.T) {
193 193
 	b := &Builder{flags: &BFlags{}, runConfig: &container.Config{}, disableCommit: true}
194
+	b.imageContexts = &imageContexts{b: b}
194 195
 
195 196
 	err := from(b, []string{"scratch"}, nil, "")
196 197
 
... ...
@@ -460,9 +461,11 @@ func TestStopSignal(t *testing.T) {
460 460
 }
461 461
 
462 462
 func TestArg(t *testing.T) {
463
+	// This is a bad test that tests implementation details and not at
464
+	// any features of the builder. Replace or remove.
463 465
 	buildOptions := &types.ImageBuildOptions{BuildArgs: make(map[string]*string)}
464 466
 
465
-	b := &Builder{flags: &BFlags{}, runConfig: &container.Config{}, disableCommit: true, allowedBuildArgs: make(map[string]bool), options: buildOptions}
467
+	b := &Builder{flags: &BFlags{}, runConfig: &container.Config{}, disableCommit: true, allowedBuildArgs: make(map[string]*string), allBuildArgs: make(map[string]struct{}), options: buildOptions}
466 468
 
467 469
 	argName := "foo"
468 470
 	argVal := "bar"
... ...
@@ -472,24 +475,14 @@ func TestArg(t *testing.T) {
472 472
 		t.Fatalf("Error should be empty, got: %s", err.Error())
473 473
 	}
474 474
 
475
-	allowed, ok := b.allowedBuildArgs[argName]
476
-
477
-	if !ok {
478
-		t.Fatalf("%s argument should be allowed as a build arg", argName)
479
-	}
480
-
481
-	if !allowed {
482
-		t.Fatalf("%s argument was present in map but disallowed as a build arg", argName)
483
-	}
484
-
485
-	val, ok := b.options.BuildArgs[argName]
475
+	value, ok := b.getBuildArg(argName)
486 476
 
487 477
 	if !ok {
488 478
 		t.Fatalf("%s argument should be a build arg", argName)
489 479
 	}
490 480
 
491
-	if *val != "bar" {
492
-		t.Fatalf("%s argument should have default value 'bar', got %s", argName, *val)
481
+	if value != "bar" {
482
+		t.Fatalf("%s argument should have default value 'bar', got %s", argName, value)
493 483
 	}
494 484
 }
495 485
 
... ...
@@ -192,17 +192,9 @@ func (b *Builder) buildArgsWithoutConfigEnv() []string {
192 192
 	envs := []string{}
193 193
 	configEnv := runconfigopts.ConvertKVStringsToMap(b.runConfig.Env)
194 194
 
195
-	for key, val := range b.options.BuildArgs {
196
-		if !b.isBuildArgAllowed(key) {
197
-			// skip build-args that are not in allowed list, meaning they have
198
-			// not been defined by an "ARG" Dockerfile command yet.
199
-			// This is an error condition but only if there is no "ARG" in the entire
200
-			// Dockerfile, so we'll generate any necessary errors after we parsed
201
-			// the entire file (see 'leftoverArgs' processing in evaluator.go )
202
-			continue
203
-		}
204
-		if _, ok := configEnv[key]; !ok && val != nil {
205
-			envs = append(envs, fmt.Sprintf("%s=%s", key, *val))
195
+	for key, val := range b.getBuildArgs() {
196
+		if _, ok := configEnv[key]; !ok {
197
+			envs = append(envs, fmt.Sprintf("%s=%s", key, val))
206 198
 		}
207 199
 	}
208 200
 	return envs
209 201
new file mode 100644
... ...
@@ -0,0 +1,121 @@
0
+package dockerfile
1
+
2
+import (
3
+	"sync"
4
+
5
+	"github.com/Sirupsen/logrus"
6
+	"github.com/docker/docker/builder"
7
+	"github.com/docker/docker/builder/remotecontext"
8
+	"github.com/pkg/errors"
9
+)
10
+
11
+// imageContexts is a helper for stacking up built image rootfs and reusing
12
+// them as contexts
13
+type imageContexts struct {
14
+	b     *Builder
15
+	list  []*imageMount
16
+	cache *pathCache
17
+}
18
+
19
+type imageMount struct {
20
+	id      string
21
+	ctx     builder.Context
22
+	release func() error
23
+}
24
+
25
+func (ic *imageContexts) new() {
26
+	ic.list = append(ic.list, &imageMount{})
27
+}
28
+
29
+func (ic *imageContexts) update(imageID string) {
30
+	ic.list[len(ic.list)-1].id = imageID
31
+}
32
+
33
+func (ic *imageContexts) validate(i int) error {
34
+	if i < 0 || i >= len(ic.list)-1 {
35
+		var extraMsg string
36
+		if i == len(ic.list)-1 {
37
+			extraMsg = " refers current build block"
38
+		}
39
+		return errors.Errorf("invalid from flag value %d%s", i, extraMsg)
40
+	}
41
+	return nil
42
+}
43
+
44
+func (ic *imageContexts) context(i int) (builder.Context, error) {
45
+	if err := ic.validate(i); err != nil {
46
+		return nil, err
47
+	}
48
+	im := ic.list[i]
49
+	if im.ctx == nil {
50
+		if im.id == "" {
51
+			return nil, errors.Errorf("could not copy from empty context")
52
+		}
53
+		p, release, err := ic.b.docker.MountImage(im.id)
54
+		if err != nil {
55
+			return nil, errors.Wrapf(err, "failed to mount %s", im.id)
56
+		}
57
+		ctx, err := remotecontext.NewLazyContext(p)
58
+		if err != nil {
59
+			return nil, errors.Wrapf(err, "failed to create lazycontext for %s", p)
60
+		}
61
+		logrus.Debugf("mounted image: %s %s", im.id, p)
62
+		im.release = release
63
+		im.ctx = ctx
64
+	}
65
+	return im.ctx, nil
66
+}
67
+
68
+func (ic *imageContexts) unmount() (retErr error) {
69
+	for _, im := range ic.list {
70
+		if im.release != nil {
71
+			if err := im.release(); err != nil {
72
+				logrus.Error(errors.Wrapf(err, "failed to unmount previous build image"))
73
+				retErr = err
74
+			}
75
+		}
76
+	}
77
+	return
78
+}
79
+
80
+func (ic *imageContexts) getCache(i int, path string) (interface{}, bool) {
81
+	if ic.cache != nil {
82
+		im := ic.list[i]
83
+		if im.id == "" {
84
+			return nil, false
85
+		}
86
+		return ic.cache.get(im.id + path)
87
+	}
88
+	return nil, false
89
+}
90
+
91
+func (ic *imageContexts) setCache(i int, path string, v interface{}) {
92
+	if ic.cache != nil {
93
+		ic.cache.set(ic.list[i].id+path, v)
94
+	}
95
+}
96
+
97
+type pathCache struct {
98
+	mu    sync.Mutex
99
+	items map[string]interface{}
100
+}
101
+
102
+func (c *pathCache) set(k string, v interface{}) {
103
+	c.mu.Lock()
104
+	if c.items == nil {
105
+		c.items = make(map[string]interface{})
106
+	}
107
+	c.items[k] = v
108
+	c.mu.Unlock()
109
+}
110
+
111
+func (c *pathCache) get(k string) (interface{}, bool) {
112
+	c.mu.Lock()
113
+	if c.items == nil {
114
+		c.mu.Unlock()
115
+		return nil, false
116
+	}
117
+	v, ok := c.items[k]
118
+	c.mu.Unlock()
119
+	return v, ok
120
+}
... ...
@@ -6,7 +6,6 @@ package dockerfile
6 6
 import (
7 7
 	"crypto/sha256"
8 8
 	"encoding/hex"
9
-	"errors"
10 9
 	"fmt"
11 10
 	"io"
12 11
 	"io/ioutil"
... ...
@@ -14,6 +13,8 @@ import (
14 14
 	"net/url"
15 15
 	"os"
16 16
 	"path/filepath"
17
+	"regexp"
18
+	"runtime"
17 19
 	"sort"
18 20
 	"strings"
19 21
 	"time"
... ...
@@ -36,6 +37,7 @@ import (
36 36
 	"github.com/docker/docker/pkg/tarsum"
37 37
 	"github.com/docker/docker/pkg/urlutil"
38 38
 	"github.com/docker/docker/runconfig/opts"
39
+	"github.com/pkg/errors"
39 40
 )
40 41
 
41 42
 func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) error {
... ...
@@ -83,6 +85,7 @@ func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) e
83 83
 	}
84 84
 
85 85
 	b.image = imageID
86
+	b.imageContexts.update(imageID)
86 87
 	return nil
87 88
 }
88 89
 
... ...
@@ -91,11 +94,7 @@ type copyInfo struct {
91 91
 	decompress bool
92 92
 }
93 93
 
94
-func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalDecompression bool, cmdName string) error {
95
-	if b.context == nil {
96
-		return fmt.Errorf("No context given. Impossible to use %s", cmdName)
97
-	}
98
-
94
+func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalDecompression bool, cmdName string, contextID *int) error {
99 95
 	if len(args) < 2 {
100 96
 		return fmt.Errorf("Invalid %s format - at least two arguments required", cmdName)
101 97
 	}
... ...
@@ -129,7 +128,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD
129 129
 			continue
130 130
 		}
131 131
 		// not a URL
132
-		subInfos, err := b.calcCopyInfo(cmdName, orig, allowLocalDecompression, true)
132
+		subInfos, err := b.calcCopyInfo(cmdName, orig, allowLocalDecompression, true, contextID)
133 133
 		if err != nil {
134 134
 			return err
135 135
 		}
... ...
@@ -299,20 +298,41 @@ func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) {
299 299
 	return &builder.HashedFileInfo{FileInfo: builder.PathFileInfo{FileInfo: tmpFileSt, FilePath: tmpFileName}, FileHash: hash}, nil
300 300
 }
301 301
 
302
-func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression, allowWildcards bool) ([]copyInfo, error) {
302
+func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression, allowWildcards bool, contextID *int) ([]copyInfo, error) {
303 303
 
304 304
 	// Work in daemon-specific OS filepath semantics
305 305
 	origPath = filepath.FromSlash(origPath)
306 306
 
307
+	// validate windows paths from other images
308
+	if contextID != nil && runtime.GOOS == "windows" {
309
+		forbid := regexp.MustCompile("(?i)^" + string(os.PathSeparator) + "?(windows(" + string(os.PathSeparator) + ".+)?)?$")
310
+		if p := filepath.Clean(origPath); p == "." || forbid.MatchString(p) {
311
+			return nil, errors.Errorf("copy from %s is not allowed on windows", origPath)
312
+		}
313
+	}
314
+
307 315
 	if origPath != "" && origPath[0] == os.PathSeparator && len(origPath) > 1 {
308 316
 		origPath = origPath[1:]
309 317
 	}
310 318
 	origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator))
311 319
 
320
+	context := b.context
321
+	var err error
322
+	if contextID != nil {
323
+		context, err = b.imageContexts.context(*contextID)
324
+		if err != nil {
325
+			return nil, err
326
+		}
327
+	}
328
+
329
+	if context == nil {
330
+		return nil, errors.Errorf("No context given. Impossible to use %s", cmdName)
331
+	}
332
+
312 333
 	// Deal with wildcards
313 334
 	if allowWildcards && containsWildcards(origPath) {
314 335
 		var copyInfos []copyInfo
315
-		if err := b.context.Walk("", func(path string, info builder.FileInfo, err error) error {
336
+		if err := context.Walk("", func(path string, info builder.FileInfo, err error) error {
316 337
 			if err != nil {
317 338
 				return err
318 339
 			}
... ...
@@ -326,7 +346,7 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
326 326
 
327 327
 			// Note we set allowWildcards to false in case the name has
328 328
 			// a * in it
329
-			subInfos, err := b.calcCopyInfo(cmdName, path, allowLocalDecompression, false)
329
+			subInfos, err := b.calcCopyInfo(cmdName, path, allowLocalDecompression, false, contextID)
330 330
 			if err != nil {
331 331
 				return err
332 332
 			}
... ...
@@ -339,8 +359,7 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
339 339
 	}
340 340
 
341 341
 	// Must be a dir or a file
342
-
343
-	statPath, fi, err := b.context.Stat(origPath)
342
+	statPath, fi, err := context.Stat(origPath)
344 343
 	if err != nil {
345 344
 		return nil, err
346 345
 	}
... ...
@@ -351,6 +370,13 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
351 351
 	if !handleHash {
352 352
 		return copyInfos, nil
353 353
 	}
354
+	if contextID != nil {
355
+		// fast-cache based on imageID
356
+		if h, ok := b.imageContexts.getCache(*contextID, origPath); ok {
357
+			hfi.SetHash(h.(string))
358
+			return copyInfos, nil
359
+		}
360
+	}
354 361
 
355 362
 	// Deal with the single file case
356 363
 	if !fi.IsDir() {
... ...
@@ -359,7 +385,7 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
359 359
 	}
360 360
 	// Must be a dir
361 361
 	var subfiles []string
362
-	err = b.context.Walk(statPath, func(path string, info builder.FileInfo, err error) error {
362
+	err = context.Walk(statPath, func(path string, info builder.FileInfo, err error) error {
363 363
 		if err != nil {
364 364
 			return err
365 365
 		}
... ...
@@ -375,6 +401,9 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
375 375
 	hasher := sha256.New()
376 376
 	hasher.Write([]byte(strings.Join(subfiles, ",")))
377 377
 	hfi.SetHash("dir:" + hex.EncodeToString(hasher.Sum(nil)))
378
+	if contextID != nil {
379
+		b.imageContexts.setCache(*contextID, origPath, hfi.Hash())
380
+	}
378 381
 
379 382
 	return copyInfos, nil
380 383
 }
... ...
@@ -468,6 +497,7 @@ func (b *Builder) probeCache() (bool, error) {
468 468
 	fmt.Fprint(b.Stdout, " ---> Using cache\n")
469 469
 	logrus.Debugf("[BUILDER] Use cached version: %s", b.runConfig.Cmd)
470 470
 	b.image = string(cache)
471
+	b.imageContexts.update(b.image)
471 472
 
472 473
 	return true, nil
473 474
 }
... ...
@@ -668,14 +698,35 @@ func (b *Builder) parseDockerfile() error {
668 668
 	return nil
669 669
 }
670 670
 
671
-// determine if build arg is part of built-in args or user
672
-// defined args in Dockerfile at any point in time.
673
-func (b *Builder) isBuildArgAllowed(arg string) bool {
674
-	if _, ok := BuiltinAllowedBuildArgs[arg]; ok {
675
-		return true
671
+func (b *Builder) getBuildArg(arg string) (string, bool) {
672
+	defaultValue, defined := b.allowedBuildArgs[arg]
673
+	_, builtin := BuiltinAllowedBuildArgs[arg]
674
+	if defined || builtin {
675
+		if v, ok := b.options.BuildArgs[arg]; ok && v != nil {
676
+			return *v, ok
677
+		}
678
+	}
679
+	if defaultValue == nil {
680
+		return "", false
681
+	}
682
+	return *defaultValue, defined
683
+}
684
+
685
+func (b *Builder) getBuildArgs() map[string]string {
686
+	m := make(map[string]string)
687
+	for arg := range b.options.BuildArgs {
688
+		v, ok := b.getBuildArg(arg)
689
+		if ok {
690
+			m[arg] = v
691
+		}
676 692
 	}
677
-	if _, ok := b.allowedBuildArgs[arg]; ok {
678
-		return true
693
+	for arg := range b.allowedBuildArgs {
694
+		if _, ok := m[arg]; !ok {
695
+			v, ok := b.getBuildArg(arg)
696
+			if ok {
697
+				m[arg] = v
698
+			}
699
+		}
679 700
 	}
680
-	return false
701
+	return m
681 702
 }
682 703
new file mode 100644
... ...
@@ -0,0 +1,34 @@
0
+package remotecontext
1
+
2
+import (
3
+	"archive/tar"
4
+	"crypto/sha256"
5
+	"hash"
6
+	"os"
7
+
8
+	"github.com/docker/docker/pkg/archive"
9
+	"github.com/docker/docker/pkg/tarsum"
10
+)
11
+
12
+// NewFileHash returns new hash that is used for the builder cache keys
13
+func NewFileHash(path, name string, fi os.FileInfo) (hash.Hash, error) {
14
+	hdr, err := archive.FileInfoHeader(path, name, fi)
15
+	if err != nil {
16
+		return nil, err
17
+	}
18
+	tsh := &tarsumHash{hdr: hdr, Hash: sha256.New()}
19
+	tsh.Reset() // initialize header
20
+	return tsh, nil
21
+}
22
+
23
+type tarsumHash struct {
24
+	hash.Hash
25
+	hdr *tar.Header
26
+}
27
+
28
+// Reset resets the Hash to its initial state.
29
+func (tsh *tarsumHash) Reset() {
30
+	// comply with hash.Hash and reset to the state hash had before any writes
31
+	tsh.Hash.Reset()
32
+	tarsum.WriteV1Header(tsh.hdr, tsh.Hash)
33
+}
0 34
new file mode 100644
... ...
@@ -0,0 +1,166 @@
0
+package remotecontext
1
+
2
+import (
3
+	"encoding/hex"
4
+	"io"
5
+	"os"
6
+	"path/filepath"
7
+	"runtime"
8
+	"strings"
9
+
10
+	"github.com/docker/docker/builder"
11
+	"github.com/docker/docker/pkg/pools"
12
+	"github.com/docker/docker/pkg/symlink"
13
+	"github.com/pkg/errors"
14
+)
15
+
16
+// NewLazyContext creates a new LazyContext. LazyContext defines a hashed build
17
+// context based on a root directory. Individual files are hashed first time
18
+// they are asked. It is not safe to call methods of LazyContext concurrently.
19
+func NewLazyContext(root string) (builder.Context, error) {
20
+	return &lazyContext{
21
+		root: root,
22
+		sums: make(map[string]string),
23
+	}, nil
24
+}
25
+
26
+type lazyContext struct {
27
+	root string
28
+	sums map[string]string
29
+}
30
+
31
+func (c *lazyContext) Close() error {
32
+	return nil
33
+}
34
+
35
+func (c *lazyContext) Open(path string) (io.ReadCloser, error) {
36
+	cleanPath, fullPath, err := c.normalize(path)
37
+	if err != nil {
38
+		return nil, err
39
+	}
40
+
41
+	r, err := os.Open(fullPath)
42
+	if err != nil {
43
+		return nil, errors.WithStack(convertPathError(err, cleanPath))
44
+	}
45
+	return r, nil
46
+}
47
+
48
+func (c *lazyContext) Stat(path string) (string, builder.FileInfo, error) {
49
+	// TODO: although stat returns builder.FileInfo it builder.Context actually requires Hashed
50
+	cleanPath, fullPath, err := c.normalize(path)
51
+	if err != nil {
52
+		return "", nil, err
53
+	}
54
+
55
+	st, err := os.Lstat(fullPath)
56
+	if err != nil {
57
+		return "", nil, errors.WithStack(convertPathError(err, cleanPath))
58
+	}
59
+
60
+	relPath, err := rel(c.root, fullPath)
61
+	if err != nil {
62
+		return "", nil, errors.WithStack(convertPathError(err, cleanPath))
63
+	}
64
+
65
+	sum, ok := c.sums[relPath]
66
+	if !ok {
67
+		sum, err = c.prepareHash(relPath, st)
68
+		if err != nil {
69
+			return "", nil, err
70
+		}
71
+	}
72
+
73
+	fi := &builder.HashedFileInfo{
74
+		builder.PathFileInfo{st, fullPath, filepath.Base(cleanPath)},
75
+		sum,
76
+	}
77
+	return relPath, fi, nil
78
+}
79
+
80
+func (c *lazyContext) Walk(root string, walkFn builder.WalkFunc) error {
81
+	_, fullPath, err := c.normalize(root)
82
+	if err != nil {
83
+		return err
84
+	}
85
+	return filepath.Walk(fullPath, func(fullPath string, fi os.FileInfo, err error) error {
86
+		relPath, err := rel(c.root, fullPath)
87
+		if err != nil {
88
+			return errors.WithStack(err)
89
+		}
90
+		if relPath == "." {
91
+			return nil
92
+		}
93
+
94
+		sum, ok := c.sums[relPath]
95
+		if !ok {
96
+			sum, err = c.prepareHash(relPath, fi)
97
+			if err != nil {
98
+				return err
99
+			}
100
+		}
101
+
102
+		hfi := &builder.HashedFileInfo{
103
+			builder.PathFileInfo{FileInfo: fi, FilePath: fullPath},
104
+			sum,
105
+		}
106
+		if err := walkFn(relPath, hfi, nil); err != nil {
107
+			return err
108
+		}
109
+		return nil
110
+	})
111
+}
112
+
113
+func (c *lazyContext) prepareHash(relPath string, fi os.FileInfo) (string, error) {
114
+	p := filepath.Join(c.root, relPath)
115
+	h, err := NewFileHash(p, relPath, fi)
116
+	if err != nil {
117
+		return "", errors.Wrapf(err, "failed to create hash for %s", relPath)
118
+	}
119
+	if fi.Mode().IsRegular() && fi.Size() > 0 {
120
+		f, err := os.Open(p)
121
+		if err != nil {
122
+			return "", errors.Wrapf(err, "failed to open %s", relPath)
123
+		}
124
+		defer f.Close()
125
+		if _, err := pools.Copy(h, f); err != nil {
126
+			return "", errors.Wrapf(err, "failed to copy file data for %s", relPath)
127
+		}
128
+	}
129
+	sum := hex.EncodeToString(h.Sum(nil))
130
+	c.sums[relPath] = sum
131
+	return sum, nil
132
+}
133
+
134
+func (c *lazyContext) normalize(path string) (cleanPath, fullPath string, err error) {
135
+	// todo: combine these helpers with tarsum after they are moved to same package
136
+	cleanPath = filepath.Clean(string(os.PathSeparator) + path)[1:]
137
+	fullPath, err = symlink.FollowSymlinkInScope(filepath.Join(c.root, path), c.root)
138
+	if err != nil {
139
+		return "", "", errors.Wrapf(err, "forbidden path outside the build context: %s (%s)", path, fullPath)
140
+	}
141
+	return
142
+}
143
+
144
+func convertPathError(err error, cleanpath string) error {
145
+	if err, ok := err.(*os.PathError); ok {
146
+		err.Path = cleanpath
147
+		return err
148
+	}
149
+	return err
150
+}
151
+
152
+func rel(basepath, targpath string) (string, error) {
153
+	// filepath.Rel can't handle UUID paths in windows
154
+	if runtime.GOOS == "windows" {
155
+		pfx := basepath + `\`
156
+		if strings.HasPrefix(targpath, pfx) {
157
+			p := strings.TrimPrefix(targpath, pfx)
158
+			if p == "" {
159
+				p = "."
160
+			}
161
+			return p, nil
162
+		}
163
+	}
164
+	return filepath.Rel(basepath, targpath)
165
+}
... ...
@@ -1,7 +1,6 @@
1 1
 package daemon
2 2
 
3 3
 import (
4
-	"errors"
5 4
 	"io"
6 5
 	"os"
7 6
 	"path/filepath"
... ...
@@ -10,11 +9,14 @@ import (
10 10
 	"github.com/docker/docker/api/types"
11 11
 	"github.com/docker/docker/builder"
12 12
 	"github.com/docker/docker/container"
13
+	"github.com/docker/docker/layer"
13 14
 	"github.com/docker/docker/pkg/archive"
14 15
 	"github.com/docker/docker/pkg/chrootarchive"
15 16
 	"github.com/docker/docker/pkg/idtools"
16 17
 	"github.com/docker/docker/pkg/ioutils"
18
+	"github.com/docker/docker/pkg/stringid"
17 19
 	"github.com/docker/docker/pkg/system"
20
+	"github.com/pkg/errors"
18 21
 )
19 22
 
20 23
 // ErrExtractPointNotDirectory is used to convey that the operation to extract
... ...
@@ -454,3 +456,34 @@ func (daemon *Daemon) CopyOnBuild(cID string, destPath string, src builder.FileI
454 454
 
455 455
 	return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists)
456 456
 }
457
+
458
+// MountImage returns mounted path with rootfs of an image.
459
+func (daemon *Daemon) MountImage(name string) (string, func() error, error) {
460
+	img, err := daemon.GetImage(name)
461
+	if err != nil {
462
+		return "", nil, errors.Wrapf(err, "no such image: %s", name)
463
+	}
464
+
465
+	mountID := stringid.GenerateRandomID()
466
+	rwLayer, err := daemon.layerStore.CreateRWLayer(mountID, img.RootFS.ChainID(), nil)
467
+	if err != nil {
468
+		return "", nil, errors.Wrap(err, "failed to create rwlayer")
469
+	}
470
+
471
+	mountPath, err := rwLayer.Mount("")
472
+	if err != nil {
473
+		metadata, releaseErr := daemon.layerStore.ReleaseRWLayer(rwLayer)
474
+		if releaseErr != nil {
475
+			err = errors.Wrapf(err, "failed to release rwlayer: %s", releaseErr.Error())
476
+		}
477
+		layer.LogReleaseMetadata(metadata)
478
+		return "", nil, errors.Wrap(err, "failed to mount rwlayer")
479
+	}
480
+
481
+	return mountPath, func() error {
482
+		rwLayer.Unmount()
483
+		metadata, err := daemon.layerStore.ReleaseRWLayer(rwLayer)
484
+		layer.LogReleaseMetadata(metadata)
485
+		return err
486
+	}, nil
487
+}
... ...
@@ -4773,6 +4773,61 @@ func (s *DockerSuite) TestBuildBuildTimeArgDefintionWithNoEnvInjection(c *check.
4773 4773
 	}
4774 4774
 }
4775 4775
 
4776
+func (s *DockerSuite) TestBuildBuildTimeArgMultipleFrom(c *check.C) {
4777
+	imgName := "multifrombldargtest"
4778
+	dockerfile := `FROM busybox
4779
+    ARG foo=abc
4780
+    LABEL multifromtest=1
4781
+    RUN env > /out
4782
+    FROM busybox
4783
+    ARG bar=def
4784
+    RUN env > /out`
4785
+
4786
+	result := buildImage(imgName, withDockerfile(dockerfile))
4787
+	result.Assert(c, icmd.Success)
4788
+
4789
+	result = icmd.RunCmd(icmd.Cmd{
4790
+		Command: []string{dockerBinary, "images", "-q", "-f", "label=multifromtest=1"},
4791
+	})
4792
+	result.Assert(c, icmd.Success)
4793
+	parentID := strings.TrimSpace(result.Stdout())
4794
+
4795
+	result = icmd.RunCmd(icmd.Cmd{
4796
+		Command: []string{dockerBinary, "run", "--rm", parentID, "cat", "/out"},
4797
+	})
4798
+	result.Assert(c, icmd.Success)
4799
+	c.Assert(result.Stdout(), checker.Contains, "foo=abc")
4800
+
4801
+	result = icmd.RunCmd(icmd.Cmd{
4802
+		Command: []string{dockerBinary, "run", "--rm", imgName, "cat", "/out"},
4803
+	})
4804
+	result.Assert(c, icmd.Success)
4805
+	c.Assert(result.Stdout(), checker.Not(checker.Contains), "foo")
4806
+	c.Assert(result.Stdout(), checker.Contains, "bar=def")
4807
+}
4808
+
4809
+func (s *DockerSuite) TestBuildBuildTimeUnusedArgMultipleFrom(c *check.C) {
4810
+	imgName := "multifromunusedarg"
4811
+	dockerfile := `FROM busybox
4812
+    ARG foo
4813
+    FROM busybox
4814
+    ARG bar
4815
+    RUN env > /out`
4816
+
4817
+	result := buildImage(imgName, withDockerfile(dockerfile), withBuildFlags(
4818
+		"--build-arg", fmt.Sprintf("baz=abc")))
4819
+	result.Assert(c, icmd.Success)
4820
+	c.Assert(result.Combined(), checker.Contains, "[Warning]")
4821
+	c.Assert(result.Combined(), checker.Contains, "[baz] were not consumed")
4822
+
4823
+	result = icmd.RunCmd(icmd.Cmd{
4824
+		Command: []string{dockerBinary, "run", "--rm", imgName, "cat", "/out"},
4825
+	})
4826
+	result.Assert(c, icmd.Success)
4827
+	c.Assert(result.Stdout(), checker.Not(checker.Contains), "bar")
4828
+	c.Assert(result.Stdout(), checker.Not(checker.Contains), "baz")
4829
+}
4830
+
4776 4831
 func (s *DockerSuite) TestBuildNoNamedVolume(c *check.C) {
4777 4832
 	volName := "testname:/foo"
4778 4833
 
... ...
@@ -5572,6 +5627,30 @@ func (s *DockerSuite) TestBuildCacheFrom(c *check.C) {
5572 5572
 	c.Assert(layers1[len(layers1)-1], checker.Not(checker.Equals), layers2[len(layers1)-1])
5573 5573
 }
5574 5574
 
5575
+func (s *DockerSuite) TestBuildCacheMultipleFrom(c *check.C) {
5576
+	testRequires(c, DaemonIsLinux) // All tests that do save are skipped in windows
5577
+	dockerfile := `
5578
+		FROM busybox
5579
+		ADD baz /
5580
+		FROM busybox
5581
+    ADD baz /`
5582
+	ctx := fakeContext(c, dockerfile, map[string]string{
5583
+		"Dockerfile": dockerfile,
5584
+		"baz":        "baz",
5585
+	})
5586
+	defer ctx.Close()
5587
+
5588
+	result := buildImage("build1", withExternalBuildContext(ctx))
5589
+	result.Assert(c, icmd.Success)
5590
+	// second part of dockerfile was a repeat of first so should be cached
5591
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 1)
5592
+
5593
+	result = buildImage("build2", withBuildFlags("--cache-from=build1"), withExternalBuildContext(ctx))
5594
+	result.Assert(c, icmd.Success)
5595
+	// now both parts of dockerfile should be cached
5596
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 2)
5597
+}
5598
+
5575 5599
 func (s *DockerSuite) TestBuildNetNone(c *check.C) {
5576 5600
 	testRequires(c, DaemonIsLinux)
5577 5601
 	name := "testbuildnetnone"
... ...
@@ -5709,6 +5788,138 @@ func (s *DockerSuite) TestBuildContChar(c *check.C) {
5709 5709
 	c.Assert(result.Combined(), checker.Contains, "Step 2/2 : RUN echo hi \\\\\n")
5710 5710
 }
5711 5711
 
5712
+func (s *DockerSuite) TestBuildCopyFromPreviousRootFS(c *check.C) {
5713
+	dockerfile := `
5714
+		FROM busybox
5715
+		COPY foo bar
5716
+		FROM busybox
5717
+    %s
5718
+    COPY baz baz
5719
+    RUN echo mno > baz/cc
5720
+		FROM busybox
5721
+    COPY bar /
5722
+    COPY --from=1 baz sub/
5723
+    COPY --from=0 bar baz
5724
+    COPY --from=0 bar bay`
5725
+	ctx := fakeContext(c, fmt.Sprintf(dockerfile, ""), map[string]string{
5726
+		"Dockerfile": dockerfile,
5727
+		"foo":        "abc",
5728
+		"bar":        "def",
5729
+		"baz/aa":     "ghi",
5730
+		"baz/bb":     "jkl",
5731
+	})
5732
+	defer ctx.Close()
5733
+
5734
+	result := buildImage("build1", withExternalBuildContext(ctx))
5735
+	result.Assert(c, icmd.Success)
5736
+
5737
+	out, _ := dockerCmd(c, "run", "build1", "cat", "bar")
5738
+	c.Assert(strings.TrimSpace(out), check.Equals, "def")
5739
+	out, _ = dockerCmd(c, "run", "build1", "cat", "sub/aa")
5740
+	c.Assert(strings.TrimSpace(out), check.Equals, "ghi")
5741
+	out, _ = dockerCmd(c, "run", "build1", "cat", "sub/cc")
5742
+	c.Assert(strings.TrimSpace(out), check.Equals, "mno")
5743
+	out, _ = dockerCmd(c, "run", "build1", "cat", "baz")
5744
+	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5745
+	out, _ = dockerCmd(c, "run", "build1", "cat", "bay")
5746
+	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5747
+
5748
+	result = buildImage("build2", withExternalBuildContext(ctx))
5749
+	result.Assert(c, icmd.Success)
5750
+
5751
+	// all commands should be cached
5752
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 7)
5753
+
5754
+	err := ioutil.WriteFile(filepath.Join(ctx.Dir, "Dockerfile"), []byte(fmt.Sprintf(dockerfile, "COPY baz/aa foo")), 0644)
5755
+	c.Assert(err, checker.IsNil)
5756
+
5757
+	// changing file in parent block should not affect last block
5758
+	result = buildImage("build3", withExternalBuildContext(ctx))
5759
+	result.Assert(c, icmd.Success)
5760
+
5761
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 5)
5762
+
5763
+	c.Assert(getIDByName(c, "build1"), checker.Equals, getIDByName(c, "build2"))
5764
+
5765
+	err = ioutil.WriteFile(filepath.Join(ctx.Dir, "foo"), []byte("pqr"), 0644)
5766
+	c.Assert(err, checker.IsNil)
5767
+
5768
+	// changing file in parent block should affect both first and last block
5769
+	result = buildImage("build4", withExternalBuildContext(ctx))
5770
+	result.Assert(c, icmd.Success)
5771
+	c.Assert(strings.Count(result.Combined(), "Using cache"), checker.Equals, 5)
5772
+
5773
+	out, _ = dockerCmd(c, "run", "build4", "cat", "bay")
5774
+	c.Assert(strings.TrimSpace(out), check.Equals, "pqr")
5775
+	out, _ = dockerCmd(c, "run", "build4", "cat", "baz")
5776
+	c.Assert(strings.TrimSpace(out), check.Equals, "pqr")
5777
+}
5778
+
5779
+func (s *DockerSuite) TestBuildCopyFromPreviousRootFSErrors(c *check.C) {
5780
+	dockerfile := `
5781
+		FROM busybox
5782
+		COPY --from=foo foo bar`
5783
+
5784
+	ctx := fakeContext(c, dockerfile, map[string]string{
5785
+		"Dockerfile": dockerfile,
5786
+		"foo":        "abc",
5787
+	})
5788
+	defer ctx.Close()
5789
+
5790
+	buildImage("build1", withExternalBuildContext(ctx)).Assert(c, icmd.Expected{
5791
+		ExitCode: 1,
5792
+		Err:      "from expects an integer value corresponding to the context number",
5793
+	})
5794
+
5795
+	dockerfile = `
5796
+		FROM busybox
5797
+		COPY --from=0 foo bar`
5798
+
5799
+	ctx = fakeContext(c, dockerfile, map[string]string{
5800
+		"Dockerfile": dockerfile,
5801
+		"foo":        "abc",
5802
+	})
5803
+	defer ctx.Close()
5804
+
5805
+	buildImage("build1", withExternalBuildContext(ctx)).Assert(c, icmd.Expected{
5806
+		ExitCode: 1,
5807
+		Err:      "invalid from flag value 0 refers current build block",
5808
+	})
5809
+}
5810
+
5811
+func (s *DockerSuite) TestBuildCopyFromPreviousFrom(c *check.C) {
5812
+	dockerfile := `
5813
+		FROM busybox
5814
+		COPY foo bar`
5815
+	ctx := fakeContext(c, dockerfile, map[string]string{
5816
+		"Dockerfile": dockerfile,
5817
+		"foo":        "abc",
5818
+	})
5819
+	defer ctx.Close()
5820
+
5821
+	result := buildImage("build1", withExternalBuildContext(ctx))
5822
+	result.Assert(c, icmd.Success)
5823
+
5824
+	dockerfile = `
5825
+		FROM build1:latest
5826
+    FROM busybox
5827
+		COPY --from=0 bar /
5828
+		COPY foo /`
5829
+	ctx = fakeContext(c, dockerfile, map[string]string{
5830
+		"Dockerfile": dockerfile,
5831
+		"foo":        "def",
5832
+	})
5833
+	defer ctx.Close()
5834
+
5835
+	result = buildImage("build2", withExternalBuildContext(ctx))
5836
+	result.Assert(c, icmd.Success)
5837
+
5838
+	out, _ := dockerCmd(c, "run", "build2", "cat", "bar")
5839
+	c.Assert(strings.TrimSpace(out), check.Equals, "abc")
5840
+	out, _ = dockerCmd(c, "run", "build2", "cat", "foo")
5841
+	c.Assert(strings.TrimSpace(out), check.Equals, "def")
5842
+}
5843
+
5712 5844
 // TestBuildOpaqueDirectory tests that a build succeeds which
5713 5845
 // creates opaque directories.
5714 5846
 // See https://github.com/docker/docker/issues/25244
... ...
@@ -240,6 +240,38 @@ func (compression *Compression) Extension() string {
240 240
 	return ""
241 241
 }
242 242
 
243
+// FileInfoHeader creates a populated Header from fi.
244
+// Compared to archive pkg this function fills in more information.
245
+func FileInfoHeader(path, name string, fi os.FileInfo) (*tar.Header, error) {
246
+	var link string
247
+	if fi.Mode()&os.ModeSymlink != 0 {
248
+		var err error
249
+		link, err = os.Readlink(path)
250
+		if err != nil {
251
+			return nil, err
252
+		}
253
+	}
254
+	hdr, err := tar.FileInfoHeader(fi, link)
255
+	if err != nil {
256
+		return nil, err
257
+	}
258
+	hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
259
+	name, err = canonicalTarName(name, fi.IsDir())
260
+	if err != nil {
261
+		return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err)
262
+	}
263
+	hdr.Name = name
264
+	if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil {
265
+		return nil, err
266
+	}
267
+	capability, _ := system.Lgetxattr(path, "security.capability")
268
+	if capability != nil {
269
+		hdr.Xattrs = make(map[string]string)
270
+		hdr.Xattrs["security.capability"] = string(capability)
271
+	}
272
+	return hdr, nil
273
+}
274
+
243 275
 type tarWhiteoutConverter interface {
244 276
 	ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error)
245 277
 	ConvertRead(*tar.Header, string) (bool, error)
... ...
@@ -283,26 +315,7 @@ func (ta *tarAppender) addTarFile(path, name string) error {
283 283
 		return err
284 284
 	}
285 285
 
286
-	link := ""
287
-	if fi.Mode()&os.ModeSymlink != 0 {
288
-		if link, err = os.Readlink(path); err != nil {
289
-			return err
290
-		}
291
-	}
292
-
293
-	hdr, err := tar.FileInfoHeader(fi, link)
294
-	if err != nil {
295
-		return err
296
-	}
297
-	hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
298
-
299
-	name, err = canonicalTarName(name, fi.IsDir())
300
-	if err != nil {
301
-		return fmt.Errorf("tar: cannot canonicalize path: %v", err)
302
-	}
303
-	hdr.Name = name
304
-
305
-	inode, err := setHeaderForSpecialDevice(hdr, ta, name, fi.Sys())
286
+	hdr, err := FileInfoHeader(path, name, fi)
306 287
 	if err != nil {
307 288
 		return err
308 289
 	}
... ...
@@ -310,6 +323,10 @@ func (ta *tarAppender) addTarFile(path, name string) error {
310 310
 	// if it's not a directory and has more than 1 link,
311 311
 	// it's hard linked, so set the type flag accordingly
312 312
 	if !fi.IsDir() && hasHardlinks(fi) {
313
+		inode, err := getInodeFromStat(fi.Sys())
314
+		if err != nil {
315
+			return err
316
+		}
313 317
 		// a link should have a name that it links too
314 318
 		// and that linked name should be first in the tar archive
315 319
 		if oldpath, ok := ta.SeenFiles[inode]; ok {
... ...
@@ -321,12 +338,6 @@ func (ta *tarAppender) addTarFile(path, name string) error {
321 321
 		}
322 322
 	}
323 323
 
324
-	capability, _ := system.Lgetxattr(path, "security.capability")
325
-	if capability != nil {
326
-		hdr.Xattrs = make(map[string]string)
327
-		hdr.Xattrs["security.capability"] = string(capability)
328
-	}
329
-
330 324
 	//handle re-mapping container ID mappings back to host ID mappings before
331 325
 	//writing tar headers/files. We skip whiteout files because they were written
332 326
 	//by the kernel and already have proper ownership relative to the host
... ...
@@ -41,7 +41,7 @@ func chmodTarEntry(perm os.FileMode) os.FileMode {
41 41
 	return perm // noop for unix as golang APIs provide perm bits correctly
42 42
 }
43 43
 
44
-func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, stat interface{}) (inode uint64, err error) {
44
+func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (err error) {
45 45
 	s, ok := stat.(*syscall.Stat_t)
46 46
 
47 47
 	if !ok {
... ...
@@ -49,8 +49,6 @@ func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, st
49 49
 		return
50 50
 	}
51 51
 
52
-	inode = uint64(s.Ino)
53
-
54 52
 	// Currently go does not fill in the major/minors
55 53
 	if s.Mode&syscall.S_IFBLK != 0 ||
56 54
 		s.Mode&syscall.S_IFCHR != 0 {
... ...
@@ -61,6 +59,19 @@ func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, st
61 61
 	return
62 62
 }
63 63
 
64
+func getInodeFromStat(stat interface{}) (inode uint64, err error) {
65
+	s, ok := stat.(*syscall.Stat_t)
66
+
67
+	if !ok {
68
+		err = errors.New("cannot convert stat value to syscall.Stat_t")
69
+		return
70
+	}
71
+
72
+	inode = uint64(s.Ino)
73
+
74
+	return
75
+}
76
+
64 77
 func getFileUIDGID(stat interface{}) (int, int, error) {
65 78
 	s, ok := stat.(*syscall.Stat_t)
66 79
 
... ...
@@ -49,8 +49,13 @@ func chmodTarEntry(perm os.FileMode) os.FileMode {
49 49
 	return perm
50 50
 }
51 51
 
52
-func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, stat interface{}) (inode uint64, err error) {
53
-	// do nothing. no notion of Rdev, Inode, Nlink in stat on Windows
52
+func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (err error) {
53
+	// do nothing. no notion of Rdev, Nlink in stat on Windows
54
+	return
55
+}
56
+
57
+func getInodeFromStat(stat interface{}) (inode uint64, err error) {
58
+	// do nothing. no notion of Inode in stat on Windows
54 59
 	return
55 60
 }
56 61
 
... ...
@@ -3,6 +3,7 @@ package tarsum
3 3
 import (
4 4
 	"archive/tar"
5 5
 	"errors"
6
+	"io"
6 7
 	"sort"
7 8
 	"strconv"
8 9
 	"strings"
... ...
@@ -21,6 +22,13 @@ const (
21 21
 	VersionDev
22 22
 )
23 23
 
24
+// WriteV1Header writes a tar header to a writer in V1 tarsum format.
25
+func WriteV1Header(h *tar.Header, w io.Writer) {
26
+	for _, elem := range v1TarHeaderSelect(h) {
27
+		w.Write([]byte(elem[0] + elem[1]))
28
+	}
29
+}
30
+
24 31
 // VersionLabelForChecksum returns the label for the given tarsum
25 32
 // checksum, i.e., everything before the first `+` character in
26 33
 // the string or an empty string if no label separator is found.