Browse code

vendor: update buildkit to ae7ff174

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2020/04/14 12:31:26
Showing 36 changed files
... ...
@@ -31,13 +31,11 @@ import (
31 31
 	"github.com/moby/buildkit/cache"
32 32
 	"github.com/moby/buildkit/client/llb"
33 33
 	"github.com/moby/buildkit/session"
34
-	"github.com/moby/buildkit/session/auth"
35 34
 	"github.com/moby/buildkit/source"
36 35
 	"github.com/moby/buildkit/util/flightcontrol"
37 36
 	"github.com/moby/buildkit/util/imageutil"
38 37
 	"github.com/moby/buildkit/util/progress"
39 38
 	"github.com/moby/buildkit/util/resolver"
40
-	"github.com/moby/buildkit/util/tracing"
41 39
 	digest "github.com/opencontainers/go-digest"
42 40
 	"github.com/opencontainers/image-spec/identity"
43 41
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
... ...
@@ -54,7 +52,7 @@ type SourceOpt struct {
54 54
 	DownloadManager distribution.RootFSDownloadManager
55 55
 	MetadataStore   metadata.V2MetadataService
56 56
 	ImageStore      image.Store
57
-	ResolverOpt     resolver.ResolveOptionsFunc
57
+	RegistryHosts   docker.RegistryHosts
58 58
 	LayerStore      layer.Store
59 59
 }
60 60
 
... ...
@@ -78,44 +76,15 @@ func (is *imageSource) ID() string {
78 78
 	return source.DockerImageScheme
79 79
 }
80 80
 
81
-func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string, sm *session.Manager) remotes.Resolver {
81
+func (is *imageSource) getResolver(ctx context.Context, hosts docker.RegistryHosts, ref string, sm *session.Manager) remotes.Resolver {
82 82
 	if res := is.resolverCache.Get(ctx, ref); res != nil {
83 83
 		return res
84 84
 	}
85
-
86
-	opt := docker.ResolverOptions{
87
-		Client: tracing.DefaultClient,
88
-	}
89
-	if rfn != nil {
90
-		opt = rfn(ref)
91
-	}
92
-	opt.Credentials = is.getCredentialsFromSession(ctx, sm)
93
-	r := docker.NewResolver(opt)
85
+	r := resolver.New(ctx, hosts, sm)
94 86
 	r = is.resolverCache.Add(ctx, ref, r)
95 87
 	return r
96 88
 }
97 89
 
98
-func (is *imageSource) getCredentialsFromSession(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {
99
-	id := session.FromContext(ctx)
100
-	if id == "" {
101
-		// can be removed after containerd/containerd#2812
102
-		return func(string) (string, string, error) {
103
-			return "", "", nil
104
-		}
105
-	}
106
-	return func(host string) (string, string, error) {
107
-		timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
108
-		defer cancel()
109
-
110
-		caller, err := sm.Get(timeoutCtx, id)
111
-		if err != nil {
112
-			return "", "", err
113
-		}
114
-
115
-		return auth.CredentialsFunc(tracing.ContextWithSpanFromContext(context.TODO(), ctx), caller)(host)
116
-	}
117
-}
118
-
119 90
 func (is *imageSource) resolveLocal(refStr string) (*image.Image, error) {
120 91
 	ref, err := distreference.ParseNormalizedNamed(refStr)
121 92
 	if err != nil {
... ...
@@ -138,7 +107,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
138 138
 		dt   []byte
139 139
 	}
140 140
 	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
141
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, nil, platform)
141
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.RegistryHosts, ref, sm), is.ContentStore, nil, platform)
142 142
 		if err != nil {
143 143
 			return nil, err
144 144
 		}
... ...
@@ -208,7 +177,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier, sm *se
208 208
 	p := &puller{
209 209
 		src:      imageIdentifier,
210 210
 		is:       is,
211
-		resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String(), sm),
211
+		resolver: is.getResolver(ctx, is.RegistryHosts, imageIdentifier.Reference.String(), sm),
212 212
 		platform: platform,
213 213
 		sm:       sm,
214 214
 	}
... ...
@@ -7,6 +7,7 @@ import (
7 7
 
8 8
 	"github.com/containerd/containerd/content"
9 9
 	"github.com/containerd/containerd/images"
10
+	"github.com/containerd/containerd/remotes/docker"
10 11
 	distreference "github.com/docker/distribution/reference"
11 12
 	imagestore "github.com/docker/docker/image"
12 13
 	"github.com/docker/docker/reference"
... ...
@@ -15,7 +16,6 @@ import (
15 15
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
16 16
 	"github.com/moby/buildkit/session"
17 17
 	"github.com/moby/buildkit/solver"
18
-	"github.com/moby/buildkit/util/resolver"
19 18
 	"github.com/moby/buildkit/worker"
20 19
 	digest "github.com/opencontainers/go-digest"
21 20
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
... ...
@@ -23,9 +23,9 @@ import (
23 23
 )
24 24
 
25 25
 // ResolveCacheImporterFunc returns a resolver function for local inline cache
26
-func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
26
+func ResolveCacheImporterFunc(sm *session.Manager, resolverFunc docker.RegistryHosts, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
27 27
 
28
-	upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverOpt)
28
+	upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverFunc)
29 29
 
30 30
 	return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
31 31
 		if dt, err := tryImportLocal(rs, is, attrs["ref"]); err == nil {
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	"time"
12 12
 
13 13
 	"github.com/containerd/containerd/platforms"
14
+	"github.com/containerd/containerd/remotes/docker"
14 15
 	"github.com/docker/docker/api/types"
15 16
 	"github.com/docker/docker/api/types/backend"
16 17
 	"github.com/docker/docker/builder"
... ...
@@ -26,7 +27,6 @@ import (
26 26
 	"github.com/moby/buildkit/identity"
27 27
 	"github.com/moby/buildkit/session"
28 28
 	"github.com/moby/buildkit/util/entitlements"
29
-	"github.com/moby/buildkit/util/resolver"
30 29
 	"github.com/moby/buildkit/util/tracing"
31 30
 	"github.com/pkg/errors"
32 31
 	"golang.org/x/sync/errgroup"
... ...
@@ -70,7 +70,7 @@ type Opt struct {
70 70
 	Dist                images.DistributionServices
71 71
 	NetworkController   libnetwork.NetworkController
72 72
 	DefaultCgroupParent string
73
-	ResolverOpt         resolver.ResolveOptionsFunc
73
+	RegistryHosts       docker.RegistryHosts
74 74
 	BuilderConfig       config.BuilderConfig
75 75
 	Rootless            bool
76 76
 	IdentityMapping     *idtools.IdentityMapping
... ...
@@ -119,7 +119,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
119 119
 		MetadataStore:   dist.V2MetadataService,
120 120
 		ImageStore:      dist.ImageStore,
121 121
 		ReferenceStore:  dist.ReferenceStore,
122
-		ResolverOpt:     opt.ResolverOpt,
122
+		RegistryHosts:   opt.RegistryHosts,
123 123
 		LayerStore:      dist.LayerStore,
124 124
 	})
125 125
 	if err != nil {
... ...
@@ -210,7 +210,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
210 210
 		Frontends:        frontends,
211 211
 		CacheKeyStorage:  cacheStorage,
212 212
 		ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{
213
-			"registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt, store, dist.ReferenceStore, dist.ImageStore),
213
+			"registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.RegistryHosts, store, dist.ReferenceStore, dist.ImageStore),
214 214
 			"local":    localremotecache.ResolveCacheImporterFunc(opt.SessionManager),
215 215
 		},
216 216
 		ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
... ...
@@ -292,7 +292,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e
292 292
 		Dist:                d.DistributionServices(),
293 293
 		NetworkController:   d.NetworkController(),
294 294
 		DefaultCgroupParent: cgroupParent,
295
-		ResolverOpt:         d.NewResolveOptionsFunc(),
295
+		RegistryHosts:       d.RegistryHosts(),
296 296
 		BuilderConfig:       config.Builder,
297 297
 		Rootless:            d.Rootless(),
298 298
 		IdentityMapping:     d.IdentityMapping(),
... ...
@@ -9,7 +9,6 @@ import (
9 9
 	"context"
10 10
 	"fmt"
11 11
 	"io/ioutil"
12
-	"math/rand"
13 12
 	"net"
14 13
 	"net/url"
15 14
 	"os"
... ...
@@ -28,7 +27,6 @@ import (
28 28
 	"github.com/containerd/containerd/defaults"
29 29
 	"github.com/containerd/containerd/pkg/dialer"
30 30
 	"github.com/containerd/containerd/remotes/docker"
31
-	"github.com/docker/distribution/reference"
32 31
 	"github.com/docker/docker/api/types"
33 32
 	containertypes "github.com/docker/docker/api/types/container"
34 33
 	"github.com/docker/docker/api/types/swarm"
... ...
@@ -42,8 +40,8 @@ import (
42 42
 	"github.com/docker/docker/daemon/logger"
43 43
 	"github.com/docker/docker/daemon/network"
44 44
 	"github.com/docker/docker/errdefs"
45
+	bkconfig "github.com/moby/buildkit/cmd/buildkitd/config"
45 46
 	"github.com/moby/buildkit/util/resolver"
46
-	"github.com/moby/buildkit/util/tracing"
47 47
 	"github.com/sirupsen/logrus"
48 48
 
49 49
 	// register graph drivers
... ...
@@ -153,63 +151,46 @@ func (daemon *Daemon) Features() *map[string]bool {
153 153
 	return &daemon.configStore.Features
154 154
 }
155 155
 
156
-// NewResolveOptionsFunc returns a call back function to resolve "registry-mirrors" and
157
-// "insecure-registries" for buildkit
158
-func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc {
159
-	return func(ref string) docker.ResolverOptions {
160
-		var (
161
-			registryKey = "docker.io"
162
-			mirrors     = make([]string, len(daemon.configStore.Mirrors))
163
-			m           = map[string]resolver.RegistryConf{}
164
-		)
165
-		// must trim "https://" or "http://" prefix
166
-		for i, v := range daemon.configStore.Mirrors {
167
-			if uri, err := url.Parse(v); err == nil {
168
-				v = uri.Host
169
-			}
170
-			mirrors[i] = v
171
-		}
172
-		// set "registry-mirrors"
173
-		m[registryKey] = resolver.RegistryConf{Mirrors: mirrors}
174
-		// set "insecure-registries"
175
-		for _, v := range daemon.configStore.InsecureRegistries {
176
-			if uri, err := url.Parse(v); err == nil {
177
-				v = uri.Host
178
-			}
179
-			plainHTTP := true
180
-			m[v] = resolver.RegistryConf{
181
-				PlainHTTP: &plainHTTP,
182
-			}
183
-		}
184
-		def := docker.ResolverOptions{
185
-			Client: tracing.DefaultClient,
186
-		}
187
-
188
-		parsed, err := reference.ParseNormalizedNamed(ref)
189
-		if err != nil {
190
-			return def
156
+// RegistryHosts returns registry configuration in containerd resolvers format
157
+func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
158
+	var (
159
+		registryKey = "docker.io"
160
+		mirrors     = make([]string, len(daemon.configStore.Mirrors))
161
+		m           = map[string]bkconfig.RegistryConfig{}
162
+	)
163
+	// must trim "https://" or "http://" prefix
164
+	for i, v := range daemon.configStore.Mirrors {
165
+		if uri, err := url.Parse(v); err == nil {
166
+			v = uri.Host
191 167
 		}
192
-		host := reference.Domain(parsed)
193
-
194
-		c, ok := m[host]
195
-		if !ok {
196
-			return def
197
-		}
198
-
199
-		if len(c.Mirrors) > 0 {
200
-			// TODO ResolverOptions.Host is deprecated; ResolverOptions.Hosts should be used
201
-			def.Host = func(string) (string, error) {
202
-				return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
168
+		mirrors[i] = v
169
+	}
170
+	// set mirrors for default registry
171
+	m[registryKey] = bkconfig.RegistryConfig{Mirrors: mirrors}
172
+
173
+	for _, v := range daemon.configStore.InsecureRegistries {
174
+		u, err := url.Parse(v)
175
+		c := bkconfig.RegistryConfig{}
176
+		if err == nil {
177
+			v = u.Host
178
+			t := true
179
+			if u.Scheme == "http" {
180
+				c.PlainHTTP = &t
181
+			} else {
182
+				c.Insecure = &t
203 183
 			}
204 184
 		}
185
+		m[v] = c
186
+	}
205 187
 
206
-		// TODO ResolverOptions.PlainHTTP is deprecated; ResolverOptions.Hosts should be used
207
-		if c.PlainHTTP != nil {
208
-			def.PlainHTTP = *c.PlainHTTP
188
+	for k, v := range m {
189
+		if d, err := registry.HostCertsDir(k); err == nil {
190
+			v.TLSConfigDir = []string{d}
191
+			m[k] = v
209 192
 		}
210
-
211
-		return def
212 193
 	}
194
+
195
+	return resolver.NewRegistryConfig(m)
213 196
 }
214 197
 
215 198
 func (daemon *Daemon) restore() error {
... ...
@@ -26,6 +26,24 @@ var (
26 26
 	ErrAlreadyExists = errors.New("Image already exists")
27 27
 )
28 28
 
29
+// HostCertsDir returns the config directory for a specific host
30
+func HostCertsDir(hostname string) (string, error) {
31
+	certsDir := CertsDir
32
+
33
+	if rootless.RunningWithRootlessKit() {
34
+		configHome, err := homedir.GetConfigHome()
35
+		if err != nil {
36
+			return "", err
37
+		}
38
+
39
+		certsDir = filepath.Join(configHome, "docker/certs.d")
40
+	}
41
+
42
+	hostDir := filepath.Join(certsDir, cleanPath(hostname))
43
+
44
+	return hostDir, nil
45
+}
46
+
29 47
 func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
30 48
 	// PreferredServerCipherSuites should have no effect
31 49
 	tlsConfig := tlsconfig.ServerDefault()
... ...
@@ -33,19 +51,11 @@ func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
33 33
 	tlsConfig.InsecureSkipVerify = !isSecure
34 34
 
35 35
 	if isSecure && CertsDir != "" {
36
-		certsDir := CertsDir
37
-
38
-		if rootless.RunningWithRootlessKit() {
39
-			configHome, err := homedir.GetConfigHome()
40
-			if err != nil {
41
-				return nil, err
42
-			}
43
-
44
-			certsDir = filepath.Join(configHome, "docker/certs.d")
36
+		hostDir, err := HostCertsDir(hostname)
37
+		if err != nil {
38
+			return nil, err
45 39
 		}
46 40
 
47
-		hostDir := filepath.Join(certsDir, cleanPath(hostname))
48
-
49 41
 		logrus.Debugf("hostDir: %s", hostDir)
50 42
 		if err := ReadCertsDirectory(tlsConfig, hostDir); err != nil {
51 43
 			return nil, err
... ...
@@ -27,8 +27,8 @@ github.com/imdario/mergo                            1afb36080aec31e0d1528973ebe6
27 27
 golang.org/x/sync                                   cd5d95a43a6e21273425c7ae415d3df9ea832eeb
28 28
 
29 29
 # buildkit
30
-github.com/moby/buildkit                            4d8d91bf49c769b8458e9aa84746c842b4a0e39a
31
-github.com/tonistiigi/fsutil                        013a9fe6aee2d1658457075bf9e688bc8c0be2e0
30
+github.com/moby/buildkit                            ae7ff7174f73bcb4df89b97e1623b3fb0bfb0a0c
31
+github.com/tonistiigi/fsutil                        c2c7d7b0e1441705cd802e5699c0a10b1dfe39fd
32 32
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
33 33
 github.com/opentracing/opentracing-go               1361b9cd60be79c4c3a7fa9841b3c132e40066a7
34 34
 github.com/google/shlex                             e7afc7fbc51079733e9468cdfd1efcd7d196cd1d
... ...
@@ -128,7 +128,7 @@ See [Expose BuildKit as a TCP service](#expose-buildkit-as-a-tcp-service).
128 128
 
129 129
 :information_source: Notice to Fedora 31 users:
130 130
 
131
-* As runc still does not work on cgroup v2 environment like Fedora 31, you need to substitute runc with crun. Run `rm -f $(which buildkit-runc) && ln -s $(which crun) /usr/local/bin/buildkit-runc` .
131
+* As runc still does not work on cgroup v2 environment like Fedora 31, you need to substitute runc with crun. Run `buildkitd` with `--oci-worker-binary=crun`.
132 132
 * If you want to use runc, you need to configure the system to use cgroup v1. Run `sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=0"` and reboot.
133 133
 
134 134
 ### Exploring LLB
... ...
@@ -205,8 +205,8 @@ By default, the build result and intermediate cache will only remain internally
205 205
 buildctl build ... --output type=image,name=docker.io/username/image,push=true
206 206
 ```
207 207
 
208
-To export and import the cache along with the image, you need to specify `--export-cache type=inline` and `--import-cache type=registry,ref=...`.
209
-See [Export cache](#export-cache).
208
+To export the cache embed with the image and pushing them to registry together, type `registry` is required to import the cache, you should specify `--export-cache type=inline` and `--import-cache type=registry,ref=...`. To export the cache to a local directy, you should specify `--export-cache type=local`.
209
+Details in [Export cache](#export-cache).
210 210
 
211 211
 ```bash
212 212
 buildctl build ...\
... ...
@@ -357,7 +357,8 @@ The directory layout conforms to OCI Image Spec v1.0.
357 357
 -   `ref=docker.io/user/image:tag`: reference for `registry` cache importer
358 358
 -   `src=path/to/input-dir`: directory for `local` cache importer
359 359
 -   `digest=sha256:deadbeef`: digest of the manifest list to import for `local` cache importer.
360
-    Defaults to the digest of "latest" tag in `index.json`
360
+-   `tag=customtag`: custom tag of image for `local` cache importer.
361
+    Defaults to the digest of "latest" tag in `index.json` is for digest, not for tag
361 362
 
362 363
 ### Consistent hashing
363 364
 
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	"io"
6 6
 	"sort"
7 7
 	"strconv"
8
+	"strings"
8 9
 )
9 10
 
10 11
 // WriteV1TarsumHeaders writes a tar header to a writer in V1 tarsum format.
... ...
@@ -38,7 +39,9 @@ func v1TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
38 38
 	// Get extended attributes.
39 39
 	xAttrKeys := make([]string, len(h.Xattrs))
40 40
 	for k := range h.Xattrs {
41
-		xAttrKeys = append(xAttrKeys, k)
41
+		if !strings.HasPrefix(k, "security.") && !strings.HasPrefix(k, "system.") {
42
+			xAttrKeys = append(xAttrKeys, k)
43
+		}
42 44
 	}
43 45
 	sort.Strings(xAttrKeys)
44 46
 
... ...
@@ -2,15 +2,12 @@ package registry
2 2
 
3 3
 import (
4 4
 	"context"
5
-	"time"
6 5
 
7 6
 	"github.com/containerd/containerd/content"
8
-	"github.com/containerd/containerd/remotes"
9 7
 	"github.com/containerd/containerd/remotes/docker"
10 8
 	"github.com/docker/distribution/reference"
11 9
 	"github.com/moby/buildkit/cache/remotecache"
12 10
 	"github.com/moby/buildkit/session"
13
-	"github.com/moby/buildkit/session/auth"
14 11
 	"github.com/moby/buildkit/util/contentutil"
15 12
 	"github.com/moby/buildkit/util/resolver"
16 13
 	"github.com/opencontainers/go-digest"
... ...
@@ -34,13 +31,13 @@ const (
34 34
 	attrRef = "ref"
35 35
 )
36 36
 
37
-func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheExporterFunc {
37
+func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) remotecache.ResolveCacheExporterFunc {
38 38
 	return func(ctx context.Context, attrs map[string]string) (remotecache.Exporter, error) {
39 39
 		ref, err := canonicalizeRef(attrs[attrRef])
40 40
 		if err != nil {
41 41
 			return nil, err
42 42
 		}
43
-		remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
43
+		remote := resolver.New(ctx, hosts, sm)
44 44
 		pusher, err := remote.Pusher(ctx, ref)
45 45
 		if err != nil {
46 46
 			return nil, err
... ...
@@ -49,13 +46,13 @@ func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveO
49 49
 	}
50 50
 }
51 51
 
52
-func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
52
+func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, hosts docker.RegistryHosts) remotecache.ResolveCacheImporterFunc {
53 53
 	return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
54 54
 		ref, err := canonicalizeRef(attrs[attrRef])
55 55
 		if err != nil {
56 56
 			return nil, specs.Descriptor{}, err
57 57
 		}
58
-		remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
58
+		remote := resolver.New(ctx, hosts, sm)
59 59
 		xref, desc, err := remote.Resolve(ctx, ref)
60 60
 		if err != nil {
61 61
 			return nil, specs.Descriptor{}, err
... ...
@@ -97,27 +94,3 @@ func (dsl *withDistributionSourceLabel) SetDistributionSourceAnnotation(desc oci
97 97
 	desc.Annotations["containerd.io/distribution.source.ref"] = dsl.ref
98 98
 	return desc
99 99
 }
100
-
101
-func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver {
102
-	opt := resolverOpt(ref)
103
-	opt.Credentials = getCredentialsFunc(ctx, sm)
104
-	return docker.NewResolver(opt)
105
-}
106
-
107
-func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {
108
-	id := session.FromContext(ctx)
109
-	if id == "" {
110
-		return nil
111
-	}
112
-	return func(host string) (string, string, error) {
113
-		timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
114
-		defer cancel()
115
-
116
-		caller, err := sm.Get(timeoutCtx, id)
117
-		if err != nil {
118
-			return "", "", err
119
-		}
120
-
121
-		return auth.CredentialsFunc(context.TODO(), caller)(host)
122
-	}
123
-}
... ...
@@ -11,6 +11,8 @@ import (
11 11
 	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
12 12
 	controlapi "github.com/moby/buildkit/api/services/control"
13 13
 	"github.com/moby/buildkit/client/connhelper"
14
+	"github.com/moby/buildkit/session"
15
+	"github.com/moby/buildkit/session/grpchijack"
14 16
 	"github.com/moby/buildkit/util/appdefaults"
15 17
 	opentracing "github.com/opentracing/opentracing-go"
16 18
 	"github.com/pkg/errors"
... ...
@@ -80,6 +82,10 @@ func (c *Client) controlClient() controlapi.ControlClient {
80 80
 	return controlapi.NewControlClient(c.conn)
81 81
 }
82 82
 
83
+func (c *Client) Dialer() session.Dialer {
84
+	return grpchijack.Dialer(c.controlClient())
85
+}
86
+
83 87
 func (c *Client) Close() error {
84 88
 	return c.conn.Close()
85 89
 }
86 90
new file mode 100644
... ...
@@ -0,0 +1,98 @@
0
+package llb
1
+
2
+import (
3
+	"context"
4
+
5
+	"github.com/moby/buildkit/solver/pb"
6
+	"github.com/moby/buildkit/util/flightcontrol"
7
+	digest "github.com/opencontainers/go-digest"
8
+	"github.com/pkg/errors"
9
+)
10
+
11
+type asyncState struct {
12
+	f      func(context.Context, State) (State, error)
13
+	prev   State
14
+	target State
15
+	set    bool
16
+	err    error
17
+	g      flightcontrol.Group
18
+}
19
+
20
+func (as *asyncState) Output() Output {
21
+	return as
22
+}
23
+
24
+func (as *asyncState) Vertex(ctx context.Context) Vertex {
25
+	err := as.Do(ctx)
26
+	if err != nil {
27
+		return &errVertex{err}
28
+	}
29
+	if as.set {
30
+		out := as.target.Output()
31
+		if out == nil {
32
+			return nil
33
+		}
34
+		return out.Vertex(ctx)
35
+	}
36
+	return nil
37
+}
38
+
39
+func (as *asyncState) ToInput(ctx context.Context, c *Constraints) (*pb.Input, error) {
40
+	err := as.Do(ctx)
41
+	if err != nil {
42
+		return nil, err
43
+	}
44
+	if as.set {
45
+		out := as.target.Output()
46
+		if out == nil {
47
+			return nil, nil
48
+		}
49
+		return out.ToInput(ctx, c)
50
+	}
51
+	return nil, nil
52
+}
53
+
54
+func (as *asyncState) Do(ctx context.Context) error {
55
+	_, err := as.g.Do(ctx, "", func(ctx context.Context) (interface{}, error) {
56
+		if as.set {
57
+			return as.target, as.err
58
+		}
59
+		res, err := as.f(ctx, as.prev)
60
+		if err != nil {
61
+			select {
62
+			case <-ctx.Done():
63
+				if errors.Cause(err) == ctx.Err() {
64
+					return res, err
65
+				}
66
+			default:
67
+			}
68
+		}
69
+		as.target = res
70
+		as.err = err
71
+		as.set = true
72
+		return res, err
73
+	})
74
+	if err != nil {
75
+		return err
76
+	}
77
+	return as.err
78
+}
79
+
80
+type errVertex struct {
81
+	err error
82
+}
83
+
84
+func (v *errVertex) Validate(context.Context) error {
85
+	return v.err
86
+}
87
+func (v *errVertex) Marshal(context.Context, *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
88
+	return "", nil, nil, v.err
89
+}
90
+func (v *errVertex) Output() Output {
91
+	return nil
92
+}
93
+func (v *errVertex) Inputs() []Output {
94
+	return nil
95
+}
96
+
97
+var _ Vertex = &errVertex{}
... ...
@@ -1,6 +1,9 @@
1 1
 package llb
2 2
 
3 3
 import (
4
+	"context"
5
+	"sync"
6
+
4 7
 	"github.com/moby/buildkit/solver/pb"
5 8
 	digest "github.com/opencontainers/go-digest"
6 9
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
... ...
@@ -13,6 +16,7 @@ import (
13 13
 // LLB state can be reconstructed from the definition.
14 14
 type DefinitionOp struct {
15 15
 	MarshalCache
16
+	mu        sync.Mutex
16 17
 	ops       map[digest.Digest]*pb.Op
17 18
 	defs      map[digest.Digest][]byte
18 19
 	metas     map[digest.Digest]pb.OpMetadata
... ...
@@ -25,6 +29,7 @@ type DefinitionOp struct {
25 25
 func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
26 26
 	ops := make(map[digest.Digest]*pb.Op)
27 27
 	defs := make(map[digest.Digest][]byte)
28
+	platforms := make(map[digest.Digest]*specs.Platform)
28 29
 
29 30
 	var dgst digest.Digest
30 31
 	for _, dt := range def.Def {
... ...
@@ -35,6 +40,13 @@ func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
35 35
 		dgst = digest.FromBytes(dt)
36 36
 		ops[dgst] = &op
37 37
 		defs[dgst] = dt
38
+
39
+		var platform *specs.Platform
40
+		if op.Platform != nil {
41
+			spec := op.Platform.Spec()
42
+			platform = &spec
43
+		}
44
+		platforms[dgst] = platform
38 45
 	}
39 46
 
40 47
 	var index pb.OutputIndex
... ...
@@ -47,26 +59,29 @@ func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
47 47
 		ops:       ops,
48 48
 		defs:      defs,
49 49
 		metas:     def.Metadata,
50
-		platforms: make(map[digest.Digest]*specs.Platform),
50
+		platforms: platforms,
51 51
 		dgst:      dgst,
52 52
 		index:     index,
53 53
 	}, nil
54 54
 }
55 55
 
56
-func (d *DefinitionOp) ToInput(c *Constraints) (*pb.Input, error) {
57
-	return d.Output().ToInput(c)
56
+func (d *DefinitionOp) ToInput(ctx context.Context, c *Constraints) (*pb.Input, error) {
57
+	return d.Output().ToInput(ctx, c)
58 58
 }
59 59
 
60
-func (d *DefinitionOp) Vertex() Vertex {
60
+func (d *DefinitionOp) Vertex(context.Context) Vertex {
61 61
 	return d
62 62
 }
63 63
 
64
-func (d *DefinitionOp) Validate() error {
64
+func (d *DefinitionOp) Validate(context.Context) error {
65 65
 	// Scratch state has no digest, ops or metas.
66 66
 	if d.dgst == "" {
67 67
 		return nil
68 68
 	}
69 69
 
70
+	d.mu.Lock()
71
+	defer d.mu.Unlock()
72
+
70 73
 	if len(d.ops) == 0 || len(d.defs) == 0 || len(d.metas) == 0 {
71 74
 		return errors.Errorf("invalid definition op with no ops %d %d", len(d.ops), len(d.metas))
72 75
 	}
... ...
@@ -95,15 +110,18 @@ func (d *DefinitionOp) Validate() error {
95 95
 	return nil
96 96
 }
97 97
 
98
-func (d *DefinitionOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
98
+func (d *DefinitionOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
99 99
 	if d.dgst == "" {
100 100
 		return "", nil, nil, errors.Errorf("cannot marshal empty definition op")
101 101
 	}
102 102
 
103
-	if err := d.Validate(); err != nil {
103
+	if err := d.Validate(ctx); err != nil {
104 104
 		return "", nil, nil, err
105 105
 	}
106 106
 
107
+	d.mu.Lock()
108
+	defer d.mu.Unlock()
109
+
107 110
 	meta := d.metas[d.dgst]
108 111
 	return d.dgst, d.defs[d.dgst], &meta, nil
109 112
 
... ...
@@ -114,7 +132,11 @@ func (d *DefinitionOp) Output() Output {
114 114
 		return nil
115 115
 	}
116 116
 
117
-	return &output{vertex: d, platform: d.platform(), getIndex: func() (pb.OutputIndex, error) {
117
+	d.mu.Lock()
118
+	platform := d.platforms[d.dgst]
119
+	d.mu.Unlock()
120
+
121
+	return &output{vertex: d, platform: platform, getIndex: func() (pb.OutputIndex, error) {
118 122
 		return d.index, nil
119 123
 	}}
120 124
 }
... ...
@@ -126,7 +148,11 @@ func (d *DefinitionOp) Inputs() []Output {
126 126
 
127 127
 	var inputs []Output
128 128
 
129
+	d.mu.Lock()
129 130
 	op := d.ops[d.dgst]
131
+	platform := d.platforms[d.dgst]
132
+	d.mu.Unlock()
133
+
130 134
 	for _, input := range op.Inputs {
131 135
 		vtx := &DefinitionOp{
132 136
 			ops:       d.ops,
... ...
@@ -136,26 +162,10 @@ func (d *DefinitionOp) Inputs() []Output {
136 136
 			dgst:      input.Digest,
137 137
 			index:     input.Index,
138 138
 		}
139
-		inputs = append(inputs, &output{vertex: vtx, platform: d.platform(), getIndex: func() (pb.OutputIndex, error) {
139
+		inputs = append(inputs, &output{vertex: vtx, platform: platform, getIndex: func() (pb.OutputIndex, error) {
140 140
 			return pb.OutputIndex(vtx.index), nil
141 141
 		}})
142 142
 	}
143 143
 
144 144
 	return inputs
145 145
 }
146
-
147
-func (d *DefinitionOp) platform() *specs.Platform {
148
-	platform, ok := d.platforms[d.dgst]
149
-	if ok {
150
-		return platform
151
-	}
152
-
153
-	op := d.ops[d.dgst]
154
-	if op.Platform != nil {
155
-		spec := op.Platform.Spec()
156
-		platform = &spec
157
-	}
158
-
159
-	d.platforms[d.dgst] = platform
160
-	return platform
161
-}
... ...
@@ -1,6 +1,7 @@
1 1
 package llb
2 2
 
3 3
 import (
4
+	"context"
4 5
 	_ "crypto/sha256"
5 6
 	"fmt"
6 7
 	"net"
... ...
@@ -12,19 +13,9 @@ import (
12 12
 	"github.com/pkg/errors"
13 13
 )
14 14
 
15
-type Meta struct {
16
-	Args       []string
17
-	Env        EnvList
18
-	Cwd        string
19
-	User       string
20
-	ProxyEnv   *ProxyEnv
21
-	ExtraHosts []HostIP
22
-	Network    pb.NetMode
23
-	Security   pb.SecurityMode
24
-}
25
-
26
-func NewExecOp(root Output, meta Meta, readOnly bool, c Constraints) *ExecOp {
27
-	e := &ExecOp{meta: meta, constraints: c}
15
+func NewExecOp(base State, proxyEnv *ProxyEnv, readOnly bool, c Constraints) *ExecOp {
16
+	e := &ExecOp{base: base, constraints: c, proxyEnv: proxyEnv}
17
+	root := base.Output()
28 18
 	rootMount := &mount{
29 19
 		target:   pb.RootMount,
30 20
 		source:   root,
... ...
@@ -58,9 +49,10 @@ type mount struct {
58 58
 
59 59
 type ExecOp struct {
60 60
 	MarshalCache
61
+	proxyEnv    *ProxyEnv
61 62
 	root        Output
62 63
 	mounts      []*mount
63
-	meta        Meta
64
+	base        State
64 65
 	constraints Constraints
65 66
 	isValidated bool
66 67
 	secrets     []SecretInfo
... ...
@@ -103,19 +95,27 @@ func (e *ExecOp) GetMount(target string) Output {
103 103
 	return nil
104 104
 }
105 105
 
106
-func (e *ExecOp) Validate() error {
106
+func (e *ExecOp) Validate(ctx context.Context) error {
107 107
 	if e.isValidated {
108 108
 		return nil
109 109
 	}
110
-	if len(e.meta.Args) == 0 {
110
+	args, err := getArgs(e.base)(ctx)
111
+	if err != nil {
112
+		return err
113
+	}
114
+	if len(args) == 0 {
111 115
 		return errors.Errorf("arguments are required")
112 116
 	}
113
-	if e.meta.Cwd == "" {
117
+	cwd, err := getDir(e.base)(ctx)
118
+	if err != nil {
119
+		return err
120
+	}
121
+	if cwd == "" {
114 122
 		return errors.Errorf("working directory is required")
115 123
 	}
116 124
 	for _, m := range e.mounts {
117 125
 		if m.source != nil {
118
-			if err := m.source.Vertex().Validate(); err != nil {
126
+			if err := m.source.Vertex(ctx).Validate(ctx); err != nil {
119 127
 				return err
120 128
 			}
121 129
 		}
... ...
@@ -124,11 +124,11 @@ func (e *ExecOp) Validate() error {
124 124
 	return nil
125 125
 }
126 126
 
127
-func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
127
+func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
128 128
 	if e.Cached(c) {
129 129
 		return e.Load()
130 130
 	}
131
-	if err := e.Validate(); err != nil {
131
+	if err := e.Validate(ctx); err != nil {
132 132
 		return "", nil, nil, err
133 133
 	}
134 134
 	// make sure mounts are sorted
... ...
@@ -136,52 +136,86 @@ func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata,
136 136
 		return e.mounts[i].target < e.mounts[j].target
137 137
 	})
138 138
 
139
+	env, err := getEnv(e.base)(ctx)
140
+	if err != nil {
141
+		return "", nil, nil, err
142
+	}
143
+
139 144
 	if len(e.ssh) > 0 {
140 145
 		for i, s := range e.ssh {
141 146
 			if s.Target == "" {
142 147
 				e.ssh[i].Target = fmt.Sprintf("/run/buildkit/ssh_agent.%d", i)
143 148
 			}
144 149
 		}
145
-		if _, ok := e.meta.Env.Get("SSH_AUTH_SOCK"); !ok {
146
-			e.meta.Env = e.meta.Env.AddOrReplace("SSH_AUTH_SOCK", e.ssh[0].Target)
150
+		if _, ok := env.Get("SSH_AUTH_SOCK"); !ok {
151
+			env = env.AddOrReplace("SSH_AUTH_SOCK", e.ssh[0].Target)
147 152
 		}
148 153
 	}
149 154
 	if c.Caps != nil {
150 155
 		if err := c.Caps.Supports(pb.CapExecMetaSetsDefaultPath); err != nil {
151
-			e.meta.Env = e.meta.Env.SetDefault("PATH", system.DefaultPathEnv)
156
+			env = env.SetDefault("PATH", system.DefaultPathEnv)
152 157
 		} else {
153 158
 			addCap(&e.constraints, pb.CapExecMetaSetsDefaultPath)
154 159
 		}
155 160
 	}
156 161
 
162
+	args, err := getArgs(e.base)(ctx)
163
+	if err != nil {
164
+		return "", nil, nil, err
165
+	}
166
+
167
+	cwd, err := getDir(e.base)(ctx)
168
+	if err != nil {
169
+		return "", nil, nil, err
170
+	}
171
+
172
+	user, err := getUser(e.base)(ctx)
173
+	if err != nil {
174
+		return "", nil, nil, err
175
+	}
176
+
157 177
 	meta := &pb.Meta{
158
-		Args: e.meta.Args,
159
-		Env:  e.meta.Env.ToArray(),
160
-		Cwd:  e.meta.Cwd,
161
-		User: e.meta.User,
162
-	}
163
-	if len(e.meta.ExtraHosts) > 0 {
164
-		hosts := make([]*pb.HostIP, len(e.meta.ExtraHosts))
165
-		for i, h := range e.meta.ExtraHosts {
178
+		Args: args,
179
+		Env:  env.ToArray(),
180
+		Cwd:  cwd,
181
+		User: user,
182
+	}
183
+	extraHosts, err := getExtraHosts(e.base)(ctx)
184
+	if err != nil {
185
+		return "", nil, nil, err
186
+	}
187
+	if len(extraHosts) > 0 {
188
+		hosts := make([]*pb.HostIP, len(extraHosts))
189
+		for i, h := range extraHosts {
166 190
 			hosts[i] = &pb.HostIP{Host: h.Host, IP: h.IP.String()}
167 191
 		}
168 192
 		meta.ExtraHosts = hosts
169 193
 	}
170 194
 
195
+	network, err := getNetwork(e.base)(ctx)
196
+	if err != nil {
197
+		return "", nil, nil, err
198
+	}
199
+
200
+	security, err := getSecurity(e.base)(ctx)
201
+	if err != nil {
202
+		return "", nil, nil, err
203
+	}
204
+
171 205
 	peo := &pb.ExecOp{
172 206
 		Meta:     meta,
173
-		Network:  e.meta.Network,
174
-		Security: e.meta.Security,
207
+		Network:  network,
208
+		Security: security,
175 209
 	}
176
-	if e.meta.Network != NetModeSandbox {
210
+	if network != NetModeSandbox {
177 211
 		addCap(&e.constraints, pb.CapExecMetaNetwork)
178 212
 	}
179 213
 
180
-	if e.meta.Security != SecurityModeSandbox {
214
+	if security != SecurityModeSandbox {
181 215
 		addCap(&e.constraints, pb.CapExecMetaSecurity)
182 216
 	}
183 217
 
184
-	if p := e.meta.ProxyEnv; p != nil {
218
+	if p := e.proxyEnv; p != nil {
185 219
 		peo.Meta.ProxyEnv = &pb.ProxyEnv{
186 220
 			HttpProxy:  p.HttpProxy,
187 221
 			HttpsProxy: p.HttpsProxy,
... ...
@@ -215,6 +249,14 @@ func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata,
215 215
 		addCap(&e.constraints, pb.CapExecMountSSH)
216 216
 	}
217 217
 
218
+	if e.constraints.Platform == nil {
219
+		p, err := getPlatform(e.base)(ctx)
220
+		if err != nil {
221
+			return "", nil, nil, err
222
+		}
223
+		e.constraints.Platform = p
224
+	}
225
+
218 226
 	pop, md := MarshalConstraints(c, &e.constraints)
219 227
 	pop.Op = &pb.Op_Exec{
220 228
 		Exec: peo,
... ...
@@ -227,7 +269,7 @@ func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata,
227 227
 			if m.tmpfs {
228 228
 				return "", nil, nil, errors.Errorf("tmpfs mounts must use scratch")
229 229
 			}
230
-			inp, err := m.source.ToInput(c)
230
+			inp, err := m.source.ToInput(ctx, c)
231 231
 			if err != nil {
232 232
 				return "", nil, nil, err
233 233
 			}
... ...
@@ -414,17 +456,11 @@ func (fn runOptionFunc) SetRunOption(ei *ExecInfo) {
414 414
 	fn(ei)
415 415
 }
416 416
 
417
-func Network(n pb.NetMode) RunOption {
418
-	return runOptionFunc(func(ei *ExecInfo) {
419
-		ei.State = network(n)(ei.State)
420
-	})
417
+func (fn StateOption) SetRunOption(ei *ExecInfo) {
418
+	ei.State = ei.State.With(fn)
421 419
 }
422 420
 
423
-func Security(s pb.SecurityMode) RunOption {
424
-	return runOptionFunc(func(ei *ExecInfo) {
425
-		ei.State = security(s)(ei.State)
426
-	})
427
-}
421
+var _ RunOption = StateOption(func(_ State) State { return State{} })
428 422
 
429 423
 func Shlex(str string) RunOption {
430 424
 	return runOptionFunc(func(ei *ExecInfo) {
... ...
@@ -443,47 +479,12 @@ func Args(a []string) RunOption {
443 443
 	})
444 444
 }
445 445
 
446
-func AddEnv(key, value string) RunOption {
447
-	return runOptionFunc(func(ei *ExecInfo) {
448
-		ei.State = ei.State.AddEnv(key, value)
449
-	})
450
-}
451
-
452
-func AddEnvf(key, value string, v ...interface{}) RunOption {
453
-	return runOptionFunc(func(ei *ExecInfo) {
454
-		ei.State = ei.State.AddEnvf(key, value, v...)
455
-	})
456
-}
457
-
458
-func User(str string) RunOption {
459
-	return runOptionFunc(func(ei *ExecInfo) {
460
-		ei.State = ei.State.User(str)
461
-	})
462
-}
463
-
464
-func Dir(str string) RunOption {
465
-	return runOptionFunc(func(ei *ExecInfo) {
466
-		ei.State = ei.State.Dir(str)
467
-	})
468
-}
469
-func Dirf(str string, v ...interface{}) RunOption {
470
-	return runOptionFunc(func(ei *ExecInfo) {
471
-		ei.State = ei.State.Dirf(str, v...)
472
-	})
473
-}
474
-
475 446
 func AddExtraHost(host string, ip net.IP) RunOption {
476 447
 	return runOptionFunc(func(ei *ExecInfo) {
477 448
 		ei.State = ei.State.AddExtraHost(host, ip)
478 449
 	})
479 450
 }
480 451
 
481
-func Reset(s State) RunOption {
482
-	return runOptionFunc(func(ei *ExecInfo) {
483
-		ei.State = ei.State.Reset(s)
484
-	})
485
-}
486
-
487 452
 func With(so ...StateOption) RunOption {
488 453
 	return runOptionFunc(func(ei *ExecInfo) {
489 454
 		ei.State = ei.State.With(so...)
... ...
@@ -1,6 +1,7 @@
1 1
 package llb
2 2
 
3 3
 import (
4
+	"context"
4 5
 	_ "crypto/sha256"
5 6
 	"os"
6 7
 	"path"
... ...
@@ -52,7 +53,7 @@ type CopyInput interface {
52 52
 }
53 53
 
54 54
 type subAction interface {
55
-	toProtoAction(string, pb.InputIndex) pb.IsFileAction
55
+	toProtoAction(context.Context, string, pb.InputIndex) (pb.IsFileAction, error)
56 56
 }
57 57
 
58 58
 type FileAction struct {
... ...
@@ -146,7 +147,7 @@ type fileActionMkdir struct {
146 146
 	info MkdirInfo
147 147
 }
148 148
 
149
-func (a *fileActionMkdir) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
149
+func (a *fileActionMkdir) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
150 150
 	return &pb.FileAction_Mkdir{
151 151
 		Mkdir: &pb.FileActionMkDir{
152 152
 			Path:        normalizePath(parent, a.file, false),
... ...
@@ -155,7 +156,7 @@ func (a *fileActionMkdir) toProtoAction(parent string, base pb.InputIndex) pb.Is
155 155
 			Owner:       a.info.ChownOpt.marshal(base),
156 156
 			Timestamp:   marshalTime(a.info.CreatedTime),
157 157
 		},
158
-	}
158
+	}, nil
159 159
 }
160 160
 
161 161
 type MkdirOption interface {
... ...
@@ -315,7 +316,7 @@ type fileActionMkfile struct {
315 315
 	info MkfileInfo
316 316
 }
317 317
 
318
-func (a *fileActionMkfile) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
318
+func (a *fileActionMkfile) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
319 319
 	return &pb.FileAction_Mkfile{
320 320
 		Mkfile: &pb.FileActionMkFile{
321 321
 			Path:      normalizePath(parent, a.file, false),
... ...
@@ -324,7 +325,7 @@ func (a *fileActionMkfile) toProtoAction(parent string, base pb.InputIndex) pb.I
324 324
 			Owner:     a.info.ChownOpt.marshal(base),
325 325
 			Timestamp: marshalTime(a.info.CreatedTime),
326 326
 		},
327
-	}
327
+	}, nil
328 328
 }
329 329
 
330 330
 func Rm(p string, opts ...RmOption) *FileAction {
... ...
@@ -379,14 +380,14 @@ type fileActionRm struct {
379 379
 	info RmInfo
380 380
 }
381 381
 
382
-func (a *fileActionRm) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
382
+func (a *fileActionRm) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
383 383
 	return &pb.FileAction_Rm{
384 384
 		Rm: &pb.FileActionRm{
385 385
 			Path:          normalizePath(parent, a.file, false),
386 386
 			AllowNotFound: a.info.AllowNotFound,
387 387
 			AllowWildcard: a.info.AllowWildcard,
388 388
 		},
389
-	}
389
+	}, nil
390 390
 }
391 391
 
392 392
 func Copy(input CopyInput, src, dest string, opts ...CopyOption) *FileAction {
... ...
@@ -448,9 +449,13 @@ type fileActionCopy struct {
448 448
 	info  CopyInfo
449 449
 }
450 450
 
451
-func (a *fileActionCopy) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
451
+func (a *fileActionCopy) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
452
+	src, err := a.sourcePath(ctx)
453
+	if err != nil {
454
+		return nil, err
455
+	}
452 456
 	c := &pb.FileActionCopy{
453
-		Src:                              a.sourcePath(),
457
+		Src:                              src,
454 458
 		Dest:                             normalizePath(parent, a.dest, true),
455 459
 		Owner:                            a.info.ChownOpt.marshal(base),
456 460
 		AllowWildcard:                    a.info.AllowWildcard,
... ...
@@ -468,19 +473,27 @@ func (a *fileActionCopy) toProtoAction(parent string, base pb.InputIndex) pb.IsF
468 468
 	}
469 469
 	return &pb.FileAction_Copy{
470 470
 		Copy: c,
471
-	}
471
+	}, nil
472 472
 }
473 473
 
474
-func (c *fileActionCopy) sourcePath() string {
474
+func (c *fileActionCopy) sourcePath(ctx context.Context) (string, error) {
475 475
 	p := path.Clean(c.src)
476 476
 	if !path.IsAbs(p) {
477 477
 		if c.state != nil {
478
-			p = path.Join("/", c.state.GetDir(), p)
478
+			dir, err := c.state.GetDir(ctx)
479
+			if err != nil {
480
+				return "", err
481
+			}
482
+			p = path.Join("/", dir, p)
479 483
 		} else if c.fas != nil {
480
-			p = path.Join("/", c.fas.state.GetDir(), p)
484
+			dir, err := c.fas.state.GetDir(ctx)
485
+			if err != nil {
486
+				return "", err
487
+			}
488
+			p = path.Join("/", dir, p)
481 489
 		}
482 490
 	}
483
-	return p
491
+	return p, nil
484 492
 }
485 493
 
486 494
 type CreatedTime time.Time
... ...
@@ -517,7 +530,7 @@ type FileOp struct {
517 517
 	isValidated bool
518 518
 }
519 519
 
520
-func (f *FileOp) Validate() error {
520
+func (f *FileOp) Validate(context.Context) error {
521 521
 	if f.isValidated {
522 522
 		return nil
523 523
 	}
... ...
@@ -529,14 +542,16 @@ func (f *FileOp) Validate() error {
529 529
 }
530 530
 
531 531
 type marshalState struct {
532
+	ctx     context.Context
532 533
 	visited map[*FileAction]*fileActionState
533 534
 	inputs  []*pb.Input
534 535
 	actions []*fileActionState
535 536
 }
536 537
 
537
-func newMarshalState() *marshalState {
538
+func newMarshalState(ctx context.Context) *marshalState {
538 539
 	return &marshalState{
539 540
 		visited: map[*FileAction]*fileActionState{},
541
+		ctx:     ctx,
540 542
 	}
541 543
 }
542 544
 
... ...
@@ -552,7 +567,7 @@ type fileActionState struct {
552 552
 }
553 553
 
554 554
 func (ms *marshalState) addInput(st *fileActionState, c *Constraints, o Output) (pb.InputIndex, error) {
555
-	inp, err := o.ToInput(c)
555
+	inp, err := o.ToInput(ms.ctx, c)
556 556
 	if err != nil {
557 557
 		return 0, err
558 558
 	}
... ...
@@ -634,11 +649,11 @@ func (ms *marshalState) add(fa *FileAction, c *Constraints) (*fileActionState, e
634 634
 	return st, nil
635 635
 }
636 636
 
637
-func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
637
+func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
638 638
 	if f.Cached(c) {
639 639
 		return f.Load()
640 640
 	}
641
-	if err := f.Validate(); err != nil {
641
+	if err := f.Validate(ctx); err != nil {
642 642
 		return "", nil, nil, err
643 643
 	}
644 644
 
... ...
@@ -651,7 +666,7 @@ func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata,
651 651
 		File: pfo,
652 652
 	}
653 653
 
654
-	state := newMarshalState()
654
+	state := newMarshalState(ctx)
655 655
 	_, err := state.add(f.action, c)
656 656
 	if err != nil {
657 657
 		return "", nil, nil, err
... ...
@@ -666,14 +681,22 @@ func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata,
666 666
 
667 667
 		var parent string
668 668
 		if st.fa.state != nil {
669
-			parent = st.fa.state.GetDir()
669
+			parent, err = st.fa.state.GetDir(ctx)
670
+			if err != nil {
671
+				return "", nil, nil, err
672
+			}
673
+		}
674
+
675
+		action, err := st.action.toProtoAction(ctx, parent, st.base)
676
+		if err != nil {
677
+			return "", nil, nil, err
670 678
 		}
671 679
 
672 680
 		pfo.Actions = append(pfo.Actions, &pb.FileAction{
673 681
 			Input:          getIndex(st.input, len(state.inputs), st.inputRelative),
674 682
 			SecondaryInput: getIndex(st.input2, len(state.inputs), st.input2Relative),
675 683
 			Output:         output,
676
-			Action:         st.action.toProtoAction(parent, st.base),
684
+			Action:         action,
677 685
 		})
678 686
 	}
679 687
 
... ...
@@ -1,6 +1,7 @@
1 1
 package llb
2 2
 
3 3
 import (
4
+	"context"
4 5
 	"fmt"
5 6
 	"net"
6 7
 	"path"
... ...
@@ -24,79 +25,122 @@ var (
24 24
 	keySecurity  = contextKeyT("llb.security")
25 25
 )
26 26
 
27
+func AddEnvf(key, value string, v ...interface{}) StateOption {
28
+	return addEnvf(key, value, true, v...)
29
+}
30
+
31
+func AddEnv(key, value string) StateOption {
32
+	return addEnvf(key, value, false)
33
+}
34
+
27 35
 func addEnvf(key, value string, replace bool, v ...interface{}) StateOption {
28 36
 	if replace {
29 37
 		value = fmt.Sprintf(value, v...)
30 38
 	}
31 39
 	return func(s State) State {
32
-		return s.WithValue(keyEnv, getEnv(s).AddOrReplace(key, value))
40
+		return s.withValue(keyEnv, func(ctx context.Context) (interface{}, error) {
41
+			env, err := getEnv(s)(ctx)
42
+			if err != nil {
43
+				return nil, err
44
+			}
45
+			return env.AddOrReplace(key, value), nil
46
+		})
33 47
 	}
34 48
 }
35 49
 
36
-func dir(str string) StateOption {
50
+func Dir(str string) StateOption {
37 51
 	return dirf(str, false)
38 52
 }
39 53
 
54
+func Dirf(str string, v ...interface{}) StateOption {
55
+	return dirf(str, true, v...)
56
+}
57
+
40 58
 func dirf(value string, replace bool, v ...interface{}) StateOption {
41 59
 	if replace {
42 60
 		value = fmt.Sprintf(value, v...)
43 61
 	}
44 62
 	return func(s State) State {
45
-		if !path.IsAbs(value) {
46
-			prev := getDir(s)
47
-			if prev == "" {
48
-				prev = "/"
63
+		return s.withValue(keyDir, func(ctx context.Context) (interface{}, error) {
64
+			if !path.IsAbs(value) {
65
+				prev, err := getDir(s)(ctx)
66
+				if err != nil {
67
+					return nil, err
68
+				}
69
+				if prev == "" {
70
+					prev = "/"
71
+				}
72
+				value = path.Join(prev, value)
49 73
 			}
50
-			value = path.Join(prev, value)
51
-		}
52
-		return s.WithValue(keyDir, value)
74
+			return value, nil
75
+		})
53 76
 	}
54 77
 }
55 78
 
56
-func user(str string) StateOption {
79
+func User(str string) StateOption {
57 80
 	return func(s State) State {
58 81
 		return s.WithValue(keyUser, str)
59 82
 	}
60 83
 }
61 84
 
62
-func reset(s_ State) StateOption {
85
+func Reset(other State) StateOption {
63 86
 	return func(s State) State {
64 87
 		s = NewState(s.Output())
65
-		s.ctx = s_.ctx
88
+		s.prev = &other
66 89
 		return s
67 90
 	}
68 91
 }
69 92
 
70
-func getEnv(s State) EnvList {
71
-	v := s.Value(keyEnv)
72
-	if v != nil {
73
-		return v.(EnvList)
93
+func getEnv(s State) func(context.Context) (EnvList, error) {
94
+	return func(ctx context.Context) (EnvList, error) {
95
+		v, err := s.getValue(keyEnv)(ctx)
96
+		if err != nil {
97
+			return nil, err
98
+		}
99
+		if v != nil {
100
+			return v.(EnvList), nil
101
+		}
102
+		return EnvList{}, nil
74 103
 	}
75
-	return EnvList{}
76 104
 }
77 105
 
78
-func getDir(s State) string {
79
-	v := s.Value(keyDir)
80
-	if v != nil {
81
-		return v.(string)
106
+func getDir(s State) func(context.Context) (string, error) {
107
+	return func(ctx context.Context) (string, error) {
108
+		v, err := s.getValue(keyDir)(ctx)
109
+		if err != nil {
110
+			return "", err
111
+		}
112
+		if v != nil {
113
+			return v.(string), nil
114
+		}
115
+		return "", nil
82 116
 	}
83
-	return ""
84 117
 }
85 118
 
86
-func getArgs(s State) []string {
87
-	v := s.Value(keyArgs)
88
-	if v != nil {
89
-		return v.([]string)
119
+func getArgs(s State) func(context.Context) ([]string, error) {
120
+	return func(ctx context.Context) ([]string, error) {
121
+		v, err := s.getValue(keyArgs)(ctx)
122
+		if err != nil {
123
+			return nil, err
124
+		}
125
+		if v != nil {
126
+			return v.([]string), nil
127
+		}
128
+		return nil, nil
90 129
 	}
91
-	return nil
92 130
 }
93 131
 
94
-func getUser(s State) string {
95
-	v := s.Value(keyUser)
96
-	if v != nil {
97
-		return v.(string)
132
+func getUser(s State) func(context.Context) (string, error) {
133
+	return func(ctx context.Context) (string, error) {
134
+		v, err := s.getValue(keyUser)(ctx)
135
+		if err != nil {
136
+			return "", err
137
+		}
138
+		if v != nil {
139
+			return v.(string), nil
140
+		}
141
+		return "", nil
98 142
 	}
99
-	return ""
100 143
 }
101 144
 
102 145
 func args(args ...string) StateOption {
... ...
@@ -124,27 +168,43 @@ func platform(p specs.Platform) StateOption {
124 124
 	}
125 125
 }
126 126
 
127
-func getPlatform(s State) *specs.Platform {
128
-	v := s.Value(keyPlatform)
129
-	if v != nil {
130
-		p := v.(specs.Platform)
131
-		return &p
127
+func getPlatform(s State) func(context.Context) (*specs.Platform, error) {
128
+	return func(ctx context.Context) (*specs.Platform, error) {
129
+		v, err := s.getValue(keyPlatform)(ctx)
130
+		if err != nil {
131
+			return nil, err
132
+		}
133
+		if v != nil {
134
+			p := v.(specs.Platform)
135
+			return &p, nil
136
+		}
137
+		return nil, nil
132 138
 	}
133
-	return nil
134 139
 }
135 140
 
136 141
 func extraHost(host string, ip net.IP) StateOption {
137 142
 	return func(s State) State {
138
-		return s.WithValue(keyExtraHost, append(getExtraHosts(s), HostIP{Host: host, IP: ip}))
143
+		return s.withValue(keyExtraHost, func(ctx context.Context) (interface{}, error) {
144
+			v, err := getExtraHosts(s)(ctx)
145
+			if err != nil {
146
+				return nil, err
147
+			}
148
+			return append(v, HostIP{Host: host, IP: ip}), nil
149
+		})
139 150
 	}
140 151
 }
141 152
 
142
-func getExtraHosts(s State) []HostIP {
143
-	v := s.Value(keyExtraHost)
144
-	if v != nil {
145
-		return v.([]HostIP)
153
+func getExtraHosts(s State) func(context.Context) ([]HostIP, error) {
154
+	return func(ctx context.Context) ([]HostIP, error) {
155
+		v, err := s.getValue(keyExtraHost)(ctx)
156
+		if err != nil {
157
+			return nil, err
158
+		}
159
+		if v != nil {
160
+			return v.([]HostIP), nil
161
+		}
162
+		return nil, nil
146 163
 	}
147
-	return nil
148 164
 }
149 165
 
150 166
 type HostIP struct {
... ...
@@ -152,32 +212,42 @@ type HostIP struct {
152 152
 	IP   net.IP
153 153
 }
154 154
 
155
-func network(v pb.NetMode) StateOption {
155
+func Network(v pb.NetMode) StateOption {
156 156
 	return func(s State) State {
157 157
 		return s.WithValue(keyNetwork, v)
158 158
 	}
159 159
 }
160
-func getNetwork(s State) pb.NetMode {
161
-	v := s.Value(keyNetwork)
162
-	if v != nil {
163
-		n := v.(pb.NetMode)
164
-		return n
160
+func getNetwork(s State) func(context.Context) (pb.NetMode, error) {
161
+	return func(ctx context.Context) (pb.NetMode, error) {
162
+		v, err := s.getValue(keyNetwork)(ctx)
163
+		if err != nil {
164
+			return 0, err
165
+		}
166
+		if v != nil {
167
+			n := v.(pb.NetMode)
168
+			return n, nil
169
+		}
170
+		return NetModeSandbox, nil
165 171
 	}
166
-	return NetModeSandbox
167 172
 }
168 173
 
169
-func security(v pb.SecurityMode) StateOption {
174
+func Security(v pb.SecurityMode) StateOption {
170 175
 	return func(s State) State {
171 176
 		return s.WithValue(keySecurity, v)
172 177
 	}
173 178
 }
174
-func getSecurity(s State) pb.SecurityMode {
175
-	v := s.Value(keySecurity)
176
-	if v != nil {
177
-		n := v.(pb.SecurityMode)
178
-		return n
179
+func getSecurity(s State) func(context.Context) (pb.SecurityMode, error) {
180
+	return func(ctx context.Context) (pb.SecurityMode, error) {
181
+		v, err := s.getValue(keySecurity)(ctx)
182
+		if err != nil {
183
+			return 0, err
184
+		}
185
+		if v != nil {
186
+			n := v.(pb.SecurityMode)
187
+			return n, nil
188
+		}
189
+		return SecurityModeSandbox, nil
179 190
 	}
180
-	return SecurityModeSandbox
181 191
 }
182 192
 
183 193
 type EnvList []KeyValue
... ...
@@ -14,6 +14,15 @@ func WithMetaResolver(mr ImageMetaResolver) ImageOption {
14 14
 	})
15 15
 }
16 16
 
17
+// ResolveDigest uses the meta resolver to update the ref of image with full digest before marshaling.
18
+// This makes image ref immutable and is recommended if you want to make sure meta resolver data
19
+// matches the image used during the build.
20
+func ResolveDigest(v bool) ImageOption {
21
+	return imageOptionFunc(func(ii *ImageInfo) {
22
+		ii.resolveDigest = v
23
+	})
24
+}
25
+
17 26
 // ImageMetaResolver can resolve image config metadata from a reference
18 27
 type ImageMetaResolver interface {
19 28
 	ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (digest.Digest, []byte, error)
... ...
@@ -34,7 +34,7 @@ func NewSource(id string, attrs map[string]string, c Constraints) *SourceOp {
34 34
 	return s
35 35
 }
36 36
 
37
-func (s *SourceOp) Validate() error {
37
+func (s *SourceOp) Validate(ctx context.Context) error {
38 38
 	if s.err != nil {
39 39
 		return s.err
40 40
 	}
... ...
@@ -44,11 +44,11 @@ func (s *SourceOp) Validate() error {
44 44
 	return nil
45 45
 }
46 46
 
47
-func (s *SourceOp) Marshal(constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
47
+func (s *SourceOp) Marshal(ctx context.Context, constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
48 48
 	if s.Cached(constraints) {
49 49
 		return s.Load()
50 50
 	}
51
-	if err := s.Validate(); err != nil {
51
+	if err := s.Validate(ctx); err != nil {
52 52
 		return "", nil, nil, err
53 53
 	}
54 54
 
... ...
@@ -92,7 +92,8 @@ func (s *SourceOp) Inputs() []Output {
92 92
 func Image(ref string, opts ...ImageOption) State {
93 93
 	r, err := reference.ParseNormalizedNamed(ref)
94 94
 	if err == nil {
95
-		ref = reference.TagNameOnly(r).String()
95
+		r = reference.TagNameOnly(r)
96
+		ref = r.String()
96 97
 	}
97 98
 	var info ImageInfo
98 99
 	for _, opt := range opts {
... ...
@@ -116,21 +117,35 @@ func Image(ref string, opts ...ImageOption) State {
116 116
 	src := NewSource("docker-image://"+ref, attrs, info.Constraints) // controversial
117 117
 	if err != nil {
118 118
 		src.err = err
119
-	}
120
-	if info.metaResolver != nil {
121
-		_, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
122
-			Platform:    info.Constraints.Platform,
123
-			ResolveMode: info.resolveMode.String(),
124
-		})
125
-		if err != nil {
126
-			src.err = err
127
-		} else {
128
-			st, err := NewState(src.Output()).WithImageConfig(dt)
129
-			if err == nil {
130
-				return st
131
-			}
132
-			src.err = err
119
+	} else if info.metaResolver != nil {
120
+		if _, ok := r.(reference.Digested); ok || !info.resolveDigest {
121
+			return NewState(src.Output()).Async(func(ctx context.Context, st State) (State, error) {
122
+				_, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, ResolveImageConfigOpt{
123
+					Platform:    info.Constraints.Platform,
124
+					ResolveMode: info.resolveMode.String(),
125
+				})
126
+				if err != nil {
127
+					return State{}, err
128
+				}
129
+				return st.WithImageConfig(dt)
130
+			})
133 131
 		}
132
+		return Scratch().Async(func(ctx context.Context, _ State) (State, error) {
133
+			dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
134
+				Platform:    info.Constraints.Platform,
135
+				ResolveMode: info.resolveMode.String(),
136
+			})
137
+			if err != nil {
138
+				return State{}, err
139
+			}
140
+			if dgst != "" {
141
+				r, err = reference.WithDigest(r, dgst)
142
+				if err != nil {
143
+					return State{}, err
144
+				}
145
+			}
146
+			return NewState(NewSource("docker-image://"+r.String(), attrs, info.Constraints).Output()).WithImageConfig(dt)
147
+		})
134 148
 	}
135 149
 	return NewState(src.Output())
136 150
 }
... ...
@@ -176,9 +191,10 @@ func (r ResolveMode) String() string {
176 176
 
177 177
 type ImageInfo struct {
178 178
 	constraintsWrapper
179
-	metaResolver ImageMetaResolver
180
-	resolveMode  ResolveMode
181
-	RecordType   string
179
+	metaResolver  ImageMetaResolver
180
+	resolveDigest bool
181
+	resolveMode   ResolveMode
182
+	RecordType    string
182 183
 }
183 184
 
184 185
 func Git(remote, ref string, opts ...GitOption) State {
... ...
@@ -18,13 +18,13 @@ import (
18 18
 type StateOption func(State) State
19 19
 
20 20
 type Output interface {
21
-	ToInput(*Constraints) (*pb.Input, error)
22
-	Vertex() Vertex
21
+	ToInput(context.Context, *Constraints) (*pb.Input, error)
22
+	Vertex(context.Context) Vertex
23 23
 }
24 24
 
25 25
 type Vertex interface {
26
-	Validate() error
27
-	Marshal(*Constraints) (digest.Digest, []byte, *pb.OpMetadata, error)
26
+	Validate(context.Context) error
27
+	Marshal(context.Context, *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error)
28 28
 	Output() Output
29 29
 	Inputs() []Output
30 30
 }
... ...
@@ -32,17 +32,18 @@ type Vertex interface {
32 32
 func NewState(o Output) State {
33 33
 	s := State{
34 34
 		out: o,
35
-		ctx: context.Background(),
36
-	}
37
-	s = dir("/")(s)
35
+	}.Dir("/")
38 36
 	s = s.ensurePlatform()
39 37
 	return s
40 38
 }
41 39
 
42 40
 type State struct {
43
-	out  Output
44
-	ctx  context.Context
45
-	opts []ConstraintsOpt
41
+	out   Output
42
+	prev  *State
43
+	key   interface{}
44
+	value func(context.Context) (interface{}, error)
45
+	opts  []ConstraintsOpt
46
+	async *asyncState
46 47
 }
47 48
 
48 49
 func (s State) ensurePlatform() State {
... ...
@@ -57,14 +58,48 @@ func (s State) ensurePlatform() State {
57 57
 }
58 58
 
59 59
 func (s State) WithValue(k, v interface{}) State {
60
+	return s.withValue(k, func(context.Context) (interface{}, error) {
61
+		return v, nil
62
+	})
63
+}
64
+
65
+func (s State) withValue(k interface{}, v func(context.Context) (interface{}, error)) State {
60 66
 	return State{
61
-		out: s.out,
62
-		ctx: context.WithValue(s.ctx, k, v),
67
+		out:   s.Output(),
68
+		prev:  &s, // doesn't need to be original pointer
69
+		key:   k,
70
+		value: v,
71
+	}
72
+}
73
+
74
+func (s State) Value(ctx context.Context, k interface{}) (interface{}, error) {
75
+	return s.getValue(k)(ctx)
76
+}
77
+
78
+func (s State) getValue(k interface{}) func(context.Context) (interface{}, error) {
79
+	if s.key == k {
80
+		return s.value
81
+	}
82
+	if s.async != nil {
83
+		return func(ctx context.Context) (interface{}, error) {
84
+			err := s.async.Do(ctx)
85
+			if err != nil {
86
+				return nil, err
87
+			}
88
+			return s.async.target.getValue(k)(ctx)
89
+		}
90
+	}
91
+	if s.prev == nil {
92
+		return nilValue
63 93
 	}
94
+	return s.prev.getValue(k)
64 95
 }
65 96
 
66
-func (s State) Value(k interface{}) interface{} {
67
-	return s.ctx.Value(k)
97
+func (s State) Async(f func(context.Context, State) (State, error)) State {
98
+	s2 := State{
99
+		async: &asyncState{f: f, prev: s},
100
+	}
101
+	return s2
68 102
 }
69 103
 
70 104
 func (s State) SetMarshalDefaults(co ...ConstraintsOpt) State {
... ...
@@ -72,11 +107,11 @@ func (s State) SetMarshalDefaults(co ...ConstraintsOpt) State {
72 72
 	return s
73 73
 }
74 74
 
75
-func (s State) Marshal(co ...ConstraintsOpt) (*Definition, error) {
75
+func (s State) Marshal(ctx context.Context, co ...ConstraintsOpt) (*Definition, error) {
76 76
 	def := &Definition{
77 77
 		Metadata: make(map[digest.Digest]pb.OpMetadata, 0),
78 78
 	}
79
-	if s.Output() == nil {
79
+	if s.Output() == nil || s.Output().Vertex(ctx) == nil {
80 80
 		return def, nil
81 81
 	}
82 82
 
... ...
@@ -89,11 +124,11 @@ func (s State) Marshal(co ...ConstraintsOpt) (*Definition, error) {
89 89
 		o.SetConstraintsOption(c)
90 90
 	}
91 91
 
92
-	def, err := marshal(s.Output().Vertex(), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{}, c)
92
+	def, err := marshal(ctx, s.Output().Vertex(ctx), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{}, c)
93 93
 	if err != nil {
94 94
 		return def, err
95 95
 	}
96
-	inp, err := s.Output().ToInput(c)
96
+	inp, err := s.Output().ToInput(ctx, c)
97 97
 	if err != nil {
98 98
 		return def, err
99 99
 	}
... ...
@@ -128,19 +163,19 @@ func (s State) Marshal(co ...ConstraintsOpt) (*Definition, error) {
128 128
 	return def, nil
129 129
 }
130 130
 
131
-func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}, c *Constraints) (*Definition, error) {
131
+func marshal(ctx context.Context, v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}, c *Constraints) (*Definition, error) {
132 132
 	if _, ok := vertexCache[v]; ok {
133 133
 		return def, nil
134 134
 	}
135 135
 	for _, inp := range v.Inputs() {
136 136
 		var err error
137
-		def, err = marshal(inp.Vertex(), def, cache, vertexCache, c)
137
+		def, err = marshal(ctx, inp.Vertex(ctx), def, cache, vertexCache, c)
138 138
 		if err != nil {
139 139
 			return def, err
140 140
 		}
141 141
 	}
142 142
 
143
-	dgst, dt, opMeta, err := v.Marshal(c)
143
+	dgst, dt, opMeta, err := v.Marshal(ctx, c)
144 144
 	if err != nil {
145 145
 		return def, err
146 146
 	}
... ...
@@ -156,18 +191,22 @@ func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertex
156 156
 	return def, nil
157 157
 }
158 158
 
159
-func (s State) Validate() error {
160
-	return s.Output().Vertex().Validate()
159
+func (s State) Validate(ctx context.Context) error {
160
+	return s.Output().Vertex(ctx).Validate(ctx)
161 161
 }
162 162
 
163 163
 func (s State) Output() Output {
164
+	if s.async != nil {
165
+		return s.async.Output()
166
+	}
164 167
 	return s.out
165 168
 }
166 169
 
167 170
 func (s State) WithOutput(o Output) State {
171
+	prev := s
168 172
 	s = State{
169
-		out: o,
170
-		ctx: s.ctx,
173
+		out:  o,
174
+		prev: &prev,
171 175
 	}
172 176
 	s = s.ensurePlatform()
173 177
 	return s
... ...
@@ -200,24 +239,10 @@ func (s State) WithImageConfig(c []byte) (State, error) {
200 200
 
201 201
 func (s State) Run(ro ...RunOption) ExecState {
202 202
 	ei := &ExecInfo{State: s}
203
-	if p := s.GetPlatform(); p != nil {
204
-		ei.Constraints.Platform = p
205
-	}
206 203
 	for _, o := range ro {
207 204
 		o.SetRunOption(ei)
208 205
 	}
209
-	meta := Meta{
210
-		Args:       getArgs(ei.State),
211
-		Cwd:        getDir(ei.State),
212
-		Env:        getEnv(ei.State),
213
-		User:       getUser(ei.State),
214
-		ProxyEnv:   ei.ProxyEnv,
215
-		ExtraHosts: getExtraHosts(ei.State),
216
-		Network:    getNetwork(ei.State),
217
-		Security:   getSecurity(ei.State),
218
-	}
219
-
220
-	exec := NewExecOp(s.Output(), meta, ei.ReadonlyRootFS, ei.Constraints)
206
+	exec := NewExecOp(ei.State, ei.ProxyEnv, ei.ReadonlyRootFS, ei.Constraints)
221 207
 	for _, m := range ei.Mounts {
222 208
 		exec.AddMount(m.Target, m.Source, m.Opts...)
223 209
 	}
... ...
@@ -240,65 +265,74 @@ func (s State) File(a *FileAction, opts ...ConstraintsOpt) State {
240 240
 }
241 241
 
242 242
 func (s State) AddEnv(key, value string) State {
243
-	return addEnvf(key, value, false)(s)
243
+	return AddEnv(key, value)(s)
244 244
 }
245 245
 
246 246
 func (s State) AddEnvf(key, value string, v ...interface{}) State {
247
-	return addEnvf(key, value, true, v...)(s)
247
+	return AddEnvf(key, value, v...)(s)
248 248
 }
249 249
 
250 250
 func (s State) Dir(str string) State {
251
-	return dirf(str, false)(s)
251
+	return Dir(str)(s)
252 252
 }
253 253
 func (s State) Dirf(str string, v ...interface{}) State {
254
-	return dirf(str, true, v...)(s)
254
+	return Dirf(str, v...)(s)
255 255
 }
256 256
 
257
-func (s State) GetEnv(key string) (string, bool) {
258
-	return getEnv(s).Get(key)
257
+func (s State) GetEnv(ctx context.Context, key string) (string, bool, error) {
258
+	env, err := getEnv(s)(ctx)
259
+	if err != nil {
260
+		return "", false, err
261
+	}
262
+	v, ok := env.Get(key)
263
+	return v, ok, nil
259 264
 }
260 265
 
261
-func (s State) Env() []string {
262
-	return getEnv(s).ToArray()
266
+func (s State) Env(ctx context.Context) ([]string, error) {
267
+	env, err := getEnv(s)(ctx)
268
+	if err != nil {
269
+		return nil, err
270
+	}
271
+	return env.ToArray(), nil
263 272
 }
264 273
 
265
-func (s State) GetDir() string {
266
-	return getDir(s)
274
+func (s State) GetDir(ctx context.Context) (string, error) {
275
+	return getDir(s)(ctx)
267 276
 }
268 277
 
269
-func (s State) GetArgs() []string {
270
-	return getArgs(s)
278
+func (s State) GetArgs(ctx context.Context) ([]string, error) {
279
+	return getArgs(s)(ctx)
271 280
 }
272 281
 
273 282
 func (s State) Reset(s2 State) State {
274
-	return reset(s2)(s)
283
+	return Reset(s2)(s)
275 284
 }
276 285
 
277 286
 func (s State) User(v string) State {
278
-	return user(v)(s)
287
+	return User(v)(s)
279 288
 }
280 289
 
281 290
 func (s State) Platform(p specs.Platform) State {
282 291
 	return platform(p)(s)
283 292
 }
284 293
 
285
-func (s State) GetPlatform() *specs.Platform {
286
-	return getPlatform(s)
294
+func (s State) GetPlatform(ctx context.Context) (*specs.Platform, error) {
295
+	return getPlatform(s)(ctx)
287 296
 }
288 297
 
289 298
 func (s State) Network(n pb.NetMode) State {
290
-	return network(n)(s)
299
+	return Network(n)(s)
291 300
 }
292 301
 
293
-func (s State) GetNetwork() pb.NetMode {
294
-	return getNetwork(s)
302
+func (s State) GetNetwork(ctx context.Context) (pb.NetMode, error) {
303
+	return getNetwork(s)(ctx)
295 304
 }
296 305
 func (s State) Security(n pb.SecurityMode) State {
297
-	return security(n)(s)
306
+	return Security(n)(s)
298 307
 }
299 308
 
300
-func (s State) GetSecurity() pb.SecurityMode {
301
-	return getSecurity(s)
309
+func (s State) GetSecurity(ctx context.Context) (pb.SecurityMode, error) {
310
+	return getSecurity(s)(ctx)
302 311
 }
303 312
 
304 313
 func (s State) With(so ...StateOption) State {
... ...
@@ -321,7 +355,7 @@ type output struct {
321 321
 	platform *specs.Platform
322 322
 }
323 323
 
324
-func (o *output) ToInput(c *Constraints) (*pb.Input, error) {
324
+func (o *output) ToInput(ctx context.Context, c *Constraints) (*pb.Input, error) {
325 325
 	if o.err != nil {
326 326
 		return nil, o.err
327 327
 	}
... ...
@@ -333,14 +367,14 @@ func (o *output) ToInput(c *Constraints) (*pb.Input, error) {
333 333
 			return nil, err
334 334
 		}
335 335
 	}
336
-	dgst, _, _, err := o.vertex.Marshal(c)
336
+	dgst, _, _, err := o.vertex.Marshal(ctx, c)
337 337
 	if err != nil {
338 338
 		return nil, err
339 339
 	}
340 340
 	return &pb.Input{Digest: dgst, Index: index}, nil
341 341
 }
342 342
 
343
-func (o *output) Vertex() Vertex {
343
+func (o *output) Vertex(context.Context) Vertex {
344 344
 	return o.vertex
345 345
 }
346 346
 
... ...
@@ -513,3 +547,7 @@ func Require(filters ...string) ConstraintsOpt {
513 513
 		}
514 514
 	})
515 515
 }
516
+
517
+func nilValue(context.Context) (interface{}, error) {
518
+	return nil, nil
519
+}
... ...
@@ -115,6 +115,12 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
115 115
 	}
116 116
 
117 117
 	var ex ExportEntry
118
+	if len(opt.Exports) > 1 {
119
+		return nil, errors.New("currently only single Exports can be specified")
120
+	}
121
+	if len(opt.Exports) == 1 {
122
+		ex = opt.Exports[0]
123
+	}
118 124
 
119 125
 	if !opt.SessionPreInitialized {
120 126
 		if len(syncedDirs) > 0 {
... ...
@@ -125,13 +131,6 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
125 125
 			s.Allow(a)
126 126
 		}
127 127
 
128
-		if len(opt.Exports) > 1 {
129
-			return nil, errors.New("currently only single Exports can be specified")
130
-		}
131
-		if len(opt.Exports) == 1 {
132
-			ex = opt.Exports[0]
133
-		}
134
-
135 128
 		switch ex.Type {
136 129
 		case ExporterLocal:
137 130
 			if ex.Output != nil {
... ...
@@ -192,7 +191,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
192 192
 
193 193
 		frontendInputs := make(map[string]*pb.Definition)
194 194
 		for key, st := range opt.FrontendInputs {
195
-			def, err := st.Marshal()
195
+			def, err := st.Marshal(ctx)
196 196
 			if err != nil {
197 197
 				return err
198 198
 			}
... ...
@@ -435,13 +434,13 @@ func parseCacheOptions(opt SolveOpt) (*cacheOptions, error) {
435 435
 					continue
436 436
 				}
437 437
 				for _, m := range idx.Manifests {
438
-					if m.Annotations[ocispec.AnnotationRefName] == "latest" {
438
+					if (m.Annotations[ocispec.AnnotationRefName] == "latest" && attrs["tag"] == "") || (attrs["tag"] != "" && m.Annotations[ocispec.AnnotationRefName] == attrs["tag"]) {
439 439
 						attrs["digest"] = string(m.Digest)
440 440
 						break
441 441
 					}
442 442
 				}
443 443
 				if attrs["digest"] == "" {
444
-					return nil, errors.New("local cache importer requires either explicit digest or \"latest\" tag on index.json")
444
+					return nil, errors.New("local cache importer requires either explicit digest, \"latest\" tag or custom tag on index.json")
445 445
 				}
446 446
 			}
447 447
 			contentStores["local:"+csDir] = cs
448 448
new file mode 100644
... ...
@@ -0,0 +1,105 @@
0
+package config
1
+
2
+// Config provides containerd configuration data for the server
3
+type Config struct {
4
+	Debug bool `toml:"debug"`
5
+
6
+	// Root is the path to a directory where buildkit will store persistent data
7
+	Root string `toml:"root"`
8
+
9
+	//Entitlements e.g. security.insecure, network.host
10
+	Entitlements []string `toml:"insecure-entitlements"`
11
+	// GRPC configuration settings
12
+	GRPC GRPCConfig `toml:"grpc"`
13
+
14
+	Workers struct {
15
+		OCI        OCIConfig        `toml:"oci"`
16
+		Containerd ContainerdConfig `toml:"containerd"`
17
+	} `toml:"worker"`
18
+
19
+	Registries map[string]RegistryConfig `toml:"registry"`
20
+
21
+	DNS *DNSConfig `toml:"dns"`
22
+}
23
+
24
+type GRPCConfig struct {
25
+	Address      []string `toml:"address"`
26
+	DebugAddress string   `toml:"debugAddress"`
27
+	UID          int      `toml:"uid"`
28
+	GID          int      `toml:"gid"`
29
+
30
+	TLS TLSConfig `toml:"tls"`
31
+	// MaxRecvMsgSize int    `toml:"max_recv_message_size"`
32
+	// MaxSendMsgSize int    `toml:"max_send_message_size"`
33
+}
34
+
35
+type RegistryConfig struct {
36
+	Mirrors      []string     `toml:"mirrors"`
37
+	PlainHTTP    *bool        `toml:"http"`
38
+	Insecure     *bool        `toml:"insecure"`
39
+	RootCAs      []string     `toml:"ca"`
40
+	KeyPairs     []TLSKeyPair `toml:"keypair"`
41
+	TLSConfigDir []string     `toml:"tlsconfigdir"`
42
+}
43
+
44
+type TLSKeyPair struct {
45
+	Key         string `toml:"key"`
46
+	Certificate string `toml:"cert"`
47
+}
48
+
49
+type TLSConfig struct {
50
+	Cert string `toml:"cert"`
51
+	Key  string `toml:"key"`
52
+	CA   string `toml:"ca"`
53
+}
54
+
55
+type GCConfig struct {
56
+	GC            *bool      `toml:"gc"`
57
+	GCKeepStorage int64      `toml:"gckeepstorage"`
58
+	GCPolicy      []GCPolicy `toml:"gcpolicy"`
59
+}
60
+
61
+type NetworkConfig struct {
62
+	Mode          string `toml:"networkMode"`
63
+	CNIConfigPath string `toml:"cniConfigPath"`
64
+	CNIBinaryPath string `toml:"cniBinaryPath"`
65
+}
66
+
67
+type OCIConfig struct {
68
+	Enabled          *bool             `toml:"enabled"`
69
+	Labels           map[string]string `toml:"labels"`
70
+	Platforms        []string          `toml:"platforms"`
71
+	Snapshotter      string            `toml:"snapshotter"`
72
+	Rootless         bool              `toml:"rootless"`
73
+	NoProcessSandbox bool              `toml:"noProcessSandbox"`
74
+	GCConfig
75
+	NetworkConfig
76
+	// UserRemapUnsupported is unsupported key for testing. The feature is
77
+	// incomplete and the intention is to make it default without config.
78
+	UserRemapUnsupported string `toml:"userRemapUnsupported"`
79
+	// For use in storing the OCI worker binary name that will replace buildkit-runc
80
+	Binary string `toml:"binary"`
81
+}
82
+
83
+type ContainerdConfig struct {
84
+	Address   string            `toml:"address"`
85
+	Enabled   *bool             `toml:"enabled"`
86
+	Labels    map[string]string `toml:"labels"`
87
+	Platforms []string          `toml:"platforms"`
88
+	Namespace string            `toml:"namespace"`
89
+	GCConfig
90
+	NetworkConfig
91
+}
92
+
93
+type GCPolicy struct {
94
+	All          bool     `toml:"all"`
95
+	KeepBytes    int64    `toml:"keepBytes"`
96
+	KeepDuration int64    `toml:"keepDuration"`
97
+	Filters      []string `toml:"filters"`
98
+}
99
+
100
+type DNSConfig struct {
101
+	Nameservers   []string `toml:"nameservers"`
102
+	Options       []string `toml:"options"`
103
+	SearchDomains []string `toml:"searchDomains"`
104
+}
0 105
new file mode 100644
... ...
@@ -0,0 +1,31 @@
0
+package config
1
+
2
+const defaultCap int64 = 2e9 // 2GB
3
+
4
+func DefaultGCPolicy(p string, keep int64) []GCPolicy {
5
+	if keep == 0 {
6
+		keep = DetectDefaultGCCap(p)
7
+	}
8
+	return []GCPolicy{
9
+		// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
10
+		{
11
+			Filters:      []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
12
+			KeepDuration: 48 * 3600, // 48h
13
+			KeepBytes:    512 * 1e6, // 512MB
14
+		},
15
+		// remove any data not used for 60 days
16
+		{
17
+			KeepDuration: 60 * 24 * 3600, // 60d
18
+			KeepBytes:    keep,
19
+		},
20
+		// keep the unshared build cache under cap
21
+		{
22
+			KeepBytes: keep,
23
+		},
24
+		// if previous policies were insufficient start deleting internal data to keep build cache under cap
25
+		{
26
+			All:       true,
27
+			KeepBytes: keep,
28
+		},
29
+	}
30
+}
0 31
new file mode 100644
... ...
@@ -0,0 +1,17 @@
0
+// +build !windows
1
+
2
+package config
3
+
4
+import (
5
+	"syscall"
6
+)
7
+
8
+func DetectDefaultGCCap(root string) int64 {
9
+	var st syscall.Statfs_t
10
+	if err := syscall.Statfs(root, &st); err != nil {
11
+		return defaultCap
12
+	}
13
+	diskSize := int64(st.Bsize) * int64(st.Blocks)
14
+	avail := diskSize / 10
15
+	return (avail/(1<<30) + 1) * 1e9 // round up
16
+}
0 17
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+// +build windows
1
+
2
+package config
3
+
4
+func DetectDefaultGCCap(root string) int64 {
5
+	return defaultCap
6
+}
... ...
@@ -139,7 +139,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
139 139
 		buildContext = st
140 140
 	} else if httpPrefix.MatchString(opts[localNameContext]) {
141 141
 		httpContext := llb.HTTP(opts[localNameContext], llb.Filename("context"), dockerfile2llb.WithInternalName("load remote build context"))
142
-		def, err := httpContext.Marshal(marshalOpts...)
142
+		def, err := httpContext.Marshal(ctx, marshalOpts...)
143 143
 		if err != nil {
144 144
 			return nil, errors.Wrapf(err, "failed to marshal httpcontext")
145 145
 		}
... ...
@@ -221,7 +221,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
221 221
 		}
222 222
 	}
223 223
 
224
-	def, err := src.Marshal(marshalOpts...)
224
+	def, err := src.Marshal(ctx, marshalOpts...)
225 225
 	if err != nil {
226 226
 		return nil, errors.Wrapf(err, "failed to marshal local source")
227 227
 	}
... ...
@@ -271,7 +271,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
271 271
 				)
272 272
 				dockerignoreState = &st
273 273
 			}
274
-			def, err := dockerignoreState.Marshal(marshalOpts...)
274
+			def, err := dockerignoreState.Marshal(ctx, marshalOpts...)
275 275
 			if err != nil {
276 276
 				return err
277 277
 			}
... ...
@@ -363,7 +363,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
363 363
 					return errors.Wrapf(err, "failed to create LLB definition")
364 364
 				}
365 365
 
366
-				def, err := st.Marshal()
366
+				def, err := st.Marshal(ctx)
367 367
 				if err != nil {
368 368
 					return errors.Wrapf(err, "failed to marshal LLB definition")
369 369
 				}
... ...
@@ -456,9 +456,29 @@ func forwardGateway(ctx context.Context, c client.Client, ref string, cmdline st
456 456
 	}
457 457
 	opts["cmdline"] = cmdline
458 458
 	opts["source"] = ref
459
+
460
+	gwcaps := c.BuildOpts().Caps
461
+	var frontendInputs map[string]*pb.Definition
462
+	if (&gwcaps).Supports(gwpb.CapFrontendInputs) == nil {
463
+		inputs, err := c.Inputs(ctx)
464
+		if err != nil {
465
+			return nil, errors.Wrapf(err, "failed to get frontend inputs")
466
+		}
467
+
468
+		frontendInputs = make(map[string]*pb.Definition)
469
+		for name, state := range inputs {
470
+			def, err := state.Marshal(ctx)
471
+			if err != nil {
472
+				return nil, err
473
+			}
474
+			frontendInputs[name] = def.ToPB()
475
+		}
476
+	}
477
+
459 478
 	return c.Solve(ctx, client.SolveRequest{
460
-		Frontend:    "gateway.v0",
461
-		FrontendOpt: opts,
479
+		Frontend:       "gateway.v0",
480
+		FrontendOpt:    opts,
481
+		FrontendInputs: frontendInputs,
462 482
 	})
463 483
 }
464 484
 
... ...
@@ -462,7 +462,11 @@ type dispatchOpt struct {
462 462
 func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
463 463
 	if ex, ok := cmd.Command.(instructions.SupportsSingleWordExpansion); ok {
464 464
 		err := ex.Expand(func(word string) (string, error) {
465
-			return opt.shlex.ProcessWord(word, d.state.Env())
465
+			env, err := d.state.Env(context.TODO())
466
+			if err != nil {
467
+				return "", err
468
+			}
469
+			return opt.shlex.ProcessWord(word, env)
466 470
 		})
467 471
 		if err != nil {
468 472
 			return err
... ...
@@ -626,7 +630,10 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
626 626
 	if c.PrependShell {
627 627
 		args = withShell(d.image, args)
628 628
 	}
629
-	env := d.state.Env()
629
+	env, err := d.state.Env(context.TODO())
630
+	if err != nil {
631
+		return err
632
+	}
630 633
 	opt := []llb.RunOption{llb.Args(args), dfCmd(c)}
631 634
 	if d.ignoreCache {
632 635
 		opt = append(opt, llb.IgnoreCache)
... ...
@@ -661,7 +668,11 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
661 661
 	shlex.RawQuotes = true
662 662
 	shlex.SkipUnsetEnv = true
663 663
 
664
-	opt = append(opt, llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(&shlex, c.String(), env)), d.prefixPlatform, d.state.GetPlatform())))
664
+	pl, err := d.state.GetPlatform(context.TODO())
665
+	if err != nil {
666
+		return err
667
+	}
668
+	opt = append(opt, llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(&shlex, c.String(), env)), d.prefixPlatform, pl)))
665 669
 	for _, h := range dopt.extraHosts {
666 670
 		opt = append(opt, llb.AddExtraHost(h.Host, h.IP))
667 671
 	}
... ...
@@ -687,7 +698,11 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo
687 687
 			if d.platform != nil {
688 688
 				platform = *d.platform
689 689
 			}
690
-			d.state = d.state.File(llb.Mkdir(wd, 0755, mkdirOpt...), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, c.String(), d.state.Env())), d.prefixPlatform, &platform)))
690
+			env, err := d.state.Env(context.TODO())
691
+			if err != nil {
692
+				return err
693
+			}
694
+			d.state = d.state.File(llb.Mkdir(wd, 0755, mkdirOpt...), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, c.String(), env)), d.prefixPlatform, &platform)))
691 695
 			withLayer = true
692 696
 		}
693 697
 		return commitToHistory(&d.image, "WORKDIR "+wd, withLayer, nil)
... ...
@@ -696,7 +711,11 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo
696 696
 }
697 697
 
698 698
 func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, opt dispatchOpt) error {
699
-	dest := path.Join("/", pathRelativeToWorkingDir(d.state, c.Dest()))
699
+	pp, err := pathRelativeToWorkingDir(d.state, c.Dest())
700
+	if err != nil {
701
+		return err
702
+	}
703
+	dest := path.Join("/", pp)
700 704
 	if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
701 705
 		dest += string(filepath.Separator)
702 706
 	}
... ...
@@ -772,7 +791,12 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
772 772
 		platform = *d.platform
773 773
 	}
774 774
 
775
-	fileOpt := []llb.ConstraintsOpt{llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), d.state.Env())), d.prefixPlatform, &platform))}
775
+	env, err := d.state.Env(context.TODO())
776
+	if err != nil {
777
+		return err
778
+	}
779
+
780
+	fileOpt := []llb.ConstraintsOpt{llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform))}
776 781
 	if d.ignoreCache {
777 782
 		fileOpt = append(fileOpt, llb.IgnoreCache)
778 783
 	}
... ...
@@ -787,8 +811,11 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
787 787
 	}
788 788
 
789 789
 	img := llb.Image(opt.copyImage, llb.MarkImageInternal, llb.Platform(opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
790
-
791
-	dest := path.Join(".", pathRelativeToWorkingDir(d.state, c.Dest()))
790
+	pp, err := pathRelativeToWorkingDir(d.state, c.Dest())
791
+	if err != nil {
792
+		return err
793
+	}
794
+	dest := path.Join(".", pp)
792 795
 	if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
793 796
 		dest += string(filepath.Separator)
794 797
 	}
... ...
@@ -861,7 +888,12 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
861 861
 		platform = *d.platform
862 862
 	}
863 863
 
864
-	runOpt := []llb.RunOption{llb.Args(args), llb.Dir("/dest"), llb.ReadonlyRootFS(), dfCmd(cmdToPrint), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), d.state.Env())), d.prefixPlatform, &platform))}
864
+	env, err := d.state.Env(context.TODO())
865
+	if err != nil {
866
+		return err
867
+	}
868
+
869
+	runOpt := []llb.RunOption{llb.Args(args), llb.Dir("/dest"), llb.ReadonlyRootFS(), dfCmd(cmdToPrint), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform))}
865 870
 	if d.ignoreCache {
866 871
 		runOpt = append(runOpt, llb.IgnoreCache)
867 872
 	}
... ...
@@ -936,8 +968,12 @@ func dispatchHealthcheck(d *dispatchState, c *instructions.HealthCheckCommand) e
936 936
 
937 937
 func dispatchExpose(d *dispatchState, c *instructions.ExposeCommand, shlex *shell.Lex) error {
938 938
 	ports := []string{}
939
+	env, err := d.state.Env(context.TODO())
940
+	if err != nil {
941
+		return err
942
+	}
939 943
 	for _, p := range c.Ports {
940
-		ps, err := shlex.ProcessWords(p, d.state.Env())
944
+		ps, err := shlex.ProcessWords(p, env)
941 945
 		if err != nil {
942 946
 			return err
943 947
 		}
... ...
@@ -1018,11 +1054,15 @@ func dispatchArg(d *dispatchState, c *instructions.ArgCommand, metaArgs []instru
1018 1018
 	return commitToHistory(&d.image, commitStr, false, nil)
1019 1019
 }
1020 1020
 
1021
-func pathRelativeToWorkingDir(s llb.State, p string) string {
1021
+func pathRelativeToWorkingDir(s llb.State, p string) (string, error) {
1022 1022
 	if path.IsAbs(p) {
1023
-		return p
1023
+		return p, nil
1024
+	}
1025
+	dir, err := s.GetDir(context.TODO())
1026
+	if err != nil {
1027
+		return "", err
1024 1028
 	}
1025
-	return path.Join(s.GetDir(), p)
1029
+	return path.Join(dir, p), nil
1026 1030
 }
1027 1031
 
1028 1032
 func splitWildcards(name string) (string, string) {
... ...
@@ -3,6 +3,7 @@
3 3
 package dockerfile2llb
4 4
 
5 5
 import (
6
+	"context"
6 7
 	"fmt"
7 8
 	"os"
8 9
 	"path"
... ...
@@ -132,7 +133,11 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*
132 132
 		}
133 133
 		target := mount.Target
134 134
 		if !filepath.IsAbs(filepath.Clean(mount.Target)) {
135
-			target = filepath.Join("/", d.state.GetDir(), mount.Target)
135
+			dir, err := d.state.GetDir(context.TODO())
136
+			if err != nil {
137
+				return nil, err
138
+			}
139
+			target = filepath.Join("/", dir, mount.Target)
136 140
 		}
137 141
 		if target == "/" {
138 142
 			return nil, errors.Errorf("invalid mount target %q", target)
... ...
@@ -135,7 +135,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
135 135
 
136 136
 		src := llb.Image(sourceRef.String(), &markTypeFrontend{})
137 137
 
138
-		def, err := src.Marshal()
138
+		def, err := src.Marshal(ctx)
139 139
 		if err != nil {
140 140
 			return nil, err
141 141
 		}
... ...
@@ -3,6 +3,7 @@ module github.com/moby/buildkit
3 3
 go 1.13
4 4
 
5 5
 require (
6
+	github.com/AkihiroSuda/containerd-fuse-overlayfs v0.0.0-20200220082720-bb896865146c
6 7
 	github.com/BurntSushi/toml v0.3.1
7 8
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
8 9
 	github.com/Microsoft/hcsshim v0.8.7 // indirect
... ...
@@ -23,13 +24,12 @@ require (
23 23
 	github.com/docker/docker v0.0.0
24 24
 	github.com/docker/docker-credential-helpers v0.6.0 // indirect
25 25
 	github.com/docker/go-connections v0.3.0
26
-	github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect
27 26
 	github.com/docker/libnetwork v0.8.0-dev.2.0.20200226230617-d8334ccdb9be
28 27
 	github.com/gofrs/flock v0.7.0
29 28
 	github.com/gogo/googleapis v1.3.2
30 29
 	github.com/gogo/protobuf v1.3.1
31 30
 	github.com/golang/protobuf v1.3.3
32
-	github.com/google/go-cmp v0.3.0
31
+	github.com/google/go-cmp v0.3.1
33 32
 	github.com/google/shlex v0.0.0-20150127133951-6f45313302b9
34 33
 	github.com/google/uuid v1.1.1 // indirect
35 34
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
... ...
@@ -54,7 +54,7 @@ require (
54 54
 	github.com/sirupsen/logrus v1.4.2
55 55
 	github.com/stretchr/testify v1.4.0
56 56
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
57
-	github.com/tonistiigi/fsutil v0.0.0-20200225063759-013a9fe6aee2
57
+	github.com/tonistiigi/fsutil v0.0.0-20200326231323-c2c7d7b0e144
58 58
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
59 59
 	github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e
60 60
 	github.com/uber/jaeger-lib v1.2.1 // indirect
... ...
@@ -150,8 +150,12 @@ type LoadedResult struct {
150 150
 	CacheKey    *CacheKey
151 151
 }
152 152
 
153
-func (c *cacheManager) filterResults(m map[string]Result, ck *CacheKey) (results []LoadedResult, err error) {
153
+func (c *cacheManager) filterResults(m map[string]Result, ck *CacheKey, visited map[string]struct{}) (results []LoadedResult, err error) {
154 154
 	id := c.getID(ck)
155
+	if _, ok := visited[id]; ok {
156
+		return nil, nil
157
+	}
158
+	visited[id] = struct{}{}
155 159
 	if err := c.backend.WalkResults(id, func(cr CacheResult) error {
156 160
 		res, ok := m[id]
157 161
 		if ok {
... ...
@@ -170,7 +174,7 @@ func (c *cacheManager) filterResults(m map[string]Result, ck *CacheKey) (results
170 170
 	}
171 171
 	for _, keys := range ck.Deps() {
172 172
 		for _, key := range keys {
173
-			res, err := c.filterResults(m, key.CacheKey.CacheKey)
173
+			res, err := c.filterResults(m, key.CacheKey.CacheKey, visited)
174 174
 			if err != nil {
175 175
 				for _, r := range results {
176 176
 					r.Result.Release(context.TODO())
... ...
@@ -207,7 +211,7 @@ func (c *cacheManager) LoadWithParents(ctx context.Context, rec *CacheRecord) ([
207 207
 		return nil, err
208 208
 	}
209 209
 
210
-	results, err := c.filterResults(m, rec.key)
210
+	results, err := c.filterResults(m, rec.key, map[string]struct{}{})
211 211
 	if err != nil {
212 212
 		for _, r := range m {
213 213
 			r.Release(context.TODO())
... ...
@@ -20,11 +20,12 @@ func addBacklinks(t CacheExporterTarget, rec CacheExporterRecord, cm *cacheManag
20 20
 	if rec == nil {
21 21
 		var ok bool
22 22
 		rec, ok = bkm[id]
23
-		if ok {
23
+		if ok && rec != nil {
24 24
 			return rec, nil
25 25
 		}
26 26
 		_ = ok
27 27
 	}
28
+	bkm[id] = nil
28 29
 	if err := cm.backend.WalkBacklinks(id, func(id string, link CacheInfoLink) error {
29 30
 		if rec == nil {
30 31
 			rec = t.Add(link.Digest)
... ...
@@ -37,7 +38,9 @@ func addBacklinks(t CacheExporterTarget, rec CacheExporterRecord, cm *cacheManag
37 37
 				return err
38 38
 			}
39 39
 		}
40
-		rec.LinkFrom(r, int(link.Input), link.Selector.String())
40
+		if r != nil {
41
+			rec.LinkFrom(r, int(link.Input), link.Selector.String())
42
+		}
41 43
 		return nil
42 44
 	}); err != nil {
43 45
 		return nil, err
... ...
@@ -66,6 +69,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
66 66
 	if t.Visited(e) {
67 67
 		return e.res, nil
68 68
 	}
69
+	t.Visit(e)
69 70
 
70 71
 	deps := e.k.Deps()
71 72
 
... ...
@@ -177,7 +181,6 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
177 177
 	}
178 178
 
179 179
 	e.res = allRec
180
-	t.Visit(e)
181 180
 
182 181
 	return e.res, nil
183 182
 }
... ...
@@ -272,8 +272,9 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
272 272
 		}
273 273
 		args = append(args, "origin")
274 274
 		if !isCommitSHA(ref) {
275
-			args = append(args, ref+":tags/"+ref)
276
-			// local refs are needed so they would be advertised on next fetches
275
+			args = append(args, "--force", ref+":tags/"+ref)
276
+			// local refs are needed so they would be advertised on next fetches. Force is used
277
+			// in case the ref is a branch and it now points to a different commit sha
277 278
 			// TODO: is there a better way to do this?
278 279
 		}
279 280
 		if _, err := gitWithinDir(ctx, gitDir, "", args...); err != nil {
... ...
@@ -1,54 +1,219 @@
1 1
 package resolver
2 2
 
3 3
 import (
4
-	"math/rand"
4
+	"context"
5
+	"crypto/tls"
6
+	"crypto/x509"
7
+	"io/ioutil"
8
+	"net"
9
+	"net/http"
10
+	"os"
11
+	"path/filepath"
12
+	"runtime"
5 13
 	"strings"
14
+	"time"
6 15
 
16
+	"github.com/containerd/containerd/remotes"
7 17
 	"github.com/containerd/containerd/remotes/docker"
8
-	"github.com/docker/distribution/reference"
18
+	"github.com/moby/buildkit/cmd/buildkitd/config"
19
+	"github.com/moby/buildkit/session"
20
+	"github.com/moby/buildkit/session/auth"
9 21
 	"github.com/moby/buildkit/util/tracing"
22
+	"github.com/pkg/errors"
10 23
 )
11 24
 
12
-type RegistryConf struct {
13
-	Mirrors   []string
14
-	PlainHTTP *bool
25
+func fillInsecureOpts(host string, c config.RegistryConfig, h *docker.RegistryHost) error {
26
+	tc, err := loadTLSConfig(c)
27
+	if err != nil {
28
+		return err
29
+	}
30
+
31
+	if c.PlainHTTP != nil && *c.PlainHTTP {
32
+		h.Scheme = "http"
33
+	} else if c.Insecure != nil && *c.Insecure {
34
+		tc.InsecureSkipVerify = true
35
+	} else if c.PlainHTTP == nil {
36
+		if ok, _ := docker.MatchLocalhost(host); ok {
37
+			h.Scheme = "http"
38
+		}
39
+	}
40
+
41
+	transport := newDefaultTransport()
42
+	transport.TLSClientConfig = tc
43
+
44
+	h.Client = &http.Client{
45
+		Transport: tracing.NewTransport(transport),
46
+	}
47
+	return nil
15 48
 }
16 49
 
17
-type ResolveOptionsFunc func(string) docker.ResolverOptions
50
+func loadTLSConfig(c config.RegistryConfig) (*tls.Config, error) {
51
+	for _, d := range c.TLSConfigDir {
52
+		fs, err := ioutil.ReadDir(d)
53
+		if err != nil && !os.IsNotExist(err) && !os.IsPermission(err) {
54
+			return nil, errors.WithStack(err)
55
+		}
56
+		for _, f := range fs {
57
+			if strings.HasSuffix(f.Name(), ".crt") {
58
+				c.RootCAs = append(c.RootCAs, filepath.Join(d, f.Name()))
59
+			}
60
+			if strings.HasSuffix(f.Name(), ".cert") {
61
+				c.KeyPairs = append(c.KeyPairs, config.TLSKeyPair{
62
+					Certificate: filepath.Join(d, f.Name()),
63
+					Key:         filepath.Join(d, strings.TrimSuffix(f.Name(), ".cert")+".key"),
64
+				})
65
+			}
66
+		}
67
+	}
18 68
 
19
-func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc {
20
-	return func(ref string) docker.ResolverOptions {
21
-		def := docker.ResolverOptions{
22
-			Client: tracing.DefaultClient,
69
+	tc := &tls.Config{}
70
+	if len(c.RootCAs) > 0 {
71
+		systemPool, err := x509.SystemCertPool()
72
+		if err != nil {
73
+			if runtime.GOOS == "windows" {
74
+				systemPool = x509.NewCertPool()
75
+			} else {
76
+				return nil, errors.Wrapf(err, "unable to get system cert pool")
77
+			}
23 78
 		}
79
+		tc.RootCAs = systemPool
80
+	}
24 81
 
25
-		parsed, err := reference.ParseNormalizedNamed(ref)
82
+	for _, p := range c.RootCAs {
83
+		dt, err := ioutil.ReadFile(p)
26 84
 		if err != nil {
27
-			return def
85
+			return nil, errors.Wrapf(err, "failed to read %s", p)
28 86
 		}
29
-		host := reference.Domain(parsed)
87
+		tc.RootCAs.AppendCertsFromPEM(dt)
88
+	}
30 89
 
31
-		c, ok := m[host]
32
-		if !ok {
33
-			return def
90
+	for _, kp := range c.KeyPairs {
91
+		cert, err := tls.LoadX509KeyPair(kp.Certificate, kp.Key)
92
+		if err != nil {
93
+			return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate)
34 94
 		}
95
+		tc.Certificates = append(tc.Certificates, cert)
96
+	}
97
+	return tc, nil
98
+}
35 99
 
36
-		var mirrorHost string
37
-		if len(c.Mirrors) > 0 {
38
-			mirrorHost = c.Mirrors[rand.Intn(len(c.Mirrors))]
39
-			def.Host = func(string) (string, error) {
40
-				return mirrorHost, nil
100
+func NewRegistryConfig(m map[string]config.RegistryConfig) docker.RegistryHosts {
101
+	return docker.Registries(
102
+		func(host string) ([]docker.RegistryHost, error) {
103
+			c, ok := m[host]
104
+			if !ok {
105
+				return nil, nil
106
+			}
107
+
108
+			var out []docker.RegistryHost
109
+
110
+			for _, mirror := range c.Mirrors {
111
+				h := docker.RegistryHost{
112
+					Scheme:       "https",
113
+					Client:       newDefaultClient(),
114
+					Host:         mirror,
115
+					Path:         "/v2",
116
+					Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve,
117
+				}
118
+
119
+				if err := fillInsecureOpts(mirror, m[mirror], &h); err != nil {
120
+					return nil, err
121
+				}
122
+
123
+				out = append(out, h)
124
+			}
125
+
126
+			if host == "docker.io" {
127
+				host = "registry-1.docker.io"
41 128
 			}
42
-		}
43 129
 
44
-		if c.PlainHTTP != nil {
45
-			def.PlainHTTP = *c.PlainHTTP
46
-		} else {
47
-			if mirrorHost == "localhost" || strings.HasPrefix(mirrorHost, "localhost:") {
48
-				def.PlainHTTP = true
130
+			h := docker.RegistryHost{
131
+				Scheme:       "https",
132
+				Client:       newDefaultClient(),
133
+				Host:         host,
134
+				Path:         "/v2",
135
+				Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve,
49 136
 			}
137
+
138
+			if err := fillInsecureOpts(host, c, &h); err != nil {
139
+				return nil, err
140
+			}
141
+
142
+			out = append(out, h)
143
+			return out, nil
144
+		},
145
+		docker.ConfigureDefaultRegistries(
146
+			docker.WithClient(newDefaultClient()),
147
+			docker.WithPlainHTTP(docker.MatchLocalhost),
148
+		),
149
+	)
150
+}
151
+
152
+func New(ctx context.Context, hosts docker.RegistryHosts, sm *session.Manager) remotes.Resolver {
153
+	return docker.NewResolver(docker.ResolverOptions{
154
+		Hosts: hostsWithCredentials(ctx, hosts, sm),
155
+	})
156
+}
157
+
158
+func hostsWithCredentials(ctx context.Context, hosts docker.RegistryHosts, sm *session.Manager) docker.RegistryHosts {
159
+	id := session.FromContext(ctx)
160
+	if id == "" {
161
+		return hosts
162
+	}
163
+	return func(domain string) ([]docker.RegistryHost, error) {
164
+		res, err := hosts(domain)
165
+		if err != nil {
166
+			return nil, err
50 167
 		}
168
+		if len(res) == 0 {
169
+			return nil, nil
170
+		}
171
+
172
+		timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
173
+		defer cancel()
174
+
175
+		caller, err := sm.Get(timeoutCtx, id)
176
+		if err != nil {
177
+			return nil, err
178
+		}
179
+
180
+		a := docker.NewDockerAuthorizer(
181
+			docker.WithAuthClient(res[0].Client),
182
+			docker.WithAuthCreds(auth.CredentialsFunc(context.TODO(), caller)),
183
+		)
184
+		for i := range res {
185
+			res[i].Authorizer = a
186
+		}
187
+		return res, nil
188
+	}
189
+}
190
+
191
+func newDefaultClient() *http.Client {
192
+	return &http.Client{
193
+		Transport: newDefaultTransport(),
194
+	}
195
+}
51 196
 
52
-		return def
197
+// newDefaultTransport is for pull or push client
198
+//
199
+// NOTE: For push, there must disable http2 for https because the flow control
200
+// will limit data transfer. The net/http package doesn't provide http2 tunable
201
+// settings which limits push performance.
202
+//
203
+// REF: https://github.com/golang/go/issues/14077
204
+func newDefaultTransport() *http.Transport {
205
+	return &http.Transport{
206
+		Proxy: http.ProxyFromEnvironment,
207
+		DialContext: (&net.Dialer{
208
+			Timeout:   30 * time.Second,
209
+			KeepAlive: 30 * time.Second,
210
+			DualStack: true,
211
+		}).DialContext,
212
+		MaxIdleConns:          10,
213
+		IdleConnTimeout:       30 * time.Second,
214
+		TLSHandshakeTimeout:   10 * time.Second,
215
+		ExpectContinueTimeout: 5 * time.Second,
216
+		DisableKeepAlives:     true,
217
+		TLSNextProto:          make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
53 218
 	}
54 219
 }
... ...
@@ -74,6 +74,12 @@ type Transport struct {
74 74
 	http.RoundTripper
75 75
 }
76 76
 
77
+func NewTransport(rt http.RoundTripper) http.RoundTripper {
78
+	return &Transport{
79
+		RoundTripper: &nethttp.Transport{RoundTripper: rt},
80
+	}
81
+}
82
+
77 83
 func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
78 84
 	span := opentracing.SpanFromContext(req.Context())
79 85
 	if span == nil { // no tracer connected with either request or transport
... ...
@@ -37,7 +37,11 @@ func WriteTar(ctx context.Context, fs FS, w io.Writer) error {
37 37
 		hdr.Linkname = stat.Linkname
38 38
 		if hdr.Linkname != "" {
39 39
 			hdr.Size = 0
40
-			hdr.Typeflag = tar.TypeLink
40
+			if fi.Mode() & os.ModeSymlink != 0 {
41
+				hdr.Typeflag = tar.TypeSymlink
42
+			} else {
43
+				hdr.Typeflag = tar.TypeLink
44
+			}
41 45
 		}
42 46
 
43 47
 		if len(stat.Xattrs) > 0 {