Browse code

bump moby/buildkit f7042823e340d38d1746aa675b83d1aca431cee3

full diff: https://github.com/moby/buildkit/compare/588c73e1e4f0f3d7d3738abaaa7cf8026064b33e...f7042823e340d38d1746aa675b83d1aca431cee3

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

fix daemon for changes in containerd registry configuration

Signed-off-by: Evan Hazlett <ejhazlett@gmail.com>

Update buildernext and daemon for buildkit update

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

Derek McGowan authored on 2019/10/03 09:16:29
Showing 26 changed files
... ...
@@ -23,9 +23,9 @@ import (
23 23
 )
24 24
 
25 25
 // ResolveCacheImporterFunc returns a resolver function for local inline cache
26
-func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
26
+func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
27 27
 
28
-	upstream := registryremotecache.ResolveCacheImporterFunc(sm, resolverOpt)
28
+	upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverOpt)
29 29
 
30 30
 	return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
31 31
 		if dt, err := tryImportLocal(rs, is, attrs["ref"]); err == nil {
... ...
@@ -189,7 +189,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
189 189
 		Frontends:        frontends,
190 190
 		CacheKeyStorage:  cacheStorage,
191 191
 		ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{
192
-			"registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt, dist.ReferenceStore, dist.ImageStore),
192
+			"registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt, store, dist.ReferenceStore, dist.ImageStore),
193 193
 			"local":    localremotecache.ResolveCacheImporterFunc(opt.SessionManager),
194 194
 		},
195 195
 		ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
... ...
@@ -151,6 +151,11 @@ func (w *Worker) GCPolicy() []client.PruneInfo {
151 151
 	return w.Opt.GCPolicy
152 152
 }
153 153
 
154
+// ContentStore returns content store
155
+func (w *Worker) ContentStore() content.Store {
156
+	return w.Opt.ContentStore
157
+}
158
+
154 159
 // LoadRef loads a reference by ID
155 160
 func (w *Worker) LoadRef(id string, hidden bool) (cache.ImmutableRef, error) {
156 161
 	var opts []cache.RefOption
... ...
@@ -322,7 +327,7 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
322 322
 
323 323
 	defer func() {
324 324
 		for _, l := range rootfs {
325
-			w.ContentStore.Delete(context.TODO(), l.Blob.Digest)
325
+			w.ContentStore().Delete(context.TODO(), l.Blob.Digest)
326 326
 		}
327 327
 	}()
328 328
 
... ...
@@ -391,12 +396,12 @@ func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
391 391
 
392 392
 func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
393 393
 	done := oneOffProgress(ld.pctx, fmt.Sprintf("pulling %s", ld.desc.Digest))
394
-	if err := contentutil.Copy(ctx, ld.w.ContentStore, ld.provider, ld.desc); err != nil {
394
+	if err := contentutil.Copy(ctx, ld.w.ContentStore(), ld.provider, ld.desc); err != nil {
395 395
 		return nil, 0, done(err)
396 396
 	}
397 397
 	_ = done(nil)
398 398
 
399
-	ra, err := ld.w.ContentStore.ReaderAt(ctx, ld.desc)
399
+	ra, err := ld.w.ContentStore().ReaderAt(ctx, ld.desc)
400 400
 	if err != nil {
401 401
 		return nil, 0, err
402 402
 	}
... ...
@@ -405,7 +410,7 @@ func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogr
405 405
 }
406 406
 
407 407
 func (ld *layerDescriptor) Close() {
408
-	// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest)
408
+	// ld.is.ContentStore().Delete(context.TODO(), ld.desc.Digest)
409 409
 }
410 410
 
411 411
 func (ld *layerDescriptor) Registered(diffID layer.DiffID) {
... ...
@@ -173,8 +173,9 @@ func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc {
173 173
 			if uri, err := url.Parse(v); err == nil {
174 174
 				v = uri.Host
175 175
 			}
176
+			plainHTTP := true
176 177
 			m[v] = resolver.RegistryConf{
177
-				PlainHTTP: true,
178
+				PlainHTTP: &plainHTTP,
178 179
 			}
179 180
 		}
180 181
 		def := docker.ResolverOptions{
... ...
@@ -193,12 +194,16 @@ func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc {
193 193
 		}
194 194
 
195 195
 		if len(c.Mirrors) > 0 {
196
+			// TODO ResolverOptions.Host is deprecated; ResolverOptions.Hosts should be used
196 197
 			def.Host = func(string) (string, error) {
197 198
 				return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
198 199
 			}
199 200
 		}
200 201
 
201
-		def.PlainHTTP = c.PlainHTTP
202
+		// TODO ResolverOptions.PlainHTTP is deprecated; ResolverOptions.Hosts should be used
203
+		if c.PlainHTTP != nil {
204
+			def.PlainHTTP = *c.PlainHTTP
205
+		}
202 206
 
203 207
 		return def
204 208
 	}
... ...
@@ -26,7 +26,7 @@ github.com/imdario/mergo                            7c29201646fa3de8506f70121347
26 26
 golang.org/x/sync                                   e225da77a7e68af35c70ccbf71af2b83e6acac3c
27 27
 
28 28
 # buildkit
29
-github.com/moby/buildkit                            588c73e1e4f0f3d7d3738abaaa7cf8026064b33e
29
+github.com/moby/buildkit                            f7042823e340d38d1746aa675b83d1aca431cee3
30 30
 github.com/tonistiigi/fsutil                        3bbb99cdbd76619ab717299830c60f6f2a533a6b
31 31
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
32 32
 github.com/opentracing/opentracing-go               1361b9cd60be79c4c3a7fa9841b3c132e40066a7
... ...
@@ -38,7 +38,7 @@ BuildKit is used by the following projects:
38 38
 -   [Knative Build Templates](https://github.com/knative/build-templates)
39 39
 -   [the Sanic build tool](https://github.com/distributed-containers-inc/sanic)
40 40
 -   [vab](https://github.com/stellarproject/vab)
41
--   [Rio](https://github.com/rancher/rio) (on roadmap)
41
+-   [Rio](https://github.com/rancher/rio)
42 42
 
43 43
 ### Quick start
44 44
 
... ...
@@ -27,6 +27,11 @@ type Importer interface {
27 27
 	Resolve(ctx context.Context, desc ocispec.Descriptor, id string, w worker.Worker) (solver.CacheManager, error)
28 28
 }
29 29
 
30
+type DistributionSourceLabelSetter interface {
31
+	SetDistributionSourceLabel(context.Context, digest.Digest) error
32
+	SetDistributionSourceAnnotation(desc ocispec.Descriptor) ocispec.Descriptor
33
+}
34
+
30 35
 func NewImporter(provider content.Provider) Importer {
31 36
 	return &contentCacheImporter{provider: provider}
32 37
 }
... ...
@@ -61,6 +66,15 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispec.Descri
61 61
 		}
62 62
 	}
63 63
 
64
+	if dsls, ok := ci.provider.(DistributionSourceLabelSetter); ok {
65
+		for dgst, l := range allLayers {
66
+			err := dsls.SetDistributionSourceLabel(ctx, dgst)
67
+			_ = err // error ignored because layer may not exist
68
+			l.Descriptor = dsls.SetDistributionSourceAnnotation(l.Descriptor)
69
+			allLayers[dgst] = l
70
+		}
71
+	}
72
+
64 73
 	if configDesc.Digest == "" {
65 74
 		return ci.importInlineCache(ctx, dt, id, w)
66 75
 	}
... ...
@@ -127,6 +141,14 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
127 127
 					return nil
128 128
 				}
129 129
 
130
+				if dsls, ok := ci.provider.(DistributionSourceLabelSetter); ok {
131
+					for i, l := range m.Layers {
132
+						err := dsls.SetDistributionSourceLabel(ctx, l.Digest)
133
+						_ = err // error ignored because layer may not exist
134
+						m.Layers[i] = dsls.SetDistributionSourceAnnotation(l)
135
+					}
136
+				}
137
+
130 138
 				p, err := content.ReadBlob(ctx, ci.provider, m.Config)
131 139
 				if err != nil {
132 140
 					return errors.WithStack(err)
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"context"
5 5
 	"time"
6 6
 
7
+	"github.com/containerd/containerd/content"
7 8
 	"github.com/containerd/containerd/remotes"
8 9
 	"github.com/containerd/containerd/remotes/docker"
9 10
 	"github.com/docker/distribution/reference"
... ...
@@ -12,6 +13,8 @@ import (
12 12
 	"github.com/moby/buildkit/session/auth"
13 13
 	"github.com/moby/buildkit/util/contentutil"
14 14
 	"github.com/moby/buildkit/util/resolver"
15
+	"github.com/opencontainers/go-digest"
16
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
15 17
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
16 18
 	"github.com/pkg/errors"
17 19
 )
... ...
@@ -46,7 +49,7 @@ func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveO
46 46
 	}
47 47
 }
48 48
 
49
-func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
49
+func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
50 50
 	return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
51 51
 		ref, err := canonicalizeRef(attrs[attrRef])
52 52
 		if err != nil {
... ...
@@ -61,8 +64,38 @@ func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveO
61 61
 		if err != nil {
62 62
 			return nil, specs.Descriptor{}, err
63 63
 		}
64
-		return remotecache.NewImporter(contentutil.FromFetcher(fetcher)), desc, nil
64
+		src := &withDistributionSourceLabel{
65
+			Provider: contentutil.FromFetcher(fetcher),
66
+			ref:      ref,
67
+			source:   cs,
68
+		}
69
+		return remotecache.NewImporter(src), desc, nil
70
+	}
71
+}
72
+
73
+type withDistributionSourceLabel struct {
74
+	content.Provider
75
+	ref    string
76
+	source content.Manager
77
+}
78
+
79
+var _ remotecache.DistributionSourceLabelSetter = &withDistributionSourceLabel{}
80
+
81
+func (dsl *withDistributionSourceLabel) SetDistributionSourceLabel(ctx context.Context, dgst digest.Digest) error {
82
+	hf, err := docker.AppendDistributionSourceLabel(dsl.source, dsl.ref)
83
+	if err != nil {
84
+		return err
85
+	}
86
+	_, err = hf(ctx, ocispec.Descriptor{Digest: dgst})
87
+	return err
88
+}
89
+
90
+func (dsl *withDistributionSourceLabel) SetDistributionSourceAnnotation(desc ocispec.Descriptor) ocispec.Descriptor {
91
+	if desc.Annotations == nil {
92
+		desc.Annotations = map[string]string{}
65 93
 	}
94
+	desc.Annotations["containerd.io/distribution.source.ref"] = dsl.ref
95
+	return desc
66 96
 }
67 97
 
68 98
 func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver {
... ...
@@ -293,12 +293,11 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
293 293
 		NoPivot: w.noPivot,
294 294
 	})
295 295
 	close(done)
296
-	if err != nil {
297
-		return err
298
-	}
299 296
 
300
-	if status != 0 {
301
-		err := errors.Errorf("exit code: %d", status)
297
+	if status != 0 || err != nil {
298
+		if err == nil {
299
+			err = errors.Errorf("exit code: %d", status)
300
+		}
302 301
 		select {
303 302
 		case <-ctx.Done():
304 303
 			return errors.Wrapf(ctx.Err(), err.Error())
... ...
@@ -640,10 +640,21 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
640 640
 	}
641 641
 	opt = append(opt, runMounts...)
642 642
 
643
-	err = dispatchRunSecurity(d, c)
643
+	securityOpt, err := dispatchRunSecurity(c)
644 644
 	if err != nil {
645 645
 		return err
646 646
 	}
647
+	if securityOpt != nil {
648
+		opt = append(opt, securityOpt)
649
+	}
650
+
651
+	networkOpt, err := dispatchRunNetwork(c)
652
+	if err != nil {
653
+		return err
654
+	}
655
+	if networkOpt != nil {
656
+		opt = append(opt, networkOpt)
657
+	}
647 658
 
648 659
 	shlex := *dopt.shlex
649 660
 	shlex.RawQuotes = true
650 661
new file mode 100644
... ...
@@ -0,0 +1,12 @@
0
+// +build !dfrunnetwork
1
+
2
+package dockerfile2llb
3
+
4
+import (
5
+	"github.com/moby/buildkit/client/llb"
6
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
7
+)
8
+
9
+func dispatchRunNetwork(c *instructions.RunCommand) (llb.RunOption, error) {
10
+	return nil, nil
11
+}
... ...
@@ -3,9 +3,10 @@
3 3
 package dockerfile2llb
4 4
 
5 5
 import (
6
+	"github.com/moby/buildkit/client/llb"
6 7
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
7 8
 )
8 9
 
9
-func dispatchRunSecurity(d *dispatchState, c *instructions.RunCommand) error {
10
-	return nil
10
+func dispatchRunSecurity(c *instructions.RunCommand) (llb.RunOption, error) {
11
+	return nil, nil
11 12
 }
12 13
new file mode 100644
... ...
@@ -0,0 +1,26 @@
0
+// +build dfrunnetwork
1
+
2
+package dockerfile2llb
3
+
4
+import (
5
+	"github.com/pkg/errors"
6
+
7
+	"github.com/moby/buildkit/client/llb"
8
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
9
+	"github.com/moby/buildkit/solver/pb"
10
+)
11
+
12
+func dispatchRunNetwork(c *instructions.RunCommand) (llb.RunOption, error) {
13
+	network := instructions.GetNetwork(c)
14
+
15
+	switch network {
16
+	case instructions.NetworkDefault:
17
+		return nil, nil
18
+	case instructions.NetworkNone:
19
+		return llb.Network(pb.NetMode_NONE), nil
20
+	case instructions.NetworkHost:
21
+		return llb.Network(pb.NetMode_HOST), nil
22
+	default:
23
+		return nil, errors.Errorf("unsupported network mode %q", network)
24
+	}
25
+}
... ...
@@ -5,23 +5,20 @@ package dockerfile2llb
5 5
 import (
6 6
 	"github.com/pkg/errors"
7 7
 
8
+	"github.com/moby/buildkit/client/llb"
8 9
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
9 10
 	"github.com/moby/buildkit/solver/pb"
10 11
 )
11 12
 
12
-func dispatchRunSecurity(d *dispatchState, c *instructions.RunCommand) error {
13
+func dispatchRunSecurity(c *instructions.RunCommand) (llb.RunOption, error) {
13 14
 	security := instructions.GetSecurity(c)
14 15
 
15
-	for _, sec := range security {
16
-		switch sec {
17
-		case instructions.SecurityInsecure:
18
-			d.state = d.state.Security(pb.SecurityMode_INSECURE)
19
-		case instructions.SecuritySandbox:
20
-			d.state = d.state.Security(pb.SecurityMode_SANDBOX)
21
-		default:
22
-			return errors.Errorf("unsupported security mode %q", sec)
23
-		}
16
+	switch security {
17
+	case instructions.SecurityInsecure:
18
+		return llb.Security(pb.SecurityMode_INSECURE), nil
19
+	case instructions.SecuritySandbox:
20
+		return llb.Security(pb.SecurityMode_SANDBOX), nil
21
+	default:
22
+		return nil, errors.Errorf("unsupported security mode %q", security)
24 23
 	}
25
-
26
-	return nil
27 24
 }
28 25
new file mode 100644
... ...
@@ -0,0 +1,63 @@
0
+// +build dfrunnetwork
1
+
2
+package instructions
3
+
4
+import (
5
+	"github.com/pkg/errors"
6
+)
7
+
8
+const (
9
+	NetworkDefault = "default"
10
+	NetworkNone    = "none"
11
+	NetworkHost    = "host"
12
+)
13
+
14
+var allowedNetwork = map[string]struct{}{
15
+	NetworkDefault: {},
16
+	NetworkNone:    {},
17
+	NetworkHost:    {},
18
+}
19
+
20
+func isValidNetwork(value string) bool {
21
+	_, ok := allowedNetwork[value]
22
+	return ok
23
+}
24
+
25
+var networkKey = "dockerfile/run/network"
26
+
27
+func init() {
28
+	parseRunPreHooks = append(parseRunPreHooks, runNetworkPreHook)
29
+	parseRunPostHooks = append(parseRunPostHooks, runNetworkPostHook)
30
+}
31
+
32
+func runNetworkPreHook(cmd *RunCommand, req parseRequest) error {
33
+	st := &networkState{}
34
+	st.flag = req.flags.AddString("network", NetworkDefault)
35
+	cmd.setExternalValue(networkKey, st)
36
+	return nil
37
+}
38
+
39
+func runNetworkPostHook(cmd *RunCommand, req parseRequest) error {
40
+	st := cmd.getExternalValue(networkKey).(*networkState)
41
+	if st == nil {
42
+		return errors.Errorf("no network state")
43
+	}
44
+
45
+	value := st.flag.Value
46
+	if !isValidNetwork(value) {
47
+		return errors.Errorf("invalid network mode %q", value)
48
+	}
49
+
50
+	st.networkMode = value
51
+
52
+	return nil
53
+}
54
+
55
+func GetNetwork(cmd *RunCommand) string {
56
+	return cmd.getExternalValue(networkKey).(*networkState).networkMode
57
+}
58
+
59
+type networkState struct {
60
+	flag        *Flag
61
+	networkMode string
62
+}
... ...
@@ -3,9 +3,6 @@
3 3
 package instructions
4 4
 
5 5
 import (
6
-	"encoding/csv"
7
-	"strings"
8
-
9 6
 	"github.com/pkg/errors"
10 7
 )
11 8
 
... ...
@@ -24,9 +21,7 @@ func isValidSecurity(value string) bool {
24 24
 	return ok
25 25
 }
26 26
 
27
-type securityKeyT string
28
-
29
-var securityKey = securityKeyT("dockerfile/run/security")
27
+var securityKey = "dockerfile/run/security"
30 28
 
31 29
 func init() {
32 30
 	parseRunPreHooks = append(parseRunPreHooks, runSecurityPreHook)
... ...
@@ -35,49 +30,32 @@ func init() {
35 35
 
36 36
 func runSecurityPreHook(cmd *RunCommand, req parseRequest) error {
37 37
 	st := &securityState{}
38
-	st.flag = req.flags.AddStrings("security")
38
+	st.flag = req.flags.AddString("security", SecuritySandbox)
39 39
 	cmd.setExternalValue(securityKey, st)
40 40
 	return nil
41 41
 }
42 42
 
43 43
 func runSecurityPostHook(cmd *RunCommand, req parseRequest) error {
44
-	st := getSecurityState(cmd)
44
+	st := cmd.getExternalValue(securityKey).(*securityState)
45 45
 	if st == nil {
46 46
 		return errors.Errorf("no security state")
47 47
 	}
48 48
 
49
-	for _, value := range st.flag.StringValues {
50
-		csvReader := csv.NewReader(strings.NewReader(value))
51
-		fields, err := csvReader.Read()
52
-		if err != nil {
53
-			return errors.Wrap(err, "failed to parse csv security")
54
-		}
55
-
56
-		for _, field := range fields {
57
-			if !isValidSecurity(field) {
58
-				return errors.Errorf("security %q is not valid", field)
59
-			}
60
-
61
-			st.security = append(st.security, field)
62
-		}
49
+	value := st.flag.Value
50
+	if !isValidSecurity(value) {
51
+		return errors.Errorf("security %q is not valid", value)
63 52
 	}
64 53
 
65
-	return nil
66
-}
54
+	st.security = value
67 55
 
68
-func getSecurityState(cmd *RunCommand) *securityState {
69
-	v := cmd.getExternalValue(securityKey)
70
-	if v == nil {
71
-		return nil
72
-	}
73
-	return v.(*securityState)
56
+	return nil
74 57
 }
75 58
 
76
-func GetSecurity(cmd *RunCommand) []string {
77
-	return getSecurityState(cmd).security
59
+func GetSecurity(cmd *RunCommand) string {
60
+	return cmd.getExternalValue(securityKey).(*securityState).security
78 61
 }
79 62
 
80 63
 type securityState struct {
81 64
 	flag     *Flag
82
-	security []string
65
+	security string
83 66
 }
... ...
@@ -4,20 +4,20 @@ go 1.11
4 4
 
5 5
 require (
6 6
 	github.com/BurntSushi/toml v0.3.1
7
-	github.com/Microsoft/go-winio v0.4.13-0.20190408173621-84b4ab48a507
7
+	github.com/Microsoft/go-winio v0.4.14
8 8
 	github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7 // indirect
9 9
 	github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect
10
-	github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect
10
+	github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601 // indirect
11 11
 	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
12
-	github.com/containerd/containerd v1.3.0-0.20190507210959-7c1e88399ec0
13
-	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc
14
-	github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 // indirect
15
-	github.com/containerd/go-cni v0.0.0-20190610170741-5a4663dad645
16
-	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
17
-	github.com/containerd/ttrpc v0.0.0-20190411181408-699c4e40d1e7 // indirect
12
+	github.com/containerd/containerd v1.3.0
13
+	github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6
14
+	github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c // indirect
15
+	github.com/containerd/go-cni v0.0.0-20190813230227-49fbd9b210f3
16
+	github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda
17
+	github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8 // indirect
18 18
 	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd // indirect
19
-	github.com/containernetworking/cni v0.6.1-0.20180218032124-142cde0c766c // indirect
20
-	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
19
+	github.com/containernetworking/cni v0.7.1 // indirect
20
+	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a
21 21
 	github.com/docker/cli v0.0.0-20190321234815-f40f9c240ab0
22 22
 	github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
23 23
 	github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c
... ...
@@ -25,17 +25,18 @@ require (
25 25
 	github.com/docker/go-connections v0.3.0
26 26
 	github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect
27 27
 	github.com/docker/libnetwork v0.8.0-dev.2.0.20190604151032-3c26b4e7495e
28
-	github.com/godbus/dbus v4.1.0+incompatible // indirect
28
+	github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f // indirect
29 29
 	github.com/gofrs/flock v0.7.0
30 30
 	github.com/gogo/googleapis v1.1.0
31 31
 	github.com/gogo/protobuf v1.2.0
32 32
 	github.com/golang/protobuf v1.2.0
33
-	github.com/google/go-cmp v0.2.0
33
+	github.com/google/go-cmp v0.3.0
34 34
 	github.com/google/shlex v0.0.0-20150127133951-6f45313302b9
35 35
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
36 36
 	github.com/hashicorp/go-immutable-radix v1.0.0
37 37
 	github.com/hashicorp/golang-lru v0.0.0-20160207214719-a0d98a5f2880
38 38
 	github.com/hashicorp/uuid v0.0.0-20160311170451-ebb0a03e909c // indirect
39
+	github.com/imdario/mergo v0.3.7 // indirect
39 40
 	github.com/ishidawataru/sctp v0.0.0-20180213033435-07191f837fed // indirect
40 41
 	github.com/jaguilar/vt100 v0.0.0-20150826170717-2703a27b14ea
41 42
 	github.com/kr/pretty v0.1.0 // indirect
... ...
@@ -43,31 +44,32 @@ require (
43 43
 	github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c
44 44
 	github.com/opencontainers/go-digest v1.0.0-rc1
45 45
 	github.com/opencontainers/image-spec v1.0.1
46
-	github.com/opencontainers/runc v1.0.0-rc8
46
+	github.com/opencontainers/runc v1.0.0-rc8.0.20190621203724-f4982d86f7fd
47 47
 	github.com/opencontainers/runtime-spec v0.0.0-20180909173843-eba862dc2470
48 48
 	github.com/opentracing-contrib/go-stdlib v0.0.0-20171029140428-b1a47cfbdd75
49 49
 	github.com/opentracing/opentracing-go v0.0.0-20171003133519-1361b9cd60be
50 50
 	github.com/pkg/errors v0.8.1
51 51
 	github.com/pkg/profile v1.2.1
52
+	github.com/prometheus/procfs v0.0.3 // indirect
52 53
 	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
53
-	github.com/sirupsen/logrus v1.3.0
54
+	github.com/sirupsen/logrus v1.4.1
54 55
 	github.com/stretchr/testify v1.3.0
55 56
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
56
-	github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76
57
+	github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d
57 58
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
58 59
 	github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e
59 60
 	github.com/uber/jaeger-lib v1.2.1 // indirect
60 61
 	github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5
61 62
 	github.com/vishvananda/netlink v1.0.0 // indirect
62 63
 	github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect
63
-	go.etcd.io/bbolt v1.3.2
64
+	go.etcd.io/bbolt v1.3.3
64 65
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
65
-	golang.org/x/net v0.0.0-20190311183353-d8887717615a
66
-	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
67
-	golang.org/x/sys v0.0.0-20190303122642-d455e41777fc
66
+	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
67
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
68
+	golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e
68 69
 	golang.org/x/time v0.0.0-20161028155119-f51c12702a4d
69 70
 	google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8
70
-	google.golang.org/grpc v1.20.1
71
+	google.golang.org/grpc v1.23.0
71 72
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
72 73
 	gotest.tools v2.2.0+incompatible
73 74
 )
... ...
@@ -9,17 +9,17 @@ import (
9 9
 	"google.golang.org/grpc"
10 10
 )
11 11
 
12
-func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) error {
12
+func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream, closeStream func() error) error {
13 13
 	g, ctx := errgroup.WithContext(ctx)
14 14
 
15 15
 	g.Go(func() (retErr error) {
16 16
 		p := &BytesMessage{}
17 17
 		for {
18 18
 			if err := stream.RecvMsg(p); err != nil {
19
+				conn.Close()
19 20
 				if err == io.EOF {
20 21
 					return nil
21 22
 				}
22
-				conn.Close()
23 23
 				return errors.WithStack(err)
24 24
 			}
25 25
 			select {
... ...
@@ -42,6 +42,9 @@ func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) erro
42 42
 			n, err := conn.Read(buf)
43 43
 			switch {
44 44
 			case err == io.EOF:
45
+				if closeStream != nil {
46
+					closeStream()
47
+				}
45 48
 				return nil
46 49
 			case err != nil:
47 50
 				return errors.WithStack(err)
... ...
@@ -49,7 +49,7 @@ func (s *server) run(ctx context.Context, l net.Listener, id string) error {
49 49
 				return err
50 50
 			}
51 51
 
52
-			go Copy(ctx, conn, stream)
52
+			go Copy(ctx, conn, stream, stream.CloseSend)
53 53
 		}
54 54
 	})
55 55
 
... ...
@@ -177,6 +177,9 @@ func (e *edge) finishIncoming(req pipe.Sender) {
177 177
 
178 178
 // updateIncoming updates the current value of incoming pipe request
179 179
 func (e *edge) updateIncoming(req pipe.Sender) {
180
+	if debugScheduler {
181
+		logrus.Debugf("updateIncoming %s %#v desired=%s", e.edge.Vertex.Name(), e.edgeState, req.Request().Payload.(*edgeRequest).desiredState)
182
+	}
180 183
 	req.Update(&e.edgeState)
181 184
 }
182 185
 
... ...
@@ -11,11 +11,15 @@ import (
11 11
 type channel struct {
12 12
 	OnSendCompletion func()
13 13
 	value            atomic.Value
14
-	lastValue        interface{}
14
+	lastValue        *wrappedValue
15
+}
16
+
17
+type wrappedValue struct {
18
+	value interface{}
15 19
 }
16 20
 
17 21
 func (c *channel) Send(v interface{}) {
18
-	c.value.Store(v)
22
+	c.value.Store(&wrappedValue{value: v})
19 23
 	if c.OnSendCompletion != nil {
20 24
 		c.OnSendCompletion()
21 25
 	}
... ...
@@ -23,11 +27,11 @@ func (c *channel) Send(v interface{}) {
23 23
 
24 24
 func (c *channel) Receive() (interface{}, bool) {
25 25
 	v := c.value.Load()
26
-	if c.lastValue == v {
26
+	if v == nil || v.(*wrappedValue) == c.lastValue {
27 27
 		return nil, false
28 28
 	}
29
-	c.lastValue = v
30
-	return v, true
29
+	c.lastValue = v.(*wrappedValue)
30
+	return v.(*wrappedValue).value, true
31 31
 }
32 32
 
33 33
 type Pipe struct {
... ...
@@ -54,7 +54,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res *
54 54
 			}
55 55
 		}
56 56
 		if prevCm, ok := b.cms[cmId]; !ok {
57
-			func(cmId string) {
57
+			func(cmId string, im gw.CacheOptionsEntry) {
58 58
 				cm = newLazyCacheManager(cmId, func() (solver.CacheManager, error) {
59 59
 					var cmNew solver.CacheManager
60 60
 					if err := inVertexContext(b.builder.Context(ctx), "importing cache manifest from "+cmId, "", func(ctx context.Context) error {
... ...
@@ -74,7 +74,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res *
74 74
 					}
75 75
 					return cmNew, nil
76 76
 				})
77
-			}(cmId)
77
+			}(cmId, im)
78 78
 			b.cms[cmId] = cm
79 79
 		} else {
80 80
 			cm = prevCm
... ...
@@ -144,6 +144,11 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
144 144
 	req = req.WithContext(ctx)
145 145
 	m := map[string]*metadata.StorageItem{}
146 146
 
147
+	// If we request a single ETag in 'If-None-Match', some servers omit the
148
+	// unambiguous ETag in their response.
149
+	// See: https://github.com/moby/buildkit/issues/905
150
+	var onlyETag string
151
+
147 152
 	if len(sis) > 0 {
148 153
 		for _, si := range sis {
149 154
 			// if metaDigest := getMetaDigest(si); metaDigest == hs.formatCacheKey("") {
... ...
@@ -160,6 +165,10 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
160 160
 				etags = append(etags, t)
161 161
 			}
162 162
 			req.Header.Add("If-None-Match", strings.Join(etags, ", "))
163
+
164
+			if len(etags) == 1 {
165
+				onlyETag = etags[0]
166
+			}
163 167
 		}
164 168
 	}
165 169
 
... ...
@@ -172,6 +181,12 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
172 172
 		if err == nil {
173 173
 			if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified {
174 174
 				respETag := resp.Header.Get("ETag")
175
+
176
+				// If a 304 is returned without an ETag and we had only sent one ETag,
177
+				// the response refers to the ETag we asked about.
178
+				if respETag == "" && onlyETag != "" && resp.StatusCode == http.StatusNotModified {
179
+					respETag = onlyETag
180
+				}
175 181
 				si, ok := m[respETag]
176 182
 				if ok {
177 183
 					hs.refID = si.ID()
... ...
@@ -197,6 +212,13 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b
197 197
 	}
198 198
 	if resp.StatusCode == http.StatusNotModified {
199 199
 		respETag := resp.Header.Get("ETag")
200
+		if respETag == "" && onlyETag != "" {
201
+			respETag = onlyETag
202
+
203
+			// Set the missing ETag header on the response so that it's available
204
+			// to .save()
205
+			resp.Header.Set("ETag", onlyETag)
206
+		}
200 207
 		si, ok := m[respETag]
201 208
 		if !ok {
202 209
 			return "", false, errors.Errorf("invalid not-modified ETag: %v", respETag)
... ...
@@ -5,9 +5,7 @@ import (
5 5
 	"time"
6 6
 
7 7
 	"github.com/containerd/containerd/leases"
8
-	"github.com/containerd/containerd/metadata"
9 8
 	"github.com/containerd/containerd/namespaces"
10
-	bolt "go.etcd.io/bbolt"
11 9
 )
12 10
 
13 11
 func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (context.Context, func(context.Context) error, error) {
... ...
@@ -29,56 +27,6 @@ func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (cont
29 29
 	}, nil
30 30
 }
31 31
 
32
-func NewManager(mdb *metadata.DB) leases.Manager {
33
-	return &local{db: mdb}
34
-}
35
-
36
-type local struct {
37
-	db *metadata.DB
38
-}
39
-
40
-func (l *local) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) {
41
-	var lease leases.Lease
42
-	if err := l.db.Update(func(tx *bolt.Tx) error {
43
-		var err error
44
-		lease, err = metadata.NewLeaseManager(tx).Create(ctx, opts...)
45
-		return err
46
-	}); err != nil {
47
-		return leases.Lease{}, err
48
-	}
49
-	return lease, nil
50
-}
51
-
52
-func (l *local) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error {
53
-	var do leases.DeleteOptions
54
-	for _, opt := range opts {
55
-		if err := opt(ctx, &do); err != nil {
56
-			return err
57
-		}
58
-	}
59
-
60
-	if err := l.db.Update(func(tx *bolt.Tx) error {
61
-		return metadata.NewLeaseManager(tx).Delete(ctx, lease)
62
-	}); err != nil {
63
-		return err
64
-	}
65
-
66
-	return nil
67
-
68
-}
69
-
70
-func (l *local) List(ctx context.Context, filters ...string) ([]leases.Lease, error) {
71
-	var ll []leases.Lease
72
-	if err := l.db.View(func(tx *bolt.Tx) error {
73
-		var err error
74
-		ll, err = metadata.NewLeaseManager(tx).List(ctx, filters...)
75
-		return err
76
-	}); err != nil {
77
-		return nil, err
78
-	}
79
-	return ll, nil
80
-}
81
-
82 32
 func WithNamespace(lm leases.Manager, ns string) leases.Manager {
83 33
 	return &nsLM{Manager: lm, ns: ns}
84 34
 }
... ...
@@ -2,6 +2,7 @@ package resolver
2 2
 
3 3
 import (
4 4
 	"math/rand"
5
+	"strings"
5 6
 
6 7
 	"github.com/containerd/containerd/remotes/docker"
7 8
 	"github.com/docker/distribution/reference"
... ...
@@ -10,7 +11,7 @@ import (
10 10
 
11 11
 type RegistryConf struct {
12 12
 	Mirrors   []string
13
-	PlainHTTP bool
13
+	PlainHTTP *bool
14 14
 }
15 15
 
16 16
 type ResolveOptionsFunc func(string) docker.ResolverOptions
... ...
@@ -32,13 +33,21 @@ func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc {
32 32
 			return def
33 33
 		}
34 34
 
35
+		var mirrorHost string
35 36
 		if len(c.Mirrors) > 0 {
37
+			mirrorHost = c.Mirrors[rand.Intn(len(c.Mirrors))]
36 38
 			def.Host = func(string) (string, error) {
37
-				return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
39
+				return mirrorHost, nil
38 40
 			}
39 41
 		}
40 42
 
41
-		def.PlainHTTP = c.PlainHTTP
43
+		if c.PlainHTTP != nil {
44
+			def.PlainHTTP = *c.PlainHTTP
45
+		} else {
46
+			if mirrorHost == "localhost" || strings.HasPrefix(mirrorHost, "localhost:") {
47
+				def.PlainHTTP = true
48
+			}
49
+		}
42 50
 
43 51
 		return def
44 52
 	}
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"context"
5 5
 	"io"
6 6
 
7
+	"github.com/containerd/containerd/content"
7 8
 	"github.com/moby/buildkit/cache"
8 9
 	"github.com/moby/buildkit/client"
9 10
 	"github.com/moby/buildkit/executor"
... ...
@@ -34,6 +35,7 @@ type Worker interface {
34 34
 	GetRemote(ctx context.Context, ref cache.ImmutableRef, createIfNeeded bool) (*solver.Remote, error)
35 35
 	FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error)
36 36
 	PruneCacheMounts(ctx context.Context, ids []string) error
37
+	ContentStore() content.Store
37 38
 }
38 39
 
39 40
 // Pre-defined label keys