Browse code

vendor: github.com/moby/buildkit v0.8.3

full diff: https://github.com/moby/buildkit/compare/v0.8.2...v0.8.3

- vendor containerd (required for rootless overlayfs on kernel 5.11)
- not included to avoid depending on a fork
- Add retry on image push 5xx errors
- contenthash: include basename in content checksum for wildcards
- Fix missing mounts in execOp cache map
- Add regression test for run cache not considering mounts
- Add hack to preserve Dockerfile RUN cache compatibility after mount cache bugfix

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
(cherry picked from commit 79ee285d763dbde96ef87ab832dff57969c32254)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Sebastiaan van Stijn authored on 2021/05/26 20:33:04
Showing 8 changed files
... ...
@@ -33,7 +33,7 @@ github.com/imdario/mergo                            1afb36080aec31e0d1528973ebe6
33 33
 golang.org/x/sync                                   cd5d95a43a6e21273425c7ae415d3df9ea832eeb
34 34
 
35 35
 # buildkit
36
-github.com/moby/buildkit                            9065b18ba4633c75862befca8188de4338d9f94a # v0.8.2
36
+github.com/moby/buildkit                            81c2cbd8a418918d62b71e347a00034189eea455 # v0.8.3
37 37
 github.com/tonistiigi/fsutil                        0834f99b7b85462efb69b4f571a4fa3ca7da5ac9
38 38
 github.com/tonistiigi/units                         6950e57a87eaf136bbe44ef2ec8e75b9e3569de2
39 39
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
... ...
@@ -406,17 +406,19 @@ func (cc *cacheContext) ChecksumWildcard(ctx context.Context, mountable cache.Mo
406 406
 		return digest.FromBytes([]byte{}), nil
407 407
 	}
408 408
 
409
-	if len(wildcards) > 1 {
410
-		digester := digest.Canonical.Digester()
411
-		for i, w := range wildcards {
412
-			if i != 0 {
413
-				digester.Hash().Write([]byte{0})
414
-			}
415
-			digester.Hash().Write([]byte(w.Record.Digest))
409
+	if len(wildcards) == 1 && path.Base(p) == path.Base(wildcards[0].Path) {
410
+		return wildcards[0].Record.Digest, nil
411
+	}
412
+
413
+	digester := digest.Canonical.Digester()
414
+	for i, w := range wildcards {
415
+		if i != 0 {
416
+			digester.Hash().Write([]byte{0})
416 417
 		}
417
-		return digester.Digest(), nil
418
+		digester.Hash().Write([]byte(path.Base(w.Path)))
419
+		digester.Hash().Write([]byte(w.Record.Digest))
418 420
 	}
419
-	return wildcards[0].Record.Digest, nil
421
+	return digester.Digest(), nil
420 422
 }
421 423
 
422 424
 func (cc *cacheContext) Checksum(ctx context.Context, mountable cache.Mountable, p string, followLinks bool, s session.Group) (digest.Digest, error) {
... ...
@@ -9,6 +9,7 @@ require (
9 9
 	github.com/Microsoft/hcsshim v0.8.10
10 10
 	github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect
11 11
 	github.com/containerd/console v1.0.1
12
+	// containerd: the actual version is replaced in replace()
12 13
 	github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc
13 14
 	github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe
14 15
 	github.com/containerd/go-cni v1.0.1
... ...
@@ -71,6 +72,11 @@ require (
71 71
 )
72 72
 
73 73
 replace (
74
+	// containerd: Forked from 0edc412565dcc6e3d6125ff9e4b009ad4b89c638 (20201117) with:
75
+	// - `Adjust overlay tests to expect "index=off"`        (#4719, for ease of cherry-picking #5076)
76
+	// - `overlay: support "userxattr" option (kernel 5.11)` (#5076)
77
+	// - `docker: avoid concurrent map access panic`         (#4855)
78
+	github.com/containerd/containerd => github.com/AkihiroSuda/containerd v1.1.1-0.20210312044057-48f85a131bb8
74 79
 	// protobuf: corresponds to containerd
75 80
 	github.com/golang/protobuf => github.com/golang/protobuf v1.3.5
76 81
 	github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe
... ...
@@ -69,8 +69,8 @@ func cloneExecOp(old *pb.ExecOp) pb.ExecOp {
69 69
 	}
70 70
 	n.Meta = &meta
71 71
 	n.Mounts = nil
72
-	for i := range n.Mounts {
73
-		m := *n.Mounts[i]
72
+	for i := range old.Mounts {
73
+		m := *old.Mounts[i]
74 74
 		n.Mounts = append(n.Mounts, &m)
75 75
 	}
76 76
 	return n
... ...
@@ -97,6 +97,22 @@ func (e *execOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol
97 97
 		}
98 98
 	}
99 99
 
100
+	// Special case for cache compatibility with buggy versions that wrongly
101
+	// excluded Exec.Mounts: for the default case of one root mount (i.e. RUN
102
+	// inside a Dockerfile), do not include the mount when generating the cache
103
+	// map.
104
+	if len(op.Mounts) == 1 &&
105
+		op.Mounts[0].Dest == "/" &&
106
+		op.Mounts[0].Selector == "" &&
107
+		!op.Mounts[0].Readonly &&
108
+		op.Mounts[0].MountType == pb.MountType_BIND &&
109
+		op.Mounts[0].CacheOpt == nil &&
110
+		op.Mounts[0].SSHOpt == nil &&
111
+		op.Mounts[0].SecretOpt == nil &&
112
+		op.Mounts[0].ResultID == "" {
113
+		op.Mounts = nil
114
+	}
115
+
100 116
 	dt, err := json.Marshal(struct {
101 117
 		Type    string
102 118
 		Exec    *pb.ExecOp
... ...
@@ -65,7 +65,7 @@ func CopyChain(ctx context.Context, ingester content.Ingester, provider content.
65 65
 	handlers := []images.Handler{
66 66
 		images.ChildrenHandler(provider),
67 67
 		filterHandler,
68
-		retryhandler.New(remotes.FetchHandler(ingester, &localFetcher{provider}), nil),
68
+		retryhandler.New(remotes.FetchHandler(ingester, &localFetcher{provider}), func(_ []byte) {}),
69 69
 	}
70 70
 
71 71
 	if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
... ...
@@ -101,7 +101,7 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
101 101
 	children := childrenConfigHandler(cache, platform)
102 102
 
103 103
 	handlers := []images.Handler{
104
-		retryhandler.New(remotes.FetchHandler(cache, fetcher), nil),
104
+		retryhandler.New(remotes.FetchHandler(cache, fetcher), func(_ []byte) {}),
105 105
 		children,
106 106
 	}
107 107
 	if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
... ...
@@ -132,6 +132,7 @@ func (sw *streamWriter) Close() error {
132 132
 func LoggerFromContext(ctx context.Context) func([]byte) {
133 133
 	return func(dt []byte) {
134 134
 		pw, _, _ := progress.FromContext(ctx)
135
+		defer pw.Close()
135 136
 		pw.Write(identity.NewID(), client.VertexLog{
136 137
 			Stream: stderr,
137 138
 			Data:   []byte(dt),
... ...
@@ -9,6 +9,7 @@ import (
9 9
 	"time"
10 10
 
11 11
 	"github.com/containerd/containerd/images"
12
+	remoteserrors "github.com/containerd/containerd/remotes/errors"
12 13
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
13 14
 	"github.com/pkg/errors"
14 15
 )
... ...
@@ -47,6 +48,14 @@ func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
47 47
 }
48 48
 
49 49
 func retryError(err error) bool {
50
+	// Retry on 5xx errors
51
+	var errUnexpectedStatus remoteserrors.ErrUnexpectedStatus
52
+	if errors.As(err, &errUnexpectedStatus) &&
53
+		errUnexpectedStatus.StatusCode >= 500 &&
54
+		errUnexpectedStatus.StatusCode <= 599 {
55
+		return true
56
+	}
57
+
50 58
 	if errors.Is(err, io.EOF) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) {
51 59
 		return true
52 60
 	}