full diff: https://github.com/moby/buildkit/compare/v0.8.2...v0.8.3
- vendor containerd (required for rootless overlayfs on kernel 5.11)
- not included to avoid depending on a fork
- Add retry on image push 5xx errors
- contenthash: include basename in content checksum for wildcards
- Fix missing mounts in execOp cache map
- Add regression test for run cache not considering mounts
- Add hack to preserve Dockerfile RUN cache compatibility after mount cache bugfix
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
(cherry picked from commit 79ee285d763dbde96ef87ab832dff57969c32254)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
| ... | ... |
@@ -33,7 +33,7 @@ github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6 |
| 33 | 33 |
golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb |
| 34 | 34 |
|
| 35 | 35 |
# buildkit |
| 36 |
-github.com/moby/buildkit 9065b18ba4633c75862befca8188de4338d9f94a # v0.8.2 |
|
| 36 |
+github.com/moby/buildkit 81c2cbd8a418918d62b71e347a00034189eea455 # v0.8.3 |
|
| 37 | 37 |
github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 |
| 38 | 38 |
github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 |
| 39 | 39 |
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 |
| ... | ... |
@@ -406,17 +406,19 @@ func (cc *cacheContext) ChecksumWildcard(ctx context.Context, mountable cache.Mo |
| 406 | 406 |
return digest.FromBytes([]byte{}), nil
|
| 407 | 407 |
} |
| 408 | 408 |
|
| 409 |
- if len(wildcards) > 1 {
|
|
| 410 |
- digester := digest.Canonical.Digester() |
|
| 411 |
- for i, w := range wildcards {
|
|
| 412 |
- if i != 0 {
|
|
| 413 |
- digester.Hash().Write([]byte{0})
|
|
| 414 |
- } |
|
| 415 |
- digester.Hash().Write([]byte(w.Record.Digest)) |
|
| 409 |
+ if len(wildcards) == 1 && path.Base(p) == path.Base(wildcards[0].Path) {
|
|
| 410 |
+ return wildcards[0].Record.Digest, nil |
|
| 411 |
+ } |
|
| 412 |
+ |
|
| 413 |
+ digester := digest.Canonical.Digester() |
|
| 414 |
+ for i, w := range wildcards {
|
|
| 415 |
+ if i != 0 {
|
|
| 416 |
+ digester.Hash().Write([]byte{0})
|
|
| 416 | 417 |
} |
| 417 |
- return digester.Digest(), nil |
|
| 418 |
+ digester.Hash().Write([]byte(path.Base(w.Path))) |
|
| 419 |
+ digester.Hash().Write([]byte(w.Record.Digest)) |
|
| 418 | 420 |
} |
| 419 |
- return wildcards[0].Record.Digest, nil |
|
| 421 |
+ return digester.Digest(), nil |
|
| 420 | 422 |
} |
| 421 | 423 |
|
| 422 | 424 |
func (cc *cacheContext) Checksum(ctx context.Context, mountable cache.Mountable, p string, followLinks bool, s session.Group) (digest.Digest, error) {
|
| ... | ... |
@@ -9,6 +9,7 @@ require ( |
| 9 | 9 |
github.com/Microsoft/hcsshim v0.8.10 |
| 10 | 10 |
github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect |
| 11 | 11 |
github.com/containerd/console v1.0.1 |
| 12 |
+ // containerd: the actual version is replaced in replace() |
|
| 12 | 13 |
github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc |
| 13 | 14 |
github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe |
| 14 | 15 |
github.com/containerd/go-cni v1.0.1 |
| ... | ... |
@@ -71,6 +72,11 @@ require ( |
| 71 | 71 |
) |
| 72 | 72 |
|
| 73 | 73 |
replace ( |
| 74 |
+ // containerd: Forked from 0edc412565dcc6e3d6125ff9e4b009ad4b89c638 (20201117) with: |
|
| 75 |
+ // - `Adjust overlay tests to expect "index=off"` (#4719, for ease of cherry-picking #5076) |
|
| 76 |
+ // - `overlay: support "userxattr" option (kernel 5.11)` (#5076) |
|
| 77 |
+ // - `docker: avoid concurrent map access panic` (#4855) |
|
| 78 |
+ github.com/containerd/containerd => github.com/AkihiroSuda/containerd v1.1.1-0.20210312044057-48f85a131bb8 |
|
| 74 | 79 |
// protobuf: corresponds to containerd |
| 75 | 80 |
github.com/golang/protobuf => github.com/golang/protobuf v1.3.5 |
| 76 | 81 |
github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe |
| ... | ... |
@@ -69,8 +69,8 @@ func cloneExecOp(old *pb.ExecOp) pb.ExecOp {
|
| 69 | 69 |
} |
| 70 | 70 |
n.Meta = &meta |
| 71 | 71 |
n.Mounts = nil |
| 72 |
- for i := range n.Mounts {
|
|
| 73 |
- m := *n.Mounts[i] |
|
| 72 |
+ for i := range old.Mounts {
|
|
| 73 |
+ m := *old.Mounts[i] |
|
| 74 | 74 |
n.Mounts = append(n.Mounts, &m) |
| 75 | 75 |
} |
| 76 | 76 |
return n |
| ... | ... |
@@ -97,6 +97,22 @@ func (e *execOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol |
| 97 | 97 |
} |
| 98 | 98 |
} |
| 99 | 99 |
|
| 100 |
+ // Special case for cache compatibility with buggy versions that wrongly |
|
| 101 |
+ // excluded Exec.Mounts: for the default case of one root mount (i.e. RUN |
|
| 102 |
+ // inside a Dockerfile), do not include the mount when generating the cache |
|
| 103 |
+ // map. |
|
| 104 |
+ if len(op.Mounts) == 1 && |
|
| 105 |
+ op.Mounts[0].Dest == "/" && |
|
| 106 |
+ op.Mounts[0].Selector == "" && |
|
| 107 |
+ !op.Mounts[0].Readonly && |
|
| 108 |
+ op.Mounts[0].MountType == pb.MountType_BIND && |
|
| 109 |
+ op.Mounts[0].CacheOpt == nil && |
|
| 110 |
+ op.Mounts[0].SSHOpt == nil && |
|
| 111 |
+ op.Mounts[0].SecretOpt == nil && |
|
| 112 |
+ op.Mounts[0].ResultID == "" {
|
|
| 113 |
+ op.Mounts = nil |
|
| 114 |
+ } |
|
| 115 |
+ |
|
| 100 | 116 |
dt, err := json.Marshal(struct {
|
| 101 | 117 |
Type string |
| 102 | 118 |
Exec *pb.ExecOp |
| ... | ... |
@@ -65,7 +65,7 @@ func CopyChain(ctx context.Context, ingester content.Ingester, provider content. |
| 65 | 65 |
handlers := []images.Handler{
|
| 66 | 66 |
images.ChildrenHandler(provider), |
| 67 | 67 |
filterHandler, |
| 68 |
- retryhandler.New(remotes.FetchHandler(ingester, &localFetcher{provider}), nil),
|
|
| 68 |
+ retryhandler.New(remotes.FetchHandler(ingester, &localFetcher{provider}), func(_ []byte) {}),
|
|
| 69 | 69 |
} |
| 70 | 70 |
|
| 71 | 71 |
if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
|
| ... | ... |
@@ -101,7 +101,7 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co |
| 101 | 101 |
children := childrenConfigHandler(cache, platform) |
| 102 | 102 |
|
| 103 | 103 |
handlers := []images.Handler{
|
| 104 |
- retryhandler.New(remotes.FetchHandler(cache, fetcher), nil), |
|
| 104 |
+ retryhandler.New(remotes.FetchHandler(cache, fetcher), func(_ []byte) {}),
|
|
| 105 | 105 |
children, |
| 106 | 106 |
} |
| 107 | 107 |
if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
|
| ... | ... |
@@ -132,6 +132,7 @@ func (sw *streamWriter) Close() error {
|
| 132 | 132 |
func LoggerFromContext(ctx context.Context) func([]byte) {
|
| 133 | 133 |
return func(dt []byte) {
|
| 134 | 134 |
pw, _, _ := progress.FromContext(ctx) |
| 135 |
+ defer pw.Close() |
|
| 135 | 136 |
pw.Write(identity.NewID(), client.VertexLog{
|
| 136 | 137 |
Stream: stderr, |
| 137 | 138 |
Data: []byte(dt), |
| ... | ... |
@@ -9,6 +9,7 @@ import ( |
| 9 | 9 |
"time" |
| 10 | 10 |
|
| 11 | 11 |
"github.com/containerd/containerd/images" |
| 12 |
+ remoteserrors "github.com/containerd/containerd/remotes/errors" |
|
| 12 | 13 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 13 | 14 |
"github.com/pkg/errors" |
| 14 | 15 |
) |
| ... | ... |
@@ -47,6 +48,14 @@ func New(f images.HandlerFunc, logger func([]byte)) images.HandlerFunc {
|
| 47 | 47 |
} |
| 48 | 48 |
|
| 49 | 49 |
func retryError(err error) bool {
|
| 50 |
+ // Retry on 5xx errors |
|
| 51 |
+ var errUnexpectedStatus remoteserrors.ErrUnexpectedStatus |
|
| 52 |
+ if errors.As(err, &errUnexpectedStatus) && |
|
| 53 |
+ errUnexpectedStatus.StatusCode >= 500 && |
|
| 54 |
+ errUnexpectedStatus.StatusCode <= 599 {
|
|
| 55 |
+ return true |
|
| 56 |
+ } |
|
| 57 |
+ |
|
| 50 | 58 |
if errors.Is(err, io.EOF) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) {
|
| 51 | 59 |
return true |
| 52 | 60 |
} |