Browse code

vendor: update buildkit to 37d53758

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2019/06/04 16:34:16
Showing 19 changed files
... ...
@@ -135,7 +135,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
135 135
 		dt   []byte
136 136
 	}
137 137
 	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
138
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, platform)
138
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, nil, platform)
139 139
 		if err != nil {
140 140
 			return nil, err
141 141
 		}
... ...
@@ -27,7 +27,7 @@ github.com/imdario/mergo                            7c29201646fa3de8506f70121347
27 27
 golang.org/x/sync                                   e225da77a7e68af35c70ccbf71af2b83e6acac3c
28 28
 
29 29
 # buildkit
30
-github.com/moby/buildkit                            f238f1efb04f00bf0cc147141fda9ddb55c8bc49
30
+github.com/moby/buildkit                            37d53758a68d9f5cede1806dbb2da7c3caa8d5bc
31 31
 github.com/tonistiigi/fsutil                        3bbb99cdbd76619ab717299830c60f6f2a533a6b
32 32
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
33 33
 github.com/opentracing/opentracing-go               1361b9cd60be79c4c3a7fa9841b3c132e40066a7
... ...
@@ -184,6 +184,19 @@ The local client will copy the files directly to the client. This is useful if B
184 184
 buildctl build ... --output type=local,dest=path/to/output-dir
185 185
 ```
186 186
 
187
+To export specific files use multi-stage builds with a scratch stage and copy the needed files into that stage with `COPY --from`.
188
+```dockerfile
189
+...
190
+FROM scratch as testresult
191
+
192
+COPY --from=builder /usr/src/app/testresult.xml .
193
+...
194
+```
195
+
196
+```
197
+buildctl build ... --opt target=testresult --output type=local,dest=path/to/output-dir
198
+```
199
+
187 200
 Tar exporter is similar to local exporter but transfers the files through a tarball.
188 201
 
189 202
 ```
... ...
@@ -268,6 +281,16 @@ export BUILDKIT_HOST=tcp://0.0.0.0:1234
268 268
 buildctl build --help
269 269
 ```
270 270
 
271
+To run client and an ephemeral daemon in a single container ("daemonless mode"):
272
+
273
+```
274
+docker run -it --rm --privileged -v /path/to/dir:/tmp/work --entrypoint buildctl-daemonless.sh moby/buildkit:master build --frontend dockerfile.v0 --local context=/tmp/work --local dockerfile=/tmp/work
275
+```
276
+or
277
+```
278
+docker run -it --rm --security-opt seccomp=unconfined --security-opt apparmor=unconfined -e BUILDKITD_FLAGS=--oci-worker-no-process-sandbox -v /path/to/dir:/tmp/work --entrypoint buildctl-daemonless.sh moby/buildkit:master-rootless build --frontend dockerfile.v0 --local context=/tmp/work --local dockerfile=/tmp/work
279
+```
280
+
271 281
 The images can be also built locally using `./hack/dockerfiles/test.Dockerfile` (or `./hack/dockerfiles/test.buildkit.Dockerfile` if you already have BuildKit).
272 282
 Run `make images` to build the images as `moby/buildkit:local` and `moby/buildkit:local-rootless`.
273 283
 
... ...
@@ -60,6 +60,10 @@ func SetCacheContext(ctx context.Context, md *metadata.StorageItem, cc CacheCont
60 60
 	return getDefaultManager().SetCacheContext(ctx, md, cc)
61 61
 }
62 62
 
63
+func ClearCacheContext(md *metadata.StorageItem) {
64
+	getDefaultManager().clearCacheContext(md.ID())
65
+}
66
+
63 67
 type CacheContext interface {
64 68
 	Checksum(ctx context.Context, ref cache.Mountable, p string, followLinks bool) (digest.Digest, error)
65 69
 	ChecksumWildcard(ctx context.Context, ref cache.Mountable, p string, followLinks bool) (digest.Digest, error)
... ...
@@ -142,6 +146,12 @@ func (cm *cacheManager) SetCacheContext(ctx context.Context, md *metadata.Storag
142 142
 	return nil
143 143
 }
144 144
 
145
+func (cm *cacheManager) clearCacheContext(id string) {
146
+	cm.lruMu.Lock()
147
+	cm.lru.Remove(id)
148
+	cm.lruMu.Unlock()
149
+}
150
+
145 151
 type cacheContext struct {
146 152
 	mu    sync.RWMutex
147 153
 	md    *metadata.StorageItem
... ...
@@ -125,7 +125,7 @@ func (cm *cacheManager) GetFromSnapshotter(ctx context.Context, id string, opts
125 125
 }
126 126
 
127 127
 // get requires manager lock to be taken
128
-func (cm *cacheManager) get(ctx context.Context, id string, fromSnapshotter bool, opts ...RefOption) (ImmutableRef, error) {
128
+func (cm *cacheManager) get(ctx context.Context, id string, fromSnapshotter bool, opts ...RefOption) (*immutableRef, error) {
129 129
 	rec, err := cm.getRecord(ctx, id, fromSnapshotter, opts...)
130 130
 	if err != nil {
131 131
 		return nil, err
... ...
@@ -193,7 +193,7 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte
193 193
 		return nil, errors.Wrap(errNotFound, err.Error())
194 194
 	}
195 195
 
196
-	var parent ImmutableRef
196
+	var parent *immutableRef
197 197
 	if info.Parent != "" {
198 198
 		parent, err = cm.get(ctx, info.Parent, fromSnapshotter, append(opts, NoUpdateLastUsed)...)
199 199
 		if err != nil {
... ...
@@ -201,7 +201,9 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte
201 201
 		}
202 202
 		defer func() {
203 203
 			if retErr != nil {
204
-				parent.Release(context.TODO())
204
+				parent.mu.Lock()
205
+				parent.release(context.TODO())
206
+				parent.mu.Unlock()
205 207
 			}
206 208
 		}()
207 209
 	}
... ...
@@ -224,9 +226,6 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte
224 224
 	}
225 225
 
226 226
 	if err := initializeMetadata(rec, opts...); err != nil {
227
-		if parent != nil {
228
-			parent.Release(context.TODO())
229
-		}
230 227
 		return nil, err
231 228
 	}
232 229
 
... ...
@@ -237,18 +236,18 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte
237 237
 func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, opts ...RefOption) (MutableRef, error) {
238 238
 	id := identity.NewID()
239 239
 
240
-	var parent ImmutableRef
240
+	var parent *immutableRef
241 241
 	var parentID string
242 242
 	if s != nil {
243
-		var err error
244
-		parent, err = cm.Get(ctx, s.ID(), NoUpdateLastUsed)
243
+		p, err := cm.Get(ctx, s.ID(), NoUpdateLastUsed)
245 244
 		if err != nil {
246 245
 			return nil, err
247 246
 		}
248
-		if err := parent.Finalize(ctx, true); err != nil {
247
+		if err := p.Finalize(ctx, true); err != nil {
249 248
 			return nil, err
250 249
 		}
251
-		parentID = parent.ID()
250
+		parentID = p.ID()
251
+		parent = p.(*immutableRef)
252 252
 	}
253 253
 
254 254
 	if err := cm.Snapshotter.Prepare(ctx, id, parentID); err != nil {
... ...
@@ -737,6 +736,10 @@ func CachePolicyRetain(m withMetadata) error {
737 737
 	return queueCachePolicy(m.Metadata(), cachePolicyRetain)
738 738
 }
739 739
 
740
+func CachePolicyDefault(m withMetadata) error {
741
+	return queueCachePolicy(m.Metadata(), cachePolicyDefault)
742
+}
743
+
740 744
 func WithDescription(descr string) RefOption {
741 745
 	return func(m withMetadata) error {
742 746
 		return queueDescription(m.Metadata(), descr)
... ...
@@ -50,7 +50,7 @@ type cacheRecord struct {
50 50
 
51 51
 	mutable bool
52 52
 	refs    map[ref]struct{}
53
-	parent  ImmutableRef
53
+	parent  *immutableRef
54 54
 	md      *metadata.StorageItem
55 55
 
56 56
 	// dead means record is marked as deleted
... ...
@@ -126,14 +126,17 @@ func (cr *cacheRecord) Size(ctx context.Context) (int64, error) {
126 126
 }
127 127
 
128 128
 func (cr *cacheRecord) Parent() ImmutableRef {
129
-	return cr.parentRef(true)
129
+	if p := cr.parentRef(true); p != nil { // avoid returning typed nil pointer
130
+		return p
131
+	}
132
+	return nil
130 133
 }
131 134
 
132
-func (cr *cacheRecord) parentRef(hidden bool) ImmutableRef {
133
-	if cr.parent == nil {
135
+func (cr *cacheRecord) parentRef(hidden bool) *immutableRef {
136
+	p := cr.parent
137
+	if p == nil {
134 138
 		return nil
135 139
 	}
136
-	p := cr.parent.(*immutableRef)
137 140
 	p.mu.Lock()
138 141
 	defer p.mu.Unlock()
139 142
 	return p.ref(hidden)
... ...
@@ -181,7 +184,7 @@ func (cr *cacheRecord) Mount(ctx context.Context, readonly bool) (snapshot.Mount
181 181
 func (cr *cacheRecord) remove(ctx context.Context, removeSnapshot bool) error {
182 182
 	delete(cr.cm.records, cr.ID())
183 183
 	if cr.parent != nil {
184
-		if err := cr.parent.(*immutableRef).release(ctx); err != nil {
184
+		if err := cr.parent.release(ctx); err != nil {
185 185
 			return err
186 186
 		}
187 187
 	}
... ...
@@ -315,7 +318,7 @@ func (sr *mutableRef) updateLastUsed() bool {
315 315
 	return sr.triggerLastUsed
316 316
 }
317 317
 
318
-func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) {
318
+func (sr *mutableRef) commit(ctx context.Context) (*immutableRef, error) {
319 319
 	if !sr.mutable || len(sr.refs) == 0 {
320 320
 		return nil, errors.Wrapf(errInvalid, "invalid mutable ref %p", sr)
321 321
 	}
... ...
@@ -398,7 +401,7 @@ func (sr *mutableRef) release(ctx context.Context) error {
398 398
 			}
399 399
 		}
400 400
 		if sr.parent != nil {
401
-			if err := sr.parent.(*immutableRef).release(ctx); err != nil {
401
+			if err := sr.parent.release(ctx); err != nil {
402 402
 				return err
403 403
 			}
404 404
 		}
... ...
@@ -87,7 +87,7 @@ func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string
87 87
 		return res.dgst, res.config, nil
88 88
 	}
89 89
 
90
-	dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, platform)
90
+	dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform)
91 91
 	if err != nil {
92 92
 		return "", nil, err
93 93
 	}
... ...
@@ -3,6 +3,7 @@ package control
3 3
 import (
4 4
 	"context"
5 5
 	"sync"
6
+	"sync/atomic"
6 7
 	"time"
7 8
 
8 9
 	controlapi "github.com/moby/buildkit/api/services/control"
... ...
@@ -17,6 +18,7 @@ import (
17 17
 	"github.com/moby/buildkit/solver"
18 18
 	"github.com/moby/buildkit/solver/llbsolver"
19 19
 	"github.com/moby/buildkit/solver/pb"
20
+	"github.com/moby/buildkit/util/imageutil"
20 21
 	"github.com/moby/buildkit/util/throttle"
21 22
 	"github.com/moby/buildkit/worker"
22 23
 	"github.com/pkg/errors"
... ...
@@ -42,6 +44,7 @@ type Controller struct { // TODO: ControlService
42 42
 	gatewayForwarder *controlgateway.GatewayForwarder
43 43
 	throttledGC      func()
44 44
 	gcmu             sync.Mutex
45
+	buildCount       int64
45 46
 }
46 47
 
47 48
 func NewController(opt Opt) (*Controller, error) {
... ...
@@ -110,6 +113,10 @@ func (c *Controller) DiskUsage(ctx context.Context, r *controlapi.DiskUsageReque
110 110
 }
111 111
 
112 112
 func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Control_PruneServer) error {
113
+	if atomic.LoadInt64(&c.buildCount) == 0 {
114
+		imageutil.CancelCacheLeases()
115
+	}
116
+
113 117
 	ch := make(chan client.UsageInfo)
114 118
 
115 119
 	eg, ctx := errgroup.WithContext(stream.Context())
... ...
@@ -207,6 +214,9 @@ func translateLegacySolveRequest(req *controlapi.SolveRequest) error {
207 207
 }
208 208
 
209 209
 func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*controlapi.SolveResponse, error) {
210
+	atomic.AddInt64(&c.buildCount, 1)
211
+	defer atomic.AddInt64(&c.buildCount, -1)
212
+
210 213
 	if err := translateLegacySolveRequest(req); err != nil {
211 214
 		return nil, err
212 215
 	}
... ...
@@ -3,8 +3,12 @@
3 3
 package dockerfile2llb
4 4
 
5 5
 import (
6
+	"fmt"
7
+	"os"
6 8
 	"path"
7 9
 	"path/filepath"
10
+	"strconv"
11
+	"strings"
8 12
 
9 13
 	"github.com/moby/buildkit/client/llb"
10 14
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
... ...
@@ -40,6 +44,40 @@ func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
40 40
 	return false
41 41
 }
42 42
 
43
+func setCacheUIDGIDFileOp(m *instructions.Mount, st llb.State) llb.State {
44
+	uid := 0
45
+	gid := 0
46
+	mode := os.FileMode(0755)
47
+	if m.UID != nil {
48
+		uid = int(*m.UID)
49
+	}
50
+	if m.GID != nil {
51
+		gid = int(*m.GID)
52
+	}
53
+	if m.Mode != nil {
54
+		mode = os.FileMode(*m.Mode)
55
+	}
56
+	return st.File(llb.Mkdir("/cache", mode, llb.WithUIDGID(uid, gid)), llb.WithCustomName("[internal] settings cache mount permissions"))
57
+}
58
+
59
+func setCacheUIDGID(m *instructions.Mount, st llb.State, fileop bool) llb.State {
60
+	if fileop {
61
+		return setCacheUIDGIDFileOp(m, st)
62
+	}
63
+
64
+	var b strings.Builder
65
+	if m.UID != nil {
66
+		b.WriteString(fmt.Sprintf("chown %d /mnt/cache;", *m.UID))
67
+	}
68
+	if m.GID != nil {
69
+		b.WriteString(fmt.Sprintf("chown :%d /mnt/cache;", *m.GID))
70
+	}
71
+	if m.Mode != nil {
72
+		b.WriteString(fmt.Sprintf("chmod %s /mnt/cache;", strconv.FormatUint(*m.Mode, 8)))
73
+	}
74
+	return llb.Image("busybox").Run(llb.Shlex(fmt.Sprintf("sh -c 'mkdir -p /mnt/cache;%s'", b.String())), llb.WithCustomName("[internal] settings cache mount permissions")).AddMount("/mnt", st)
75
+}
76
+
43 77
 func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*dispatchState, opt dispatchOpt) ([]llb.RunOption, error) {
44 78
 	var out []llb.RunOption
45 79
 	mounts := instructions.GetMounts(c)
... ...
@@ -97,7 +135,13 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*
97 97
 		}
98 98
 		if src := path.Join("/", mount.Source); src != "/" {
99 99
 			mountOpts = append(mountOpts, llb.SourcePath(src))
100
+		} else {
101
+			if mount.UID != nil || mount.GID != nil || mount.Mode != nil {
102
+				st = setCacheUIDGID(mount, st, useFileOp(opt.buildArgValues, opt.llbCaps))
103
+				mountOpts = append(mountOpts, llb.SourcePath("/cache"))
104
+			}
100 105
 		}
106
+
101 107
 		out = append(out, llb.AddMount(target, st, mountOpts...))
102 108
 
103 109
 		d.ctxPaths[path.Join("/", filepath.ToSlash(mount.Source))] = struct{}{}
... ...
@@ -206,18 +206,18 @@ func parseMount(value string) (*Mount, error) {
206 206
 		}
207 207
 	}
208 208
 
209
-	fileInfoAllowed := m.Type == MountTypeSecret || m.Type == MountTypeSSH
209
+	fileInfoAllowed := m.Type == MountTypeSecret || m.Type == MountTypeSSH || m.Type == MountTypeCache
210 210
 
211 211
 	if m.Mode != nil && !fileInfoAllowed {
212
-		return nil, errors.Errorf("mode not allowed for %q type mounts")
212
+		return nil, errors.Errorf("mode not allowed for %q type mounts", m.Type)
213 213
 	}
214 214
 
215 215
 	if m.UID != nil && !fileInfoAllowed {
216
-		return nil, errors.Errorf("uid not allowed for %q type mounts")
216
+		return nil, errors.Errorf("uid not allowed for %q type mounts", m.Type)
217 217
 	}
218 218
 
219 219
 	if m.GID != nil && !fileInfoAllowed {
220
-		return nil, errors.Errorf("gid not allowed for %q type mounts")
220
+		return nil, errors.Errorf("gid not allowed for %q type mounts", m.Type)
221 221
 	}
222 222
 
223 223
 	if roAuto {
... ...
@@ -35,7 +35,7 @@ type Node struct {
35 35
 	Original   string          // original line used before parsing
36 36
 	Flags      []string        // only top Node should have this set
37 37
 	StartLine  int             // the line in the original dockerfile where the node begins
38
-	endLine    int             // the line in the original dockerfile where the node ends
38
+	EndLine    int             // the line in the original dockerfile where the node ends
39 39
 }
40 40
 
41 41
 // Dump dumps the AST defined by `node` as a list of sexps.
... ...
@@ -65,7 +65,7 @@ func (node *Node) Dump() string {
65 65
 
66 66
 func (node *Node) lines(start, end int) {
67 67
 	node.StartLine = start
68
-	node.endLine = end
68
+	node.EndLine = end
69 69
 }
70 70
 
71 71
 // AddChild adds a new child node, and updates line information
... ...
@@ -74,7 +74,7 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) {
74 74
 	if node.StartLine < 0 {
75 75
 		node.StartLine = startLine
76 76
 	}
77
-	node.endLine = endLine
77
+	node.EndLine = endLine
78 78
 	node.Children = append(node.Children, child)
79 79
 }
80 80
 
... ...
@@ -2,6 +2,7 @@ package blobmapping
2 2
 
3 3
 import (
4 4
 	"context"
5
+	"time"
5 6
 
6 7
 	"github.com/containerd/containerd/content"
7 8
 	"github.com/containerd/containerd/snapshots"
... ...
@@ -122,6 +123,16 @@ func (s *Snapshotter) SetBlob(ctx context.Context, key string, diffID, blobsum d
122 122
 			return err
123 123
 		}
124 124
 	}
125
+	// update gc.root cause blob might be held by lease only
126
+	if _, err := s.opt.Content.Update(ctx, content.Info{
127
+		Digest: blobsum,
128
+		Labels: map[string]string{
129
+			"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
130
+		},
131
+	}, "labels.containerd.io/gc.root"); err != nil {
132
+		return err
133
+	}
134
+
125 135
 	md, _ := s.opt.MetadataStore.Get(key)
126 136
 
127 137
 	v, err := metadata.NewValue(DiffPair{DiffID: diffID, Blobsum: blobsum})
... ...
@@ -133,6 +133,12 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable
133 133
 
134 134
 	defer func() {
135 135
 		if retErr != nil && mutable != nil {
136
+			// on error remove the record as checksum update is in undefined state
137
+			cache.CachePolicyDefault(mutable)
138
+			if err := mutable.Metadata().Commit(); err != nil {
139
+				logrus.Errorf("failed to reset mutable cachepolicy: %v", err)
140
+			}
141
+			contenthash.ClearCacheContext(mutable.Metadata())
136 142
 			go mutable.Release(context.TODO())
137 143
 		}
138 144
 	}()
... ...
@@ -21,6 +21,9 @@ func SupportedPlatforms() []string {
21 21
 		if p := "linux/arm64"; def != p && arm64Supported() == nil {
22 22
 			arr = append(arr, p)
23 23
 		}
24
+		if p := "linux/riscv64"; def != p && riscv64Supported() == nil {
25
+			arr = append(arr, p)
26
+		}
24 27
 		if !strings.HasPrefix(def, "linux/arm/") && armSupported() == nil {
25 28
 			arr = append(arr, "linux/arm/v7", "linux/arm/v6")
26 29
 		} else if def == "linux/arm/v7" {
... ...
@@ -47,6 +50,11 @@ func WarnIfUnsupported(pfs []string) {
47 47
 					printPlatfromWarning(p, err)
48 48
 				}
49 49
 			}
50
+			if p == "linux/riscv64" {
51
+				if err := riscv64Supported(); err != nil {
52
+					printPlatfromWarning(p, err)
53
+				}
54
+			}
50 55
 			if strings.HasPrefix(p, "linux/arm/v6") || strings.HasPrefix(p, "linux/arm/v7") {
51 56
 				if err := armSupported(); err != nil {
52 57
 					printPlatfromWarning(p, err)
53 58
new file mode 100644
... ...
@@ -0,0 +1,8 @@
0
+// +build !riscv64
1
+
2
+package binfmt_misc
3
+
4
+// This file is generated by running make inside the binfmt_misc package.
5
+// Do not edit manually.
6
+
7
+const Binaryriscv64 = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xaa\x77\xf5\x71\x63\x62\x64\x64\x80\x01\x26\x86\xcf\x0c\x20\x5e\x05\x03\x44\xcc\x01\x2a\x3e\x03\x4a\xb3\x80\xc5\x2c\x18\x18\x19\x1c\x18\x98\x19\x98\xc0\xaa\x58\x19\x90\x01\x23\x0a\xdd\x02\xe5\xc1\x68\x06\x01\x08\x25\xcc\xca\xc0\x30\x99\xe3\x02\x6b\x31\x88\xa3\x57\x9c\x51\x5c\x52\x54\x92\x98\xc4\xa0\x57\x92\x5a\x51\xc2\x40\x05\xc0\x0d\x75\x01\x1b\x94\x0f\xf3\x4f\x05\x94\xcf\x83\xa6\x9e\x05\x8d\x0f\x52\xcd\x8c\xc5\x5c\x98\x3f\x04\xb1\xa8\x47\x06\x80\x00\x00\x00\xff\xff\x39\x41\xdf\xa1\x58\x01\x00\x00"
0 8
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+// +build !riscv64
1
+
2
+package binfmt_misc
3
+
4
+func riscv64Supported() error {
5
+	return check(Binaryriscv64)
6
+}
0 7
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+// +build riscv64
1
+
2
+package binfmt_misc
3
+
4
+func riscv64Supported() error {
5
+	return nil
6
+}
... ...
@@ -3,12 +3,17 @@ package imageutil
3 3
 import (
4 4
 	"context"
5 5
 	"encoding/json"
6
+	"fmt"
7
+	"sync"
8
+	"time"
6 9
 
7 10
 	"github.com/containerd/containerd/content"
8 11
 	"github.com/containerd/containerd/images"
12
+	"github.com/containerd/containerd/leases"
9 13
 	"github.com/containerd/containerd/platforms"
10 14
 	"github.com/containerd/containerd/reference"
11 15
 	"github.com/containerd/containerd/remotes"
16
+	"github.com/moby/buildkit/util/leaseutil"
12 17
 	digest "github.com/opencontainers/go-digest"
13 18
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
14 19
 	"github.com/pkg/errors"
... ...
@@ -19,7 +24,19 @@ type ContentCache interface {
19 19
 	content.Provider
20 20
 }
21 21
 
22
-func Config(ctx context.Context, str string, resolver remotes.Resolver, cache ContentCache, p *specs.Platform) (digest.Digest, []byte, error) {
22
+var leasesMu sync.Mutex
23
+var leasesF []func(context.Context) error
24
+
25
+func CancelCacheLeases() {
26
+	leasesMu.Lock()
27
+	for _, f := range leasesF {
28
+		f(context.TODO())
29
+	}
30
+	leasesF = nil
31
+	leasesMu.Unlock()
32
+}
33
+
34
+func Config(ctx context.Context, str string, resolver remotes.Resolver, cache ContentCache, leaseManager leases.Manager, p *specs.Platform) (digest.Digest, []byte, error) {
23 35
 	// TODO: fix buildkit to take interface instead of struct
24 36
 	var platform platforms.MatchComparer
25 37
 	if p != nil {
... ...
@@ -32,6 +49,20 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
32 32
 		return "", nil, errors.WithStack(err)
33 33
 	}
34 34
 
35
+	if leaseManager != nil {
36
+		ctx2, done, err := leaseutil.WithLease(ctx, leaseManager, leases.WithExpiration(5*time.Minute))
37
+		if err != nil {
38
+			return "", nil, errors.WithStack(err)
39
+		}
40
+		ctx = ctx2
41
+		defer func() {
42
+			// this lease is not deleted to allow other components to access manifest/config from cache. It will be deleted after 5 min deadline or on pruning inactive builder
43
+			leasesMu.Lock()
44
+			leasesF = append(leasesF, done)
45
+			leasesMu.Unlock()
46
+		}()
47
+	}
48
+
35 49
 	desc := specs.Descriptor{
36 50
 		Digest: ref.Digest(),
37 51
 	}
... ...
@@ -62,9 +93,14 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co
62 62
 		return readSchema1Config(ctx, ref.String(), desc, fetcher, cache)
63 63
 	}
64 64
 
65
+	children := childrenConfigHandler(cache, platform)
66
+	if m, ok := cache.(content.Manager); ok {
67
+		children = SetChildrenLabelsNonBlobs(m, children)
68
+	}
69
+
65 70
 	handlers := []images.Handler{
66 71
 		fetchWithoutRoot(remotes.FetchHandler(cache, fetcher)),
67
-		childrenConfigHandler(cache, platform),
72
+		children,
68 73
 	}
69 74
 	if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
70 75
 		return "", nil, err
... ...
@@ -171,3 +207,39 @@ func DetectManifestBlobMediaType(dt []byte) (string, error) {
171 171
 	}
172 172
 	return images.MediaTypeDockerSchema2ManifestList, nil
173 173
 }
174
+
175
+func SetChildrenLabelsNonBlobs(manager content.Manager, f images.HandlerFunc) images.HandlerFunc {
176
+	return func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) {
177
+		children, err := f(ctx, desc)
178
+		if err != nil {
179
+			return children, err
180
+		}
181
+
182
+		if len(children) > 0 {
183
+			info := content.Info{
184
+				Digest: desc.Digest,
185
+				Labels: map[string]string{},
186
+			}
187
+			fields := []string{}
188
+			for i, ch := range children {
189
+				switch ch.MediaType {
190
+				case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip, specs.MediaTypeImageLayer, specs.MediaTypeImageLayerGzip:
191
+					continue
192
+				default:
193
+				}
194
+
195
+				info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = ch.Digest.String()
196
+				fields = append(fields, fmt.Sprintf("labels.containerd.io/gc.ref.content.%d", i))
197
+			}
198
+
199
+			if len(info.Labels) > 0 {
200
+				_, err := manager.Update(ctx, info, fields...)
201
+				if err != nil {
202
+					return nil, err
203
+				}
204
+			}
205
+		}
206
+
207
+		return children, err
208
+	}
209
+}
174 210
new file mode 100644
... ...
@@ -0,0 +1,104 @@
0
+package leaseutil
1
+
2
+import (
3
+	"context"
4
+	"time"
5
+
6
+	"github.com/containerd/containerd/leases"
7
+	"github.com/containerd/containerd/metadata"
8
+	"github.com/containerd/containerd/namespaces"
9
+	bolt "go.etcd.io/bbolt"
10
+)
11
+
12
+func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (context.Context, func(context.Context) error, error) {
13
+	_, ok := leases.FromContext(ctx)
14
+	if ok {
15
+		return ctx, func(context.Context) error {
16
+			return nil
17
+		}, nil
18
+	}
19
+
20
+	l, err := ls.Create(ctx, append([]leases.Opt{leases.WithRandomID(), leases.WithExpiration(time.Hour)}, opts...)...)
21
+	if err != nil {
22
+		return nil, nil, err
23
+	}
24
+
25
+	ctx = leases.WithLease(ctx, l.ID)
26
+	return ctx, func(ctx context.Context) error {
27
+		return ls.Delete(ctx, l)
28
+	}, nil
29
+}
30
+
31
+func NewManager(mdb *metadata.DB) leases.Manager {
32
+	return &local{db: mdb}
33
+}
34
+
35
+type local struct {
36
+	db *metadata.DB
37
+}
38
+
39
+func (l *local) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) {
40
+	var lease leases.Lease
41
+	if err := l.db.Update(func(tx *bolt.Tx) error {
42
+		var err error
43
+		lease, err = metadata.NewLeaseManager(tx).Create(ctx, opts...)
44
+		return err
45
+	}); err != nil {
46
+		return leases.Lease{}, err
47
+	}
48
+	return lease, nil
49
+}
50
+
51
+func (l *local) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error {
52
+	var do leases.DeleteOptions
53
+	for _, opt := range opts {
54
+		if err := opt(ctx, &do); err != nil {
55
+			return err
56
+		}
57
+	}
58
+
59
+	if err := l.db.Update(func(tx *bolt.Tx) error {
60
+		return metadata.NewLeaseManager(tx).Delete(ctx, lease)
61
+	}); err != nil {
62
+		return err
63
+	}
64
+
65
+	return nil
66
+
67
+}
68
+
69
+func (l *local) List(ctx context.Context, filters ...string) ([]leases.Lease, error) {
70
+	var ll []leases.Lease
71
+	if err := l.db.View(func(tx *bolt.Tx) error {
72
+		var err error
73
+		ll, err = metadata.NewLeaseManager(tx).List(ctx, filters...)
74
+		return err
75
+	}); err != nil {
76
+		return nil, err
77
+	}
78
+	return ll, nil
79
+}
80
+
81
+func WithNamespace(lm leases.Manager, ns string) leases.Manager {
82
+	return &nsLM{Manager: lm, ns: ns}
83
+}
84
+
85
+type nsLM struct {
86
+	leases.Manager
87
+	ns string
88
+}
89
+
90
+func (l *nsLM) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) {
91
+	ctx = namespaces.WithNamespace(ctx, l.ns)
92
+	return l.Manager.Create(ctx, opts...)
93
+}
94
+
95
+func (l *nsLM) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error {
96
+	ctx = namespaces.WithNamespace(ctx, l.ns)
97
+	return l.Manager.Delete(ctx, lease, opts...)
98
+}
99
+
100
+func (l *nsLM) List(ctx context.Context, filters ...string) ([]leases.Lease, error) {
101
+	ctx = namespaces.WithNamespace(ctx, l.ns)
102
+	return l.Manager.List(ctx, filters...)
103
+}