Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
| ... | ... |
@@ -135,7 +135,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform * |
| 135 | 135 |
dt []byte |
| 136 | 136 |
} |
| 137 | 137 |
res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
|
| 138 |
- dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, platform) |
|
| 138 |
+ dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, nil, platform) |
|
| 139 | 139 |
if err != nil {
|
| 140 | 140 |
return nil, err |
| 141 | 141 |
} |
| ... | ... |
@@ -27,7 +27,7 @@ github.com/imdario/mergo 7c29201646fa3de8506f70121347 |
| 27 | 27 |
golang.org/x/sync e225da77a7e68af35c70ccbf71af2b83e6acac3c |
| 28 | 28 |
|
| 29 | 29 |
# buildkit |
| 30 |
-github.com/moby/buildkit f238f1efb04f00bf0cc147141fda9ddb55c8bc49 |
|
| 30 |
+github.com/moby/buildkit 37d53758a68d9f5cede1806dbb2da7c3caa8d5bc |
|
| 31 | 31 |
github.com/tonistiigi/fsutil 3bbb99cdbd76619ab717299830c60f6f2a533a6b |
| 32 | 32 |
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 |
| 33 | 33 |
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 |
| ... | ... |
@@ -184,6 +184,19 @@ The local client will copy the files directly to the client. This is useful if B |
| 184 | 184 |
buildctl build ... --output type=local,dest=path/to/output-dir |
| 185 | 185 |
``` |
| 186 | 186 |
|
| 187 |
+To export specific files use multi-stage builds with a scratch stage and copy the needed files into that stage with `COPY --from`. |
|
| 188 |
+```dockerfile |
|
| 189 |
+... |
|
| 190 |
+FROM scratch as testresult |
|
| 191 |
+ |
|
| 192 |
+COPY --from=builder /usr/src/app/testresult.xml . |
|
| 193 |
+... |
|
| 194 |
+``` |
|
| 195 |
+ |
|
| 196 |
+``` |
|
| 197 |
+buildctl build ... --opt target=testresult --output type=local,dest=path/to/output-dir |
|
| 198 |
+``` |
|
| 199 |
+ |
|
| 187 | 200 |
Tar exporter is similar to local exporter but transfers the files through a tarball. |
| 188 | 201 |
|
| 189 | 202 |
``` |
| ... | ... |
@@ -268,6 +281,16 @@ export BUILDKIT_HOST=tcp://0.0.0.0:1234 |
| 268 | 268 |
buildctl build --help |
| 269 | 269 |
``` |
| 270 | 270 |
|
| 271 |
+To run client and an ephemeral daemon in a single container ("daemonless mode"):
|
|
| 272 |
+ |
|
| 273 |
+``` |
|
| 274 |
+docker run -it --rm --privileged -v /path/to/dir:/tmp/work --entrypoint buildctl-daemonless.sh moby/buildkit:master build --frontend dockerfile.v0 --local context=/tmp/work --local dockerfile=/tmp/work |
|
| 275 |
+``` |
|
| 276 |
+or |
|
| 277 |
+``` |
|
| 278 |
+docker run -it --rm --security-opt seccomp=unconfined --security-opt apparmor=unconfined -e BUILDKITD_FLAGS=--oci-worker-no-process-sandbox -v /path/to/dir:/tmp/work --entrypoint buildctl-daemonless.sh moby/buildkit:master-rootless build --frontend dockerfile.v0 --local context=/tmp/work --local dockerfile=/tmp/work |
|
| 279 |
+``` |
|
| 280 |
+ |
|
| 271 | 281 |
The images can be also built locally using `./hack/dockerfiles/test.Dockerfile` (or `./hack/dockerfiles/test.buildkit.Dockerfile` if you already have BuildKit). |
| 272 | 282 |
Run `make images` to build the images as `moby/buildkit:local` and `moby/buildkit:local-rootless`. |
| 273 | 283 |
|
| ... | ... |
@@ -60,6 +60,10 @@ func SetCacheContext(ctx context.Context, md *metadata.StorageItem, cc CacheCont |
| 60 | 60 |
return getDefaultManager().SetCacheContext(ctx, md, cc) |
| 61 | 61 |
} |
| 62 | 62 |
|
| 63 |
+func ClearCacheContext(md *metadata.StorageItem) {
|
|
| 64 |
+ getDefaultManager().clearCacheContext(md.ID()) |
|
| 65 |
+} |
|
| 66 |
+ |
|
| 63 | 67 |
type CacheContext interface {
|
| 64 | 68 |
Checksum(ctx context.Context, ref cache.Mountable, p string, followLinks bool) (digest.Digest, error) |
| 65 | 69 |
ChecksumWildcard(ctx context.Context, ref cache.Mountable, p string, followLinks bool) (digest.Digest, error) |
| ... | ... |
@@ -142,6 +146,12 @@ func (cm *cacheManager) SetCacheContext(ctx context.Context, md *metadata.Storag |
| 142 | 142 |
return nil |
| 143 | 143 |
} |
| 144 | 144 |
|
| 145 |
+func (cm *cacheManager) clearCacheContext(id string) {
|
|
| 146 |
+ cm.lruMu.Lock() |
|
| 147 |
+ cm.lru.Remove(id) |
|
| 148 |
+ cm.lruMu.Unlock() |
|
| 149 |
+} |
|
| 150 |
+ |
|
| 145 | 151 |
type cacheContext struct {
|
| 146 | 152 |
mu sync.RWMutex |
| 147 | 153 |
md *metadata.StorageItem |
| ... | ... |
@@ -125,7 +125,7 @@ func (cm *cacheManager) GetFromSnapshotter(ctx context.Context, id string, opts |
| 125 | 125 |
} |
| 126 | 126 |
|
| 127 | 127 |
// get requires manager lock to be taken |
| 128 |
-func (cm *cacheManager) get(ctx context.Context, id string, fromSnapshotter bool, opts ...RefOption) (ImmutableRef, error) {
|
|
| 128 |
+func (cm *cacheManager) get(ctx context.Context, id string, fromSnapshotter bool, opts ...RefOption) (*immutableRef, error) {
|
|
| 129 | 129 |
rec, err := cm.getRecord(ctx, id, fromSnapshotter, opts...) |
| 130 | 130 |
if err != nil {
|
| 131 | 131 |
return nil, err |
| ... | ... |
@@ -193,7 +193,7 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte |
| 193 | 193 |
return nil, errors.Wrap(errNotFound, err.Error()) |
| 194 | 194 |
} |
| 195 | 195 |
|
| 196 |
- var parent ImmutableRef |
|
| 196 |
+ var parent *immutableRef |
|
| 197 | 197 |
if info.Parent != "" {
|
| 198 | 198 |
parent, err = cm.get(ctx, info.Parent, fromSnapshotter, append(opts, NoUpdateLastUsed)...) |
| 199 | 199 |
if err != nil {
|
| ... | ... |
@@ -201,7 +201,9 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte |
| 201 | 201 |
} |
| 202 | 202 |
defer func() {
|
| 203 | 203 |
if retErr != nil {
|
| 204 |
- parent.Release(context.TODO()) |
|
| 204 |
+ parent.mu.Lock() |
|
| 205 |
+ parent.release(context.TODO()) |
|
| 206 |
+ parent.mu.Unlock() |
|
| 205 | 207 |
} |
| 206 | 208 |
}() |
| 207 | 209 |
} |
| ... | ... |
@@ -224,9 +226,6 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte |
| 224 | 224 |
} |
| 225 | 225 |
|
| 226 | 226 |
if err := initializeMetadata(rec, opts...); err != nil {
|
| 227 |
- if parent != nil {
|
|
| 228 |
- parent.Release(context.TODO()) |
|
| 229 |
- } |
|
| 230 | 227 |
return nil, err |
| 231 | 228 |
} |
| 232 | 229 |
|
| ... | ... |
@@ -237,18 +236,18 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, fromSnapshotte |
| 237 | 237 |
func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, opts ...RefOption) (MutableRef, error) {
|
| 238 | 238 |
id := identity.NewID() |
| 239 | 239 |
|
| 240 |
- var parent ImmutableRef |
|
| 240 |
+ var parent *immutableRef |
|
| 241 | 241 |
var parentID string |
| 242 | 242 |
if s != nil {
|
| 243 |
- var err error |
|
| 244 |
- parent, err = cm.Get(ctx, s.ID(), NoUpdateLastUsed) |
|
| 243 |
+ p, err := cm.Get(ctx, s.ID(), NoUpdateLastUsed) |
|
| 245 | 244 |
if err != nil {
|
| 246 | 245 |
return nil, err |
| 247 | 246 |
} |
| 248 |
- if err := parent.Finalize(ctx, true); err != nil {
|
|
| 247 |
+ if err := p.Finalize(ctx, true); err != nil {
|
|
| 249 | 248 |
return nil, err |
| 250 | 249 |
} |
| 251 |
- parentID = parent.ID() |
|
| 250 |
+ parentID = p.ID() |
|
| 251 |
+ parent = p.(*immutableRef) |
|
| 252 | 252 |
} |
| 253 | 253 |
|
| 254 | 254 |
if err := cm.Snapshotter.Prepare(ctx, id, parentID); err != nil {
|
| ... | ... |
@@ -737,6 +736,10 @@ func CachePolicyRetain(m withMetadata) error {
|
| 737 | 737 |
return queueCachePolicy(m.Metadata(), cachePolicyRetain) |
| 738 | 738 |
} |
| 739 | 739 |
|
| 740 |
+func CachePolicyDefault(m withMetadata) error {
|
|
| 741 |
+ return queueCachePolicy(m.Metadata(), cachePolicyDefault) |
|
| 742 |
+} |
|
| 743 |
+ |
|
| 740 | 744 |
func WithDescription(descr string) RefOption {
|
| 741 | 745 |
return func(m withMetadata) error {
|
| 742 | 746 |
return queueDescription(m.Metadata(), descr) |
| ... | ... |
@@ -50,7 +50,7 @@ type cacheRecord struct {
|
| 50 | 50 |
|
| 51 | 51 |
mutable bool |
| 52 | 52 |
refs map[ref]struct{}
|
| 53 |
- parent ImmutableRef |
|
| 53 |
+ parent *immutableRef |
|
| 54 | 54 |
md *metadata.StorageItem |
| 55 | 55 |
|
| 56 | 56 |
// dead means record is marked as deleted |
| ... | ... |
@@ -126,14 +126,17 @@ func (cr *cacheRecord) Size(ctx context.Context) (int64, error) {
|
| 126 | 126 |
} |
| 127 | 127 |
|
| 128 | 128 |
func (cr *cacheRecord) Parent() ImmutableRef {
|
| 129 |
- return cr.parentRef(true) |
|
| 129 |
+ if p := cr.parentRef(true); p != nil { // avoid returning typed nil pointer
|
|
| 130 |
+ return p |
|
| 131 |
+ } |
|
| 132 |
+ return nil |
|
| 130 | 133 |
} |
| 131 | 134 |
|
| 132 |
-func (cr *cacheRecord) parentRef(hidden bool) ImmutableRef {
|
|
| 133 |
- if cr.parent == nil {
|
|
| 135 |
+func (cr *cacheRecord) parentRef(hidden bool) *immutableRef {
|
|
| 136 |
+ p := cr.parent |
|
| 137 |
+ if p == nil {
|
|
| 134 | 138 |
return nil |
| 135 | 139 |
} |
| 136 |
- p := cr.parent.(*immutableRef) |
|
| 137 | 140 |
p.mu.Lock() |
| 138 | 141 |
defer p.mu.Unlock() |
| 139 | 142 |
return p.ref(hidden) |
| ... | ... |
@@ -181,7 +184,7 @@ func (cr *cacheRecord) Mount(ctx context.Context, readonly bool) (snapshot.Mount |
| 181 | 181 |
func (cr *cacheRecord) remove(ctx context.Context, removeSnapshot bool) error {
|
| 182 | 182 |
delete(cr.cm.records, cr.ID()) |
| 183 | 183 |
if cr.parent != nil {
|
| 184 |
- if err := cr.parent.(*immutableRef).release(ctx); err != nil {
|
|
| 184 |
+ if err := cr.parent.release(ctx); err != nil {
|
|
| 185 | 185 |
return err |
| 186 | 186 |
} |
| 187 | 187 |
} |
| ... | ... |
@@ -315,7 +318,7 @@ func (sr *mutableRef) updateLastUsed() bool {
|
| 315 | 315 |
return sr.triggerLastUsed |
| 316 | 316 |
} |
| 317 | 317 |
|
| 318 |
-func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) {
|
|
| 318 |
+func (sr *mutableRef) commit(ctx context.Context) (*immutableRef, error) {
|
|
| 319 | 319 |
if !sr.mutable || len(sr.refs) == 0 {
|
| 320 | 320 |
return nil, errors.Wrapf(errInvalid, "invalid mutable ref %p", sr) |
| 321 | 321 |
} |
| ... | ... |
@@ -398,7 +401,7 @@ func (sr *mutableRef) release(ctx context.Context) error {
|
| 398 | 398 |
} |
| 399 | 399 |
} |
| 400 | 400 |
if sr.parent != nil {
|
| 401 |
- if err := sr.parent.(*immutableRef).release(ctx); err != nil {
|
|
| 401 |
+ if err := sr.parent.release(ctx); err != nil {
|
|
| 402 | 402 |
return err |
| 403 | 403 |
} |
| 404 | 404 |
} |
| ... | ... |
@@ -87,7 +87,7 @@ func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string |
| 87 | 87 |
return res.dgst, res.config, nil |
| 88 | 88 |
} |
| 89 | 89 |
|
| 90 |
- dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, platform) |
|
| 90 |
+ dgst, config, err := imageutil.Config(ctx, ref, imr.resolver, imr.buffer, nil, platform) |
|
| 91 | 91 |
if err != nil {
|
| 92 | 92 |
return "", nil, err |
| 93 | 93 |
} |
| ... | ... |
@@ -3,6 +3,7 @@ package control |
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 | 5 |
"sync" |
| 6 |
+ "sync/atomic" |
|
| 6 | 7 |
"time" |
| 7 | 8 |
|
| 8 | 9 |
controlapi "github.com/moby/buildkit/api/services/control" |
| ... | ... |
@@ -17,6 +18,7 @@ import ( |
| 17 | 17 |
"github.com/moby/buildkit/solver" |
| 18 | 18 |
"github.com/moby/buildkit/solver/llbsolver" |
| 19 | 19 |
"github.com/moby/buildkit/solver/pb" |
| 20 |
+ "github.com/moby/buildkit/util/imageutil" |
|
| 20 | 21 |
"github.com/moby/buildkit/util/throttle" |
| 21 | 22 |
"github.com/moby/buildkit/worker" |
| 22 | 23 |
"github.com/pkg/errors" |
| ... | ... |
@@ -42,6 +44,7 @@ type Controller struct { // TODO: ControlService
|
| 42 | 42 |
gatewayForwarder *controlgateway.GatewayForwarder |
| 43 | 43 |
throttledGC func() |
| 44 | 44 |
gcmu sync.Mutex |
| 45 |
+ buildCount int64 |
|
| 45 | 46 |
} |
| 46 | 47 |
|
| 47 | 48 |
func NewController(opt Opt) (*Controller, error) {
|
| ... | ... |
@@ -110,6 +113,10 @@ func (c *Controller) DiskUsage(ctx context.Context, r *controlapi.DiskUsageReque |
| 110 | 110 |
} |
| 111 | 111 |
|
| 112 | 112 |
func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Control_PruneServer) error {
|
| 113 |
+ if atomic.LoadInt64(&c.buildCount) == 0 {
|
|
| 114 |
+ imageutil.CancelCacheLeases() |
|
| 115 |
+ } |
|
| 116 |
+ |
|
| 113 | 117 |
ch := make(chan client.UsageInfo) |
| 114 | 118 |
|
| 115 | 119 |
eg, ctx := errgroup.WithContext(stream.Context()) |
| ... | ... |
@@ -207,6 +214,9 @@ func translateLegacySolveRequest(req *controlapi.SolveRequest) error {
|
| 207 | 207 |
} |
| 208 | 208 |
|
| 209 | 209 |
func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*controlapi.SolveResponse, error) {
|
| 210 |
+ atomic.AddInt64(&c.buildCount, 1) |
|
| 211 |
+ defer atomic.AddInt64(&c.buildCount, -1) |
|
| 212 |
+ |
|
| 210 | 213 |
if err := translateLegacySolveRequest(req); err != nil {
|
| 211 | 214 |
return nil, err |
| 212 | 215 |
} |
| ... | ... |
@@ -3,8 +3,12 @@ |
| 3 | 3 |
package dockerfile2llb |
| 4 | 4 |
|
| 5 | 5 |
import ( |
| 6 |
+ "fmt" |
|
| 7 |
+ "os" |
|
| 6 | 8 |
"path" |
| 7 | 9 |
"path/filepath" |
| 10 |
+ "strconv" |
|
| 11 |
+ "strings" |
|
| 8 | 12 |
|
| 9 | 13 |
"github.com/moby/buildkit/client/llb" |
| 10 | 14 |
"github.com/moby/buildkit/frontend/dockerfile/instructions" |
| ... | ... |
@@ -40,6 +44,40 @@ func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
|
| 40 | 40 |
return false |
| 41 | 41 |
} |
| 42 | 42 |
|
| 43 |
+func setCacheUIDGIDFileOp(m *instructions.Mount, st llb.State) llb.State {
|
|
| 44 |
+ uid := 0 |
|
| 45 |
+ gid := 0 |
|
| 46 |
+ mode := os.FileMode(0755) |
|
| 47 |
+ if m.UID != nil {
|
|
| 48 |
+ uid = int(*m.UID) |
|
| 49 |
+ } |
|
| 50 |
+ if m.GID != nil {
|
|
| 51 |
+ gid = int(*m.GID) |
|
| 52 |
+ } |
|
| 53 |
+ if m.Mode != nil {
|
|
| 54 |
+ mode = os.FileMode(*m.Mode) |
|
| 55 |
+ } |
|
| 56 |
+ return st.File(llb.Mkdir("/cache", mode, llb.WithUIDGID(uid, gid)), llb.WithCustomName("[internal] settings cache mount permissions"))
|
|
| 57 |
+} |
|
| 58 |
+ |
|
| 59 |
+func setCacheUIDGID(m *instructions.Mount, st llb.State, fileop bool) llb.State {
|
|
| 60 |
+ if fileop {
|
|
| 61 |
+ return setCacheUIDGIDFileOp(m, st) |
|
| 62 |
+ } |
|
| 63 |
+ |
|
| 64 |
+ var b strings.Builder |
|
| 65 |
+ if m.UID != nil {
|
|
| 66 |
+ b.WriteString(fmt.Sprintf("chown %d /mnt/cache;", *m.UID))
|
|
| 67 |
+ } |
|
| 68 |
+ if m.GID != nil {
|
|
| 69 |
+ b.WriteString(fmt.Sprintf("chown :%d /mnt/cache;", *m.GID))
|
|
| 70 |
+ } |
|
| 71 |
+ if m.Mode != nil {
|
|
| 72 |
+ b.WriteString(fmt.Sprintf("chmod %s /mnt/cache;", strconv.FormatUint(*m.Mode, 8)))
|
|
| 73 |
+ } |
|
| 74 |
+ return llb.Image("busybox").Run(llb.Shlex(fmt.Sprintf("sh -c 'mkdir -p /mnt/cache;%s'", b.String())), llb.WithCustomName("[internal] settings cache mount permissions")).AddMount("/mnt", st)
|
|
| 75 |
+} |
|
| 76 |
+ |
|
| 43 | 77 |
func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*dispatchState, opt dispatchOpt) ([]llb.RunOption, error) {
|
| 44 | 78 |
var out []llb.RunOption |
| 45 | 79 |
mounts := instructions.GetMounts(c) |
| ... | ... |
@@ -97,7 +135,13 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []* |
| 97 | 97 |
} |
| 98 | 98 |
if src := path.Join("/", mount.Source); src != "/" {
|
| 99 | 99 |
mountOpts = append(mountOpts, llb.SourcePath(src)) |
| 100 |
+ } else {
|
|
| 101 |
+ if mount.UID != nil || mount.GID != nil || mount.Mode != nil {
|
|
| 102 |
+ st = setCacheUIDGID(mount, st, useFileOp(opt.buildArgValues, opt.llbCaps)) |
|
| 103 |
+ mountOpts = append(mountOpts, llb.SourcePath("/cache"))
|
|
| 104 |
+ } |
|
| 100 | 105 |
} |
| 106 |
+ |
|
| 101 | 107 |
out = append(out, llb.AddMount(target, st, mountOpts...)) |
| 102 | 108 |
|
| 103 | 109 |
d.ctxPaths[path.Join("/", filepath.ToSlash(mount.Source))] = struct{}{}
|
| ... | ... |
@@ -206,18 +206,18 @@ func parseMount(value string) (*Mount, error) {
|
| 206 | 206 |
} |
| 207 | 207 |
} |
| 208 | 208 |
|
| 209 |
- fileInfoAllowed := m.Type == MountTypeSecret || m.Type == MountTypeSSH |
|
| 209 |
+ fileInfoAllowed := m.Type == MountTypeSecret || m.Type == MountTypeSSH || m.Type == MountTypeCache |
|
| 210 | 210 |
|
| 211 | 211 |
if m.Mode != nil && !fileInfoAllowed {
|
| 212 |
- return nil, errors.Errorf("mode not allowed for %q type mounts")
|
|
| 212 |
+ return nil, errors.Errorf("mode not allowed for %q type mounts", m.Type)
|
|
| 213 | 213 |
} |
| 214 | 214 |
|
| 215 | 215 |
if m.UID != nil && !fileInfoAllowed {
|
| 216 |
- return nil, errors.Errorf("uid not allowed for %q type mounts")
|
|
| 216 |
+ return nil, errors.Errorf("uid not allowed for %q type mounts", m.Type)
|
|
| 217 | 217 |
} |
| 218 | 218 |
|
| 219 | 219 |
if m.GID != nil && !fileInfoAllowed {
|
| 220 |
- return nil, errors.Errorf("gid not allowed for %q type mounts")
|
|
| 220 |
+ return nil, errors.Errorf("gid not allowed for %q type mounts", m.Type)
|
|
| 221 | 221 |
} |
| 222 | 222 |
|
| 223 | 223 |
if roAuto {
|
| ... | ... |
@@ -35,7 +35,7 @@ type Node struct {
|
| 35 | 35 |
Original string // original line used before parsing |
| 36 | 36 |
Flags []string // only top Node should have this set |
| 37 | 37 |
StartLine int // the line in the original dockerfile where the node begins |
| 38 |
- endLine int // the line in the original dockerfile where the node ends |
|
| 38 |
+ EndLine int // the line in the original dockerfile where the node ends |
|
| 39 | 39 |
} |
| 40 | 40 |
|
| 41 | 41 |
// Dump dumps the AST defined by `node` as a list of sexps. |
| ... | ... |
@@ -65,7 +65,7 @@ func (node *Node) Dump() string {
|
| 65 | 65 |
|
| 66 | 66 |
func (node *Node) lines(start, end int) {
|
| 67 | 67 |
node.StartLine = start |
| 68 |
- node.endLine = end |
|
| 68 |
+ node.EndLine = end |
|
| 69 | 69 |
} |
| 70 | 70 |
|
| 71 | 71 |
// AddChild adds a new child node, and updates line information |
| ... | ... |
@@ -74,7 +74,7 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) {
|
| 74 | 74 |
if node.StartLine < 0 {
|
| 75 | 75 |
node.StartLine = startLine |
| 76 | 76 |
} |
| 77 |
- node.endLine = endLine |
|
| 77 |
+ node.EndLine = endLine |
|
| 78 | 78 |
node.Children = append(node.Children, child) |
| 79 | 79 |
} |
| 80 | 80 |
|
| ... | ... |
@@ -2,6 +2,7 @@ package blobmapping |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 |
+ "time" |
|
| 5 | 6 |
|
| 6 | 7 |
"github.com/containerd/containerd/content" |
| 7 | 8 |
"github.com/containerd/containerd/snapshots" |
| ... | ... |
@@ -122,6 +123,16 @@ func (s *Snapshotter) SetBlob(ctx context.Context, key string, diffID, blobsum d |
| 122 | 122 |
return err |
| 123 | 123 |
} |
| 124 | 124 |
} |
| 125 |
+ // update gc.root cause blob might be held by lease only |
|
| 126 |
+ if _, err := s.opt.Content.Update(ctx, content.Info{
|
|
| 127 |
+ Digest: blobsum, |
|
| 128 |
+ Labels: map[string]string{
|
|
| 129 |
+ "containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano), |
|
| 130 |
+ }, |
|
| 131 |
+ }, "labels.containerd.io/gc.root"); err != nil {
|
|
| 132 |
+ return err |
|
| 133 |
+ } |
|
| 134 |
+ |
|
| 125 | 135 |
md, _ := s.opt.MetadataStore.Get(key) |
| 126 | 136 |
|
| 127 | 137 |
v, err := metadata.NewValue(DiffPair{DiffID: diffID, Blobsum: blobsum})
|
| ... | ... |
@@ -133,6 +133,12 @@ func (ls *localSourceHandler) Snapshot(ctx context.Context) (out cache.Immutable |
| 133 | 133 |
|
| 134 | 134 |
defer func() {
|
| 135 | 135 |
if retErr != nil && mutable != nil {
|
| 136 |
+ // on error remove the record as checksum update is in undefined state |
|
| 137 |
+ cache.CachePolicyDefault(mutable) |
|
| 138 |
+ if err := mutable.Metadata().Commit(); err != nil {
|
|
| 139 |
+ logrus.Errorf("failed to reset mutable cachepolicy: %v", err)
|
|
| 140 |
+ } |
|
| 141 |
+ contenthash.ClearCacheContext(mutable.Metadata()) |
|
| 136 | 142 |
go mutable.Release(context.TODO()) |
| 137 | 143 |
} |
| 138 | 144 |
}() |
| ... | ... |
@@ -21,6 +21,9 @@ func SupportedPlatforms() []string {
|
| 21 | 21 |
if p := "linux/arm64"; def != p && arm64Supported() == nil {
|
| 22 | 22 |
arr = append(arr, p) |
| 23 | 23 |
} |
| 24 |
+ if p := "linux/riscv64"; def != p && riscv64Supported() == nil {
|
|
| 25 |
+ arr = append(arr, p) |
|
| 26 |
+ } |
|
| 24 | 27 |
if !strings.HasPrefix(def, "linux/arm/") && armSupported() == nil {
|
| 25 | 28 |
arr = append(arr, "linux/arm/v7", "linux/arm/v6") |
| 26 | 29 |
} else if def == "linux/arm/v7" {
|
| ... | ... |
@@ -47,6 +50,11 @@ func WarnIfUnsupported(pfs []string) {
|
| 47 | 47 |
printPlatfromWarning(p, err) |
| 48 | 48 |
} |
| 49 | 49 |
} |
| 50 |
+ if p == "linux/riscv64" {
|
|
| 51 |
+ if err := riscv64Supported(); err != nil {
|
|
| 52 |
+ printPlatfromWarning(p, err) |
|
| 53 |
+ } |
|
| 54 |
+ } |
|
| 50 | 55 |
if strings.HasPrefix(p, "linux/arm/v6") || strings.HasPrefix(p, "linux/arm/v7") {
|
| 51 | 56 |
if err := armSupported(); err != nil {
|
| 52 | 57 |
printPlatfromWarning(p, err) |
| 53 | 58 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,8 @@ |
| 0 |
+// +build !riscv64 |
|
| 1 |
+ |
|
| 2 |
+package binfmt_misc |
|
| 3 |
+ |
|
| 4 |
+// This file is generated by running make inside the binfmt_misc package. |
|
| 5 |
+// Do not edit manually. |
|
| 6 |
+ |
|
| 7 |
+const Binaryriscv64 = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xaa\x77\xf5\x71\x63\x62\x64\x64\x80\x01\x26\x86\xcf\x0c\x20\x5e\x05\x03\x44\xcc\x01\x2a\x3e\x03\x4a\xb3\x80\xc5\x2c\x18\x18\x19\x1c\x18\x98\x19\x98\xc0\xaa\x58\x19\x90\x01\x23\x0a\xdd\x02\xe5\xc1\x68\x06\x01\x08\x25\xcc\xca\xc0\x30\x99\xe3\x02\x6b\x31\x88\xa3\x57\x9c\x51\x5c\x52\x54\x92\x98\xc4\xa0\x57\x92\x5a\x51\xc2\x40\x05\xc0\x0d\x75\x01\x1b\x94\x0f\xf3\x4f\x05\x94\xcf\x83\xa6\x9e\x05\x8d\x0f\x52\xcd\x8c\xc5\x5c\x98\x3f\x04\xb1\xa8\x47\x06\x80\x00\x00\x00\xff\xff\x39\x41\xdf\xa1\x58\x01\x00\x00" |
| ... | ... |
@@ -3,12 +3,17 @@ package imageutil |
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 | 5 |
"encoding/json" |
| 6 |
+ "fmt" |
|
| 7 |
+ "sync" |
|
| 8 |
+ "time" |
|
| 6 | 9 |
|
| 7 | 10 |
"github.com/containerd/containerd/content" |
| 8 | 11 |
"github.com/containerd/containerd/images" |
| 12 |
+ "github.com/containerd/containerd/leases" |
|
| 9 | 13 |
"github.com/containerd/containerd/platforms" |
| 10 | 14 |
"github.com/containerd/containerd/reference" |
| 11 | 15 |
"github.com/containerd/containerd/remotes" |
| 16 |
+ "github.com/moby/buildkit/util/leaseutil" |
|
| 12 | 17 |
digest "github.com/opencontainers/go-digest" |
| 13 | 18 |
specs "github.com/opencontainers/image-spec/specs-go/v1" |
| 14 | 19 |
"github.com/pkg/errors" |
| ... | ... |
@@ -19,7 +24,19 @@ type ContentCache interface {
|
| 19 | 19 |
content.Provider |
| 20 | 20 |
} |
| 21 | 21 |
|
| 22 |
-func Config(ctx context.Context, str string, resolver remotes.Resolver, cache ContentCache, p *specs.Platform) (digest.Digest, []byte, error) {
|
|
| 22 |
+var leasesMu sync.Mutex |
|
| 23 |
+var leasesF []func(context.Context) error |
|
| 24 |
+ |
|
| 25 |
+func CancelCacheLeases() {
|
|
| 26 |
+ leasesMu.Lock() |
|
| 27 |
+ for _, f := range leasesF {
|
|
| 28 |
+ f(context.TODO()) |
|
| 29 |
+ } |
|
| 30 |
+ leasesF = nil |
|
| 31 |
+ leasesMu.Unlock() |
|
| 32 |
+} |
|
| 33 |
+ |
|
| 34 |
+func Config(ctx context.Context, str string, resolver remotes.Resolver, cache ContentCache, leaseManager leases.Manager, p *specs.Platform) (digest.Digest, []byte, error) {
|
|
| 23 | 35 |
// TODO: fix buildkit to take interface instead of struct |
| 24 | 36 |
var platform platforms.MatchComparer |
| 25 | 37 |
if p != nil {
|
| ... | ... |
@@ -32,6 +49,20 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co |
| 32 | 32 |
return "", nil, errors.WithStack(err) |
| 33 | 33 |
} |
| 34 | 34 |
|
| 35 |
+ if leaseManager != nil {
|
|
| 36 |
+ ctx2, done, err := leaseutil.WithLease(ctx, leaseManager, leases.WithExpiration(5*time.Minute)) |
|
| 37 |
+ if err != nil {
|
|
| 38 |
+ return "", nil, errors.WithStack(err) |
|
| 39 |
+ } |
|
| 40 |
+ ctx = ctx2 |
|
| 41 |
+ defer func() {
|
|
| 42 |
+ // this lease is not deleted to allow other components to access manifest/config from cache. It will be deleted after 5 min deadline or on pruning inactive builder |
|
| 43 |
+ leasesMu.Lock() |
|
| 44 |
+ leasesF = append(leasesF, done) |
|
| 45 |
+ leasesMu.Unlock() |
|
| 46 |
+ }() |
|
| 47 |
+ } |
|
| 48 |
+ |
|
| 35 | 49 |
desc := specs.Descriptor{
|
| 36 | 50 |
Digest: ref.Digest(), |
| 37 | 51 |
} |
| ... | ... |
@@ -62,9 +93,14 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co |
| 62 | 62 |
return readSchema1Config(ctx, ref.String(), desc, fetcher, cache) |
| 63 | 63 |
} |
| 64 | 64 |
|
| 65 |
+ children := childrenConfigHandler(cache, platform) |
|
| 66 |
+ if m, ok := cache.(content.Manager); ok {
|
|
| 67 |
+ children = SetChildrenLabelsNonBlobs(m, children) |
|
| 68 |
+ } |
|
| 69 |
+ |
|
| 65 | 70 |
handlers := []images.Handler{
|
| 66 | 71 |
fetchWithoutRoot(remotes.FetchHandler(cache, fetcher)), |
| 67 |
- childrenConfigHandler(cache, platform), |
|
| 72 |
+ children, |
|
| 68 | 73 |
} |
| 69 | 74 |
if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil {
|
| 70 | 75 |
return "", nil, err |
| ... | ... |
@@ -171,3 +207,39 @@ func DetectManifestBlobMediaType(dt []byte) (string, error) {
|
| 171 | 171 |
} |
| 172 | 172 |
return images.MediaTypeDockerSchema2ManifestList, nil |
| 173 | 173 |
} |
| 174 |
+ |
|
| 175 |
+func SetChildrenLabelsNonBlobs(manager content.Manager, f images.HandlerFunc) images.HandlerFunc {
|
|
| 176 |
+ return func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) {
|
|
| 177 |
+ children, err := f(ctx, desc) |
|
| 178 |
+ if err != nil {
|
|
| 179 |
+ return children, err |
|
| 180 |
+ } |
|
| 181 |
+ |
|
| 182 |
+ if len(children) > 0 {
|
|
| 183 |
+ info := content.Info{
|
|
| 184 |
+ Digest: desc.Digest, |
|
| 185 |
+ Labels: map[string]string{},
|
|
| 186 |
+ } |
|
| 187 |
+ fields := []string{}
|
|
| 188 |
+ for i, ch := range children {
|
|
| 189 |
+ switch ch.MediaType {
|
|
| 190 |
+ case images.MediaTypeDockerSchema2Layer, images.MediaTypeDockerSchema2LayerGzip, specs.MediaTypeImageLayer, specs.MediaTypeImageLayerGzip: |
|
| 191 |
+ continue |
|
| 192 |
+ default: |
|
| 193 |
+ } |
|
| 194 |
+ |
|
| 195 |
+ info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = ch.Digest.String()
|
|
| 196 |
+ fields = append(fields, fmt.Sprintf("labels.containerd.io/gc.ref.content.%d", i))
|
|
| 197 |
+ } |
|
| 198 |
+ |
|
| 199 |
+ if len(info.Labels) > 0 {
|
|
| 200 |
+ _, err := manager.Update(ctx, info, fields...) |
|
| 201 |
+ if err != nil {
|
|
| 202 |
+ return nil, err |
|
| 203 |
+ } |
|
| 204 |
+ } |
|
| 205 |
+ } |
|
| 206 |
+ |
|
| 207 |
+ return children, err |
|
| 208 |
+ } |
|
| 209 |
+} |
| 174 | 210 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,104 @@ |
| 0 |
+package leaseutil |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "time" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/containerd/containerd/leases" |
|
| 7 |
+ "github.com/containerd/containerd/metadata" |
|
| 8 |
+ "github.com/containerd/containerd/namespaces" |
|
| 9 |
+ bolt "go.etcd.io/bbolt" |
|
| 10 |
+) |
|
| 11 |
+ |
|
| 12 |
+func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (context.Context, func(context.Context) error, error) {
|
|
| 13 |
+ _, ok := leases.FromContext(ctx) |
|
| 14 |
+ if ok {
|
|
| 15 |
+ return ctx, func(context.Context) error {
|
|
| 16 |
+ return nil |
|
| 17 |
+ }, nil |
|
| 18 |
+ } |
|
| 19 |
+ |
|
| 20 |
+ l, err := ls.Create(ctx, append([]leases.Opt{leases.WithRandomID(), leases.WithExpiration(time.Hour)}, opts...)...)
|
|
| 21 |
+ if err != nil {
|
|
| 22 |
+ return nil, nil, err |
|
| 23 |
+ } |
|
| 24 |
+ |
|
| 25 |
+ ctx = leases.WithLease(ctx, l.ID) |
|
| 26 |
+ return ctx, func(ctx context.Context) error {
|
|
| 27 |
+ return ls.Delete(ctx, l) |
|
| 28 |
+ }, nil |
|
| 29 |
+} |
|
| 30 |
+ |
|
| 31 |
+func NewManager(mdb *metadata.DB) leases.Manager {
|
|
| 32 |
+ return &local{db: mdb}
|
|
| 33 |
+} |
|
| 34 |
+ |
|
| 35 |
+type local struct {
|
|
| 36 |
+ db *metadata.DB |
|
| 37 |
+} |
|
| 38 |
+ |
|
| 39 |
+func (l *local) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) {
|
|
| 40 |
+ var lease leases.Lease |
|
| 41 |
+ if err := l.db.Update(func(tx *bolt.Tx) error {
|
|
| 42 |
+ var err error |
|
| 43 |
+ lease, err = metadata.NewLeaseManager(tx).Create(ctx, opts...) |
|
| 44 |
+ return err |
|
| 45 |
+ }); err != nil {
|
|
| 46 |
+ return leases.Lease{}, err
|
|
| 47 |
+ } |
|
| 48 |
+ return lease, nil |
|
| 49 |
+} |
|
| 50 |
+ |
|
| 51 |
+func (l *local) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error {
|
|
| 52 |
+ var do leases.DeleteOptions |
|
| 53 |
+ for _, opt := range opts {
|
|
| 54 |
+ if err := opt(ctx, &do); err != nil {
|
|
| 55 |
+ return err |
|
| 56 |
+ } |
|
| 57 |
+ } |
|
| 58 |
+ |
|
| 59 |
+ if err := l.db.Update(func(tx *bolt.Tx) error {
|
|
| 60 |
+ return metadata.NewLeaseManager(tx).Delete(ctx, lease) |
|
| 61 |
+ }); err != nil {
|
|
| 62 |
+ return err |
|
| 63 |
+ } |
|
| 64 |
+ |
|
| 65 |
+ return nil |
|
| 66 |
+ |
|
| 67 |
+} |
|
| 68 |
+ |
|
| 69 |
+func (l *local) List(ctx context.Context, filters ...string) ([]leases.Lease, error) {
|
|
| 70 |
+ var ll []leases.Lease |
|
| 71 |
+ if err := l.db.View(func(tx *bolt.Tx) error {
|
|
| 72 |
+ var err error |
|
| 73 |
+ ll, err = metadata.NewLeaseManager(tx).List(ctx, filters...) |
|
| 74 |
+ return err |
|
| 75 |
+ }); err != nil {
|
|
| 76 |
+ return nil, err |
|
| 77 |
+ } |
|
| 78 |
+ return ll, nil |
|
| 79 |
+} |
|
| 80 |
+ |
|
| 81 |
+func WithNamespace(lm leases.Manager, ns string) leases.Manager {
|
|
| 82 |
+ return &nsLM{Manager: lm, ns: ns}
|
|
| 83 |
+} |
|
| 84 |
+ |
|
| 85 |
+type nsLM struct {
|
|
| 86 |
+ leases.Manager |
|
| 87 |
+ ns string |
|
| 88 |
+} |
|
| 89 |
+ |
|
| 90 |
+func (l *nsLM) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) {
|
|
| 91 |
+ ctx = namespaces.WithNamespace(ctx, l.ns) |
|
| 92 |
+ return l.Manager.Create(ctx, opts...) |
|
| 93 |
+} |
|
| 94 |
+ |
|
| 95 |
+func (l *nsLM) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error {
|
|
| 96 |
+ ctx = namespaces.WithNamespace(ctx, l.ns) |
|
| 97 |
+ return l.Manager.Delete(ctx, lease, opts...) |
|
| 98 |
+} |
|
| 99 |
+ |
|
| 100 |
+func (l *nsLM) List(ctx context.Context, filters ...string) ([]leases.Lease, error) {
|
|
| 101 |
+ ctx = namespaces.WithNamespace(ctx, l.ns) |
|
| 102 |
+ return l.Manager.List(ctx, filters...) |
|
| 103 |
+} |