Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
(cherry picked from commit e59b26087fffe0357d34b400740e2fb15c062cb5)
... | ... |
@@ -27,7 +27,7 @@ github.com/imdario/mergo 7c29201646fa3de8506f70121347 |
27 | 27 |
golang.org/x/sync e225da77a7e68af35c70ccbf71af2b83e6acac3c |
28 | 28 |
|
29 | 29 |
# buildkit |
30 |
-github.com/moby/buildkit f5a55a9516d1c6e2ade9bec22b83259caeed3a84 |
|
30 |
+github.com/moby/buildkit be0d75f074e7a4b0f5b5877c719213a3f5057e60 # v0.6.1 |
|
31 | 31 |
github.com/tonistiigi/fsutil 3bbb99cdbd76619ab717299830c60f6f2a533a6b |
32 | 32 |
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 |
33 | 33 |
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 |
... | ... |
@@ -111,7 +111,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte |
111 | 111 |
} |
112 | 112 |
|
113 | 113 |
var mu sync.Mutex |
114 |
- cc := v1.NewCacheChains() |
|
114 |
+ var cMap = map[digest.Digest]*v1.CacheChains{} |
|
115 | 115 |
|
116 | 116 |
eg, ctx := errgroup.WithContext(ctx) |
117 | 117 |
for dgst, dt := range m { |
... | ... |
@@ -183,11 +183,12 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte |
183 | 183 |
if err != nil { |
184 | 184 |
return errors.WithStack(err) |
185 | 185 |
} |
186 |
- |
|
187 |
- mu.Lock() |
|
186 |
+ cc := v1.NewCacheChains() |
|
188 | 187 |
if err := v1.ParseConfig(config, layers, cc); err != nil { |
189 | 188 |
return err |
190 | 189 |
} |
190 |
+ mu.Lock() |
|
191 |
+ cMap[dgst] = cc |
|
191 | 192 |
mu.Unlock() |
192 | 193 |
return nil |
193 | 194 |
}) |
... | ... |
@@ -198,11 +199,17 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte |
198 | 198 |
return nil, err |
199 | 199 |
} |
200 | 200 |
|
201 |
- keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w) |
|
202 |
- if err != nil { |
|
203 |
- return nil, err |
|
201 |
+ cms := make([]solver.CacheManager, 0, len(cMap)) |
|
202 |
+ |
|
203 |
+ for _, cc := range cMap { |
|
204 |
+ keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w) |
|
205 |
+ if err != nil { |
|
206 |
+ return nil, err |
|
207 |
+ } |
|
208 |
+ cms = append(cms, solver.NewCacheManager(id, keysStorage, resultStorage)) |
|
204 | 209 |
} |
205 |
- return solver.NewCacheManager(id, keysStorage, resultStorage), nil |
|
210 |
+ |
|
211 |
+ return solver.NewCombinedCacheManager(cms, nil), nil |
|
206 | 212 |
} |
207 | 213 |
|
208 | 214 |
func (ci *contentCacheImporter) allDistributionManifests(ctx context.Context, dt []byte, m map[digest.Digest][]byte) error { |
... | ... |
@@ -31,6 +31,12 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) { |
31 | 31 |
return nil, nil |
32 | 32 |
} |
33 | 33 |
|
34 |
+func (ce *exporter) reset() { |
|
35 |
+ cc := v1.NewCacheChains() |
|
36 |
+ ce.CacheExporterTarget = cc |
|
37 |
+ ce.chains = cc |
|
38 |
+} |
|
39 |
+ |
|
34 | 40 |
func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) { |
35 | 41 |
config, descs, err := ce.chains.Marshal() |
36 | 42 |
if err != nil { |
... | ... |
@@ -82,6 +88,7 @@ func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) { |
82 | 82 |
if err != nil { |
83 | 83 |
return nil, err |
84 | 84 |
} |
85 |
+ ce.reset() |
|
85 | 86 |
|
86 | 87 |
return dt, nil |
87 | 88 |
} |
... | ... |
@@ -78,10 +78,14 @@ func sortConfig(cc *CacheConfig) { |
78 | 78 |
if ri.Inputs[i][j].Selector != rj.Inputs[i][j].Selector { |
79 | 79 |
return ri.Inputs[i][j].Selector < rj.Inputs[i][j].Selector |
80 | 80 |
} |
81 |
- return cc.Records[ri.Inputs[i][j].LinkIndex].Digest < cc.Records[rj.Inputs[i][j].LinkIndex].Digest |
|
81 |
+ inputDigesti := cc.Records[ri.Inputs[i][j].LinkIndex].Digest |
|
82 |
+ inputDigestj := cc.Records[rj.Inputs[i][j].LinkIndex].Digest |
|
83 |
+ if inputDigesti != inputDigestj { |
|
84 |
+ return inputDigesti < inputDigestj |
|
85 |
+ } |
|
82 | 86 |
} |
83 | 87 |
} |
84 |
- return ri.Digest < rj.Digest |
|
88 |
+ return false |
|
85 | 89 |
}) |
86 | 90 |
for i, l := range sortedRecords { |
87 | 91 |
l.newIndex = i |
... | ... |
@@ -49,6 +49,7 @@ const ( |
49 | 49 |
keyNameContext = "contextkey" |
50 | 50 |
keyNameDockerfile = "dockerfilekey" |
51 | 51 |
keyContextSubDir = "contextsubdir" |
52 |
+ keyContextKeepGitDir = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR" |
|
52 | 53 |
) |
53 | 54 |
|
54 | 55 |
var httpPrefix = regexp.MustCompile(`^https?://`) |
... | ... |
@@ -129,7 +130,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { |
129 | 129 |
|
130 | 130 |
var buildContext *llb.State |
131 | 131 |
isScratchContext := false |
132 |
- if st, ok := detectGitContext(opts[localNameContext]); ok { |
|
132 |
+ if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDir]); ok { |
|
133 | 133 |
if !forceLocalDockerfile { |
134 | 134 |
src = *st |
135 | 135 |
} |
... | ... |
@@ -451,12 +452,19 @@ func filter(opt map[string]string, key string) map[string]string { |
451 | 451 |
return m |
452 | 452 |
} |
453 | 453 |
|
454 |
-func detectGitContext(ref string) (*llb.State, bool) { |
|
454 |
+func detectGitContext(ref, gitContext string) (*llb.State, bool) { |
|
455 | 455 |
found := false |
456 | 456 |
if httpPrefix.MatchString(ref) && gitUrlPathWithFragmentSuffix.MatchString(ref) { |
457 | 457 |
found = true |
458 | 458 |
} |
459 | 459 |
|
460 |
+ keepGit := false |
|
461 |
+ if gitContext != "" { |
|
462 |
+ if v, err := strconv.ParseBool(gitContext); err == nil { |
|
463 |
+ keepGit = v |
|
464 |
+ } |
|
465 |
+ } |
|
466 |
+ |
|
460 | 467 |
for _, prefix := range []string{"git://", "github.com/", "git@"} { |
461 | 468 |
if strings.HasPrefix(ref, prefix) { |
462 | 469 |
found = true |
... | ... |
@@ -472,7 +480,12 @@ func detectGitContext(ref string) (*llb.State, bool) { |
472 | 472 |
if len(parts) > 1 { |
473 | 473 |
branch = parts[1] |
474 | 474 |
} |
475 |
- st := llb.Git(parts[0], branch, dockerfile2llb.WithInternalName("load git source "+ref)) |
|
475 |
+ gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)} |
|
476 |
+ if keepGit { |
|
477 |
+ gitOpts = append(gitOpts, llb.KeepGitDir()) |
|
478 |
+ } |
|
479 |
+ |
|
480 |
+ st := llb.Git(parts[0], branch, gitOpts...) |
|
476 | 481 |
return &st, true |
477 | 482 |
} |
478 | 483 |
|
... | ... |
@@ -746,9 +746,9 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS |
746 | 746 |
}}, copyOpt...) |
747 | 747 |
|
748 | 748 |
if a == nil { |
749 |
- a = llb.Copy(sourceState, src, dest, opts...) |
|
749 |
+ a = llb.Copy(sourceState, filepath.Join("/", src), dest, opts...) |
|
750 | 750 |
} else { |
751 |
- a = a.Copy(sourceState, src, dest, opts...) |
|
751 |
+ a = a.Copy(sourceState, filepath.Join("/", src), dest, opts...) |
|
752 | 752 |
} |
753 | 753 |
} |
754 | 754 |
} |
... | ... |
@@ -12,6 +12,7 @@ import ( |
12 | 12 |
|
13 | 13 |
"github.com/moby/buildkit/client/llb" |
14 | 14 |
"github.com/moby/buildkit/frontend/dockerfile/instructions" |
15 |
+ "github.com/moby/buildkit/solver/pb" |
|
15 | 16 |
"github.com/pkg/errors" |
16 | 17 |
) |
17 | 18 |
|
... | ... |
@@ -113,7 +114,7 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []* |
113 | 113 |
} |
114 | 114 |
if mount.ReadOnly { |
115 | 115 |
mountOpts = append(mountOpts, llb.Readonly) |
116 |
- } else if mount.Type == instructions.MountTypeBind { |
|
116 |
+ } else if mount.Type == instructions.MountTypeBind && opt.llbCaps.Supports(pb.CapExecMountBindReadWriteNoOuput) == nil { |
|
117 | 117 |
mountOpts = append(mountOpts, llb.ForceNoOutput) |
118 | 118 |
} |
119 | 119 |
if mount.Type == instructions.MountTypeCache { |
... | ... |
@@ -11,7 +11,7 @@ import ( |
11 | 11 |
"golang.org/x/sync/errgroup" |
12 | 12 |
) |
13 | 13 |
|
14 |
-func newCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager { |
|
14 |
+func NewCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager { |
|
15 | 15 |
return &combinedCacheManager{cms: cms, main: main} |
16 | 16 |
} |
17 | 17 |
|
... | ... |
@@ -80,7 +80,7 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res |
80 | 80 |
res.Result.Release(context.TODO()) |
81 | 81 |
} |
82 | 82 |
}() |
83 |
- if rec.cacheManager != cm.main { |
|
83 |
+ if rec.cacheManager != cm.main && cm.main != nil { |
|
84 | 84 |
for _, res := range results { |
85 | 85 |
if _, err := cm.main.Save(res.CacheKey, res.Result, res.CacheResult.CreatedAt); err != nil { |
86 | 86 |
return nil, err |
... | ... |
@@ -91,6 +91,9 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res |
91 | 91 |
} |
92 | 92 |
|
93 | 93 |
func (cm *combinedCacheManager) Save(key *CacheKey, s Result, createdAt time.Time) (*ExportableCacheKey, error) { |
94 |
+ if cm.main == nil { |
|
95 |
+ return nil, nil |
|
96 |
+ } |
|
94 | 97 |
return cm.main.Save(key, s, createdAt) |
95 | 98 |
} |
96 | 99 |
|
... | ... |
@@ -256,7 +256,7 @@ func (e *execOp) getRefCacheDir(ctx context.Context, ref cache.ImmutableRef, id |
256 | 256 |
} |
257 | 257 |
|
258 | 258 |
func (e *execOp) getRefCacheDirNoCache(ctx context.Context, key string, ref cache.ImmutableRef, id string, m *pb.Mount, block bool) (cache.MutableRef, error) { |
259 |
- makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) { |
|
259 |
+ makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) { |
|
260 | 260 |
desc := fmt.Sprintf("cached mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " ")) |
261 | 261 |
return e.cm.New(ctx, ref, cache.WithRecordType(client.UsageRecordTypeCacheMount), cache.WithDescription(desc), cache.CachePolicyRetain) |
262 | 262 |
} |
... | ... |
@@ -585,7 +585,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res |
585 | 585 |
mountable = ref |
586 | 586 |
} |
587 | 587 |
|
588 |
- makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) { |
|
588 |
+ makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) { |
|
589 | 589 |
desc := fmt.Sprintf("mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " ")) |
590 | 590 |
return e.cm.New(ctx, ref, cache.WithDescription(desc)) |
591 | 591 |
} |
... | ... |
@@ -606,7 +606,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res |
606 | 606 |
outputs = append(outputs, active) |
607 | 607 |
mountable = active |
608 | 608 |
} |
609 |
- } else if ref == nil { |
|
609 |
+ } else if (!m.Readonly || ref == nil) && m.Dest != pb.RootMount { |
|
610 | 610 |
// this case is empty readonly scratch without output that is not really useful for anything but don't error |
611 | 611 |
active, err := makeMutable(ref) |
612 | 612 |
if err != nil { |
... | ... |
@@ -30,19 +30,20 @@ const ( |
30 | 30 |
|
31 | 31 |
CapBuildOpLLBFileName apicaps.CapID = "source.buildop.llbfilename" |
32 | 32 |
|
33 |
- CapExecMetaBase apicaps.CapID = "exec.meta.base" |
|
34 |
- CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv" |
|
35 |
- CapExecMetaNetwork apicaps.CapID = "exec.meta.network" |
|
36 |
- CapExecMetaSecurity apicaps.CapID = "exec.meta.security" |
|
37 |
- CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath" |
|
38 |
- CapExecMountBind apicaps.CapID = "exec.mount.bind" |
|
39 |
- CapExecMountCache apicaps.CapID = "exec.mount.cache" |
|
40 |
- CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing" |
|
41 |
- CapExecMountSelector apicaps.CapID = "exec.mount.selector" |
|
42 |
- CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs" |
|
43 |
- CapExecMountSecret apicaps.CapID = "exec.mount.secret" |
|
44 |
- CapExecMountSSH apicaps.CapID = "exec.mount.ssh" |
|
45 |
- CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" |
|
33 |
+ CapExecMetaBase apicaps.CapID = "exec.meta.base" |
|
34 |
+ CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv" |
|
35 |
+ CapExecMetaNetwork apicaps.CapID = "exec.meta.network" |
|
36 |
+ CapExecMetaSecurity apicaps.CapID = "exec.meta.security" |
|
37 |
+ CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath" |
|
38 |
+ CapExecMountBind apicaps.CapID = "exec.mount.bind" |
|
39 |
+ CapExecMountBindReadWriteNoOuput apicaps.CapID = "exec.mount.bind.readwrite-nooutput" |
|
40 |
+ CapExecMountCache apicaps.CapID = "exec.mount.cache" |
|
41 |
+ CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing" |
|
42 |
+ CapExecMountSelector apicaps.CapID = "exec.mount.selector" |
|
43 |
+ CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs" |
|
44 |
+ CapExecMountSecret apicaps.CapID = "exec.mount.secret" |
|
45 |
+ CapExecMountSSH apicaps.CapID = "exec.mount.ssh" |
|
46 |
+ CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" |
|
46 | 47 |
|
47 | 48 |
CapFileBase apicaps.CapID = "file.base" |
48 | 49 |
|
... | ... |
@@ -194,6 +195,12 @@ func init() { |
194 | 194 |
}) |
195 | 195 |
|
196 | 196 |
Caps.Init(apicaps.Cap{ |
197 |
+ ID: CapExecMountBindReadWriteNoOuput, |
|
198 |
+ Enabled: true, |
|
199 |
+ Status: apicaps.CapStatusExperimental, |
|
200 |
+ }) |
|
201 |
+ |
|
202 |
+ Caps.Init(apicaps.Cap{ |
|
197 | 203 |
ID: CapExecMountCache, |
198 | 204 |
Enabled: true, |
199 | 205 |
Status: apicaps.CapStatusExperimental, |
... | ... |
@@ -153,6 +153,14 @@ type gitSourceHandler struct { |
153 | 153 |
cacheKey string |
154 | 154 |
} |
155 | 155 |
|
156 |
+func (gs *gitSourceHandler) shaToCacheKey(sha string) string { |
|
157 |
+ key := sha |
|
158 |
+ if gs.src.KeepGitDir { |
|
159 |
+ key += ".git" |
|
160 |
+ } |
|
161 |
+ return key |
|
162 |
+} |
|
163 |
+ |
|
156 | 164 |
func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) { |
157 | 165 |
gitIdentifier, ok := id.(*source.GitIdentifier) |
158 | 166 |
if !ok { |
... | ... |
@@ -175,6 +183,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo |
175 | 175 |
defer gs.locker.Unlock(remote) |
176 | 176 |
|
177 | 177 |
if isCommitSHA(ref) { |
178 |
+ ref = gs.shaToCacheKey(ref) |
|
178 | 179 |
gs.cacheKey = ref |
179 | 180 |
return ref, true, nil |
180 | 181 |
} |
... | ... |
@@ -201,6 +210,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo |
201 | 201 |
if !isCommitSHA(sha) { |
202 | 202 |
return "", false, errors.Errorf("invalid commit sha %q", sha) |
203 | 203 |
} |
204 |
+ sha = gs.shaToCacheKey(sha) |
|
204 | 205 |
gs.cacheKey = sha |
205 | 206 |
return sha, true, nil |
206 | 207 |
} |
... | ... |
@@ -298,11 +308,15 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe |
298 | 298 |
}() |
299 | 299 |
|
300 | 300 |
if gs.src.KeepGitDir { |
301 |
- _, err = gitWithinDir(ctx, checkoutDir, "", "init") |
|
301 |
+ checkoutDirGit := filepath.Join(checkoutDir, ".git") |
|
302 |
+ if err := os.MkdirAll(checkoutDir, 0711); err != nil { |
|
303 |
+ return nil, err |
|
304 |
+ } |
|
305 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, "", "init") |
|
302 | 306 |
if err != nil { |
303 | 307 |
return nil, err |
304 | 308 |
} |
305 |
- _, err = gitWithinDir(ctx, checkoutDir, "", "remote", "add", "origin", gitDir) |
|
309 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir) |
|
306 | 310 |
if err != nil { |
307 | 311 |
return nil, err |
308 | 312 |
} |
... | ... |
@@ -313,16 +327,18 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe |
313 | 313 |
if err != nil { |
314 | 314 |
return nil, err |
315 | 315 |
} |
316 |
+ } else { |
|
317 |
+ pullref += ":" + pullref |
|
316 | 318 |
} |
317 |
- _, err = gitWithinDir(ctx, checkoutDir, "", "fetch", "--depth=1", "origin", pullref) |
|
319 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref) |
|
318 | 320 |
if err != nil { |
319 | 321 |
return nil, err |
320 | 322 |
} |
321 |
- _, err = gitWithinDir(ctx, checkoutDir, checkoutDir, "checkout", "FETCH_HEAD") |
|
323 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD") |
|
322 | 324 |
if err != nil { |
323 | 325 |
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) |
324 | 326 |
} |
325 |
- gitDir = checkoutDir |
|
327 |
+ gitDir = checkoutDirGit |
|
326 | 328 |
} else { |
327 | 329 |
_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".") |
328 | 330 |
if err != nil { |