Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
| ... | ... |
@@ -27,7 +27,7 @@ github.com/imdario/mergo 7c29201646fa3de8506f70121347 |
| 27 | 27 |
golang.org/x/sync e225da77a7e68af35c70ccbf71af2b83e6acac3c |
| 28 | 28 |
|
| 29 | 29 |
# buildkit |
| 30 |
-github.com/moby/buildkit f5a55a9516d1c6e2ade9bec22b83259caeed3a84 |
|
| 30 |
+github.com/moby/buildkit be0d75f074e7a4b0f5b5877c719213a3f5057e60 # v0.6.1 |
|
| 31 | 31 |
github.com/tonistiigi/fsutil 3bbb99cdbd76619ab717299830c60f6f2a533a6b |
| 32 | 32 |
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 |
| 33 | 33 |
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 |
| ... | ... |
@@ -111,7 +111,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte |
| 111 | 111 |
} |
| 112 | 112 |
|
| 113 | 113 |
var mu sync.Mutex |
| 114 |
- cc := v1.NewCacheChains() |
|
| 114 |
+ var cMap = map[digest.Digest]*v1.CacheChains{}
|
|
| 115 | 115 |
|
| 116 | 116 |
eg, ctx := errgroup.WithContext(ctx) |
| 117 | 117 |
for dgst, dt := range m {
|
| ... | ... |
@@ -183,11 +183,12 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte |
| 183 | 183 |
if err != nil {
|
| 184 | 184 |
return errors.WithStack(err) |
| 185 | 185 |
} |
| 186 |
- |
|
| 187 |
- mu.Lock() |
|
| 186 |
+ cc := v1.NewCacheChains() |
|
| 188 | 187 |
if err := v1.ParseConfig(config, layers, cc); err != nil {
|
| 189 | 188 |
return err |
| 190 | 189 |
} |
| 190 |
+ mu.Lock() |
|
| 191 |
+ cMap[dgst] = cc |
|
| 191 | 192 |
mu.Unlock() |
| 192 | 193 |
return nil |
| 193 | 194 |
}) |
| ... | ... |
@@ -198,11 +199,17 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte |
| 198 | 198 |
return nil, err |
| 199 | 199 |
} |
| 200 | 200 |
|
| 201 |
- keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w) |
|
| 202 |
- if err != nil {
|
|
| 203 |
- return nil, err |
|
| 201 |
+ cms := make([]solver.CacheManager, 0, len(cMap)) |
|
| 202 |
+ |
|
| 203 |
+ for _, cc := range cMap {
|
|
| 204 |
+ keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w) |
|
| 205 |
+ if err != nil {
|
|
| 206 |
+ return nil, err |
|
| 207 |
+ } |
|
| 208 |
+ cms = append(cms, solver.NewCacheManager(id, keysStorage, resultStorage)) |
|
| 204 | 209 |
} |
| 205 |
- return solver.NewCacheManager(id, keysStorage, resultStorage), nil |
|
| 210 |
+ |
|
| 211 |
+ return solver.NewCombinedCacheManager(cms, nil), nil |
|
| 206 | 212 |
} |
| 207 | 213 |
|
| 208 | 214 |
func (ci *contentCacheImporter) allDistributionManifests(ctx context.Context, dt []byte, m map[digest.Digest][]byte) error {
|
| ... | ... |
@@ -31,6 +31,12 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
|
| 31 | 31 |
return nil, nil |
| 32 | 32 |
} |
| 33 | 33 |
|
| 34 |
+func (ce *exporter) reset() {
|
|
| 35 |
+ cc := v1.NewCacheChains() |
|
| 36 |
+ ce.CacheExporterTarget = cc |
|
| 37 |
+ ce.chains = cc |
|
| 38 |
+} |
|
| 39 |
+ |
|
| 34 | 40 |
func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) {
|
| 35 | 41 |
config, descs, err := ce.chains.Marshal() |
| 36 | 42 |
if err != nil {
|
| ... | ... |
@@ -82,6 +88,7 @@ func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) {
|
| 82 | 82 |
if err != nil {
|
| 83 | 83 |
return nil, err |
| 84 | 84 |
} |
| 85 |
+ ce.reset() |
|
| 85 | 86 |
|
| 86 | 87 |
return dt, nil |
| 87 | 88 |
} |
| ... | ... |
@@ -78,10 +78,14 @@ func sortConfig(cc *CacheConfig) {
|
| 78 | 78 |
if ri.Inputs[i][j].Selector != rj.Inputs[i][j].Selector {
|
| 79 | 79 |
return ri.Inputs[i][j].Selector < rj.Inputs[i][j].Selector |
| 80 | 80 |
} |
| 81 |
- return cc.Records[ri.Inputs[i][j].LinkIndex].Digest < cc.Records[rj.Inputs[i][j].LinkIndex].Digest |
|
| 81 |
+ inputDigesti := cc.Records[ri.Inputs[i][j].LinkIndex].Digest |
|
| 82 |
+ inputDigestj := cc.Records[rj.Inputs[i][j].LinkIndex].Digest |
|
| 83 |
+ if inputDigesti != inputDigestj {
|
|
| 84 |
+ return inputDigesti < inputDigestj |
|
| 85 |
+ } |
|
| 82 | 86 |
} |
| 83 | 87 |
} |
| 84 |
- return ri.Digest < rj.Digest |
|
| 88 |
+ return false |
|
| 85 | 89 |
}) |
| 86 | 90 |
for i, l := range sortedRecords {
|
| 87 | 91 |
l.newIndex = i |
| ... | ... |
@@ -49,6 +49,7 @@ const ( |
| 49 | 49 |
keyNameContext = "contextkey" |
| 50 | 50 |
keyNameDockerfile = "dockerfilekey" |
| 51 | 51 |
keyContextSubDir = "contextsubdir" |
| 52 |
+ keyContextKeepGitDir = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR" |
|
| 52 | 53 |
) |
| 53 | 54 |
|
| 54 | 55 |
var httpPrefix = regexp.MustCompile(`^https?://`) |
| ... | ... |
@@ -129,7 +130,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
|
| 129 | 129 |
|
| 130 | 130 |
var buildContext *llb.State |
| 131 | 131 |
isScratchContext := false |
| 132 |
- if st, ok := detectGitContext(opts[localNameContext]); ok {
|
|
| 132 |
+ if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDir]); ok {
|
|
| 133 | 133 |
if !forceLocalDockerfile {
|
| 134 | 134 |
src = *st |
| 135 | 135 |
} |
| ... | ... |
@@ -451,12 +452,19 @@ func filter(opt map[string]string, key string) map[string]string {
|
| 451 | 451 |
return m |
| 452 | 452 |
} |
| 453 | 453 |
|
| 454 |
-func detectGitContext(ref string) (*llb.State, bool) {
|
|
| 454 |
+func detectGitContext(ref, gitContext string) (*llb.State, bool) {
|
|
| 455 | 455 |
found := false |
| 456 | 456 |
if httpPrefix.MatchString(ref) && gitUrlPathWithFragmentSuffix.MatchString(ref) {
|
| 457 | 457 |
found = true |
| 458 | 458 |
} |
| 459 | 459 |
|
| 460 |
+ keepGit := false |
|
| 461 |
+ if gitContext != "" {
|
|
| 462 |
+ if v, err := strconv.ParseBool(gitContext); err == nil {
|
|
| 463 |
+ keepGit = v |
|
| 464 |
+ } |
|
| 465 |
+ } |
|
| 466 |
+ |
|
| 460 | 467 |
for _, prefix := range []string{"git://", "github.com/", "git@"} {
|
| 461 | 468 |
if strings.HasPrefix(ref, prefix) {
|
| 462 | 469 |
found = true |
| ... | ... |
@@ -472,7 +480,12 @@ func detectGitContext(ref string) (*llb.State, bool) {
|
| 472 | 472 |
if len(parts) > 1 {
|
| 473 | 473 |
branch = parts[1] |
| 474 | 474 |
} |
| 475 |
- st := llb.Git(parts[0], branch, dockerfile2llb.WithInternalName("load git source "+ref))
|
|
| 475 |
+ gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)}
|
|
| 476 |
+ if keepGit {
|
|
| 477 |
+ gitOpts = append(gitOpts, llb.KeepGitDir()) |
|
| 478 |
+ } |
|
| 479 |
+ |
|
| 480 |
+ st := llb.Git(parts[0], branch, gitOpts...) |
|
| 476 | 481 |
return &st, true |
| 477 | 482 |
} |
| 478 | 483 |
|
| ... | ... |
@@ -746,9 +746,9 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS |
| 746 | 746 |
}}, copyOpt...) |
| 747 | 747 |
|
| 748 | 748 |
if a == nil {
|
| 749 |
- a = llb.Copy(sourceState, src, dest, opts...) |
|
| 749 |
+ a = llb.Copy(sourceState, filepath.Join("/", src), dest, opts...)
|
|
| 750 | 750 |
} else {
|
| 751 |
- a = a.Copy(sourceState, src, dest, opts...) |
|
| 751 |
+ a = a.Copy(sourceState, filepath.Join("/", src), dest, opts...)
|
|
| 752 | 752 |
} |
| 753 | 753 |
} |
| 754 | 754 |
} |
| ... | ... |
@@ -12,6 +12,7 @@ import ( |
| 12 | 12 |
|
| 13 | 13 |
"github.com/moby/buildkit/client/llb" |
| 14 | 14 |
"github.com/moby/buildkit/frontend/dockerfile/instructions" |
| 15 |
+ "github.com/moby/buildkit/solver/pb" |
|
| 15 | 16 |
"github.com/pkg/errors" |
| 16 | 17 |
) |
| 17 | 18 |
|
| ... | ... |
@@ -113,7 +114,7 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []* |
| 113 | 113 |
} |
| 114 | 114 |
if mount.ReadOnly {
|
| 115 | 115 |
mountOpts = append(mountOpts, llb.Readonly) |
| 116 |
- } else if mount.Type == instructions.MountTypeBind {
|
|
| 116 |
+ } else if mount.Type == instructions.MountTypeBind && opt.llbCaps.Supports(pb.CapExecMountBindReadWriteNoOuput) == nil {
|
|
| 117 | 117 |
mountOpts = append(mountOpts, llb.ForceNoOutput) |
| 118 | 118 |
} |
| 119 | 119 |
if mount.Type == instructions.MountTypeCache {
|
| ... | ... |
@@ -11,7 +11,7 @@ import ( |
| 11 | 11 |
"golang.org/x/sync/errgroup" |
| 12 | 12 |
) |
| 13 | 13 |
|
| 14 |
-func newCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager {
|
|
| 14 |
+func NewCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager {
|
|
| 15 | 15 |
return &combinedCacheManager{cms: cms, main: main}
|
| 16 | 16 |
} |
| 17 | 17 |
|
| ... | ... |
@@ -80,7 +80,7 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res |
| 80 | 80 |
res.Result.Release(context.TODO()) |
| 81 | 81 |
} |
| 82 | 82 |
}() |
| 83 |
- if rec.cacheManager != cm.main {
|
|
| 83 |
+ if rec.cacheManager != cm.main && cm.main != nil {
|
|
| 84 | 84 |
for _, res := range results {
|
| 85 | 85 |
if _, err := cm.main.Save(res.CacheKey, res.Result, res.CacheResult.CreatedAt); err != nil {
|
| 86 | 86 |
return nil, err |
| ... | ... |
@@ -91,6 +91,9 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res |
| 91 | 91 |
} |
| 92 | 92 |
|
| 93 | 93 |
func (cm *combinedCacheManager) Save(key *CacheKey, s Result, createdAt time.Time) (*ExportableCacheKey, error) {
|
| 94 |
+ if cm.main == nil {
|
|
| 95 |
+ return nil, nil |
|
| 96 |
+ } |
|
| 94 | 97 |
return cm.main.Save(key, s, createdAt) |
| 95 | 98 |
} |
| 96 | 99 |
|
| ... | ... |
@@ -256,7 +256,7 @@ func (e *execOp) getRefCacheDir(ctx context.Context, ref cache.ImmutableRef, id |
| 256 | 256 |
} |
| 257 | 257 |
|
| 258 | 258 |
func (e *execOp) getRefCacheDirNoCache(ctx context.Context, key string, ref cache.ImmutableRef, id string, m *pb.Mount, block bool) (cache.MutableRef, error) {
|
| 259 |
- makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) {
|
|
| 259 |
+ makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) {
|
|
| 260 | 260 |
desc := fmt.Sprintf("cached mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " "))
|
| 261 | 261 |
return e.cm.New(ctx, ref, cache.WithRecordType(client.UsageRecordTypeCacheMount), cache.WithDescription(desc), cache.CachePolicyRetain) |
| 262 | 262 |
} |
| ... | ... |
@@ -585,7 +585,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res |
| 585 | 585 |
mountable = ref |
| 586 | 586 |
} |
| 587 | 587 |
|
| 588 |
- makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) {
|
|
| 588 |
+ makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) {
|
|
| 589 | 589 |
desc := fmt.Sprintf("mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " "))
|
| 590 | 590 |
return e.cm.New(ctx, ref, cache.WithDescription(desc)) |
| 591 | 591 |
} |
| ... | ... |
@@ -606,7 +606,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res |
| 606 | 606 |
outputs = append(outputs, active) |
| 607 | 607 |
mountable = active |
| 608 | 608 |
} |
| 609 |
- } else if ref == nil {
|
|
| 609 |
+ } else if (!m.Readonly || ref == nil) && m.Dest != pb.RootMount {
|
|
| 610 | 610 |
// this case is empty readonly scratch without output that is not really useful for anything but don't error |
| 611 | 611 |
active, err := makeMutable(ref) |
| 612 | 612 |
if err != nil {
|
| ... | ... |
@@ -30,19 +30,20 @@ const ( |
| 30 | 30 |
|
| 31 | 31 |
CapBuildOpLLBFileName apicaps.CapID = "source.buildop.llbfilename" |
| 32 | 32 |
|
| 33 |
- CapExecMetaBase apicaps.CapID = "exec.meta.base" |
|
| 34 |
- CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv" |
|
| 35 |
- CapExecMetaNetwork apicaps.CapID = "exec.meta.network" |
|
| 36 |
- CapExecMetaSecurity apicaps.CapID = "exec.meta.security" |
|
| 37 |
- CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath" |
|
| 38 |
- CapExecMountBind apicaps.CapID = "exec.mount.bind" |
|
| 39 |
- CapExecMountCache apicaps.CapID = "exec.mount.cache" |
|
| 40 |
- CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing" |
|
| 41 |
- CapExecMountSelector apicaps.CapID = "exec.mount.selector" |
|
| 42 |
- CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs" |
|
| 43 |
- CapExecMountSecret apicaps.CapID = "exec.mount.secret" |
|
| 44 |
- CapExecMountSSH apicaps.CapID = "exec.mount.ssh" |
|
| 45 |
- CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" |
|
| 33 |
+ CapExecMetaBase apicaps.CapID = "exec.meta.base" |
|
| 34 |
+ CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv" |
|
| 35 |
+ CapExecMetaNetwork apicaps.CapID = "exec.meta.network" |
|
| 36 |
+ CapExecMetaSecurity apicaps.CapID = "exec.meta.security" |
|
| 37 |
+ CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath" |
|
| 38 |
+ CapExecMountBind apicaps.CapID = "exec.mount.bind" |
|
| 39 |
+ CapExecMountBindReadWriteNoOuput apicaps.CapID = "exec.mount.bind.readwrite-nooutput" |
|
| 40 |
+ CapExecMountCache apicaps.CapID = "exec.mount.cache" |
|
| 41 |
+ CapExecMountCacheSharing apicaps.CapID = "exec.mount.cache.sharing" |
|
| 42 |
+ CapExecMountSelector apicaps.CapID = "exec.mount.selector" |
|
| 43 |
+ CapExecMountTmpfs apicaps.CapID = "exec.mount.tmpfs" |
|
| 44 |
+ CapExecMountSecret apicaps.CapID = "exec.mount.secret" |
|
| 45 |
+ CapExecMountSSH apicaps.CapID = "exec.mount.ssh" |
|
| 46 |
+ CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" |
|
| 46 | 47 |
|
| 47 | 48 |
CapFileBase apicaps.CapID = "file.base" |
| 48 | 49 |
|
| ... | ... |
@@ -194,6 +195,12 @@ func init() {
|
| 194 | 194 |
}) |
| 195 | 195 |
|
| 196 | 196 |
Caps.Init(apicaps.Cap{
|
| 197 |
+ ID: CapExecMountBindReadWriteNoOuput, |
|
| 198 |
+ Enabled: true, |
|
| 199 |
+ Status: apicaps.CapStatusExperimental, |
|
| 200 |
+ }) |
|
| 201 |
+ |
|
| 202 |
+ Caps.Init(apicaps.Cap{
|
|
| 197 | 203 |
ID: CapExecMountCache, |
| 198 | 204 |
Enabled: true, |
| 199 | 205 |
Status: apicaps.CapStatusExperimental, |
| ... | ... |
@@ -153,6 +153,14 @@ type gitSourceHandler struct {
|
| 153 | 153 |
cacheKey string |
| 154 | 154 |
} |
| 155 | 155 |
|
| 156 |
+func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
|
|
| 157 |
+ key := sha |
|
| 158 |
+ if gs.src.KeepGitDir {
|
|
| 159 |
+ key += ".git" |
|
| 160 |
+ } |
|
| 161 |
+ return key |
|
| 162 |
+} |
|
| 163 |
+ |
|
| 156 | 164 |
func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) {
|
| 157 | 165 |
gitIdentifier, ok := id.(*source.GitIdentifier) |
| 158 | 166 |
if !ok {
|
| ... | ... |
@@ -175,6 +183,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo |
| 175 | 175 |
defer gs.locker.Unlock(remote) |
| 176 | 176 |
|
| 177 | 177 |
if isCommitSHA(ref) {
|
| 178 |
+ ref = gs.shaToCacheKey(ref) |
|
| 178 | 179 |
gs.cacheKey = ref |
| 179 | 180 |
return ref, true, nil |
| 180 | 181 |
} |
| ... | ... |
@@ -201,6 +210,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo |
| 201 | 201 |
if !isCommitSHA(sha) {
|
| 202 | 202 |
return "", false, errors.Errorf("invalid commit sha %q", sha)
|
| 203 | 203 |
} |
| 204 |
+ sha = gs.shaToCacheKey(sha) |
|
| 204 | 205 |
gs.cacheKey = sha |
| 205 | 206 |
return sha, true, nil |
| 206 | 207 |
} |
| ... | ... |
@@ -298,11 +308,15 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe |
| 298 | 298 |
}() |
| 299 | 299 |
|
| 300 | 300 |
if gs.src.KeepGitDir {
|
| 301 |
- _, err = gitWithinDir(ctx, checkoutDir, "", "init") |
|
| 301 |
+ checkoutDirGit := filepath.Join(checkoutDir, ".git") |
|
| 302 |
+ if err := os.MkdirAll(checkoutDir, 0711); err != nil {
|
|
| 303 |
+ return nil, err |
|
| 304 |
+ } |
|
| 305 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, "", "init") |
|
| 302 | 306 |
if err != nil {
|
| 303 | 307 |
return nil, err |
| 304 | 308 |
} |
| 305 |
- _, err = gitWithinDir(ctx, checkoutDir, "", "remote", "add", "origin", gitDir) |
|
| 309 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir) |
|
| 306 | 310 |
if err != nil {
|
| 307 | 311 |
return nil, err |
| 308 | 312 |
} |
| ... | ... |
@@ -313,16 +327,18 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe |
| 313 | 313 |
if err != nil {
|
| 314 | 314 |
return nil, err |
| 315 | 315 |
} |
| 316 |
+ } else {
|
|
| 317 |
+ pullref += ":" + pullref |
|
| 316 | 318 |
} |
| 317 |
- _, err = gitWithinDir(ctx, checkoutDir, "", "fetch", "--depth=1", "origin", pullref) |
|
| 319 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref) |
|
| 318 | 320 |
if err != nil {
|
| 319 | 321 |
return nil, err |
| 320 | 322 |
} |
| 321 |
- _, err = gitWithinDir(ctx, checkoutDir, checkoutDir, "checkout", "FETCH_HEAD") |
|
| 323 |
+ _, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD") |
|
| 322 | 324 |
if err != nil {
|
| 323 | 325 |
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote) |
| 324 | 326 |
} |
| 325 |
- gitDir = checkoutDir |
|
| 327 |
+ gitDir = checkoutDirGit |
|
| 326 | 328 |
} else {
|
| 327 | 329 |
_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".") |
| 328 | 330 |
if err != nil {
|