Browse code

vendor: update buildkit to v0.6.1

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
(cherry picked from commit e59b26087fffe0357d34b400740e2fb15c062cb5)

Tonis Tiigi authored on 2019/08/15 09:46:46
Showing 12 changed files
... ...
@@ -27,7 +27,7 @@ github.com/imdario/mergo                            7c29201646fa3de8506f70121347
27 27
 golang.org/x/sync                                   e225da77a7e68af35c70ccbf71af2b83e6acac3c
28 28
 
29 29
 # buildkit
30
-github.com/moby/buildkit                            f5a55a9516d1c6e2ade9bec22b83259caeed3a84 
30
+github.com/moby/buildkit                            be0d75f074e7a4b0f5b5877c719213a3f5057e60 # v0.6.1
31 31
 github.com/tonistiigi/fsutil                        3bbb99cdbd76619ab717299830c60f6f2a533a6b
32 32
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
33 33
 github.com/opentracing/opentracing-go               1361b9cd60be79c4c3a7fa9841b3c132e40066a7
... ...
@@ -111,7 +111,7 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
111 111
 	}
112 112
 
113 113
 	var mu sync.Mutex
114
-	cc := v1.NewCacheChains()
114
+	var cMap = map[digest.Digest]*v1.CacheChains{}
115 115
 
116 116
 	eg, ctx := errgroup.WithContext(ctx)
117 117
 	for dgst, dt := range m {
... ...
@@ -183,11 +183,12 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
183 183
 				if err != nil {
184 184
 					return errors.WithStack(err)
185 185
 				}
186
-
187
-				mu.Lock()
186
+				cc := v1.NewCacheChains()
188 187
 				if err := v1.ParseConfig(config, layers, cc); err != nil {
189 188
 					return err
190 189
 				}
190
+				mu.Lock()
191
+				cMap[dgst] = cc
191 192
 				mu.Unlock()
192 193
 				return nil
193 194
 			})
... ...
@@ -198,11 +199,17 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte
198 198
 		return nil, err
199 199
 	}
200 200
 
201
-	keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w)
202
-	if err != nil {
203
-		return nil, err
201
+	cms := make([]solver.CacheManager, 0, len(cMap))
202
+
203
+	for _, cc := range cMap {
204
+		keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w)
205
+		if err != nil {
206
+			return nil, err
207
+		}
208
+		cms = append(cms, solver.NewCacheManager(id, keysStorage, resultStorage))
204 209
 	}
205
-	return solver.NewCacheManager(id, keysStorage, resultStorage), nil
210
+
211
+	return solver.NewCombinedCacheManager(cms, nil), nil
206 212
 }
207 213
 
208 214
 func (ci *contentCacheImporter) allDistributionManifests(ctx context.Context, dt []byte, m map[digest.Digest][]byte) error {
... ...
@@ -31,6 +31,12 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
31 31
 	return nil, nil
32 32
 }
33 33
 
34
+func (ce *exporter) reset() {
35
+	cc := v1.NewCacheChains()
36
+	ce.CacheExporterTarget = cc
37
+	ce.chains = cc
38
+}
39
+
34 40
 func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) {
35 41
 	config, descs, err := ce.chains.Marshal()
36 42
 	if err != nil {
... ...
@@ -82,6 +88,7 @@ func (ce *exporter) ExportForLayers(layers []digest.Digest) ([]byte, error) {
82 82
 	if err != nil {
83 83
 		return nil, err
84 84
 	}
85
+	ce.reset()
85 86
 
86 87
 	return dt, nil
87 88
 }
... ...
@@ -78,10 +78,14 @@ func sortConfig(cc *CacheConfig) {
78 78
 				if ri.Inputs[i][j].Selector != rj.Inputs[i][j].Selector {
79 79
 					return ri.Inputs[i][j].Selector < rj.Inputs[i][j].Selector
80 80
 				}
81
-				return cc.Records[ri.Inputs[i][j].LinkIndex].Digest < cc.Records[rj.Inputs[i][j].LinkIndex].Digest
81
+				inputDigesti := cc.Records[ri.Inputs[i][j].LinkIndex].Digest
82
+				inputDigestj := cc.Records[rj.Inputs[i][j].LinkIndex].Digest
83
+				if inputDigesti != inputDigestj {
84
+					return inputDigesti < inputDigestj
85
+				}
82 86
 			}
83 87
 		}
84
-		return ri.Digest < rj.Digest
88
+		return false
85 89
 	})
86 90
 	for i, l := range sortedRecords {
87 91
 		l.newIndex = i
... ...
@@ -49,6 +49,7 @@ const (
49 49
 	keyNameContext             = "contextkey"
50 50
 	keyNameDockerfile          = "dockerfilekey"
51 51
 	keyContextSubDir           = "contextsubdir"
52
+	keyContextKeepGitDir       = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR"
52 53
 )
53 54
 
54 55
 var httpPrefix = regexp.MustCompile(`^https?://`)
... ...
@@ -129,7 +130,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
129 129
 
130 130
 	var buildContext *llb.State
131 131
 	isScratchContext := false
132
-	if st, ok := detectGitContext(opts[localNameContext]); ok {
132
+	if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDir]); ok {
133 133
 		if !forceLocalDockerfile {
134 134
 			src = *st
135 135
 		}
... ...
@@ -451,12 +452,19 @@ func filter(opt map[string]string, key string) map[string]string {
451 451
 	return m
452 452
 }
453 453
 
454
-func detectGitContext(ref string) (*llb.State, bool) {
454
+func detectGitContext(ref, gitContext string) (*llb.State, bool) {
455 455
 	found := false
456 456
 	if httpPrefix.MatchString(ref) && gitUrlPathWithFragmentSuffix.MatchString(ref) {
457 457
 		found = true
458 458
 	}
459 459
 
460
+	keepGit := false
461
+	if gitContext != "" {
462
+		if v, err := strconv.ParseBool(gitContext); err == nil {
463
+			keepGit = v
464
+		}
465
+	}
466
+
460 467
 	for _, prefix := range []string{"git://", "github.com/", "git@"} {
461 468
 		if strings.HasPrefix(ref, prefix) {
462 469
 			found = true
... ...
@@ -472,7 +480,12 @@ func detectGitContext(ref string) (*llb.State, bool) {
472 472
 	if len(parts) > 1 {
473 473
 		branch = parts[1]
474 474
 	}
475
-	st := llb.Git(parts[0], branch, dockerfile2llb.WithInternalName("load git source "+ref))
475
+	gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)}
476
+	if keepGit {
477
+		gitOpts = append(gitOpts, llb.KeepGitDir())
478
+	}
479
+
480
+	st := llb.Git(parts[0], branch, gitOpts...)
476 481
 	return &st, true
477 482
 }
478 483
 
... ...
@@ -746,9 +746,9 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS
746 746
 			}}, copyOpt...)
747 747
 
748 748
 			if a == nil {
749
-				a = llb.Copy(sourceState, src, dest, opts...)
749
+				a = llb.Copy(sourceState, filepath.Join("/", src), dest, opts...)
750 750
 			} else {
751
-				a = a.Copy(sourceState, src, dest, opts...)
751
+				a = a.Copy(sourceState, filepath.Join("/", src), dest, opts...)
752 752
 			}
753 753
 		}
754 754
 	}
... ...
@@ -12,6 +12,7 @@ import (
12 12
 
13 13
 	"github.com/moby/buildkit/client/llb"
14 14
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
15
+	"github.com/moby/buildkit/solver/pb"
15 16
 	"github.com/pkg/errors"
16 17
 )
17 18
 
... ...
@@ -113,7 +114,7 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*
113 113
 		}
114 114
 		if mount.ReadOnly {
115 115
 			mountOpts = append(mountOpts, llb.Readonly)
116
-		} else if mount.Type == instructions.MountTypeBind {
116
+		} else if mount.Type == instructions.MountTypeBind && opt.llbCaps.Supports(pb.CapExecMountBindReadWriteNoOuput) == nil {
117 117
 			mountOpts = append(mountOpts, llb.ForceNoOutput)
118 118
 		}
119 119
 		if mount.Type == instructions.MountTypeCache {
... ...
@@ -11,7 +11,7 @@ import (
11 11
 	"golang.org/x/sync/errgroup"
12 12
 )
13 13
 
14
-func newCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager {
14
+func NewCombinedCacheManager(cms []CacheManager, main CacheManager) CacheManager {
15 15
 	return &combinedCacheManager{cms: cms, main: main}
16 16
 }
17 17
 
... ...
@@ -80,7 +80,7 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res
80 80
 			res.Result.Release(context.TODO())
81 81
 		}
82 82
 	}()
83
-	if rec.cacheManager != cm.main {
83
+	if rec.cacheManager != cm.main && cm.main != nil {
84 84
 		for _, res := range results {
85 85
 			if _, err := cm.main.Save(res.CacheKey, res.Result, res.CacheResult.CreatedAt); err != nil {
86 86
 				return nil, err
... ...
@@ -91,6 +91,9 @@ func (cm *combinedCacheManager) Load(ctx context.Context, rec *CacheRecord) (res
91 91
 }
92 92
 
93 93
 func (cm *combinedCacheManager) Save(key *CacheKey, s Result, createdAt time.Time) (*ExportableCacheKey, error) {
94
+	if cm.main == nil {
95
+		return nil, nil
96
+	}
94 97
 	return cm.main.Save(key, s, createdAt)
95 98
 }
96 99
 
... ...
@@ -141,7 +141,7 @@ func (s *state) combinedCacheManager() CacheManager {
141 141
 		return s.mainCache
142 142
 	}
143 143
 
144
-	return newCombinedCacheManager(cms, s.mainCache)
144
+	return NewCombinedCacheManager(cms, s.mainCache)
145 145
 }
146 146
 
147 147
 func (s *state) Release() {
... ...
@@ -256,7 +256,7 @@ func (e *execOp) getRefCacheDir(ctx context.Context, ref cache.ImmutableRef, id
256 256
 }
257 257
 
258 258
 func (e *execOp) getRefCacheDirNoCache(ctx context.Context, key string, ref cache.ImmutableRef, id string, m *pb.Mount, block bool) (cache.MutableRef, error) {
259
-	makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) {
259
+	makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) {
260 260
 		desc := fmt.Sprintf("cached mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " "))
261 261
 		return e.cm.New(ctx, ref, cache.WithRecordType(client.UsageRecordTypeCacheMount), cache.WithDescription(desc), cache.CachePolicyRetain)
262 262
 	}
... ...
@@ -585,7 +585,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res
585 585
 			mountable = ref
586 586
 		}
587 587
 
588
-		makeMutable := func(cache.ImmutableRef) (cache.MutableRef, error) {
588
+		makeMutable := func(ref cache.ImmutableRef) (cache.MutableRef, error) {
589 589
 			desc := fmt.Sprintf("mount %s from exec %s", m.Dest, strings.Join(e.op.Meta.Args, " "))
590 590
 			return e.cm.New(ctx, ref, cache.WithDescription(desc))
591 591
 		}
... ...
@@ -606,7 +606,7 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res
606 606
 					outputs = append(outputs, active)
607 607
 					mountable = active
608 608
 				}
609
-			} else if ref == nil {
609
+			} else if (!m.Readonly || ref == nil) && m.Dest != pb.RootMount {
610 610
 				// this case is empty readonly scratch without output that is not really useful for anything but don't error
611 611
 				active, err := makeMutable(ref)
612 612
 				if err != nil {
... ...
@@ -30,19 +30,20 @@ const (
30 30
 
31 31
 	CapBuildOpLLBFileName apicaps.CapID = "source.buildop.llbfilename"
32 32
 
33
-	CapExecMetaBase            apicaps.CapID = "exec.meta.base"
34
-	CapExecMetaProxy           apicaps.CapID = "exec.meta.proxyenv"
35
-	CapExecMetaNetwork         apicaps.CapID = "exec.meta.network"
36
-	CapExecMetaSecurity        apicaps.CapID = "exec.meta.security"
37
-	CapExecMetaSetsDefaultPath apicaps.CapID = "exec.meta.setsdefaultpath"
38
-	CapExecMountBind           apicaps.CapID = "exec.mount.bind"
39
-	CapExecMountCache          apicaps.CapID = "exec.mount.cache"
40
-	CapExecMountCacheSharing   apicaps.CapID = "exec.mount.cache.sharing"
41
-	CapExecMountSelector       apicaps.CapID = "exec.mount.selector"
42
-	CapExecMountTmpfs          apicaps.CapID = "exec.mount.tmpfs"
43
-	CapExecMountSecret         apicaps.CapID = "exec.mount.secret"
44
-	CapExecMountSSH            apicaps.CapID = "exec.mount.ssh"
45
-	CapExecCgroupsMounted      apicaps.CapID = "exec.cgroup"
33
+	CapExecMetaBase                  apicaps.CapID = "exec.meta.base"
34
+	CapExecMetaProxy                 apicaps.CapID = "exec.meta.proxyenv"
35
+	CapExecMetaNetwork               apicaps.CapID = "exec.meta.network"
36
+	CapExecMetaSecurity              apicaps.CapID = "exec.meta.security"
37
+	CapExecMetaSetsDefaultPath       apicaps.CapID = "exec.meta.setsdefaultpath"
38
+	CapExecMountBind                 apicaps.CapID = "exec.mount.bind"
39
+	CapExecMountBindReadWriteNoOuput apicaps.CapID = "exec.mount.bind.readwrite-nooutput"
40
+	CapExecMountCache                apicaps.CapID = "exec.mount.cache"
41
+	CapExecMountCacheSharing         apicaps.CapID = "exec.mount.cache.sharing"
42
+	CapExecMountSelector             apicaps.CapID = "exec.mount.selector"
43
+	CapExecMountTmpfs                apicaps.CapID = "exec.mount.tmpfs"
44
+	CapExecMountSecret               apicaps.CapID = "exec.mount.secret"
45
+	CapExecMountSSH                  apicaps.CapID = "exec.mount.ssh"
46
+	CapExecCgroupsMounted            apicaps.CapID = "exec.cgroup"
46 47
 
47 48
 	CapFileBase apicaps.CapID = "file.base"
48 49
 
... ...
@@ -194,6 +195,12 @@ func init() {
194 194
 	})
195 195
 
196 196
 	Caps.Init(apicaps.Cap{
197
+		ID:      CapExecMountBindReadWriteNoOuput,
198
+		Enabled: true,
199
+		Status:  apicaps.CapStatusExperimental,
200
+	})
201
+
202
+	Caps.Init(apicaps.Cap{
197 203
 		ID:      CapExecMountCache,
198 204
 		Enabled: true,
199 205
 		Status:  apicaps.CapStatusExperimental,
... ...
@@ -153,6 +153,14 @@ type gitSourceHandler struct {
153 153
 	cacheKey string
154 154
 }
155 155
 
156
+func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
157
+	key := sha
158
+	if gs.src.KeepGitDir {
159
+		key += ".git"
160
+	}
161
+	return key
162
+}
163
+
156 164
 func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) {
157 165
 	gitIdentifier, ok := id.(*source.GitIdentifier)
158 166
 	if !ok {
... ...
@@ -175,6 +183,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo
175 175
 	defer gs.locker.Unlock(remote)
176 176
 
177 177
 	if isCommitSHA(ref) {
178
+		ref = gs.shaToCacheKey(ref)
178 179
 		gs.cacheKey = ref
179 180
 		return ref, true, nil
180 181
 	}
... ...
@@ -201,6 +210,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo
201 201
 	if !isCommitSHA(sha) {
202 202
 		return "", false, errors.Errorf("invalid commit sha %q", sha)
203 203
 	}
204
+	sha = gs.shaToCacheKey(sha)
204 205
 	gs.cacheKey = sha
205 206
 	return sha, true, nil
206 207
 }
... ...
@@ -298,11 +308,15 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
298 298
 	}()
299 299
 
300 300
 	if gs.src.KeepGitDir {
301
-		_, err = gitWithinDir(ctx, checkoutDir, "", "init")
301
+		checkoutDirGit := filepath.Join(checkoutDir, ".git")
302
+		if err := os.MkdirAll(checkoutDir, 0711); err != nil {
303
+			return nil, err
304
+		}
305
+		_, err = gitWithinDir(ctx, checkoutDirGit, "", "init")
302 306
 		if err != nil {
303 307
 			return nil, err
304 308
 		}
305
-		_, err = gitWithinDir(ctx, checkoutDir, "", "remote", "add", "origin", gitDir)
309
+		_, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir)
306 310
 		if err != nil {
307 311
 			return nil, err
308 312
 		}
... ...
@@ -313,16 +327,18 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
313 313
 			if err != nil {
314 314
 				return nil, err
315 315
 			}
316
+		} else {
317
+			pullref += ":" + pullref
316 318
 		}
317
-		_, err = gitWithinDir(ctx, checkoutDir, "", "fetch", "--depth=1", "origin", pullref)
319
+		_, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref)
318 320
 		if err != nil {
319 321
 			return nil, err
320 322
 		}
321
-		_, err = gitWithinDir(ctx, checkoutDir, checkoutDir, "checkout", "FETCH_HEAD")
323
+		_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD")
322 324
 		if err != nil {
323 325
 			return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
324 326
 		}
325
-		gitDir = checkoutDir
327
+		gitDir = checkoutDirGit
326 328
 	} else {
327 329
 		_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
328 330
 		if err != nil {