Vendor buildkit to v0.8.1
Tibor Vass authored on 2020/12/15 12:37:35... | ... |
@@ -33,7 +33,7 @@ github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6 |
33 | 33 |
golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb |
34 | 34 |
|
35 | 35 |
# buildkit |
36 |
-github.com/moby/buildkit 950603da215ae03b843f3f66fbe86c4876a6f5a1 |
|
36 |
+github.com/moby/buildkit 8142d66b5ebde79846b869fba30d9d30633e74aa # v0.8.1 |
|
37 | 37 |
github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 |
38 | 38 |
github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 |
39 | 39 |
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 |
... | ... |
@@ -3,7 +3,7 @@ |
3 | 3 |
# BuildKit |
4 | 4 |
|
5 | 5 |
[![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb) |
6 |
-[![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit) |
|
6 |
+[![Build Status](https://github.com/moby/buildkit/workflows/build/badge.svg)](https://github.com/moby/buildkit/actions?query=workflow%3Abuild) |
|
7 | 7 |
[![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit) |
8 | 8 |
[![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit) |
9 | 9 |
|
... | ... |
@@ -28,7 +28,7 @@ Introductory blog post https://blog.mobyproject.org/introducing-buildkit-17e056c |
28 | 28 |
|
29 | 29 |
Join `#buildkit` channel on [Docker Community Slack](http://dockr.ly/slack) |
30 | 30 |
|
31 |
-:information_source: If you are visiting this repo for the usage of experimental Dockerfile features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)`, please refer to [`frontend/dockerfile/docs/experimental.md`](frontend/dockerfile/docs/experimental.md). |
|
31 |
+:information_source: If you are visiting this repo for the usage of BuildKit-only Dockerfile features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)`, please refer to [`frontend/dockerfile/docs/syntax.md`](frontend/dockerfile/docs/syntax.md). |
|
32 | 32 |
|
33 | 33 |
:information_source: [BuildKit has been integrated to `docker build` since Docker 18.06 .](https://docs.docker.com/develop/develop-images/build_enhancements/) |
34 | 34 |
You don't need to read this document unless you want to use the full-featured standalone version of BuildKit. |
... | ... |
@@ -178,7 +178,7 @@ buildctl build \ |
178 | 178 |
|
179 | 179 |
#### Building a Dockerfile using external frontend: |
180 | 180 |
|
181 |
-External versions of the Dockerfile frontend are pushed to https://hub.docker.com/r/docker/dockerfile-upstream and https://hub.docker.com/r/docker/dockerfile and can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `docker/dockerfile-upsteam:master` or `docker/dockerfile-upstream:master-experimental` image can be used. |
|
181 |
+External versions of the Dockerfile frontend are pushed to https://hub.docker.com/r/docker/dockerfile-upstream and https://hub.docker.com/r/docker/dockerfile and can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `docker/dockerfile-upstream:master` or `docker/dockerfile-upstream:master-labs` image can be used. |
|
182 | 182 |
|
183 | 183 |
```bash |
184 | 184 |
buildctl build \ |
... | ... |
@@ -435,7 +435,7 @@ For Kubernetes deployments, see [`examples/kubernetes`](./examples/kubernetes). |
435 | 435 |
|
436 | 436 |
### Daemonless |
437 | 437 |
|
438 |
-To run client and an ephemeral daemon in a single container ("daemonless mode"): |
|
438 |
+To run the client and an ephemeral daemon in a single container ("daemonless mode"): |
|
439 | 439 |
|
440 | 440 |
```bash |
441 | 441 |
docker run \ |
... | ... |
@@ -661,6 +661,14 @@ func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, [] |
661 | 661 |
|
662 | 662 |
pfo := &pb.FileOp{} |
663 | 663 |
|
664 |
+ if f.constraints.Platform == nil { |
|
665 |
+ p, err := getPlatform(*f.action.state)(ctx) |
|
666 |
+ if err != nil { |
|
667 |
+ return "", nil, nil, nil, err |
|
668 |
+ } |
|
669 |
+ f.constraints.Platform = p |
|
670 |
+ } |
|
671 |
+ |
|
664 | 672 |
pop, md := MarshalConstraints(c, &f.constraints) |
665 | 673 |
pop.Op = &pb.Op_File{ |
666 | 674 |
File: pfo, |
... | ... |
@@ -5,7 +5,6 @@ import ( |
5 | 5 |
_ "crypto/sha256" // for opencontainers/go-digest |
6 | 6 |
"encoding/json" |
7 | 7 |
"os" |
8 |
- "regexp" |
|
9 | 8 |
"strconv" |
10 | 9 |
"strings" |
11 | 10 |
|
... | ... |
@@ -207,8 +206,6 @@ const ( |
207 | 207 |
gitProtocolUnknown |
208 | 208 |
) |
209 | 209 |
|
210 |
-var gitSSHRegex = regexp.MustCompile("^([a-z0-9]+@)?[^:]+:.*$") |
|
211 |
- |
|
212 | 210 |
func getGitProtocol(remote string) (string, int) { |
213 | 211 |
prefixes := map[string]int{ |
214 | 212 |
"http://": gitProtocolHTTP, |
... | ... |
@@ -224,7 +221,7 @@ func getGitProtocol(remote string) (string, int) { |
224 | 224 |
} |
225 | 225 |
} |
226 | 226 |
|
227 |
- if protocolType == gitProtocolUnknown && gitSSHRegex.MatchString(remote) { |
|
227 |
+ if protocolType == gitProtocolUnknown && sshutil.IsSSHTransport(remote) { |
|
228 | 228 |
protocolType = gitProtocolSSH |
229 | 229 |
} |
230 | 230 |
|
... | ... |
@@ -254,6 +251,9 @@ func Git(remote, ref string, opts ...GitOption) State { |
254 | 254 |
remote = parts[0] + "/" + parts[1] |
255 | 255 |
} |
256 | 256 |
} |
257 |
+ if protocolType == gitProtocolUnknown { |
|
258 |
+ url = "https://" + url |
|
259 |
+ } |
|
257 | 260 |
|
258 | 261 |
id := remote |
259 | 262 |
|
... | ... |
@@ -595,6 +595,7 @@ func (e *edge) recalcCurrentState() { |
595 | 595 |
stHigh := edgeStatusCacheSlow // maximum possible state |
596 | 596 |
if e.cacheMap != nil { |
597 | 597 |
for _, dep := range e.deps { |
598 |
+ isSlowCacheIncomplete := e.slowCacheFunc(dep) != nil && (dep.state == edgeStatusCacheSlow || (dep.state == edgeStatusComplete && !dep.slowCacheComplete)) |
|
598 | 599 |
isSlowIncomplete := (e.slowCacheFunc(dep) != nil || e.preprocessFunc(dep) != nil) && (dep.state == edgeStatusCacheSlow || (dep.state == edgeStatusComplete && !dep.slowCacheComplete)) |
599 | 600 |
|
600 | 601 |
if dep.state > stLow && len(dep.keyMap) == 0 && !isSlowIncomplete { |
... | ... |
@@ -604,10 +605,10 @@ func (e *edge) recalcCurrentState() { |
604 | 604 |
} |
605 | 605 |
} |
606 | 606 |
effectiveState := dep.state |
607 |
- if dep.state == edgeStatusCacheSlow && isSlowIncomplete { |
|
607 |
+ if dep.state == edgeStatusCacheSlow && isSlowCacheIncomplete { |
|
608 | 608 |
effectiveState = edgeStatusCacheFast |
609 | 609 |
} |
610 |
- if dep.state == edgeStatusComplete && isSlowIncomplete { |
|
610 |
+ if dep.state == edgeStatusComplete && isSlowCacheIncomplete { |
|
611 | 611 |
effectiveState = edgeStatusCacheFast |
612 | 612 |
} |
613 | 613 |
if effectiveState < stHigh { |
... | ... |
@@ -619,7 +620,7 @@ func (e *edge) recalcCurrentState() { |
619 | 619 |
if dep.state < edgeStatusCacheFast { |
620 | 620 |
allDepsCompletedCacheFast = false |
621 | 621 |
} |
622 |
- if isSlowIncomplete || dep.state < edgeStatusCacheSlow { |
|
622 |
+ if isSlowCacheIncomplete || dep.state < edgeStatusCacheSlow { |
|
623 | 623 |
allDepsCompletedCacheSlow = false |
624 | 624 |
} |
625 | 625 |
if dep.state < edgeStatusCacheSlow && len(dep.keyMap) == 0 { |
... | ... |
@@ -527,7 +527,13 @@ func (j *Job) Discard() error { |
527 | 527 |
st.mu.Unlock() |
528 | 528 |
} |
529 | 529 |
|
530 |
- delete(j.list.jobs, j.id) |
|
530 |
+ go func() { |
|
531 |
+ // don't clear job right away. there might still be a status request coming to read progress |
|
532 |
+ time.Sleep(10 * time.Second) |
|
533 |
+ j.list.mu.Lock() |
|
534 |
+ defer j.list.mu.Unlock() |
|
535 |
+ delete(j.list.jobs, j.id) |
|
536 |
+ }() |
|
531 | 537 |
return nil |
532 | 538 |
} |
533 | 539 |
|
... | ... |
@@ -131,6 +131,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro |
131 | 131 |
} |
132 | 132 |
} |
133 | 133 |
|
134 |
+ if res == nil { |
|
135 |
+ res = &frontend.Result{} |
|
136 |
+ } |
|
137 |
+ |
|
134 | 138 |
defer func() { |
135 | 139 |
res.EachRef(func(ref solver.ResultProxy) error { |
136 | 140 |
go ref.Release(context.TODO()) |
... | ... |
@@ -4,6 +4,7 @@ import ( |
4 | 4 |
"net/url" |
5 | 5 |
"strings" |
6 | 6 |
|
7 |
+ "github.com/moby/buildkit/util/sshutil" |
|
7 | 8 |
"github.com/pkg/errors" |
8 | 9 |
) |
9 | 10 |
|
... | ... |
@@ -58,7 +59,7 @@ func (i *GitIdentifier) ID() string { |
58 | 58 |
// isGitTransport returns true if the provided str is a git transport by inspecting |
59 | 59 |
// the prefix of the string for known protocols used in git. |
60 | 60 |
func isGitTransport(str string) bool { |
61 |
- return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "git@") |
|
61 |
+ return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "git://") || sshutil.IsSSHTransport(str) |
|
62 | 62 |
} |
63 | 63 |
|
64 | 64 |
func getRefAndSubdir(fragment string) (ref string, subdir string) { |
... | ... |
@@ -102,6 +102,9 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) { |
102 | 102 |
id.KeepGitDir = true |
103 | 103 |
} |
104 | 104 |
case pb.AttrFullRemoteURL: |
105 |
+ if !isGitTransport(v) { |
|
106 |
+ v = "https://" + v |
|
107 |
+ } |
|
105 | 108 |
id.Remote = v |
106 | 109 |
case pb.AttrAuthHeaderSecret: |
107 | 110 |
id.AuthHeaderSecret = v |
... | ... |
@@ -99,17 +99,21 @@ func detectCompressionType(cr io.Reader) (Type, error) { |
99 | 99 |
} |
100 | 100 |
|
101 | 101 |
var toDockerLayerType = map[string]string{ |
102 |
- ocispec.MediaTypeImageLayer: images.MediaTypeDockerSchema2Layer, |
|
103 |
- images.MediaTypeDockerSchema2Layer: images.MediaTypeDockerSchema2Layer, |
|
104 |
- ocispec.MediaTypeImageLayerGzip: images.MediaTypeDockerSchema2LayerGzip, |
|
105 |
- images.MediaTypeDockerSchema2LayerGzip: images.MediaTypeDockerSchema2LayerGzip, |
|
102 |
+ ocispec.MediaTypeImageLayer: images.MediaTypeDockerSchema2Layer, |
|
103 |
+ images.MediaTypeDockerSchema2Layer: images.MediaTypeDockerSchema2Layer, |
|
104 |
+ ocispec.MediaTypeImageLayerGzip: images.MediaTypeDockerSchema2LayerGzip, |
|
105 |
+ images.MediaTypeDockerSchema2LayerGzip: images.MediaTypeDockerSchema2LayerGzip, |
|
106 |
+ images.MediaTypeDockerSchema2LayerForeign: images.MediaTypeDockerSchema2Layer, |
|
107 |
+ images.MediaTypeDockerSchema2LayerForeignGzip: images.MediaTypeDockerSchema2LayerGzip, |
|
106 | 108 |
} |
107 | 109 |
|
108 | 110 |
var toOCILayerType = map[string]string{ |
109 |
- ocispec.MediaTypeImageLayer: ocispec.MediaTypeImageLayer, |
|
110 |
- images.MediaTypeDockerSchema2Layer: ocispec.MediaTypeImageLayer, |
|
111 |
- ocispec.MediaTypeImageLayerGzip: ocispec.MediaTypeImageLayerGzip, |
|
112 |
- images.MediaTypeDockerSchema2LayerGzip: ocispec.MediaTypeImageLayerGzip, |
|
111 |
+ ocispec.MediaTypeImageLayer: ocispec.MediaTypeImageLayer, |
|
112 |
+ images.MediaTypeDockerSchema2Layer: ocispec.MediaTypeImageLayer, |
|
113 |
+ ocispec.MediaTypeImageLayerGzip: ocispec.MediaTypeImageLayerGzip, |
|
114 |
+ images.MediaTypeDockerSchema2LayerGzip: ocispec.MediaTypeImageLayerGzip, |
|
115 |
+ images.MediaTypeDockerSchema2LayerForeign: ocispec.MediaTypeImageLayer, |
|
116 |
+ images.MediaTypeDockerSchema2LayerForeignGzip: ocispec.MediaTypeImageLayerGzip, |
|
113 | 117 |
} |
114 | 118 |
|
115 | 119 |
func convertLayerMediaType(mediaType string, oci bool) string { |
... | ... |
@@ -130,7 +130,11 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) { |
130 | 130 |
c.mu.Lock() |
131 | 131 |
// detect case where caller has just returned, let it clean up before |
132 | 132 |
select { |
133 |
- case <-c.ready: // could return if no error |
|
133 |
+ case <-c.ready: |
|
134 |
+ c.mu.Unlock() |
|
135 |
+ <-c.cleaned |
|
136 |
+ return nil, errRetry |
|
137 |
+ case <-c.ctx.done: // could return if no error |
|
134 | 138 |
c.mu.Unlock() |
135 | 139 |
<-c.cleaned |
136 | 140 |
return nil, errRetry |
... | ... |
@@ -141,6 +145,10 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) { |
141 | 141 |
if ok { |
142 | 142 |
c.progressState.add(pw) |
143 | 143 |
} |
144 |
+ |
|
145 |
+ ctx, cancel := context.WithCancel(ctx) |
|
146 |
+ defer cancel() |
|
147 |
+ |
|
144 | 148 |
c.ctxs = append(c.ctxs, ctx) |
145 | 149 |
|
146 | 150 |
c.mu.Unlock() |
... | ... |
@@ -149,18 +157,16 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) { |
149 | 149 |
|
150 | 150 |
select { |
151 | 151 |
case <-ctx.Done(): |
152 |
- select { |
|
153 |
- case <-c.ctx.Done(): |
|
152 |
+ if c.ctx.checkDone() { |
|
154 | 153 |
// if this cancelled the last context, then wait for function to shut down |
155 | 154 |
// and don't accept any more callers |
156 | 155 |
<-c.ready |
157 | 156 |
return c.result, c.err |
158 |
- default: |
|
159 |
- if ok { |
|
160 |
- c.progressState.close(pw) |
|
161 |
- } |
|
162 |
- return nil, ctx.Err() |
|
163 | 157 |
} |
158 |
+ if ok { |
|
159 |
+ c.progressState.close(pw) |
|
160 |
+ } |
|
161 |
+ return nil, ctx.Err() |
|
164 | 162 |
case <-c.ready: |
165 | 163 |
return c.result, c.err // shared not implemented yet |
166 | 164 |
} |
... | ... |
@@ -183,9 +189,6 @@ func (c *call) Deadline() (deadline time.Time, ok bool) { |
183 | 183 |
} |
184 | 184 |
|
185 | 185 |
func (c *call) Done() <-chan struct{} { |
186 |
- c.mu.Lock() |
|
187 |
- c.ctx.signal() |
|
188 |
- c.mu.Unlock() |
|
189 | 186 |
return c.ctx.done |
190 | 187 |
} |
191 | 188 |
|
... | ... |
@@ -238,23 +241,28 @@ func newContext(c *call) *sharedContext { |
238 | 238 |
return &sharedContext{call: c, done: make(chan struct{})} |
239 | 239 |
} |
240 | 240 |
|
241 |
-// call with lock |
|
242 |
-func (c *sharedContext) signal() { |
|
241 |
+func (sc *sharedContext) checkDone() bool { |
|
242 |
+ sc.mu.Lock() |
|
243 | 243 |
select { |
244 |
- case <-c.done: |
|
244 |
+ case <-sc.done: |
|
245 |
+ sc.mu.Unlock() |
|
246 |
+ return true |
|
245 | 247 |
default: |
246 |
- var err error |
|
247 |
- for _, ctx := range c.ctxs { |
|
248 |
- select { |
|
249 |
- case <-ctx.Done(): |
|
250 |
- err = ctx.Err() |
|
251 |
- default: |
|
252 |
- return |
|
253 |
- } |
|
248 |
+ } |
|
249 |
+ var err error |
|
250 |
+ for _, ctx := range sc.ctxs { |
|
251 |
+ select { |
|
252 |
+ case <-ctx.Done(): |
|
253 |
+ err = ctx.Err() |
|
254 |
+ default: |
|
255 |
+ sc.mu.Unlock() |
|
256 |
+ return false |
|
254 | 257 |
} |
255 |
- c.err = err |
|
256 |
- close(c.done) |
|
257 | 258 |
} |
259 |
+ sc.err = err |
|
260 |
+ close(sc.done) |
|
261 |
+ sc.mu.Unlock() |
|
262 |
+ return true |
|
258 | 263 |
} |
259 | 264 |
|
260 | 265 |
type rawProgressWriter interface { |