vendor: BuildKit 950603da215ae03b843f3f66fbe86c4876a6f5a1
Tibor Vass authored on 2020/12/05 12:36:42... | ... |
@@ -33,7 +33,7 @@ github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6 |
33 | 33 |
golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb |
34 | 34 |
|
35 | 35 |
# buildkit |
36 |
-github.com/moby/buildkit fcb87e6b8ccf3631a65799cc56caa76f9117816e # v0.8.0-rc2 |
|
36 |
+github.com/moby/buildkit 950603da215ae03b843f3f66fbe86c4876a6f5a1 |
|
37 | 37 |
github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 |
38 | 38 |
github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 |
39 | 39 |
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 |
... | ... |
@@ -3,8 +3,9 @@ |
3 | 3 |
# BuildKit |
4 | 4 |
|
5 | 5 |
[![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb) |
6 |
-[![Build Status](https://travis-ci.org/moby/buildkit.svg?branch=master)](https://travis-ci.org/moby/buildkit) |
|
6 |
+[![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit) |
|
7 | 7 |
[![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit) |
8 |
+[![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit) |
|
8 | 9 |
|
9 | 10 |
BuildKit is a toolkit for converting source code to build artifacts in an efficient, expressive and repeatable manner. |
10 | 11 |
|
... | ... |
@@ -399,8 +399,10 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro |
399 | 399 |
ctx = winlayers.UseWindowsLayerMode(ctx) |
400 | 400 |
} |
401 | 401 |
|
402 |
- if _, err := sr.prepareRemoteSnapshots(ctx, sr.descHandlers); err != nil { |
|
403 |
- return err |
|
402 |
+ if sr.cm.Snapshotter.Name() == "stargz" { |
|
403 |
+ if _, err := sr.prepareRemoteSnapshots(ctx, sr.descHandlers); err != nil { |
|
404 |
+ return err |
|
405 |
+ } |
|
404 | 406 |
} |
405 | 407 |
|
406 | 408 |
return sr.extract(ctx, sr.descHandlers, s) |
... | ... |
@@ -454,6 +456,9 @@ func (sr *immutableRef) prepareRemoteSnapshots(ctx context.Context, dhs DescHand |
454 | 454 |
// This layer cannot be prepared without unlazying. |
455 | 455 |
return false, nil |
456 | 456 |
}) |
457 |
+ if err != nil { |
|
458 |
+ return false, err |
|
459 |
+ } |
|
457 | 460 |
return ok.(bool), err |
458 | 461 |
} |
459 | 462 |
|
... | ... |
@@ -130,8 +130,15 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { |
130 | 130 |
|
131 | 131 |
name := "load build definition from " + filename |
132 | 132 |
|
133 |
+ filenames := []string{filename, filename + ".dockerignore"} |
|
134 |
+ |
|
135 |
+ // dockerfile is also supported casing moby/moby#10858 |
|
136 |
+ if path.Base(filename) == defaultDockerfileName { |
|
137 |
+ filenames = append(filenames, path.Join(path.Dir(filename), strings.ToLower(defaultDockerfileName))) |
|
138 |
+ } |
|
139 |
+ |
|
133 | 140 |
src := llb.Local(localNameDockerfile, |
134 |
- llb.FollowPaths([]string{filename, filename + ".dockerignore"}), |
|
141 |
+ llb.FollowPaths(filenames), |
|
135 | 142 |
llb.SessionID(c.BuildOpts().SessionID), |
136 | 143 |
llb.SharedKeyHint(localNameDockerfile), |
137 | 144 |
dockerfile2llb.WithInternalName(name), |
... | ... |
@@ -258,7 +265,19 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { |
258 | 258 |
Filename: filename, |
259 | 259 |
}) |
260 | 260 |
if err != nil { |
261 |
- return errors.Wrapf(err, "failed to read dockerfile") |
|
261 |
+ fallback := false |
|
262 |
+ if path.Base(filename) == defaultDockerfileName { |
|
263 |
+ var err1 error |
|
264 |
+ dtDockerfile, err1 = ref.ReadFile(ctx2, client.ReadRequest{ |
|
265 |
+ Filename: path.Join(path.Dir(filename), strings.ToLower(defaultDockerfileName)), |
|
266 |
+ }) |
|
267 |
+ if err1 == nil { |
|
268 |
+ fallback = true |
|
269 |
+ } |
|
270 |
+ } |
|
271 |
+ if !fallback { |
|
272 |
+ return errors.Wrapf(err, "failed to read dockerfile") |
|
273 |
+ } |
|
262 | 274 |
} |
263 | 275 |
|
264 | 276 |
sourceMap = llb.NewSourceMap(&src, filename, dtDockerfile) |
... | ... |
@@ -255,7 +255,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten |
255 | 255 |
err = w.Executor().Run(ctx, "", mountWithSession(rootFS, session.NewGroup(sid)), nil, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil) |
256 | 256 |
|
257 | 257 |
if err != nil { |
258 |
- if errors.Is(err, context.Canceled) && lbf.isErrServerClosed { |
|
258 |
+ if errdefs.IsCanceled(err) && lbf.isErrServerClosed { |
|
259 | 259 |
err = errors.Errorf("frontend grpc server closed unexpectedly") |
260 | 260 |
} |
261 | 261 |
// An existing error (set via Return rpc) takes |
... | ... |
@@ -359,6 +359,7 @@ func (e *edge) unpark(incoming []pipe.Sender, updates, allPipes []pipe.Receiver, |
359 | 359 |
if e.execReq == nil { |
360 | 360 |
if added := e.createInputRequests(desiredState, f, false); !added && !e.hasActiveOutgoing && !cacheMapReq { |
361 | 361 |
logrus.Errorf("buildkit scheluding error: leaving incoming open. forcing solve. Please report this with BUILDKIT_SCHEDULER_DEBUG=1") |
362 |
+ debugSchedulerPreUnpark(e, incoming, updates, allPipes) |
|
362 | 363 |
e.createInputRequests(desiredState, f, true) |
363 | 364 |
} |
364 | 365 |
} |
365 | 366 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,13 @@ |
0 |
+package errdefs |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "context" |
|
4 |
+ "errors" |
|
5 |
+ |
|
6 |
+ "github.com/moby/buildkit/util/grpcerrors" |
|
7 |
+ "google.golang.org/grpc/codes" |
|
8 |
+) |
|
9 |
+ |
|
10 |
+func IsCanceled(err error) bool { |
|
11 |
+ return errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled |
|
12 |
+} |
... | ... |
@@ -659,6 +659,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF |
659 | 659 |
case <-ctx.Done(): |
660 | 660 |
if strings.Contains(err.Error(), context.Canceled.Error()) { |
661 | 661 |
complete = false |
662 |
+ releaseError(err) |
|
662 | 663 |
err = errors.Wrap(ctx.Err(), err.Error()) |
663 | 664 |
} |
664 | 665 |
default: |
... | ... |
@@ -717,6 +718,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp, |
717 | 717 |
case <-ctx.Done(): |
718 | 718 |
if strings.Contains(err.Error(), context.Canceled.Error()) { |
719 | 719 |
complete = false |
720 |
+ releaseError(err) |
|
720 | 721 |
err = errors.Wrap(ctx.Err(), err.Error()) |
721 | 722 |
} |
722 | 723 |
default: |
... | ... |
@@ -774,6 +776,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, |
774 | 774 |
case <-ctx.Done(): |
775 | 775 |
if strings.Contains(err.Error(), context.Canceled.Error()) { |
776 | 776 |
complete = false |
777 |
+ releaseError(err) |
|
777 | 778 |
err = errors.Wrap(ctx.Err(), err.Error()) |
778 | 779 |
} |
779 | 780 |
default: |
... | ... |
@@ -911,3 +914,15 @@ func WrapSlowCache(err error, index Index, res Result) error { |
911 | 911 |
} |
912 | 912 |
return &SlowCacheError{Index: index, Result: res, error: err} |
913 | 913 |
} |
914 |
+ |
|
915 |
+func releaseError(err error) { |
|
916 |
+ if err == nil { |
|
917 |
+ return |
|
918 |
+ } |
|
919 |
+ if re, ok := err.(interface { |
|
920 |
+ Release() error |
|
921 |
+ }); ok { |
|
922 |
+ re.Release() |
|
923 |
+ } |
|
924 |
+ releaseError(errors.Unwrap(err)) |
|
925 |
+} |
... | ... |
@@ -16,6 +16,7 @@ import ( |
16 | 16 |
"github.com/moby/buildkit/session" |
17 | 17 |
"github.com/moby/buildkit/solver" |
18 | 18 |
"github.com/moby/buildkit/solver/errdefs" |
19 |
+ llberrdefs "github.com/moby/buildkit/solver/llbsolver/errdefs" |
|
19 | 20 |
"github.com/moby/buildkit/solver/pb" |
20 | 21 |
"github.com/moby/buildkit/util/flightcontrol" |
21 | 22 |
"github.com/moby/buildkit/worker" |
... | ... |
@@ -144,41 +145,60 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid st |
144 | 144 |
} |
145 | 145 |
|
146 | 146 |
type resultProxy struct { |
147 |
- cb func(context.Context) (solver.CachedResult, error) |
|
148 |
- def *pb.Definition |
|
149 |
- g flightcontrol.Group |
|
150 |
- mu sync.Mutex |
|
151 |
- released bool |
|
152 |
- v solver.CachedResult |
|
153 |
- err error |
|
147 |
+ cb func(context.Context) (solver.CachedResult, error) |
|
148 |
+ def *pb.Definition |
|
149 |
+ g flightcontrol.Group |
|
150 |
+ mu sync.Mutex |
|
151 |
+ released bool |
|
152 |
+ v solver.CachedResult |
|
153 |
+ err error |
|
154 |
+ errResults []solver.Result |
|
154 | 155 |
} |
155 | 156 |
|
156 | 157 |
func newResultProxy(b *llbBridge, req frontend.SolveRequest) *resultProxy { |
157 |
- return &resultProxy{ |
|
158 |
+ rp := &resultProxy{ |
|
158 | 159 |
def: req.Definition, |
159 |
- cb: func(ctx context.Context) (solver.CachedResult, error) { |
|
160 |
- return b.loadResult(ctx, req.Definition, req.CacheImports) |
|
161 |
- }, |
|
162 | 160 |
} |
161 |
+ rp.cb = func(ctx context.Context) (solver.CachedResult, error) { |
|
162 |
+ res, err := b.loadResult(ctx, req.Definition, req.CacheImports) |
|
163 |
+ var ee *llberrdefs.ExecError |
|
164 |
+ if errors.As(err, &ee) { |
|
165 |
+ ee.EachRef(func(res solver.Result) error { |
|
166 |
+ rp.errResults = append(rp.errResults, res) |
|
167 |
+ return nil |
|
168 |
+ }) |
|
169 |
+ // acquire ownership so ExecError finalizer doesn't attempt to release as well |
|
170 |
+ ee.OwnerBorrowed = true |
|
171 |
+ } |
|
172 |
+ return res, err |
|
173 |
+ } |
|
174 |
+ return rp |
|
163 | 175 |
} |
164 | 176 |
|
165 | 177 |
func (rp *resultProxy) Definition() *pb.Definition { |
166 | 178 |
return rp.def |
167 | 179 |
} |
168 | 180 |
|
169 |
-func (rp *resultProxy) Release(ctx context.Context) error { |
|
181 |
+func (rp *resultProxy) Release(ctx context.Context) (err error) { |
|
170 | 182 |
rp.mu.Lock() |
171 | 183 |
defer rp.mu.Unlock() |
184 |
+ for _, res := range rp.errResults { |
|
185 |
+ rerr := res.Release(ctx) |
|
186 |
+ if rerr != nil { |
|
187 |
+ err = rerr |
|
188 |
+ } |
|
189 |
+ } |
|
172 | 190 |
if rp.v != nil { |
173 | 191 |
if rp.released { |
174 | 192 |
logrus.Warnf("release of already released result") |
175 | 193 |
} |
176 |
- if err := rp.v.Release(ctx); err != nil { |
|
177 |
- return err |
|
194 |
+ rerr := rp.v.Release(ctx) |
|
195 |
+ if err != nil { |
|
196 |
+ return rerr |
|
178 | 197 |
} |
179 | 198 |
} |
180 | 199 |
rp.released = true |
181 |
- return nil |
|
200 |
+ return |
|
182 | 201 |
} |
183 | 202 |
|
184 | 203 |
func (rp *resultProxy) wrapError(err error) error { |
... | ... |
@@ -1,14 +1,19 @@ |
1 | 1 |
package errdefs |
2 | 2 |
|
3 | 3 |
import ( |
4 |
+ "context" |
|
5 |
+ "runtime" |
|
6 |
+ |
|
4 | 7 |
"github.com/moby/buildkit/solver" |
8 |
+ "github.com/sirupsen/logrus" |
|
5 | 9 |
) |
6 | 10 |
|
7 | 11 |
// ExecError will be returned when an error is encountered when evaluating an op. |
8 | 12 |
type ExecError struct { |
9 | 13 |
error |
10 |
- Inputs []solver.Result |
|
11 |
- Mounts []solver.Result |
|
14 |
+ Inputs []solver.Result |
|
15 |
+ Mounts []solver.Result |
|
16 |
+ OwnerBorrowed bool |
|
12 | 17 |
} |
13 | 18 |
|
14 | 19 |
func (e *ExecError) Unwrap() error { |
... | ... |
@@ -35,13 +40,35 @@ func (e *ExecError) EachRef(fn func(solver.Result) error) (err error) { |
35 | 35 |
return err |
36 | 36 |
} |
37 | 37 |
|
38 |
+func (e *ExecError) Release() error { |
|
39 |
+ if e.OwnerBorrowed { |
|
40 |
+ return nil |
|
41 |
+ } |
|
42 |
+ err := e.EachRef(func(r solver.Result) error { |
|
43 |
+ r.Release(context.TODO()) |
|
44 |
+ return nil |
|
45 |
+ }) |
|
46 |
+ e.OwnerBorrowed = true |
|
47 |
+ return err |
|
48 |
+} |
|
49 |
+ |
|
38 | 50 |
func WithExecError(err error, inputs, mounts []solver.Result) error { |
39 | 51 |
if err == nil { |
40 | 52 |
return nil |
41 | 53 |
} |
42 |
- return &ExecError{ |
|
54 |
+ ee := &ExecError{ |
|
43 | 55 |
error: err, |
44 | 56 |
Inputs: inputs, |
45 | 57 |
Mounts: mounts, |
46 | 58 |
} |
59 |
+ runtime.SetFinalizer(ee, func(e *ExecError) { |
|
60 |
+ if !e.OwnerBorrowed { |
|
61 |
+ e.EachRef(func(r solver.Result) error { |
|
62 |
+ logrus.Warn("leaked execError detected and released") |
|
63 |
+ r.Release(context.TODO()) |
|
64 |
+ return nil |
|
65 |
+ }) |
|
66 |
+ } |
|
67 |
+ }) |
|
68 |
+ return ee |
|
47 | 69 |
} |
... | ... |
@@ -178,6 +178,9 @@ func (mm *MountManager) getSSHMountable(ctx context.Context, m *pb.Mount, g sess |
178 | 178 |
if err != nil { |
179 | 179 |
return nil, err |
180 | 180 |
} |
181 |
+ if caller == nil { |
|
182 |
+ return nil, nil |
|
183 |
+ } |
|
181 | 184 |
// because ssh socket remains active, to actually handle session disconnecting ssh error |
182 | 185 |
// should restart the whole exec with new session |
183 | 186 |
return &sshMount{mount: m, caller: caller, idmap: mm.cm.IdentityMapping()}, nil |
... | ... |
@@ -16,7 +16,6 @@ import ( |
16 | 16 |
"github.com/moby/buildkit/frontend/gateway" |
17 | 17 |
"github.com/moby/buildkit/session" |
18 | 18 |
"github.com/moby/buildkit/solver" |
19 |
- "github.com/moby/buildkit/solver/llbsolver/errdefs" |
|
20 | 19 |
"github.com/moby/buildkit/util/compression" |
21 | 20 |
"github.com/moby/buildkit/util/entitlements" |
22 | 21 |
"github.com/moby/buildkit/util/progress" |
... | ... |
@@ -143,20 +142,6 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro |
143 | 143 |
res.EachRef(func(ref solver.ResultProxy) error { |
144 | 144 |
eg.Go(func() error { |
145 | 145 |
_, err := ref.Result(ctx2) |
146 |
- if err != nil { |
|
147 |
- // Also release any results referenced by exec errors. |
|
148 |
- var ee *errdefs.ExecError |
|
149 |
- if errors.As(err, &ee) { |
|
150 |
- ee.EachRef(func(res solver.Result) error { |
|
151 |
- |
|
152 |
- workerRef, ok := res.Sys().(*worker.WorkerRef) |
|
153 |
- if !ok { |
|
154 |
- return nil |
|
155 |
- } |
|
156 |
- return workerRef.ImmutableRef.Release(ctx) |
|
157 |
- }) |
|
158 |
- } |
|
159 |
- } |
|
160 | 146 |
return err |
161 | 147 |
}) |
162 | 148 |
return nil |
... | ... |
@@ -130,9 +130,13 @@ func (s *scheduler) dispatch(e *edge) { |
130 | 130 |
pf := &pipeFactory{s: s, e: e} |
131 | 131 |
|
132 | 132 |
// unpark the edge |
133 |
- debugSchedulerPreUnpark(e, inc, updates, out) |
|
133 |
+ if debugScheduler { |
|
134 |
+ debugSchedulerPreUnpark(e, inc, updates, out) |
|
135 |
+ } |
|
134 | 136 |
e.unpark(inc, updates, out, pf) |
135 |
- debugSchedulerPostUnpark(e, inc) |
|
137 |
+ if debugScheduler { |
|
138 |
+ debugSchedulerPostUnpark(e, inc) |
|
139 |
+ } |
|
136 | 140 |
|
137 | 141 |
postUnpark: |
138 | 142 |
// set up new requests that didn't complete/were added by this run |
... | ... |
@@ -361,9 +365,6 @@ func (pf *pipeFactory) NewFuncRequest(f func(context.Context) (interface{}, erro |
361 | 361 |
} |
362 | 362 |
|
363 | 363 |
func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pipe.Receiver) { |
364 |
- if !debugScheduler { |
|
365 |
- return |
|
366 |
- } |
|
367 | 364 |
logrus.Debugf(">> unpark %s req=%d upt=%d out=%d state=%s %s", e.edge.Vertex.Name(), len(inc), len(updates), len(allPipes), e.state, e.edge.Vertex.Digest()) |
368 | 365 |
|
369 | 366 |
for i, dep := range e.deps { |
... | ... |
@@ -371,7 +372,7 @@ func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pip |
371 | 371 |
if dep.req != nil { |
372 | 372 |
des = dep.req.Request().(*edgeRequest).desiredState |
373 | 373 |
} |
374 |
- logrus.Debugf(":: dep%d %s state=%s des=%s keys=%d hasslowcache=%v", i, e.edge.Vertex.Inputs()[i].Vertex.Name(), dep.state, des, len(dep.keys), e.slowCacheFunc(dep) != nil) |
|
374 |
+ logrus.Debugf(":: dep%d %s state=%s des=%s keys=%d hasslowcache=%v preprocessfunc=%v", i, e.edge.Vertex.Inputs()[i].Vertex.Name(), dep.state, des, len(dep.keys), e.slowCacheFunc(dep) != nil, e.preprocessFunc(dep) != nil) |
|
375 | 375 |
} |
376 | 376 |
|
377 | 377 |
for i, in := range inc { |
... | ... |
@@ -400,9 +401,6 @@ func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pip |
400 | 400 |
} |
401 | 401 |
|
402 | 402 |
func debugSchedulerPostUnpark(e *edge, inc []pipe.Sender) { |
403 |
- if !debugScheduler { |
|
404 |
- return |
|
405 |
- } |
|
406 | 403 |
for i, in := range inc { |
407 | 404 |
logrus.Debugf("< incoming-%d: %p completed=%v", i, in, in.Status().Completed) |
408 | 405 |
} |