Browse code

vendor: github.com/moby/buildkit v0.10.4

release notes: https://github.com/moby/buildkit/releases/tag/v0.10.4

full diff: https://github.com/moby/buildkit/compare/8e2d9b9006ca...v0.10.4

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Sebastiaan van Stijn authored on 2022/08/24 23:52:51
Showing 13 changed files
... ...
@@ -50,7 +50,7 @@ require (
50 50
 	github.com/klauspost/compress v1.15.1
51 51
 	github.com/miekg/dns v1.1.27
52 52
 	github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible
53
-	github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca // v0.10 branch
53
+	github.com/moby/buildkit v0.10.4
54 54
 	github.com/moby/ipvs v1.0.2
55 55
 	github.com/moby/locker v1.0.1
56 56
 	github.com/moby/swarmkit/v2 v2.0.0-20220721174824-48dd89375d0a
... ...
@@ -753,8 +753,8 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
753 753
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
754 754
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
755 755
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
756
-github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca h1:w/07TXrn/GqvjmFoWv8ISHa+Z3YLCDrErIOmi9JeFbE=
757
-github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca/go.mod h1:hSExepqMIdfcKis7f7V1YzweBB7zt9DnlK+PUDCKuSI=
756
+github.com/moby/buildkit v0.10.4 h1:FvC+buO8isGpUFZ1abdSLdGHZVqg9sqI4BbFL8tlzP4=
757
+github.com/moby/buildkit v0.10.4/go.mod h1:Yajz9vt1Zw5q9Pp4pdb3TCSUXJBIroIQGQ3TTs/sLug=
758 758
 github.com/moby/ipvs v1.0.2 h1:NSbzuRTvfneftLU3VwPU5QuA6NZ0IUmqq9+VHcQxqHw=
759 759
 github.com/moby/ipvs v1.0.2/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
760 760
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
... ...
@@ -1678,8 +1678,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
1678 1678
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
1679 1679
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
1680 1680
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
1681
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
1682 1681
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
1682
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
1683 1683
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
1684 1684
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
1685 1685
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
... ...
@@ -7,7 +7,6 @@ import (
7 7
 	"path/filepath"
8 8
 
9 9
 	"github.com/docker/docker/libnetwork/resolvconf"
10
-	"github.com/docker/docker/libnetwork/types"
11 10
 	"github.com/docker/docker/pkg/idtools"
12 11
 	"github.com/moby/buildkit/util/flightcontrol"
13 12
 	"github.com/pkg/errors"
... ...
@@ -74,7 +73,7 @@ func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.Identity
74 74
 
75 75
 		if dns != nil {
76 76
 			var (
77
-				dnsNameservers   = resolvconf.GetNameservers(dt, types.IP)
77
+				dnsNameservers   = resolvconf.GetNameservers(dt, resolvconf.IP)
78 78
 				dnsSearchDomains = resolvconf.GetSearchDomains(dt)
79 79
 				dnsOptions       = resolvconf.GetOptions(dt)
80 80
 			)
... ...
@@ -463,7 +463,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
463 463
 						}
464 464
 						c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
465 465
 					},
466
-					ContextByName: contextByNameFunc(c, tp),
466
+					ContextByName: contextByNameFunc(c),
467 467
 				})
468 468
 
469 469
 				if err != nil {
... ...
@@ -812,8 +812,8 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
812 812
 	return opts
813 813
 }
814 814
 
815
-func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
816
-	return func(ctx context.Context, name, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
815
+func contextByNameFunc(c client.Client) func(context.Context, string, string, *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
816
+	return func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
817 817
 		named, err := reference.ParseNormalizedNamed(name)
818 818
 		if err != nil {
819 819
 			return nil, nil, nil, errors.Wrapf(err, "invalid context name %s", name)
... ...
@@ -879,6 +879,7 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
879 879
 		if err := json.Unmarshal(data, &img); err != nil {
880 880
 			return nil, nil, nil, err
881 881
 		}
882
+		img.Created = nil
882 883
 
883 884
 		st := llb.Image(ref, imgOpt...)
884 885
 		st, err = st.WithImageConfig(data)
... ...
@@ -70,16 +70,19 @@ type ConvertOpt struct {
70 70
 	SourceMap         *llb.SourceMap
71 71
 	Hostname          string
72 72
 	Warn              func(short, url string, detail [][]byte, location *parser.Range)
73
-	ContextByName     func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error)
73
+	ContextByName     func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error)
74 74
 }
75 75
 
76 76
 func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
77 77
 	buildInfo := &binfotypes.BuildInfo{}
78 78
 	contextByName := opt.ContextByName
79
-	opt.ContextByName = func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
79
+	opt.ContextByName = func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
80 80
 		if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
81 81
 			if contextByName != nil {
82
-				st, img, bi, err := contextByName(ctx, name, resolveMode)
82
+				if p == nil {
83
+					p = opt.TargetPlatform
84
+				}
85
+				st, img, bi, err := contextByName(ctx, name, resolveMode, p)
83 86
 				if err != nil {
84 87
 					return nil, nil, nil, err
85 88
 				}
... ...
@@ -165,8 +168,21 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
165 165
 			prefixPlatform: opt.PrefixPlatform,
166 166
 		}
167 167
 
168
+		if v := st.Platform; v != "" {
169
+			v, err := shlex.ProcessWordWithMap(v, metaArgsToMap(optMetaArgs))
170
+			if err != nil {
171
+				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location)
172
+			}
173
+
174
+			p, err := platforms.Parse(v)
175
+			if err != nil {
176
+				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location)
177
+			}
178
+			ds.platform = &p
179
+		}
180
+
168 181
 		if st.Name != "" {
169
-			s, img, bi, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String())
182
+			s, img, bi, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String(), ds.platform)
170 183
 			if err != nil {
171 184
 				return nil, nil, nil, err
172 185
 			}
... ...
@@ -195,18 +211,6 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
195 195
 			ds.stageName = fmt.Sprintf("stage-%d", i)
196 196
 		}
197 197
 
198
-		if v := st.Platform; v != "" {
199
-			v, err := shlex.ProcessWordWithMap(v, metaArgsToMap(optMetaArgs))
200
-			if err != nil {
201
-				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location)
202
-			}
203
-
204
-			p, err := platforms.Parse(v)
205
-			if err != nil {
206
-				return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location)
207
-			}
208
-			ds.platform = &p
209
-		}
210 198
 		allDispatchStates.addState(ds)
211 199
 
212 200
 		total := 0
... ...
@@ -313,7 +317,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
313 313
 					d.stage.BaseName = reference.TagNameOnly(ref).String()
314 314
 
315 315
 					var isScratch bool
316
-					st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String())
316
+					st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String(), platform)
317 317
 					if err != nil {
318 318
 						return err
319 319
 					}
... ...
@@ -361,6 +361,8 @@ func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartReques
361 361
 }
362 362
 
363 363
 func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
364
+	gwCtr.mu.Lock()
365
+	defer gwCtr.mu.Unlock()
364 366
 	gwCtr.cancel()
365 367
 	err1 := gwCtr.errGroup.Wait()
366 368
 
... ...
@@ -371,6 +373,7 @@ func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
371 371
 			err2 = err
372 372
 		}
373 373
 	}
374
+	gwCtr.cleanup = nil
374 375
 
375 376
 	if err1 != nil {
376 377
 		return stack.Enable(err1)
... ...
@@ -52,6 +52,7 @@ type bridgeClient struct {
52 52
 	workers       worker.Infos
53 53
 	workerRefByID map[string]*worker.WorkerRef
54 54
 	buildOpts     client.BuildOpts
55
+	ctrs          []client.Container
55 56
 }
56 57
 
57 58
 func (c *bridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*client.Result, error) {
... ...
@@ -212,6 +213,10 @@ func (c *bridgeClient) toFrontendResult(r *client.Result) (*frontend.Result, err
212 212
 }
213 213
 
214 214
 func (c *bridgeClient) discard(err error) {
215
+	for _, ctr := range c.ctrs {
216
+		ctr.Release(context.TODO())
217
+	}
218
+
215 219
 	for id, workerRef := range c.workerRefByID {
216 220
 		workerRef.ImmutableRef.Release(context.TODO())
217 221
 		delete(c.workerRefByID, id)
... ...
@@ -300,6 +305,7 @@ func (c *bridgeClient) NewContainer(ctx context.Context, req client.NewContainer
300 300
 	if err != nil {
301 301
 		return nil, err
302 302
 	}
303
+	c.ctrs = append(c.ctrs, ctr)
303 304
 	return ctr, nil
304 305
 }
305 306
 
... ...
@@ -278,7 +278,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
278 278
 	err = w.Executor().Run(ctx, "", mountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)
279 279
 
280 280
 	if err != nil {
281
-		if errdefs.IsCanceled(err) && lbf.isErrServerClosed {
281
+		if errdefs.IsCanceled(ctx, err) && lbf.isErrServerClosed {
282 282
 			err = errors.Errorf("frontend grpc server closed unexpectedly")
283 283
 		}
284 284
 		// An existing error (set via Return rpc) takes
... ...
@@ -345,6 +345,13 @@ func (b *bindMount) IdentityMapping() *idtools.IdentityMapping {
345 345
 func (lbf *llbBridgeForwarder) Discard() {
346 346
 	lbf.mu.Lock()
347 347
 	defer lbf.mu.Unlock()
348
+
349
+	for ctr := range lbf.ctrs {
350
+		lbf.ReleaseContainer(context.TODO(), &pb.ReleaseContainerRequest{
351
+			ContainerID: ctr,
352
+		})
353
+	}
354
+
348 355
 	for id, workerRef := range lbf.workerRefByID {
349 356
 		workerRef.ImmutableRef.Release(context.TODO())
350 357
 		delete(lbf.workerRefByID, id)
... ...
@@ -2,6 +2,7 @@ package session
2 2
 
3 3
 import (
4 4
 	"context"
5
+	"math"
5 6
 	"net"
6 7
 	"sync/atomic"
7 8
 	"time"
... ...
@@ -10,6 +11,7 @@ import (
10 10
 	"github.com/moby/buildkit/util/bklog"
11 11
 	"github.com/moby/buildkit/util/grpcerrors"
12 12
 	"github.com/pkg/errors"
13
+	"github.com/sirupsen/logrus"
13 14
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
14 15
 	"go.opentelemetry.io/otel/trace"
15 16
 	"golang.org/x/net/http2"
... ...
@@ -79,21 +81,55 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func())
79 79
 	defer cancelConn()
80 80
 	defer cc.Close()
81 81
 
82
-	ticker := time.NewTicker(1 * time.Second)
82
+	ticker := time.NewTicker(5 * time.Second)
83 83
 	defer ticker.Stop()
84 84
 	healthClient := grpc_health_v1.NewHealthClient(cc)
85 85
 
86
+	failedBefore := false
87
+	consecutiveSuccessful := 0
88
+	defaultHealthcheckDuration := 30 * time.Second
89
+	lastHealthcheckDuration := time.Duration(0)
90
+
86 91
 	for {
87 92
 		select {
88 93
 		case <-ctx.Done():
89 94
 			return
90 95
 		case <-ticker.C:
91
-			ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
96
+			// This healthcheck can erroneously fail in some instances, such as receiving lots of data in a low-bandwidth scenario or too many concurrent builds.
97
+			// So, this healthcheck is purposely long, and can tolerate some failures on purpose.
98
+
99
+			healthcheckStart := time.Now()
100
+
101
+			timeout := time.Duration(math.Max(float64(defaultHealthcheckDuration), float64(lastHealthcheckDuration)*1.5))
102
+			ctx, cancel := context.WithTimeout(ctx, timeout)
92 103
 			_, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
93 104
 			cancel()
105
+
106
+			lastHealthcheckDuration = time.Since(healthcheckStart)
107
+			logFields := logrus.Fields{
108
+				"timeout":        timeout,
109
+				"actualDuration": lastHealthcheckDuration,
110
+			}
111
+
94 112
 			if err != nil {
95
-				return
113
+				if failedBefore {
114
+					bklog.G(ctx).Error("healthcheck failed fatally")
115
+					return
116
+				}
117
+
118
+				failedBefore = true
119
+				consecutiveSuccessful = 0
120
+				bklog.G(ctx).WithFields(logFields).Warn("healthcheck failed")
121
+			} else {
122
+				consecutiveSuccessful++
123
+
124
+				if consecutiveSuccessful >= 5 && failedBefore {
125
+					failedBefore = false
126
+					bklog.G(ctx).WithFields(logFields).Debug("reset healthcheck failure")
127
+				}
96 128
 			}
129
+
130
+			bklog.G(ctx).WithFields(logFields).Debug("healthcheck completed")
97 131
 		}
98 132
 	}
99 133
 }
... ...
@@ -3,11 +3,25 @@ package errdefs
3 3
 import (
4 4
 	"context"
5 5
 	"errors"
6
+	"strings"
6 7
 
7 8
 	"github.com/moby/buildkit/util/grpcerrors"
8 9
 	"google.golang.org/grpc/codes"
9 10
 )
10 11
 
11
-func IsCanceled(err error) bool {
12
-	return errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled
12
+func IsCanceled(ctx context.Context, err error) bool {
13
+	if errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled {
14
+		return true
15
+	}
16
+	// grpc does not set cancel correctly when stream gets cancelled and then Recv is called
17
+	if err != nil && ctx.Err() == context.Canceled {
18
+		// when this error comes from containerd it is not typed at all, just concatenated string
19
+		if strings.Contains(err.Error(), "EOF") {
20
+			return true
21
+		}
22
+		if strings.Contains(err.Error(), context.Canceled.Error()) {
23
+			return true
24
+		}
25
+	}
26
+	return false
13 27
 }
... ...
@@ -3,7 +3,6 @@ package solver
3 3
 import (
4 4
 	"context"
5 5
 	"fmt"
6
-	"strings"
7 6
 	"sync"
8 7
 	"time"
9 8
 
... ...
@@ -705,7 +704,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF
705 705
 		if err != nil {
706 706
 			select {
707 707
 			case <-ctx.Done():
708
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
708
+				if errdefs.IsCanceled(ctx, err) {
709 709
 					complete = false
710 710
 					releaseError(err)
711 711
 					err = errors.Wrap(ctx.Err(), err.Error())
... ...
@@ -771,7 +770,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp,
771 771
 		if err != nil {
772 772
 			select {
773 773
 			case <-ctx.Done():
774
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
774
+				if errdefs.IsCanceled(ctx, err) {
775 775
 					complete = false
776 776
 					releaseError(err)
777 777
 					err = errors.Wrap(ctx.Err(), err.Error())
... ...
@@ -810,8 +809,11 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
810 810
 	}
811 811
 	flightControlKey := "exec"
812 812
 	res, err := s.g.Do(ctx, flightControlKey, func(ctx context.Context) (ret interface{}, retErr error) {
813
-		if s.execRes != nil || s.execErr != nil {
814
-			return s.execRes, s.execErr
813
+		if s.execErr != nil {
814
+			return nil, s.execErr
815
+		}
816
+		if s.execRes != nil {
817
+			return s.execRes, nil
815 818
 		}
816 819
 		release, err := op.Acquire(ctx)
817 820
 		if err != nil {
... ...
@@ -838,7 +840,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
838 838
 		if err != nil {
839 839
 			select {
840 840
 			case <-ctx.Done():
841
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
841
+				if errdefs.IsCanceled(ctx, err) {
842 842
 					complete = false
843 843
 					releaseError(err)
844 844
 					err = errors.Wrap(ctx.Err(), err.Error())
... ...
@@ -859,9 +861,12 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
859 859
 			}
860 860
 			s.execErr = err
861 861
 		}
862
-		return s.execRes, err
862
+		if s.execRes == nil || err != nil {
863
+			return nil, err
864
+		}
865
+		return s.execRes, nil
863 866
 	})
864
-	if err != nil {
867
+	if res == nil || err != nil {
865 868
 		return nil, nil, err
866 869
 	}
867 870
 	r := res.(*execRes)
... ...
@@ -3,7 +3,6 @@ package llbsolver
3 3
 import (
4 4
 	"context"
5 5
 	"fmt"
6
-	"strings"
7 6
 	"sync"
8 7
 	"time"
9 8
 
... ...
@@ -290,7 +289,7 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err
290 290
 		if err != nil {
291 291
 			select {
292 292
 			case <-ctx.Done():
293
-				if strings.Contains(err.Error(), context.Canceled.Error()) {
293
+				if errdefs.IsCanceled(ctx, err) {
294 294
 					return v, err
295 295
 				}
296 296
 			default:
... ...
@@ -474,7 +474,7 @@ github.com/mistifyio/go-zfs
474 474
 # github.com/mitchellh/hashstructure/v2 v2.0.2
475 475
 ## explicit; go 1.14
476 476
 github.com/mitchellh/hashstructure/v2
477
-# github.com/moby/buildkit v0.10.4-0.20220719175648-8e2d9b9006ca
477
+# github.com/moby/buildkit v0.10.4
478 478
 ## explicit; go 1.17
479 479
 github.com/moby/buildkit/api/services/control
480 480
 github.com/moby/buildkit/api/types