- full diff: https://github.com/moby/buildkit/compare/v0.13.1...v0.14.0-rc2
Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
| ... | ... |
@@ -61,7 +61,7 @@ require ( |
| 61 | 61 |
github.com/miekg/dns v1.1.57 |
| 62 | 62 |
github.com/mistifyio/go-zfs/v3 v3.0.1 |
| 63 | 63 |
github.com/mitchellh/copystructure v1.2.0 |
| 64 |
- github.com/moby/buildkit v0.13.1 |
|
| 64 |
+ github.com/moby/buildkit v0.14.0-rc1.0.20240605195929-c1f5352a1b7e |
|
| 65 | 65 |
github.com/moby/docker-image-spec v1.3.1 |
| 66 | 66 |
github.com/moby/ipvs v1.1.0 |
| 67 | 67 |
github.com/moby/locker v1.0.1 |
| ... | ... |
@@ -88,7 +88,7 @@ require ( |
| 88 | 88 |
github.com/sirupsen/logrus v1.9.3 |
| 89 | 89 |
github.com/spf13/cobra v1.8.0 |
| 90 | 90 |
github.com/spf13/pflag v1.0.5 |
| 91 |
- github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 |
|
| 91 |
+ github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c |
|
| 92 | 92 |
github.com/tonistiigi/go-archvariant v1.0.0 |
| 93 | 93 |
github.com/vbatts/tar-split v0.11.5 |
| 94 | 94 |
github.com/vishvananda/netlink v1.2.1-beta.2 |
| ... | ... |
@@ -193,9 +193,9 @@ require ( |
| 193 | 193 |
github.com/stretchr/testify v1.8.4 // indirect |
| 194 | 194 |
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect |
| 195 | 195 |
github.com/tinylib/msgp v1.1.8 // indirect |
| 196 |
- github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 // indirect |
|
| 196 |
+ github.com/tonistiigi/go-actions-cache v0.0.0-20240320205438-9794bdbb2fb4 // indirect |
|
| 197 | 197 |
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect |
| 198 |
- github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 // indirect |
|
| 198 |
+ github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect |
|
| 199 | 199 |
github.com/weppos/publicsuffix-go v0.15.1-0.20210511084619-b1f36a2d6c0b // indirect |
| 200 | 200 |
github.com/zmap/zcrypto v0.0.0-20210511125630-18f1e0152cfc // indirect |
| 201 | 201 |
github.com/zmap/zlint/v3 v3.1.0 // indirect |
| ... | ... |
@@ -210,7 +210,6 @@ require ( |
| 210 | 210 |
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 // indirect |
| 211 | 211 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect |
| 212 | 212 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect |
| 213 |
- go.opentelemetry.io/otel/exporters/prometheus v0.42.0 // indirect |
|
| 214 | 213 |
go.opentelemetry.io/otel/metric v1.21.0 // indirect |
| 215 | 214 |
go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect |
| 216 | 215 |
go.opentelemetry.io/proto/otlp v1.0.0 // indirect |
| ... | ... |
@@ -480,8 +480,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh |
| 480 | 480 |
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= |
| 481 | 481 |
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= |
| 482 | 482 |
github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= |
| 483 |
-github.com/moby/buildkit v0.13.1 h1:L8afOFhPq2RPJJSr/VyzbufwID7jquZVB7oFHbPRcPE= |
|
| 484 |
-github.com/moby/buildkit v0.13.1/go.mod h1:aNmNQKLBFYAOFuzQjR3VA27/FijlvtBD1pjNwTSN37k= |
|
| 483 |
+github.com/moby/buildkit v0.14.0-rc1.0.20240605195929-c1f5352a1b7e h1:/qEvjulGGA7ykkCzK+5HHsA/TyFkwpDeOXQNYXbSYZc= |
|
| 484 |
+github.com/moby/buildkit v0.14.0-rc1.0.20240605195929-c1f5352a1b7e/go.mod h1:qeJJeU1GhOhzPJVn2opBGLjR005LpG5KlB8FXMoY46Y= |
|
| 485 | 485 |
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= |
| 486 | 486 |
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= |
| 487 | 487 |
github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= |
| ... | ... |
@@ -675,16 +675,16 @@ github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= |
| 675 | 675 |
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= |
| 676 | 676 |
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= |
| 677 | 677 |
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= |
| 678 |
-github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 h1:oZS8KCqAg62sxJkEq/Ppzqrb6EooqzWtL8Oaex7bc5c= |
|
| 679 |
-github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= |
|
| 680 |
-github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 h1:DA/NDC0YbMdnfcOSUzAnbUZE6dSM54d+0hrBqG+bOfs= |
|
| 681 |
-github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598/go.mod h1:anhKd3mnC1shAbQj1Q4IJ+w6xqezxnyDYlx/yKa7IXM= |
|
| 678 |
+github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c h1:+6wg/4ORAbnSoGDzg2Q1i3CeMcT/jjhye/ZfnBHy7/M= |
|
| 679 |
+github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= |
|
| 680 |
+github.com/tonistiigi/go-actions-cache v0.0.0-20240320205438-9794bdbb2fb4 h1:R0lM8Jo3aZL95yjQHWQti7nszllleKBxCs9uyFbykII= |
|
| 681 |
+github.com/tonistiigi/go-actions-cache v0.0.0-20240320205438-9794bdbb2fb4/go.mod h1:anhKd3mnC1shAbQj1Q4IJ+w6xqezxnyDYlx/yKa7IXM= |
|
| 682 | 682 |
github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0= |
| 683 | 683 |
github.com/tonistiigi/go-archvariant v1.0.0/go.mod h1:TxFmO5VS6vMq2kvs3ht04iPXtu2rUT/erOnGFYfk5Ho= |
| 684 | 684 |
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= |
| 685 | 685 |
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= |
| 686 |
-github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 h1:Y/M5lygoNPKwVNLMPXgVfsRT40CSFKXCxuU8LoHySjs= |
|
| 687 |
-github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= |
|
| 686 |
+github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Qb2z5hI1UHxSQt4JMyxebFR15KnApw= |
|
| 687 |
+github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= |
|
| 688 | 688 |
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= |
| 689 | 689 |
github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= |
| 690 | 690 |
github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= |
| ... | ... |
@@ -759,8 +759,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqhe |
| 759 | 759 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= |
| 760 | 760 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= |
| 761 | 761 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= |
| 762 |
-go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA= |
|
| 763 |
-go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA= |
|
| 764 | 762 |
go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= |
| 765 | 763 |
go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= |
| 766 | 764 |
go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= |
| ... | ... |
@@ -9,9 +9,9 @@ import ( |
| 9 | 9 |
"io" |
| 10 | 10 |
|
| 11 | 11 |
"github.com/containerd/containerd/content" |
| 12 |
- "github.com/containerd/containerd/errdefs" |
|
| 13 | 12 |
labelspkg "github.com/containerd/containerd/labels" |
| 14 | 13 |
"github.com/containerd/containerd/mount" |
| 14 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 15 | 15 |
"github.com/moby/buildkit/util/bklog" |
| 16 | 16 |
"github.com/moby/buildkit/util/compression" |
| 17 | 17 |
"github.com/moby/buildkit/util/overlay" |
| ... | ... |
@@ -80,10 +80,9 @@ func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper |
| 80 | 80 |
if err := compressed.Close(); err != nil {
|
| 81 | 81 |
return emptyDesc, false, errors.Wrap(err, "failed to close compressed diff writer") |
| 82 | 82 |
} |
| 83 |
- if labels == nil {
|
|
| 84 |
- labels = map[string]string{}
|
|
| 83 |
+ labels = map[string]string{
|
|
| 84 |
+ labelspkg.LabelUncompressed: dgstr.Digest().String(), |
|
| 85 | 85 |
} |
| 86 |
- labels[labelspkg.LabelUncompressed] = dgstr.Digest().String() |
|
| 87 | 86 |
} else {
|
| 88 | 87 |
if err = overlay.WriteUpperdir(ctx, bufW, upperdir, lower); err != nil {
|
| 89 | 88 |
return emptyDesc, false, errors.Wrap(err, "failed to write diff") |
| ... | ... |
@@ -99,7 +98,7 @@ func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper |
| 99 | 99 |
} |
| 100 | 100 |
dgst := cw.Digest() |
| 101 | 101 |
if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {
|
| 102 |
- if !errdefs.IsAlreadyExists(err) {
|
|
| 102 |
+ if !cerrdefs.IsAlreadyExists(err) {
|
|
| 103 | 103 |
return emptyDesc, false, errors.Wrap(err, "failed to commit") |
| 104 | 104 |
} |
| 105 | 105 |
} |
| ... | ... |
@@ -12,6 +12,6 @@ import ( |
| 12 | 12 |
"github.com/pkg/errors" |
| 13 | 13 |
) |
| 14 | 14 |
|
| 15 |
-func (sr *immutableRef) tryComputeOverlayBlob(ctx context.Context, lower, upper []mount.Mount, mediaType string, ref string, compressorFunc compression.Compressor) (_ ocispecs.Descriptor, ok bool, err error) {
|
|
| 15 |
+func (sr *immutableRef) tryComputeOverlayBlob(_ context.Context, _, _ []mount.Mount, _ string, _ string, _ compression.Compressor) (_ ocispecs.Descriptor, ok bool, err error) {
|
|
| 16 | 16 |
return ocispecs.Descriptor{}, true, errors.Errorf("overlayfs-based diff computing is unsupported")
|
| 17 | 17 |
} |
| ... | ... |
@@ -9,8 +9,8 @@ import ( |
| 9 | 9 |
"io" |
| 10 | 10 |
|
| 11 | 11 |
"github.com/containerd/containerd/content" |
| 12 |
- "github.com/containerd/containerd/errdefs" |
|
| 13 | 12 |
"github.com/containerd/containerd/labels" |
| 13 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 14 | 14 |
"github.com/moby/buildkit/session" |
| 15 | 15 |
"github.com/moby/buildkit/util/compression" |
| 16 | 16 |
digest "github.com/opencontainers/go-digest" |
| ... | ... |
@@ -95,7 +95,7 @@ func MergeNydus(ctx context.Context, ref ImmutableRef, comp compression.Config, |
| 95 | 95 |
if err := cw.Commit(ctx, 0, compressedDgst, content.WithLabels(map[string]string{
|
| 96 | 96 |
labels.LabelUncompressed: uncompressedDgst.Digest().String(), |
| 97 | 97 |
})); err != nil {
|
| 98 |
- if !errdefs.IsAlreadyExists(err) {
|
|
| 98 |
+ if !cerrdefs.IsAlreadyExists(err) {
|
|
| 99 | 99 |
return nil, errors.Wrap(err, "commit to content store") |
| 100 | 100 |
} |
| 101 | 101 |
} |
| ... | ... |
@@ -290,7 +290,7 @@ func keyPath(p string) string {
|
| 290 | 290 |
// HandleChange notifies the source about a modification operation |
| 291 | 291 |
func (cc *cacheContext) HandleChange(kind fsutil.ChangeKind, p string, fi os.FileInfo, err error) (retErr error) {
|
| 292 | 292 |
p = keyPath(p) |
| 293 |
- k := convertPathToKey([]byte(p)) |
|
| 293 |
+ k := convertPathToKey(p) |
|
| 294 | 294 |
|
| 295 | 295 |
deleteDir := func(cr *CacheRecord) {
|
| 296 | 296 |
if cr.Type == CacheRecordTypeDir {
|
| ... | ... |
@@ -369,7 +369,7 @@ func (cc *cacheContext) HandleChange(kind fsutil.ChangeKind, p string, fi os.Fil |
| 369 | 369 |
// note that the source may be called later because data writing is async |
| 370 | 370 |
if fi.Mode()&os.ModeSymlink == 0 && stat.Linkname != "" {
|
| 371 | 371 |
ln := path.Join("/", filepath.ToSlash(stat.Linkname))
|
| 372 |
- v, ok := cc.txn.Get(convertPathToKey([]byte(ln))) |
|
| 372 |
+ v, ok := cc.txn.Get(convertPathToKey(ln)) |
|
| 373 | 373 |
if ok {
|
| 374 | 374 |
cp := *v.(*CacheRecord) |
| 375 | 375 |
cr = &cp |
| ... | ... |
@@ -536,7 +536,7 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o |
| 536 | 536 |
} |
| 537 | 537 |
} else {
|
| 538 | 538 |
origPrefix = p |
| 539 |
- k = convertPathToKey([]byte(origPrefix)) |
|
| 539 |
+ k = convertPathToKey(origPrefix) |
|
| 540 | 540 |
|
| 541 | 541 |
// We need to resolve symlinks here, in case the base path |
| 542 | 542 |
// involves a symlink. That will match fsutil behavior of |
| ... | ... |
@@ -554,7 +554,7 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o |
| 554 | 554 |
iter.SeekLowerBound(append(append([]byte{}, k...), 0))
|
| 555 | 555 |
} |
| 556 | 556 |
|
| 557 |
- resolvedPrefix = string(convertKeyToPath(k)) |
|
| 557 |
+ resolvedPrefix = convertKeyToPath(k) |
|
| 558 | 558 |
} else {
|
| 559 | 559 |
k, _, keyOk = iter.Next() |
| 560 | 560 |
} |
| ... | ... |
@@ -565,7 +565,7 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o |
| 565 | 565 |
) |
| 566 | 566 |
|
| 567 | 567 |
for keyOk {
|
| 568 |
- fn := string(convertKeyToPath(k)) |
|
| 568 |
+ fn := convertKeyToPath(k) |
|
| 569 | 569 |
|
| 570 | 570 |
// Convert the path prefix from what we found in the prefix |
| 571 | 571 |
// tree to what the argument specified. |
| ... | ... |
@@ -752,7 +752,7 @@ func wildcardPrefix(root *iradix.Node, p string) (string, []byte, bool, error) {
|
| 752 | 752 |
} |
| 753 | 753 |
|
| 754 | 754 |
linksWalked := 0 |
| 755 |
- k, cr, err := getFollowLinksWalk(root, convertPathToKey([]byte(d1)), true, &linksWalked) |
|
| 755 |
+ k, cr, err := getFollowLinksWalk(root, convertPathToKey(d1), true, &linksWalked) |
|
| 756 | 756 |
if err != nil {
|
| 757 | 757 |
return "", k, false, err |
| 758 | 758 |
} |
| ... | ... |
@@ -761,7 +761,7 @@ func wildcardPrefix(root *iradix.Node, p string) (string, []byte, bool, error) {
|
| 761 | 761 |
// getFollowLinks only handles symlinks in path |
| 762 | 762 |
// components before the last component, so |
| 763 | 763 |
// handle last component in d1 specially. |
| 764 |
- resolved := string(convertKeyToPath(k)) |
|
| 764 |
+ resolved := convertKeyToPath(k) |
|
| 765 | 765 |
for {
|
| 766 | 766 |
v, ok := root.Get(k) |
| 767 | 767 |
|
| ... | ... |
@@ -778,7 +778,7 @@ func wildcardPrefix(root *iradix.Node, p string) (string, []byte, bool, error) {
|
| 778 | 778 |
} |
| 779 | 779 |
|
| 780 | 780 |
resolved := cleanLink(resolved, v.(*CacheRecord).Linkname) |
| 781 |
- k = convertPathToKey([]byte(resolved)) |
|
| 781 |
+ k = convertPathToKey(resolved) |
|
| 782 | 782 |
} |
| 783 | 783 |
} |
| 784 | 784 |
return d1, k, cr != nil, nil |
| ... | ... |
@@ -823,7 +823,7 @@ func (cc *cacheContext) checksumNoFollow(ctx context.Context, m *mount, p string |
| 823 | 823 |
if cc.txn == nil {
|
| 824 | 824 |
root := cc.tree.Root() |
| 825 | 825 |
cc.mu.RUnlock() |
| 826 |
- v, ok := root.Get(convertPathToKey([]byte(p))) |
|
| 826 |
+ v, ok := root.Get(convertPathToKey(p)) |
|
| 827 | 827 |
if ok {
|
| 828 | 828 |
cr := v.(*CacheRecord) |
| 829 | 829 |
if cr.Digest != "" {
|
| ... | ... |
@@ -856,7 +856,7 @@ func (cc *cacheContext) commitActiveTransaction() {
|
| 856 | 856 |
addParentToMap(d, cc.dirtyMap) |
| 857 | 857 |
} |
| 858 | 858 |
for d := range cc.dirtyMap {
|
| 859 |
- k := convertPathToKey([]byte(d)) |
|
| 859 |
+ k := convertPathToKey(d) |
|
| 860 | 860 |
if _, ok := cc.txn.Get(k); ok {
|
| 861 | 861 |
cc.txn.Insert(k, &CacheRecord{Type: CacheRecordTypeDir})
|
| 862 | 862 |
} |
| ... | ... |
@@ -878,7 +878,7 @@ func (cc *cacheContext) lazyChecksum(ctx context.Context, m *mount, p string) (* |
| 878 | 878 |
return nil, err |
| 879 | 879 |
} |
| 880 | 880 |
} |
| 881 |
- k := convertPathToKey([]byte(p)) |
|
| 881 |
+ k := convertPathToKey(p) |
|
| 882 | 882 |
txn := cc.tree.Txn() |
| 883 | 883 |
root = txn.Root() |
| 884 | 884 |
cr, updated, err := cc.checksum(ctx, root, txn, m, k, true) |
| ... | ... |
@@ -935,7 +935,7 @@ func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node, txn *ir |
| 935 | 935 |
dgst = digest.NewDigest(digest.SHA256, h) |
| 936 | 936 |
|
| 937 | 937 |
default: |
| 938 |
- p := string(convertKeyToPath(bytes.TrimSuffix(k, []byte{0})))
|
|
| 938 |
+ p := convertKeyToPath(bytes.TrimSuffix(k, []byte{0}))
|
|
| 939 | 939 |
|
| 940 | 940 |
target, err := m.mount(ctx) |
| 941 | 941 |
if err != nil {
|
| ... | ... |
@@ -978,7 +978,7 @@ func (cc *cacheContext) needsScanFollow(root *iradix.Node, p string, linksWalked |
| 978 | 978 |
if p == "/" {
|
| 979 | 979 |
p = "" |
| 980 | 980 |
} |
| 981 |
- v, ok := root.Get(convertPathToKey([]byte(p))) |
|
| 981 |
+ v, ok := root.Get(convertPathToKey(p)) |
|
| 982 | 982 |
if !ok {
|
| 983 | 983 |
if p == "" {
|
| 984 | 984 |
return true, nil |
| ... | ... |
@@ -1017,9 +1017,8 @@ func (cc *cacheContext) scanPath(ctx context.Context, m *mount, p string) (retEr |
| 1017 | 1017 |
Type: CacheRecordTypeSymlink, |
| 1018 | 1018 |
Linkname: filepath.ToSlash(link), |
| 1019 | 1019 |
} |
| 1020 |
- k := []byte(path.Join("/", filepath.ToSlash(p)))
|
|
| 1021 |
- k = convertPathToKey(k) |
|
| 1022 |
- txn.Insert(k, cr) |
|
| 1020 |
+ p = path.Join("/", filepath.ToSlash(p))
|
|
| 1021 |
+ txn.Insert(convertPathToKey(p), cr) |
|
| 1023 | 1022 |
return nil |
| 1024 | 1023 |
}) |
| 1025 | 1024 |
if err != nil {
|
| ... | ... |
@@ -1034,11 +1033,11 @@ func (cc *cacheContext) scanPath(ctx context.Context, m *mount, p string) (retEr |
| 1034 | 1034 |
if err != nil {
|
| 1035 | 1035 |
return err |
| 1036 | 1036 |
} |
| 1037 |
- k := []byte(path.Join("/", filepath.ToSlash(rel)))
|
|
| 1038 |
- if string(k) == "/" {
|
|
| 1039 |
- k = []byte{}
|
|
| 1037 |
+ p := path.Join("/", filepath.ToSlash(rel))
|
|
| 1038 |
+ if p == "/" {
|
|
| 1039 |
+ p = "" |
|
| 1040 | 1040 |
} |
| 1041 |
- k = convertPathToKey(k) |
|
| 1041 |
+ k := convertPathToKey(p) |
|
| 1042 | 1042 |
if _, ok := n.Get(k); !ok {
|
| 1043 | 1043 |
cr := &CacheRecord{
|
| 1044 | 1044 |
Type: CacheRecordTypeFile, |
| ... | ... |
@@ -1098,8 +1097,8 @@ func getFollowLinksWalk(root *iradix.Node, k []byte, follow bool, linksWalked *i |
| 1098 | 1098 |
return nil, nil, errors.Errorf("too many links")
|
| 1099 | 1099 |
} |
| 1100 | 1100 |
|
| 1101 |
- link := cleanLink(string(convertKeyToPath(dir)), parent.Linkname) |
|
| 1102 |
- return getFollowLinksWalk(root, append(convertPathToKey([]byte(link)), file...), follow, linksWalked) |
|
| 1101 |
+ link := cleanLink(convertKeyToPath(dir), parent.Linkname) |
|
| 1102 |
+ return getFollowLinksWalk(root, append(convertPathToKey(link), file...), follow, linksWalked) |
|
| 1103 | 1103 |
} |
| 1104 | 1104 |
} |
| 1105 | 1105 |
k = append(k, file...) |
| ... | ... |
@@ -1176,12 +1175,12 @@ func poolsCopy(dst io.Writer, src io.Reader) (written int64, err error) {
|
| 1176 | 1176 |
return |
| 1177 | 1177 |
} |
| 1178 | 1178 |
|
| 1179 |
-func convertPathToKey(p []byte) []byte {
|
|
| 1180 |
- return bytes.Replace([]byte(p), []byte("/"), []byte{0}, -1)
|
|
| 1179 |
+func convertPathToKey(p string) []byte {
|
|
| 1180 |
+ return bytes.ReplaceAll([]byte(p), []byte("/"), []byte{0})
|
|
| 1181 | 1181 |
} |
| 1182 | 1182 |
|
| 1183 |
-func convertKeyToPath(p []byte) []byte {
|
|
| 1184 |
- return bytes.Replace([]byte(p), []byte{0}, []byte("/"), -1)
|
|
| 1183 |
+func convertKeyToPath(p []byte) string {
|
|
| 1184 |
+ return string(bytes.ReplaceAll(p, []byte{0}, []byte("/")))
|
|
| 1185 | 1185 |
} |
| 1186 | 1186 |
|
| 1187 | 1187 |
func splitKey(k []byte) ([]byte, []byte) {
|
| ... | ... |
@@ -10,11 +10,11 @@ import ( |
| 10 | 10 |
|
| 11 | 11 |
"github.com/containerd/containerd/content" |
| 12 | 12 |
"github.com/containerd/containerd/diff" |
| 13 |
- "github.com/containerd/containerd/errdefs" |
|
| 14 | 13 |
"github.com/containerd/containerd/filters" |
| 15 | 14 |
"github.com/containerd/containerd/gc" |
| 16 | 15 |
"github.com/containerd/containerd/labels" |
| 17 | 16 |
"github.com/containerd/containerd/leases" |
| 17 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 18 | 18 |
"github.com/docker/docker/pkg/idtools" |
| 19 | 19 |
"github.com/moby/buildkit/cache/metadata" |
| 20 | 20 |
"github.com/moby/buildkit/client" |
| ... | ... |
@@ -137,7 +137,7 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor, |
| 137 | 137 |
|
| 138 | 138 |
descHandlers := descHandlersOf(opts...) |
| 139 | 139 |
if desc.Digest != "" && (descHandlers == nil || descHandlers[desc.Digest] == nil) {
|
| 140 |
- if _, err := cm.ContentStore.Info(ctx, desc.Digest); errors.Is(err, errdefs.ErrNotFound) {
|
|
| 140 |
+ if _, err := cm.ContentStore.Info(ctx, desc.Digest); errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 141 | 141 |
return nil, NeedsRemoteProviderError([]digest.Digest{desc.Digest})
|
| 142 | 142 |
} else if err != nil {
|
| 143 | 143 |
return nil, err |
| ... | ... |
@@ -253,7 +253,7 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor, |
| 253 | 253 |
if err := cm.LeaseManager.AddResource(ctx, l, leases.Resource{
|
| 254 | 254 |
ID: snapshotID, |
| 255 | 255 |
Type: "snapshots/" + cm.Snapshotter.Name(), |
| 256 |
- }); err != nil && !errdefs.IsAlreadyExists(err) {
|
|
| 256 |
+ }); err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 257 | 257 |
return nil, errors.Wrapf(err, "failed to add snapshot %s to lease", id) |
| 258 | 258 |
} |
| 259 | 259 |
|
| ... | ... |
@@ -374,7 +374,7 @@ func (cm *cacheManager) get(ctx context.Context, id string, pg progress.Controll |
| 374 | 374 |
if rec.equalImmutable != nil {
|
| 375 | 375 |
return rec.equalImmutable.ref(triggerUpdate, descHandlers, pg), nil |
| 376 | 376 |
} |
| 377 |
- return rec.mref(triggerUpdate, descHandlers).commit(ctx) |
|
| 377 |
+ return rec.mref(triggerUpdate, descHandlers).commit() |
|
| 378 | 378 |
} |
| 379 | 379 |
|
| 380 | 380 |
return rec.ref(triggerUpdate, descHandlers, pg), nil |
| ... | ... |
@@ -484,7 +484,7 @@ func (cm *cacheManager) getRecord(ctx context.Context, id string, opts ...RefOpt |
| 484 | 484 |
if rec.mutable {
|
| 485 | 485 |
// If the record is mutable, then the snapshot must exist |
| 486 | 486 |
if _, err := cm.Snapshotter.Stat(ctx, rec.ID()); err != nil {
|
| 487 |
- if !errdefs.IsNotFound(err) {
|
|
| 487 |
+ if !cerrdefs.IsNotFound(err) {
|
|
| 488 | 488 |
return nil, errors.Wrap(err, "failed to check mutable ref snapshot") |
| 489 | 489 |
} |
| 490 | 490 |
// the snapshot doesn't exist, clear this record |
| ... | ... |
@@ -609,7 +609,7 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, sess session.Gr |
| 609 | 609 |
if err := cm.LeaseManager.AddResource(ctx, l, leases.Resource{
|
| 610 | 610 |
ID: snapshotID, |
| 611 | 611 |
Type: "snapshots/" + cm.Snapshotter.Name(), |
| 612 |
- }); err != nil && !errdefs.IsAlreadyExists(err) {
|
|
| 612 |
+ }); err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 613 | 613 |
return nil, errors.Wrapf(err, "failed to add snapshot %s to lease", snapshotID) |
| 614 | 614 |
} |
| 615 | 615 |
|
| ... | ... |
@@ -1109,10 +1109,11 @@ func (cm *cacheManager) prune(ctx context.Context, ch chan client.UsageInfo, opt |
| 1109 | 1109 |
} |
| 1110 | 1110 |
|
| 1111 | 1111 |
c := &client.UsageInfo{
|
| 1112 |
- ID: cr.ID(), |
|
| 1113 |
- Mutable: cr.mutable, |
|
| 1114 |
- RecordType: recordType, |
|
| 1115 |
- Shared: shared, |
|
| 1112 |
+ ID: cr.ID(), |
|
| 1113 |
+ Mutable: cr.mutable, |
|
| 1114 |
+ RecordType: recordType, |
|
| 1115 |
+ Shared: shared, |
|
| 1116 |
+ Description: cr.GetDescription(), |
|
| 1116 | 1117 |
} |
| 1117 | 1118 |
|
| 1118 | 1119 |
usageCount, lastUsedAt := cr.getLastUsed() |
| ... | ... |
@@ -7,10 +7,10 @@ import ( |
| 7 | 7 |
"time" |
| 8 | 8 |
|
| 9 | 9 |
"github.com/containerd/containerd/content" |
| 10 |
- "github.com/containerd/containerd/errdefs" |
|
| 11 | 10 |
"github.com/containerd/containerd/images" |
| 12 | 11 |
"github.com/containerd/containerd/leases" |
| 13 | 12 |
"github.com/containerd/containerd/snapshots" |
| 13 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 14 | 14 |
"github.com/moby/buildkit/cache/metadata" |
| 15 | 15 |
"github.com/moby/buildkit/snapshot" |
| 16 | 16 |
"github.com/moby/buildkit/util/bklog" |
| ... | ... |
@@ -185,7 +185,7 @@ func MigrateV2(ctx context.Context, from, to string, cs content.Store, s snapsho |
| 185 | 185 |
}) |
| 186 | 186 |
if err != nil {
|
| 187 | 187 |
// if we are running the migration twice |
| 188 |
- if errors.Is(err, errdefs.ErrAlreadyExists) {
|
|
| 188 |
+ if errors.Is(err, cerrdefs.ErrAlreadyExists) {
|
|
| 189 | 189 |
continue |
| 190 | 190 |
} |
| 191 | 191 |
return errors.Wrap(err, "failed to create lease") |
| ... | ... |
@@ -216,7 +216,7 @@ func MigrateV2(ctx context.Context, from, to string, cs content.Store, s snapsho |
| 216 | 216 |
if _, err := s.Update(ctx, snapshots.Info{
|
| 217 | 217 |
Name: md.getSnapshotID(), |
| 218 | 218 |
}, "labels.containerd.io/gc.root"); err != nil {
|
| 219 |
- if !errors.Is(err, errdefs.ErrNotFound) {
|
|
| 219 |
+ if !errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 220 | 220 |
return err |
| 221 | 221 |
} |
| 222 | 222 |
} |
| ... | ... |
@@ -237,7 +237,7 @@ func MigrateV2(ctx context.Context, from, to string, cs content.Store, s snapsho |
| 237 | 237 |
if _, err := s.Update(ctx, snapshots.Info{
|
| 238 | 238 |
Name: info.Name, |
| 239 | 239 |
}, "labels.containerd.io/gc.root"); err != nil {
|
| 240 |
- if !errors.Is(err, errdefs.ErrNotFound) {
|
|
| 240 |
+ if !errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 241 | 241 |
return err |
| 242 | 242 |
} |
| 243 | 243 |
} |
| ... | ... |
@@ -10,13 +10,13 @@ import ( |
| 10 | 10 |
"time" |
| 11 | 11 |
|
| 12 | 12 |
"github.com/containerd/containerd/content" |
| 13 |
- "github.com/containerd/containerd/errdefs" |
|
| 14 | 13 |
"github.com/containerd/containerd/images" |
| 15 | 14 |
"github.com/containerd/containerd/labels" |
| 16 | 15 |
"github.com/containerd/containerd/leases" |
| 17 | 16 |
"github.com/containerd/containerd/mount" |
| 18 | 17 |
"github.com/containerd/containerd/pkg/userns" |
| 19 | 18 |
"github.com/containerd/containerd/snapshots" |
| 19 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 20 | 20 |
"github.com/docker/docker/pkg/idtools" |
| 21 | 21 |
"github.com/hashicorp/go-multierror" |
| 22 | 22 |
"github.com/moby/buildkit/cache/config" |
| ... | ... |
@@ -292,7 +292,7 @@ func (cr *cacheRecord) isLazy(ctx context.Context) (bool, error) {
|
| 292 | 292 |
return false, nil |
| 293 | 293 |
} |
| 294 | 294 |
_, err := cr.cm.ContentStore.Info(ctx, dgst) |
| 295 |
- if errors.Is(err, errdefs.ErrNotFound) {
|
|
| 295 |
+ if errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 296 | 296 |
return true, nil |
| 297 | 297 |
} else if err != nil {
|
| 298 | 298 |
return false, err |
| ... | ... |
@@ -349,7 +349,7 @@ func (cr *cacheRecord) size(ctx context.Context) (int64, error) {
|
| 349 | 349 |
if isDead {
|
| 350 | 350 |
return 0, nil |
| 351 | 351 |
} |
| 352 |
- if !errors.Is(err, errdefs.ErrNotFound) {
|
|
| 352 |
+ if !errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 353 | 353 |
return s, errors.Wrapf(err, "failed to get usage for %s", cr.ID()) |
| 354 | 354 |
} |
| 355 | 355 |
} |
| ... | ... |
@@ -383,7 +383,7 @@ func (cr *cacheRecord) size(ctx context.Context) (int64, error) {
|
| 383 | 383 |
} |
| 384 | 384 |
|
| 385 | 385 |
// caller must hold cr.mu |
| 386 |
-func (cr *cacheRecord) mount(ctx context.Context, s session.Group) (_ snapshot.Mountable, rerr error) {
|
|
| 386 |
+func (cr *cacheRecord) mount(ctx context.Context) (_ snapshot.Mountable, rerr error) {
|
|
| 387 | 387 |
if cr.mountCache != nil {
|
| 388 | 388 |
return cr.mountCache, nil |
| 389 | 389 |
} |
| ... | ... |
@@ -401,7 +401,7 @@ func (cr *cacheRecord) mount(ctx context.Context, s session.Group) (_ snapshot.M |
| 401 | 401 |
"containerd.io/gc.flat": time.Now().UTC().Format(time.RFC3339Nano), |
| 402 | 402 |
} |
| 403 | 403 |
return nil |
| 404 |
- }, leaseutil.MakeTemporary); err != nil && !errdefs.IsAlreadyExists(err) {
|
|
| 404 |
+ }, leaseutil.MakeTemporary); err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 405 | 405 |
return nil, err |
| 406 | 406 |
} |
| 407 | 407 |
defer func() {
|
| ... | ... |
@@ -412,14 +412,14 @@ func (cr *cacheRecord) mount(ctx context.Context, s session.Group) (_ snapshot.M |
| 412 | 412 |
if err := cr.cm.LeaseManager.AddResource(ctx, leases.Lease{ID: cr.viewLeaseID()}, leases.Resource{
|
| 413 | 413 |
ID: mountSnapshotID, |
| 414 | 414 |
Type: "snapshots/" + cr.cm.Snapshotter.Name(), |
| 415 |
- }); err != nil && !errdefs.IsAlreadyExists(err) {
|
|
| 415 |
+ }); err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 416 | 416 |
return nil, err |
| 417 | 417 |
} |
| 418 | 418 |
// Return the mount direct from View rather than setting it using the Mounts call below. |
| 419 | 419 |
// The two are equivalent for containerd snapshotters but the moby snapshotter requires |
| 420 | 420 |
// the use of the mountable returned by View in this case. |
| 421 | 421 |
mnts, err := cr.cm.Snapshotter.View(ctx, mountSnapshotID, cr.getSnapshotID()) |
| 422 |
- if err != nil && !errdefs.IsAlreadyExists(err) {
|
|
| 422 |
+ if err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 423 | 423 |
return nil, err |
| 424 | 424 |
} |
| 425 | 425 |
cr.mountCache = mnts |
| ... | ... |
@@ -455,12 +455,12 @@ func (cr *cacheRecord) remove(ctx context.Context, removeSnapshot bool) (rerr er |
| 455 | 455 |
if removeSnapshot {
|
| 456 | 456 |
if err := cr.cm.LeaseManager.Delete(ctx, leases.Lease{
|
| 457 | 457 |
ID: cr.ID(), |
| 458 |
- }); err != nil && !errdefs.IsNotFound(err) {
|
|
| 458 |
+ }); err != nil && !cerrdefs.IsNotFound(err) {
|
|
| 459 | 459 |
return errors.Wrapf(err, "failed to delete lease for %s", cr.ID()) |
| 460 | 460 |
} |
| 461 | 461 |
if err := cr.cm.LeaseManager.Delete(ctx, leases.Lease{
|
| 462 | 462 |
ID: cr.compressionVariantsLeaseID(), |
| 463 |
- }); err != nil && !errdefs.IsNotFound(err) {
|
|
| 463 |
+ }); err != nil && !cerrdefs.IsNotFound(err) {
|
|
| 464 | 464 |
return errors.Wrapf(err, "failed to delete compression variant lease for %s", cr.ID()) |
| 465 | 465 |
} |
| 466 | 466 |
} |
| ... | ... |
@@ -764,7 +764,7 @@ func (sr *immutableRef) linkBlob(ctx context.Context, desc ocispecs.Descriptor) |
| 764 | 764 |
l.ID = sr.compressionVariantsLeaseID() |
| 765 | 765 |
// do not make it flat lease to allow linking blobs using gc label |
| 766 | 766 |
return nil |
| 767 |
- }); err != nil && !errdefs.IsAlreadyExists(err) {
|
|
| 767 |
+ }); err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 768 | 768 |
return err |
| 769 | 769 |
} |
| 770 | 770 |
if err := sr.cm.LeaseManager.AddResource(ctx, leases.Lease{ID: sr.compressionVariantsLeaseID()}, leases.Resource{
|
| ... | ... |
@@ -828,7 +828,7 @@ func getBlobWithCompression(ctx context.Context, cs content.Store, desc ocispecs |
| 828 | 828 |
} |
| 829 | 829 |
return true |
| 830 | 830 |
}); err != nil || target == nil {
|
| 831 |
- return ocispecs.Descriptor{}, errdefs.ErrNotFound
|
|
| 831 |
+ return ocispecs.Descriptor{}, cerrdefs.ErrNotFound
|
|
| 832 | 832 |
} |
| 833 | 833 |
return *target, nil |
| 834 | 834 |
} |
| ... | ... |
@@ -849,7 +849,7 @@ func walkBlobVariantsOnly(ctx context.Context, cs content.Store, dgst digest.Dig |
| 849 | 849 |
} |
| 850 | 850 |
visited[dgst] = struct{}{}
|
| 851 | 851 |
info, err := cs.Info(ctx, dgst) |
| 852 |
- if errors.Is(err, errdefs.ErrNotFound) {
|
|
| 852 |
+ if errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 853 | 853 |
return true, nil |
| 854 | 854 |
} else if err != nil {
|
| 855 | 855 |
return false, err |
| ... | ... |
@@ -975,12 +975,12 @@ func (sr *immutableRef) Mount(ctx context.Context, readonly bool, s session.Grou |
| 975 | 975 |
var mnt snapshot.Mountable |
| 976 | 976 |
if sr.cm.Snapshotter.Name() == "stargz" {
|
| 977 | 977 |
if err := sr.withRemoteSnapshotLabelsStargzMode(ctx, s, func() {
|
| 978 |
- mnt, rerr = sr.mount(ctx, s) |
|
| 978 |
+ mnt, rerr = sr.mount(ctx) |
|
| 979 | 979 |
}); err != nil {
|
| 980 | 980 |
return nil, err |
| 981 | 981 |
} |
| 982 | 982 |
} else {
|
| 983 |
- mnt, rerr = sr.mount(ctx, s) |
|
| 983 |
+ mnt, rerr = sr.mount(ctx) |
|
| 984 | 984 |
} |
| 985 | 985 |
if rerr != nil {
|
| 986 | 986 |
return nil, rerr |
| ... | ... |
@@ -1025,9 +1025,9 @@ func (sr *immutableRef) withRemoteSnapshotLabelsStargzMode(ctx context.Context, |
| 1025 | 1025 |
for _, r := range sr.layerChain() {
|
| 1026 | 1026 |
r := r |
| 1027 | 1027 |
info, err := r.cm.Snapshotter.Stat(ctx, r.getSnapshotID()) |
| 1028 |
- if err != nil && !errdefs.IsNotFound(err) {
|
|
| 1028 |
+ if err != nil && !cerrdefs.IsNotFound(err) {
|
|
| 1029 | 1029 |
return err |
| 1030 |
- } else if errdefs.IsNotFound(err) {
|
|
| 1030 |
+ } else if cerrdefs.IsNotFound(err) {
|
|
| 1031 | 1031 |
continue // This snpashot doesn't exist; skip |
| 1032 | 1032 |
} else if _, ok := info.Labels["containerd.io/snapshot/remote"]; !ok {
|
| 1033 | 1033 |
continue // This isn't a remote snapshot; skip |
| ... | ... |
@@ -1100,7 +1100,7 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s |
| 1100 | 1100 |
parentID = r.layerParent.getSnapshotID() |
| 1101 | 1101 |
} |
| 1102 | 1102 |
if err := r.cm.Snapshotter.Prepare(ctx, key, parentID, opts...); err != nil {
|
| 1103 |
- if errdefs.IsAlreadyExists(err) {
|
|
| 1103 |
+ if cerrdefs.IsAlreadyExists(err) {
|
|
| 1104 | 1104 |
// Check if the targeting snapshot ID has been prepared as |
| 1105 | 1105 |
// a remote snapshot in the snapshotter. |
| 1106 | 1106 |
info, err := r.cm.Snapshotter.Stat(ctx, snapshotID) |
| ... | ... |
@@ -1332,7 +1332,7 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr |
| 1332 | 1332 |
return err |
| 1333 | 1333 |
} |
| 1334 | 1334 |
if err := sr.cm.Snapshotter.Commit(ctx, sr.getSnapshotID(), key); err != nil {
|
| 1335 |
- if !errors.Is(err, errdefs.ErrAlreadyExists) {
|
|
| 1335 |
+ if !errors.Is(err, cerrdefs.ErrAlreadyExists) {
|
|
| 1336 | 1336 |
return err |
| 1337 | 1337 |
} |
| 1338 | 1338 |
} |
| ... | ... |
@@ -1391,7 +1391,7 @@ func (sr *immutableRef) release(ctx context.Context) (rerr error) {
|
| 1391 | 1391 |
if sr.equalMutable != nil {
|
| 1392 | 1392 |
sr.equalMutable.release(ctx) |
| 1393 | 1393 |
} else {
|
| 1394 |
- if err := sr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: sr.viewLeaseID()}); err != nil && !errdefs.IsNotFound(err) {
|
|
| 1394 |
+ if err := sr.cm.LeaseManager.Delete(ctx, leases.Lease{ID: sr.viewLeaseID()}); err != nil && !cerrdefs.IsNotFound(err) {
|
|
| 1395 | 1395 |
return err |
| 1396 | 1396 |
} |
| 1397 | 1397 |
sr.mountCache = nil |
| ... | ... |
@@ -1422,7 +1422,7 @@ func (cr *cacheRecord) finalize(ctx context.Context) error {
|
| 1422 | 1422 |
return nil |
| 1423 | 1423 |
}) |
| 1424 | 1424 |
if err != nil {
|
| 1425 |
- if !errors.Is(err, errdefs.ErrAlreadyExists) { // migrator adds leases for everything
|
|
| 1425 |
+ if !errors.Is(err, cerrdefs.ErrAlreadyExists) { // migrator adds leases for everything
|
|
| 1426 | 1426 |
return errors.Wrap(err, "failed to create lease") |
| 1427 | 1427 |
} |
| 1428 | 1428 |
} |
| ... | ... |
@@ -1459,7 +1459,7 @@ func (sr *mutableRef) shouldUpdateLastUsed() bool {
|
| 1459 | 1459 |
return sr.triggerLastUsed |
| 1460 | 1460 |
} |
| 1461 | 1461 |
|
| 1462 |
-func (sr *mutableRef) commit(ctx context.Context) (_ *immutableRef, rerr error) {
|
|
| 1462 |
+func (sr *mutableRef) commit() (_ *immutableRef, rerr error) {
|
|
| 1463 | 1463 |
if !sr.mutable || len(sr.refs) == 0 {
|
| 1464 | 1464 |
return nil, errors.Wrapf(errInvalid, "invalid mutable ref %p", sr) |
| 1465 | 1465 |
} |
| ... | ... |
@@ -1518,12 +1518,12 @@ func (sr *mutableRef) Mount(ctx context.Context, readonly bool, s session.Group) |
| 1518 | 1518 |
var mnt snapshot.Mountable |
| 1519 | 1519 |
if sr.cm.Snapshotter.Name() == "stargz" && sr.layerParent != nil {
|
| 1520 | 1520 |
if err := sr.layerParent.withRemoteSnapshotLabelsStargzMode(ctx, s, func() {
|
| 1521 |
- mnt, rerr = sr.mount(ctx, s) |
|
| 1521 |
+ mnt, rerr = sr.mount(ctx) |
|
| 1522 | 1522 |
}); err != nil {
|
| 1523 | 1523 |
return nil, err |
| 1524 | 1524 |
} |
| 1525 | 1525 |
} else {
|
| 1526 |
- mnt, rerr = sr.mount(ctx, s) |
|
| 1526 |
+ mnt, rerr = sr.mount(ctx) |
|
| 1527 | 1527 |
} |
| 1528 | 1528 |
if rerr != nil {
|
| 1529 | 1529 |
return nil, rerr |
| ... | ... |
@@ -1546,7 +1546,7 @@ func (sr *mutableRef) Commit(ctx context.Context) (ImmutableRef, error) {
|
| 1546 | 1546 |
sr.mu.Lock() |
| 1547 | 1547 |
defer sr.mu.Unlock() |
| 1548 | 1548 |
|
| 1549 |
- return sr.commit(ctx) |
|
| 1549 |
+ return sr.commit() |
|
| 1550 | 1550 |
} |
| 1551 | 1551 |
|
| 1552 | 1552 |
func (sr *mutableRef) Release(ctx context.Context) error {
|
| ... | ... |
@@ -7,8 +7,8 @@ import ( |
| 7 | 7 |
"strings" |
| 8 | 8 |
|
| 9 | 9 |
"github.com/containerd/containerd/content" |
| 10 |
- "github.com/containerd/containerd/errdefs" |
|
| 11 | 10 |
"github.com/containerd/containerd/reference" |
| 11 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 12 | 12 |
"github.com/moby/buildkit/cache/config" |
| 13 | 13 |
"github.com/moby/buildkit/session" |
| 14 | 14 |
"github.com/moby/buildkit/solver" |
| ... | ... |
@@ -52,7 +52,7 @@ func (sr *immutableRef) GetRemotes(ctx context.Context, createIfNeeded bool, ref |
| 52 | 52 |
} |
| 53 | 53 |
|
| 54 | 54 |
// Search all available remotes that has the topmost blob with the specified |
| 55 |
- // compression with all combination of copmressions |
|
| 55 |
+ // compression with all combination of compressions |
|
| 56 | 56 |
res := []*solver.Remote{remote}
|
| 57 | 57 |
topmost, parentChain := remote.Descriptors[len(remote.Descriptors)-1], remote.Descriptors[:len(remote.Descriptors)-1] |
| 58 | 58 |
vDesc, err := getBlobWithCompression(ctx, sr.cm.ContentStore, topmost, refCfg.Compression.Type) |
| ... | ... |
@@ -301,7 +301,7 @@ type lazyRefProvider struct {
|
| 301 | 301 |
|
| 302 | 302 |
func (p lazyRefProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
|
| 303 | 303 |
if desc.Digest != p.desc.Digest {
|
| 304 |
- return nil, errdefs.ErrNotFound |
|
| 304 |
+ return nil, cerrdefs.ErrNotFound |
|
| 305 | 305 |
} |
| 306 | 306 |
if err := p.Unlazy(ctx); err != nil {
|
| 307 | 307 |
return nil, err |
| ... | ... |
@@ -311,12 +311,24 @@ func (p lazyRefProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) |
| 311 | 311 |
|
| 312 | 312 |
func (p lazyRefProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
|
| 313 | 313 |
if dgst != p.desc.Digest {
|
| 314 |
- return content.Info{}, errdefs.ErrNotFound
|
|
| 314 |
+ return content.Info{}, cerrdefs.ErrNotFound
|
|
| 315 | 315 |
} |
| 316 |
- if err := p.Unlazy(ctx); err != nil {
|
|
| 317 |
- return content.Info{}, errdefs.ErrNotFound
|
|
| 316 |
+ info, err := p.ref.cm.ContentStore.Info(ctx, dgst) |
|
| 317 |
+ if err == nil {
|
|
| 318 |
+ return info, nil |
|
| 319 |
+ } |
|
| 320 |
+ |
|
| 321 |
+ if isLazy, err1 := p.ref.isLazy(ctx); err1 != nil {
|
|
| 322 |
+ return content.Info{}, err1
|
|
| 323 |
+ } else if !isLazy {
|
|
| 324 |
+ return content.Info{}, err
|
|
| 318 | 325 |
} |
| 319 |
- return p.ref.cm.ContentStore.Info(ctx, dgst) |
|
| 326 |
+ |
|
| 327 |
+ // for lazy records don't unlazy without read request |
|
| 328 |
+ return content.Info{
|
|
| 329 |
+ Digest: p.desc.Digest, |
|
| 330 |
+ Size: p.desc.Size, |
|
| 331 |
+ }, nil |
|
| 320 | 332 |
} |
| 321 | 333 |
|
| 322 | 334 |
func (p lazyRefProvider) Unlazy(ctx context.Context) error {
|
| ... | ... |
@@ -33,20 +33,24 @@ func init() {
|
| 33 | 33 |
} |
| 34 | 34 |
|
| 35 | 35 |
const ( |
| 36 |
- attrScope = "scope" |
|
| 37 |
- attrTimeout = "timeout" |
|
| 38 |
- attrToken = "token" |
|
| 39 |
- attrURL = "url" |
|
| 40 |
- version = "1" |
|
| 36 |
+ attrScope = "scope" |
|
| 37 |
+ attrTimeout = "timeout" |
|
| 38 |
+ attrToken = "token" |
|
| 39 |
+ attrURL = "url" |
|
| 40 |
+ attrRepository = "repository" |
|
| 41 |
+ attrGHToken = "ghtoken" |
|
| 42 |
+ version = "1" |
|
| 41 | 43 |
|
| 42 | 44 |
defaultTimeout = 10 * time.Minute |
| 43 | 45 |
) |
| 44 | 46 |
|
| 45 | 47 |
type Config struct {
|
| 46 |
- Scope string |
|
| 47 |
- URL string |
|
| 48 |
- Token string |
|
| 49 |
- Timeout time.Duration |
|
| 48 |
+ Scope string |
|
| 49 |
+ URL string |
|
| 50 |
+ Token string // token for the Github Cache runtime API |
|
| 51 |
+ GHToken string // token for the Github REST API |
|
| 52 |
+ Repository string |
|
| 53 |
+ Timeout time.Duration |
|
| 50 | 54 |
} |
| 51 | 55 |
|
| 52 | 56 |
func getConfig(attrs map[string]string) (*Config, error) {
|
| ... | ... |
@@ -62,6 +66,7 @@ func getConfig(attrs map[string]string) (*Config, error) {
|
| 62 | 62 |
if !ok {
|
| 63 | 63 |
return nil, errors.Errorf("token not set for github actions cache")
|
| 64 | 64 |
} |
| 65 |
+ |
|
| 65 | 66 |
timeout := defaultTimeout |
| 66 | 67 |
if v, ok := attrs[attrTimeout]; ok {
|
| 67 | 68 |
var err error |
| ... | ... |
@@ -71,10 +76,12 @@ func getConfig(attrs map[string]string) (*Config, error) {
|
| 71 | 71 |
} |
| 72 | 72 |
} |
| 73 | 73 |
return &Config{
|
| 74 |
- Scope: scope, |
|
| 75 |
- URL: url, |
|
| 76 |
- Token: token, |
|
| 77 |
- Timeout: timeout, |
|
| 74 |
+ Scope: scope, |
|
| 75 |
+ URL: url, |
|
| 76 |
+ Token: token, |
|
| 77 |
+ Timeout: timeout, |
|
| 78 |
+ GHToken: attrs[attrGHToken], |
|
| 79 |
+ Repository: attrs[attrRepository], |
|
| 78 | 80 |
}, nil |
| 79 | 81 |
} |
| 80 | 82 |
|
| ... | ... |
@@ -91,9 +98,11 @@ func ResolveCacheExporterFunc() remotecache.ResolveCacheExporterFunc {
|
| 91 | 91 |
|
| 92 | 92 |
type exporter struct {
|
| 93 | 93 |
solver.CacheExporterTarget |
| 94 |
- chains *v1.CacheChains |
|
| 95 |
- cache *actionscache.Cache |
|
| 96 |
- config *Config |
|
| 94 |
+ chains *v1.CacheChains |
|
| 95 |
+ cache *actionscache.Cache |
|
| 96 |
+ config *Config |
|
| 97 |
+ keyMapOnce sync.Once |
|
| 98 |
+ keyMap map[string]struct{}
|
|
| 97 | 99 |
} |
| 98 | 100 |
|
| 99 | 101 |
func NewExporter(c *Config) (remotecache.Exporter, error) {
|
| ... | ... |
@@ -118,8 +127,12 @@ func (ce *exporter) Config() remotecache.Config {
|
| 118 | 118 |
} |
| 119 | 119 |
} |
| 120 | 120 |
|
| 121 |
+func (ce *exporter) blobKeyPrefix() string {
|
|
| 122 |
+ return "buildkit-blob-" + version + "-" |
|
| 123 |
+} |
|
| 124 |
+ |
|
| 121 | 125 |
func (ce *exporter) blobKey(dgst digest.Digest) string {
|
| 122 |
- return "buildkit-blob-" + version + "-" + dgst.String() |
|
| 126 |
+ return ce.blobKeyPrefix() + dgst.String() |
|
| 123 | 127 |
} |
| 124 | 128 |
|
| 125 | 129 |
func (ce *exporter) indexKey() string {
|
| ... | ... |
@@ -133,6 +146,35 @@ func (ce *exporter) indexKey() string {
|
| 133 | 133 |
return "index-" + ce.config.Scope + "-" + version + "-" + scope |
| 134 | 134 |
} |
| 135 | 135 |
|
| 136 |
+func (ce *exporter) initActiveKeyMap(ctx context.Context) {
|
|
| 137 |
+ ce.keyMapOnce.Do(func() {
|
|
| 138 |
+ if ce.config.Repository == "" || ce.config.GHToken == "" {
|
|
| 139 |
+ return |
|
| 140 |
+ } |
|
| 141 |
+ m, err := ce.initActiveKeyMapOnce(ctx) |
|
| 142 |
+ if err != nil {
|
|
| 143 |
+ bklog.G(ctx).Errorf("error initializing active key map: %v", err)
|
|
| 144 |
+ return |
|
| 145 |
+ } |
|
| 146 |
+ ce.keyMap = m |
|
| 147 |
+ }) |
|
| 148 |
+} |
|
| 149 |
+ |
|
| 150 |
+func (ce *exporter) initActiveKeyMapOnce(ctx context.Context) (map[string]struct{}, error) {
|
|
| 151 |
+ api, err := actionscache.NewRestAPI(ce.config.Repository, ce.config.GHToken, actionscache.Opt{
|
|
| 152 |
+ Client: tracing.DefaultClient, |
|
| 153 |
+ Timeout: ce.config.Timeout, |
|
| 154 |
+ }) |
|
| 155 |
+ if err != nil {
|
|
| 156 |
+ return nil, err |
|
| 157 |
+ } |
|
| 158 |
+ keys, err := ce.cache.AllKeys(ctx, api, ce.blobKeyPrefix()) |
|
| 159 |
+ if err != nil {
|
|
| 160 |
+ return nil, err |
|
| 161 |
+ } |
|
| 162 |
+ return keys, nil |
|
| 163 |
+} |
|
| 164 |
+ |
|
| 136 | 165 |
func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
|
| 137 | 166 |
// res := make(map[string]string) |
| 138 | 167 |
config, descs, err := ce.chains.Marshal(ctx) |
| ... | ... |
@@ -159,13 +201,25 @@ func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) {
|
| 159 | 159 |
return nil, errors.Wrapf(err, "failed to parse uncompressed annotation") |
| 160 | 160 |
} |
| 161 | 161 |
diffID = dgst |
| 162 |
+ ce.initActiveKeyMap(ctx) |
|
| 162 | 163 |
|
| 163 | 164 |
key := ce.blobKey(dgstPair.Descriptor.Digest) |
| 164 |
- b, err := ce.cache.Load(ctx, key) |
|
| 165 |
- if err != nil {
|
|
| 166 |
- return nil, err |
|
| 165 |
+ |
|
| 166 |
+ exists := false |
|
| 167 |
+ if ce.keyMap != nil {
|
|
| 168 |
+ if _, ok := ce.keyMap[key]; ok {
|
|
| 169 |
+ exists = true |
|
| 170 |
+ } |
|
| 171 |
+ } else {
|
|
| 172 |
+ b, err := ce.cache.Load(ctx, key) |
|
| 173 |
+ if err != nil {
|
|
| 174 |
+ return nil, err |
|
| 175 |
+ } |
|
| 176 |
+ if b != nil {
|
|
| 177 |
+ exists = true |
|
| 178 |
+ } |
|
| 167 | 179 |
} |
| 168 |
- if b == nil {
|
|
| 180 |
+ if !exists {
|
|
| 169 | 181 |
layerDone := progress.OneOff(ctx, fmt.Sprintf("writing layer %s", l.Blob))
|
| 170 | 182 |
ra, err := dgstPair.Provider.ReaderAt(ctx, dgstPair.Descriptor) |
| 171 | 183 |
if err != nil {
|
| ... | ... |
@@ -260,9 +314,11 @@ func (ci *importer) makeDescriptorProviderPair(l v1.CacheLayer) (*v1.DescriptorP |
| 260 | 260 |
Size: l.Annotations.Size, |
| 261 | 261 |
Annotations: annotations, |
| 262 | 262 |
} |
| 263 |
+ p := &ciProvider{desc: desc, ci: ci}
|
|
| 263 | 264 |
return &v1.DescriptorProviderPair{
|
| 264 |
- Descriptor: desc, |
|
| 265 |
- Provider: &ciProvider{desc: desc, ci: ci},
|
|
| 265 |
+ Descriptor: desc, |
|
| 266 |
+ Provider: p, |
|
| 267 |
+ InfoProvider: p, |
|
| 266 | 268 |
}, nil |
| 267 | 269 |
} |
| 268 | 270 |
|
| ... | ... |
@@ -347,13 +403,18 @@ type ciProvider struct {
|
| 347 | 347 |
entries map[digest.Digest]*actionscache.Entry |
| 348 | 348 |
} |
| 349 | 349 |
|
| 350 |
-func (p *ciProvider) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
|
|
| 351 |
- if desc.Digest != p.desc.Digest {
|
|
| 352 |
- return nil |
|
| 350 |
+func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
|
|
| 351 |
+ if dgst != p.desc.Digest {
|
|
| 352 |
+ return content.Info{}, errors.Errorf("content not found %s", dgst)
|
|
| 353 | 353 |
} |
| 354 | 354 |
|
| 355 |
- _, err := p.loadEntry(ctx, desc) |
|
| 356 |
- return err |
|
| 355 |
+ if _, err := p.loadEntry(ctx, p.desc); err != nil {
|
|
| 356 |
+ return content.Info{}, err
|
|
| 357 |
+ } |
|
| 358 |
+ return content.Info{
|
|
| 359 |
+ Digest: p.desc.Digest, |
|
| 360 |
+ Size: p.desc.Size, |
|
| 361 |
+ }, nil |
|
| 357 | 362 |
} |
| 358 | 363 |
|
| 359 | 364 |
func (p *ciProvider) loadEntry(ctx context.Context, desc ocispecs.Descriptor) (*actionscache.Entry, error) {
|
| ... | ... |
@@ -57,20 +57,19 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest) |
| 57 | 57 |
return nil, err |
| 58 | 58 |
} |
| 59 | 59 |
|
| 60 |
- layerBlobDigests := make([]digest.Digest, len(layers)) |
|
| 60 |
+ layerBlobDigests := make([][]digest.Digest, len(layers)) |
|
| 61 | 61 |
|
| 62 | 62 |
descs2 := map[digest.Digest]v1.DescriptorProviderPair{}
|
| 63 | 63 |
for i, k := range layers {
|
| 64 | 64 |
if v, ok := descs[k]; ok {
|
| 65 | 65 |
descs2[k] = v |
| 66 |
- layerBlobDigests[i] = k |
|
| 67 |
- continue |
|
| 66 |
+ layerBlobDigests[i] = append(layerBlobDigests[i], k) |
|
| 68 | 67 |
} |
| 69 | 68 |
// fallback for uncompressed digests |
| 70 | 69 |
for _, v := range descs {
|
| 71 | 70 |
if uc := v.Descriptor.Annotations[labels.LabelUncompressed]; uc == string(k) {
|
| 72 | 71 |
descs2[v.Descriptor.Digest] = v |
| 73 |
- layerBlobDigests[i] = v.Descriptor.Digest |
|
| 72 |
+ layerBlobDigests[i] = append(layerBlobDigests[i], v.Descriptor.Digest) |
|
| 74 | 73 |
} |
| 75 | 74 |
} |
| 76 | 75 |
} |
| ... | ... |
@@ -92,8 +91,10 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest) |
| 92 | 92 |
|
| 93 | 93 |
// reorder layers based on the order in the image |
| 94 | 94 |
blobIndexes := make(map[digest.Digest]int, len(layers)) |
| 95 |
- for i, blob := range layerBlobDigests {
|
|
| 96 |
- blobIndexes[blob] = i |
|
| 95 |
+ for i, blobs := range layerBlobDigests {
|
|
| 96 |
+ for _, blob := range blobs {
|
|
| 97 |
+ blobIndexes[blob] = i |
|
| 98 |
+ } |
|
| 97 | 99 |
} |
| 98 | 100 |
|
| 99 | 101 |
for i, r := range cfg.Records {
|
| ... | ... |
@@ -104,8 +105,14 @@ func (ce *exporter) ExportForLayers(ctx context.Context, layers []digest.Digest) |
| 104 | 104 |
if len(resultBlobs) <= len(layers) {
|
| 105 | 105 |
match = true |
| 106 | 106 |
for k, resultBlob := range resultBlobs {
|
| 107 |
- layerBlob := layers[k] |
|
| 108 |
- if resultBlob != layerBlob {
|
|
| 107 |
+ matchesBlob := false |
|
| 108 |
+ for _, layerBlob := range layerBlobDigests[k] {
|
|
| 109 |
+ if layerBlob == resultBlob {
|
|
| 110 |
+ matchesBlob = true |
|
| 111 |
+ break |
|
| 112 |
+ } |
|
| 113 |
+ } |
|
| 114 |
+ if !matchesBlob {
|
|
| 109 | 115 |
match = false |
| 110 | 116 |
break |
| 111 | 117 |
} |
| ... | ... |
@@ -105,7 +105,7 @@ func getContentStore(ctx context.Context, sm *session.Manager, g session.Group, |
| 105 | 105 |
if sessionID == "" {
|
| 106 | 106 |
return nil, errors.New("local cache exporter/importer requires session")
|
| 107 | 107 |
} |
| 108 |
- timeoutCtx, cancel := context.WithCancelCause(context.Background()) |
|
| 108 |
+ timeoutCtx, cancel := context.WithCancelCause(ctx) |
|
| 109 | 109 |
timeoutCtx, _ = context.WithTimeoutCause(timeoutCtx, 5*time.Second, errors.WithStack(context.DeadlineExceeded)) |
| 110 | 110 |
defer cancel(errors.WithStack(context.Canceled)) |
| 111 | 111 |
|
| ... | ... |
@@ -118,17 +118,19 @@ func (c *CacheChains) Marshal(ctx context.Context) (*CacheConfig, DescriptorProv |
| 118 | 118 |
type DescriptorProvider map[digest.Digest]DescriptorProviderPair |
| 119 | 119 |
|
| 120 | 120 |
type DescriptorProviderPair struct {
|
| 121 |
- Descriptor ocispecs.Descriptor |
|
| 122 |
- Provider content.Provider |
|
| 121 |
+ Descriptor ocispecs.Descriptor |
|
| 122 |
+ Provider content.Provider |
|
| 123 |
+ InfoProvider content.InfoProvider |
|
| 123 | 124 |
} |
| 124 | 125 |
|
| 125 |
-var _ withCheckDescriptor = DescriptorProviderPair{}
|
|
| 126 |
- |
|
| 127 | 126 |
func (p DescriptorProviderPair) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
|
| 128 | 127 |
return p.Provider.ReaderAt(ctx, desc) |
| 129 | 128 |
} |
| 130 | 129 |
|
| 131 | 130 |
func (p DescriptorProviderPair) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
|
| 131 |
+ if p.InfoProvider != nil {
|
|
| 132 |
+ return p.InfoProvider.Info(ctx, dgst) |
|
| 133 |
+ } |
|
| 132 | 134 |
if dgst != p.Descriptor.Digest {
|
| 133 | 135 |
return content.Info{}, errors.Errorf("content not found %s", dgst)
|
| 134 | 136 |
} |
| ... | ... |
@@ -158,13 +160,6 @@ func (p DescriptorProviderPair) SnapshotLabels(descs []ocispecs.Descriptor, inde |
| 158 | 158 |
return nil |
| 159 | 159 |
} |
| 160 | 160 |
|
| 161 |
-func (p DescriptorProviderPair) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
|
|
| 162 |
- if cd, ok := p.Provider.(withCheckDescriptor); ok {
|
|
| 163 |
- return cd.CheckDescriptor(ctx, desc) |
|
| 164 |
- } |
|
| 165 |
- return nil |
|
| 166 |
-} |
|
| 167 |
- |
|
| 168 | 161 |
// item is an implementation of a record in the cache chain. After validation, |
| 169 | 162 |
// normalization and marshalling into the cache config, the item results form |
| 170 | 163 |
// into the "layers", while the digests and the links form into the "records". |
| ... | ... |
@@ -8,15 +8,9 @@ import ( |
| 8 | 8 |
"github.com/moby/buildkit/solver" |
| 9 | 9 |
"github.com/moby/buildkit/util/bklog" |
| 10 | 10 |
digest "github.com/opencontainers/go-digest" |
| 11 |
- ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
|
| 12 | 11 |
"github.com/pkg/errors" |
| 13 | 12 |
) |
| 14 | 13 |
|
| 15 |
-type withCheckDescriptor interface {
|
|
| 16 |
- // CheckDescriptor is additional method on Provider to check if the descriptor is available without opening the reader |
|
| 17 |
- CheckDescriptor(context.Context, ocispecs.Descriptor) error |
|
| 18 |
-} |
|
| 19 |
- |
|
| 20 | 14 |
// sortConfig sorts the config structure to make sure it is deterministic |
| 21 | 15 |
func sortConfig(cc *CacheConfig) {
|
| 22 | 16 |
type indexedLayer struct {
|
| ... | ... |
@@ -284,13 +278,14 @@ func marshalRemote(ctx context.Context, r *solver.Remote, state *marshalState) s |
| 284 | 284 |
return "" |
| 285 | 285 |
} |
| 286 | 286 |
|
| 287 |
- if cd, ok := r.Provider.(withCheckDescriptor); ok && len(r.Descriptors) > 0 {
|
|
| 287 |
+ if r.Provider != nil {
|
|
| 288 | 288 |
for _, d := range r.Descriptors {
|
| 289 |
- if cd.CheckDescriptor(ctx, d) != nil {
|
|
| 289 |
+ if _, err := r.Provider.Info(ctx, d.Digest); err != nil {
|
|
| 290 | 290 |
return "" |
| 291 | 291 |
} |
| 292 | 292 |
} |
| 293 | 293 |
} |
| 294 |
+ |
|
| 294 | 295 |
var parentID string |
| 295 | 296 |
if len(r.Descriptors) > 1 {
|
| 296 | 297 |
r2 := &solver.Remote{
|
| ... | ... |
@@ -23,7 +23,7 @@ type FileRange struct {
|
| 23 | 23 |
Length int |
| 24 | 24 |
} |
| 25 | 25 |
|
| 26 |
-func withMount(ctx context.Context, mount snapshot.Mountable, cb func(string) error) error {
|
|
| 26 |
+func withMount(mount snapshot.Mountable, cb func(string) error) error {
|
|
| 27 | 27 |
lm := snapshot.LocalMounter(mount) |
| 28 | 28 |
|
| 29 | 29 |
root, err := lm.Mount() |
| ... | ... |
@@ -51,7 +51,7 @@ func withMount(ctx context.Context, mount snapshot.Mountable, cb func(string) er |
| 51 | 51 |
func ReadFile(ctx context.Context, mount snapshot.Mountable, req ReadRequest) ([]byte, error) {
|
| 52 | 52 |
var dt []byte |
| 53 | 53 |
|
| 54 |
- err := withMount(ctx, mount, func(root string) error {
|
|
| 54 |
+ err := withMount(mount, func(root string) error {
|
|
| 55 | 55 |
fp, err := fs.RootPath(root, req.Filename) |
| 56 | 56 |
if err != nil {
|
| 57 | 57 |
return errors.WithStack(err) |
| ... | ... |
@@ -95,7 +95,7 @@ func ReadDir(ctx context.Context, mount snapshot.Mountable, req ReadDirRequest) |
| 95 | 95 |
if req.IncludePattern != "" {
|
| 96 | 96 |
fo.IncludePatterns = append(fo.IncludePatterns, req.IncludePattern) |
| 97 | 97 |
} |
| 98 |
- err := withMount(ctx, mount, func(root string) error {
|
|
| 98 |
+ err := withMount(mount, func(root string) error {
|
|
| 99 | 99 |
fp, err := fs.RootPath(root, req.Path) |
| 100 | 100 |
if err != nil {
|
| 101 | 101 |
return errors.WithStack(err) |
| ... | ... |
@@ -122,7 +122,7 @@ func ReadDir(ctx context.Context, mount snapshot.Mountable, req ReadDirRequest) |
| 122 | 122 |
|
| 123 | 123 |
func StatFile(ctx context.Context, mount snapshot.Mountable, path string) (*fstypes.Stat, error) {
|
| 124 | 124 |
var st *fstypes.Stat |
| 125 |
- err := withMount(ctx, mount, func(root string) error {
|
|
| 125 |
+ err := withMount(mount, func(root string) error {
|
|
| 126 | 126 |
fp, err := fs.RootPath(root, path) |
| 127 | 127 |
if err != nil {
|
| 128 | 128 |
return errors.WithStack(err) |
| ... | ... |
@@ -7,7 +7,6 @@ import ( |
| 7 | 7 |
"net" |
| 8 | 8 |
"net/url" |
| 9 | 9 |
"os" |
| 10 |
- "strings" |
|
| 11 | 10 |
"time" |
| 12 | 11 |
|
| 13 | 12 |
contentapi "github.com/containerd/containerd/api/services/content/v1" |
| ... | ... |
@@ -18,6 +17,7 @@ import ( |
| 18 | 18 |
"github.com/moby/buildkit/session/grpchijack" |
| 19 | 19 |
"github.com/moby/buildkit/util/appdefaults" |
| 20 | 20 |
"github.com/moby/buildkit/util/grpcerrors" |
| 21 |
+ "github.com/moby/buildkit/util/tracing" |
|
| 21 | 22 |
"github.com/moby/buildkit/util/tracing/otlptracegrpc" |
| 22 | 23 |
"github.com/pkg/errors" |
| 23 | 24 |
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" |
| ... | ... |
@@ -48,9 +48,6 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error |
| 48 | 48 |
} |
| 49 | 49 |
needDialer := true |
| 50 | 50 |
|
| 51 |
- var unary []grpc.UnaryClientInterceptor |
|
| 52 |
- var stream []grpc.StreamClientInterceptor |
|
| 53 |
- |
|
| 54 | 51 |
var customTracer bool // allows manually setting disabling tracing even if tracer in context |
| 55 | 52 |
var tracerProvider trace.TracerProvider |
| 56 | 53 |
var tracerDelegate TracerDelegate |
| ... | ... |
@@ -101,9 +98,14 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error |
| 101 | 101 |
} |
| 102 | 102 |
|
| 103 | 103 |
if tracerProvider != nil {
|
| 104 |
- var propagators = propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
|
|
| 105 |
- unary = append(unary, filterInterceptor(otelgrpc.UnaryClientInterceptor(otelgrpc.WithTracerProvider(tracerProvider), otelgrpc.WithPropagators(propagators)))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681 |
|
| 106 |
- stream = append(stream, otelgrpc.StreamClientInterceptor(otelgrpc.WithTracerProvider(tracerProvider), otelgrpc.WithPropagators(propagators))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681 |
|
| 104 |
+ gopts = append(gopts, grpc.WithStatsHandler( |
|
| 105 |
+ tracing.ClientStatsHandler( |
|
| 106 |
+ otelgrpc.WithTracerProvider(tracerProvider), |
|
| 107 |
+ otelgrpc.WithPropagators( |
|
| 108 |
+ propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}),
|
|
| 109 |
+ ), |
|
| 110 |
+ ), |
|
| 111 |
+ )) |
|
| 107 | 112 |
} |
| 108 | 113 |
|
| 109 | 114 |
if needDialer {
|
| ... | ... |
@@ -111,11 +113,17 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error |
| 111 | 111 |
if err != nil {
|
| 112 | 112 |
return nil, err |
| 113 | 113 |
} |
| 114 |
- gopts = append(gopts, grpc.WithContextDialer(dialFn)) |
|
| 114 |
+ if dialFn != nil {
|
|
| 115 |
+ gopts = append(gopts, grpc.WithContextDialer(dialFn)) |
|
| 116 |
+ } |
|
| 115 | 117 |
} |
| 116 | 118 |
if address == "" {
|
| 117 | 119 |
address = appdefaults.Address |
| 118 | 120 |
} |
| 121 |
+ uri, err := url.Parse(address) |
|
| 122 |
+ if err != nil {
|
|
| 123 |
+ return nil, err |
|
| 124 |
+ } |
|
| 119 | 125 |
|
| 120 | 126 |
// Setting :authority pseudo header |
| 121 | 127 |
// - HTTP/2 (RFC7540) defines :authority pseudo header includes |
| ... | ... |
@@ -130,19 +138,17 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error |
| 130 | 130 |
} |
| 131 | 131 |
if authority == "" {
|
| 132 | 132 |
// authority as hostname from target address |
| 133 |
- uri, err := url.Parse(address) |
|
| 134 |
- if err != nil {
|
|
| 135 |
- return nil, err |
|
| 136 |
- } |
|
| 137 | 133 |
authority = uri.Host |
| 138 | 134 |
} |
| 139 |
- gopts = append(gopts, grpc.WithAuthority(authority)) |
|
| 140 |
- |
|
| 141 |
- unary = append(unary, grpcerrors.UnaryClientInterceptor) |
|
| 142 |
- stream = append(stream, grpcerrors.StreamClientInterceptor) |
|
| 135 |
+ if uri.Scheme == "tcp" {
|
|
| 136 |
+ // remove tcp scheme from address, since default dialer doesn't expect that |
|
| 137 |
+ // name resolution is done by grpc according to the following spec: https://github.com/grpc/grpc/blob/master/doc/naming.md |
|
| 138 |
+ address = uri.Host |
|
| 139 |
+ } |
|
| 143 | 140 |
|
| 144 |
- gopts = append(gopts, grpc.WithChainUnaryInterceptor(unary...)) |
|
| 145 |
- gopts = append(gopts, grpc.WithChainStreamInterceptor(stream...)) |
|
| 141 |
+ gopts = append(gopts, grpc.WithAuthority(authority)) |
|
| 142 |
+ gopts = append(gopts, grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor)) |
|
| 143 |
+ gopts = append(gopts, grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor)) |
|
| 146 | 144 |
gopts = append(gopts, customDialOptions...) |
| 147 | 145 |
|
| 148 | 146 |
conn, err := grpc.DialContext(ctx, address, gopts...) |
| ... | ... |
@@ -375,17 +381,7 @@ func resolveDialer(address string) (func(context.Context, string) (net.Conn, err |
| 375 | 375 |
if ch != nil {
|
| 376 | 376 |
return ch.ContextDialer, nil |
| 377 | 377 |
} |
| 378 |
- // basic dialer |
|
| 379 |
- return dialer, nil |
|
| 380 |
-} |
|
| 381 |
- |
|
| 382 |
-func filterInterceptor(intercept grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor {
|
|
| 383 |
- return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
|
| 384 |
- if strings.HasSuffix(method, "opentelemetry.proto.collector.trace.v1.TraceService/Export") {
|
|
| 385 |
- return invoker(ctx, method, req, reply, cc, opts...) |
|
| 386 |
- } |
|
| 387 |
- return intercept(ctx, method, req, reply, cc, invoker, opts...) |
|
| 388 |
- } |
|
| 378 |
+ return nil, nil |
|
| 389 | 379 |
} |
| 390 | 380 |
|
| 391 | 381 |
type withGRPCDialOption struct {
|
| 392 | 382 |
deleted file mode 100644 |
| ... | ... |
@@ -1,21 +0,0 @@ |
| 1 |
-//go:build !windows |
|
| 2 |
-// +build !windows |
|
| 3 |
- |
|
| 4 |
-package client |
|
| 5 |
- |
|
| 6 |
-import ( |
|
| 7 |
- "context" |
|
| 8 |
- "net" |
|
| 9 |
- "strings" |
|
| 10 |
- |
|
| 11 |
- "github.com/pkg/errors" |
|
| 12 |
-) |
|
| 13 |
- |
|
| 14 |
-func dialer(ctx context.Context, address string) (net.Conn, error) {
|
|
| 15 |
- addrParts := strings.SplitN(address, "://", 2) |
|
| 16 |
- if len(addrParts) != 2 {
|
|
| 17 |
- return nil, errors.Errorf("invalid address %s", address)
|
|
| 18 |
- } |
|
| 19 |
- var d net.Dialer |
|
| 20 |
- return d.DialContext(ctx, addrParts[0], addrParts[1]) |
|
| 21 |
-} |
| 22 | 1 |
deleted file mode 100644 |
| ... | ... |
@@ -1,25 +0,0 @@ |
| 1 |
-package client |
|
| 2 |
- |
|
| 3 |
-import ( |
|
| 4 |
- "context" |
|
| 5 |
- "net" |
|
| 6 |
- "strings" |
|
| 7 |
- |
|
| 8 |
- winio "github.com/Microsoft/go-winio" |
|
| 9 |
- "github.com/pkg/errors" |
|
| 10 |
-) |
|
| 11 |
- |
|
| 12 |
-func dialer(ctx context.Context, address string) (net.Conn, error) {
|
|
| 13 |
- addrParts := strings.SplitN(address, "://", 2) |
|
| 14 |
- if len(addrParts) != 2 {
|
|
| 15 |
- return nil, errors.Errorf("invalid address %s", address)
|
|
| 16 |
- } |
|
| 17 |
- switch addrParts[0] {
|
|
| 18 |
- case "npipe": |
|
| 19 |
- address = strings.Replace(addrParts[1], "/", "\\", -1) |
|
| 20 |
- return winio.DialPipeContext(ctx, address) |
|
| 21 |
- default: |
|
| 22 |
- var d net.Dialer |
|
| 23 |
- return d.DialContext(ctx, addrParts[0], addrParts[1]) |
|
| 24 |
- } |
|
| 25 |
-} |
| ... | ... |
@@ -438,15 +438,23 @@ func (e *ExecOp) Output() Output {
|
| 438 | 438 |
} |
| 439 | 439 |
|
| 440 | 440 |
func (e *ExecOp) Inputs() (inputs []Output) {
|
| 441 |
- mm := map[Output]struct{}{}
|
|
| 441 |
+ // make sure mounts are sorted |
|
| 442 |
+ // the same sort occurs in (*ExecOp).Marshal, and this |
|
| 443 |
+ // sort must be the same |
|
| 444 |
+ sort.Slice(e.mounts, func(i int, j int) bool {
|
|
| 445 |
+ return e.mounts[i].target < e.mounts[j].target |
|
| 446 |
+ }) |
|
| 447 |
+ |
|
| 448 |
+ seen := map[Output]struct{}{}
|
|
| 442 | 449 |
for _, m := range e.mounts {
|
| 443 | 450 |
if m.source != nil {
|
| 444 |
- mm[m.source] = struct{}{}
|
|
| 451 |
+ if _, ok := seen[m.source]; !ok {
|
|
| 452 |
+ inputs = append(inputs, m.source) |
|
| 453 |
+ seen[m.source] = struct{}{}
|
|
| 454 |
+ } |
|
| 445 | 455 |
} |
| 446 | 456 |
} |
| 447 |
- for o := range mm {
|
|
| 448 |
- inputs = append(inputs, o) |
|
| 449 |
- } |
|
| 457 |
+ |
|
| 450 | 458 |
return |
| 451 | 459 |
} |
| 452 | 460 |
|
| ... | ... |
@@ -96,24 +96,35 @@ func (fa *FileAction) Copy(input CopyInput, src, dest string, opt ...CopyOption) |
| 96 | 96 |
return a |
| 97 | 97 |
} |
| 98 | 98 |
|
| 99 |
-func (fa *FileAction) allOutputs(m map[Output]struct{}) {
|
|
| 99 |
+func (fa *FileAction) allOutputs(seen map[Output]struct{}, outputs []Output) []Output {
|
|
| 100 | 100 |
if fa == nil {
|
| 101 |
- return |
|
| 101 |
+ return outputs |
|
| 102 | 102 |
} |
| 103 |
- if fa.state != nil && fa.state.Output() != nil {
|
|
| 104 |
- m[fa.state.Output()] = struct{}{}
|
|
| 103 |
+ |
|
| 104 |
+ if fa.state != nil {
|
|
| 105 |
+ out := fa.state.Output() |
|
| 106 |
+ if out != nil {
|
|
| 107 |
+ if _, ok := seen[out]; !ok {
|
|
| 108 |
+ outputs = append(outputs, out) |
|
| 109 |
+ seen[out] = struct{}{}
|
|
| 110 |
+ } |
|
| 111 |
+ } |
|
| 105 | 112 |
} |
| 106 | 113 |
|
| 107 | 114 |
if a, ok := fa.action.(*fileActionCopy); ok {
|
| 108 | 115 |
if a.state != nil {
|
| 109 |
- if out := a.state.Output(); out != nil {
|
|
| 110 |
- m[out] = struct{}{}
|
|
| 116 |
+ out := a.state.Output() |
|
| 117 |
+ if out != nil {
|
|
| 118 |
+ if _, ok := seen[out]; !ok {
|
|
| 119 |
+ outputs = append(outputs, out) |
|
| 120 |
+ seen[out] = struct{}{}
|
|
| 121 |
+ } |
|
| 111 | 122 |
} |
| 112 | 123 |
} else if a.fas != nil {
|
| 113 |
- a.fas.allOutputs(m) |
|
| 124 |
+ outputs = a.fas.allOutputs(seen, outputs) |
|
| 114 | 125 |
} |
| 115 | 126 |
} |
| 116 |
- fa.prev.allOutputs(m) |
|
| 127 |
+ return fa.prev.allOutputs(seen, outputs) |
|
| 117 | 128 |
} |
| 118 | 129 |
|
| 119 | 130 |
func (fa *FileAction) bind(s State) *FileAction {
|
| ... | ... |
@@ -477,17 +488,18 @@ type CopyOption interface {
|
| 477 | 477 |
} |
| 478 | 478 |
|
| 479 | 479 |
type CopyInfo struct {
|
| 480 |
- Mode *os.FileMode |
|
| 481 |
- FollowSymlinks bool |
|
| 482 |
- CopyDirContentsOnly bool |
|
| 483 |
- IncludePatterns []string |
|
| 484 |
- ExcludePatterns []string |
|
| 485 |
- AttemptUnpack bool |
|
| 486 |
- CreateDestPath bool |
|
| 487 |
- AllowWildcard bool |
|
| 488 |
- AllowEmptyWildcard bool |
|
| 489 |
- ChownOpt *ChownOpt |
|
| 490 |
- CreatedTime *time.Time |
|
| 480 |
+ Mode *os.FileMode |
|
| 481 |
+ FollowSymlinks bool |
|
| 482 |
+ CopyDirContentsOnly bool |
|
| 483 |
+ IncludePatterns []string |
|
| 484 |
+ ExcludePatterns []string |
|
| 485 |
+ AttemptUnpack bool |
|
| 486 |
+ CreateDestPath bool |
|
| 487 |
+ AllowWildcard bool |
|
| 488 |
+ AllowEmptyWildcard bool |
|
| 489 |
+ ChownOpt *ChownOpt |
|
| 490 |
+ CreatedTime *time.Time |
|
| 491 |
+ AlwaysReplaceExistingDestPaths bool |
|
| 491 | 492 |
} |
| 492 | 493 |
|
| 493 | 494 |
func (mi *CopyInfo) SetCopyOption(mi2 *CopyInfo) {
|
| ... | ... |
@@ -522,6 +534,7 @@ func (a *fileActionCopy) toProtoAction(ctx context.Context, parent string, base |
| 522 | 522 |
AttemptUnpackDockerCompatibility: a.info.AttemptUnpack, |
| 523 | 523 |
CreateDestPath: a.info.CreateDestPath, |
| 524 | 524 |
Timestamp: marshalTime(a.info.CreatedTime), |
| 525 |
+ AlwaysReplaceExistingDestPaths: a.info.AlwaysReplaceExistingDestPaths, |
|
| 525 | 526 |
} |
| 526 | 527 |
if a.info.Mode != nil {
|
| 527 | 528 |
c.Mode = int32(*a.info.Mode) |
| ... | ... |
@@ -554,6 +567,9 @@ func (a *fileActionCopy) addCaps(f *FileOp) {
|
| 554 | 554 |
if len(a.info.IncludePatterns) != 0 || len(a.info.ExcludePatterns) != 0 {
|
| 555 | 555 |
addCap(&f.constraints, pb.CapFileCopyIncludeExcludePatterns) |
| 556 | 556 |
} |
| 557 |
+ if a.info.AlwaysReplaceExistingDestPaths {
|
|
| 558 |
+ addCap(&f.constraints, pb.CapFileCopyAlwaysReplaceExistingDestPaths) |
|
| 559 |
+ } |
|
| 557 | 560 |
} |
| 558 | 561 |
|
| 559 | 562 |
type CreatedTime time.Time |
| ... | ... |
@@ -626,7 +642,7 @@ type fileActionState struct {
|
| 626 | 626 |
fa *FileAction |
| 627 | 627 |
} |
| 628 | 628 |
|
| 629 |
-func (ms *marshalState) addInput(st *fileActionState, c *Constraints, o Output) (pb.InputIndex, error) {
|
|
| 629 |
+func (ms *marshalState) addInput(c *Constraints, o Output) (pb.InputIndex, error) {
|
|
| 630 | 630 |
inp, err := o.ToInput(ms.ctx, c) |
| 631 | 631 |
if err != nil {
|
| 632 | 632 |
return 0, err |
| ... | ... |
@@ -668,7 +684,7 @@ func (ms *marshalState) add(fa *FileAction, c *Constraints) (*fileActionState, e |
| 668 | 668 |
} |
| 669 | 669 |
|
| 670 | 670 |
if source := fa.state.Output(); source != nil {
|
| 671 |
- inp, err := ms.addInput(st, c, source) |
|
| 671 |
+ inp, err := ms.addInput(c, source) |
|
| 672 | 672 |
if err != nil {
|
| 673 | 673 |
return nil, err |
| 674 | 674 |
} |
| ... | ... |
@@ -684,7 +700,7 @@ func (ms *marshalState) add(fa *FileAction, c *Constraints) (*fileActionState, e |
| 684 | 684 |
if a, ok := fa.action.(*fileActionCopy); ok {
|
| 685 | 685 |
if a.state != nil {
|
| 686 | 686 |
if out := a.state.Output(); out != nil {
|
| 687 |
- inp, err := ms.addInput(st, c, out) |
|
| 687 |
+ inp, err := ms.addInput(c, out) |
|
| 688 | 688 |
if err != nil {
|
| 689 | 689 |
return nil, err |
| 690 | 690 |
} |
| ... | ... |
@@ -806,15 +822,8 @@ func (f *FileOp) Output() Output {
|
| 806 | 806 |
return f.output |
| 807 | 807 |
} |
| 808 | 808 |
|
| 809 |
-func (f *FileOp) Inputs() (inputs []Output) {
|
|
| 810 |
- mm := map[Output]struct{}{}
|
|
| 811 |
- |
|
| 812 |
- f.action.allOutputs(mm) |
|
| 813 |
- |
|
| 814 |
- for o := range mm {
|
|
| 815 |
- inputs = append(inputs, o) |
|
| 816 |
- } |
|
| 817 |
- return inputs |
|
| 809 |
+func (f *FileOp) Inputs() []Output {
|
|
| 810 |
+ return f.action.allOutputs(map[Output]struct{}{}, []Output{})
|
|
| 818 | 811 |
} |
| 819 | 812 |
|
| 820 | 813 |
func getIndex(input pb.InputIndex, len int, relative *int) pb.InputIndex {
|
| ... | ... |
@@ -227,6 +227,11 @@ type ImageInfo struct {
|
| 227 | 227 |
RecordType string |
| 228 | 228 |
} |
| 229 | 229 |
|
| 230 |
+const ( |
|
| 231 |
+ GitAuthHeaderKey = "GIT_AUTH_HEADER" |
|
| 232 |
+ GitAuthTokenKey = "GIT_AUTH_TOKEN" |
|
| 233 |
+) |
|
| 234 |
+ |
|
| 230 | 235 |
// Git returns a state that represents a git repository. |
| 231 | 236 |
// Example: |
| 232 | 237 |
// |
| ... | ... |
@@ -267,8 +272,8 @@ func Git(url, ref string, opts ...GitOption) State {
|
| 267 | 267 |
} |
| 268 | 268 |
|
| 269 | 269 |
gi := &GitInfo{
|
| 270 |
- AuthHeaderSecret: "GIT_AUTH_HEADER", |
|
| 271 |
- AuthTokenSecret: "GIT_AUTH_TOKEN", |
|
| 270 |
+ AuthHeaderSecret: GitAuthHeaderKey, |
|
| 271 |
+ AuthTokenSecret: GitAuthTokenKey, |
|
| 272 | 272 |
} |
| 273 | 273 |
for _, o := range opts {
|
| 274 | 274 |
o.SetGitOption(gi) |
| ... | ... |
@@ -102,6 +102,7 @@ func (s StoreIndex) Put(tag string, desc ocispecs.Descriptor) error {
|
| 102 | 102 |
} |
| 103 | 103 |
} |
| 104 | 104 |
|
| 105 |
+ setOCIIndexDefaults(&idx) |
|
| 105 | 106 |
if err = insertDesc(&idx, desc, tag); err != nil {
|
| 106 | 107 |
return err |
| 107 | 108 |
} |
| ... | ... |
@@ -145,6 +146,19 @@ func (s StoreIndex) GetSingle() (*ocispecs.Descriptor, error) {
|
| 145 | 145 |
return nil, nil |
| 146 | 146 |
} |
| 147 | 147 |
|
| 148 |
+// setOCIIndexDefaults updates zero values in index to their default values. |
|
| 149 |
+func setOCIIndexDefaults(index *ocispecs.Index) {
|
|
| 150 |
+ if index == nil {
|
|
| 151 |
+ return |
|
| 152 |
+ } |
|
| 153 |
+ if index.SchemaVersion == 0 {
|
|
| 154 |
+ index.SchemaVersion = 2 |
|
| 155 |
+ } |
|
| 156 |
+ if index.MediaType == "" {
|
|
| 157 |
+ index.MediaType = ocispecs.MediaTypeImageIndex |
|
| 158 |
+ } |
|
| 159 |
+} |
|
| 160 |
+ |
|
| 148 | 161 |
// insertDesc puts desc to index with tag. |
| 149 | 162 |
// Existing manifests with the same tag will be removed from the index. |
| 150 | 163 |
func insertDesc(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) error {
|
| ... | ... |
@@ -152,9 +166,6 @@ func insertDesc(index *ocispecs.Index, desc ocispecs.Descriptor, tag string) err |
| 152 | 152 |
return nil |
| 153 | 153 |
} |
| 154 | 154 |
|
| 155 |
- if index.SchemaVersion == 0 {
|
|
| 156 |
- index.SchemaVersion = 2 |
|
| 157 |
- } |
|
| 158 | 155 |
if tag != "" {
|
| 159 | 156 |
if desc.Annotations == nil {
|
| 160 | 157 |
desc.Annotations = make(map[string]string) |
| ... | ... |
@@ -33,6 +33,19 @@ type Config struct {
|
| 33 | 33 |
DNS *DNSConfig `toml:"dns"` |
| 34 | 34 |
|
| 35 | 35 |
History *HistoryConfig `toml:"history"` |
| 36 |
+ |
|
| 37 |
+ Frontends struct {
|
|
| 38 |
+ Dockerfile DockerfileFrontendConfig `toml:"dockerfile.v0"` |
|
| 39 |
+ Gateway GatewayFrontendConfig `toml:"gateway.v0"` |
|
| 40 |
+ } `toml:"frontend"` |
|
| 41 |
+ |
|
| 42 |
+ System *SystemConfig `toml:"system"` |
|
| 43 |
+} |
|
| 44 |
+ |
|
| 45 |
+type SystemConfig struct {
|
|
| 46 |
+ // PlatformCacheMaxAge controls how often supported platforms |
|
| 47 |
+ // are refreshed by rescanning the system. |
|
| 48 |
+ PlatformsCacheMaxAge *Duration `toml:"platformsCacheMaxAge"` |
|
| 36 | 49 |
} |
| 37 | 50 |
|
| 38 | 51 |
type LogConfig struct {
|
| ... | ... |
@@ -40,10 +53,11 @@ type LogConfig struct {
|
| 40 | 40 |
} |
| 41 | 41 |
|
| 42 | 42 |
type GRPCConfig struct {
|
| 43 |
- Address []string `toml:"address"` |
|
| 44 |
- DebugAddress string `toml:"debugAddress"` |
|
| 45 |
- UID *int `toml:"uid"` |
|
| 46 |
- GID *int `toml:"gid"` |
|
| 43 |
+ Address []string `toml:"address"` |
|
| 44 |
+ DebugAddress string `toml:"debugAddress"` |
|
| 45 |
+ UID *int `toml:"uid"` |
|
| 46 |
+ GID *int `toml:"gid"` |
|
| 47 |
+ SecurityDescriptor string `toml:"securityDescriptor"` |
|
| 47 | 48 |
|
| 48 | 49 |
TLS TLSConfig `toml:"tls"` |
| 49 | 50 |
// MaxRecvMsgSize int `toml:"max_recv_message_size"` |
| ... | ... |
@@ -154,3 +168,12 @@ type HistoryConfig struct {
|
| 154 | 154 |
MaxAge Duration `toml:"maxAge"` |
| 155 | 155 |
MaxEntries int64 `toml:"maxEntries"` |
| 156 | 156 |
} |
| 157 |
+ |
|
| 158 |
+type DockerfileFrontendConfig struct {
|
|
| 159 |
+ Enabled *bool `toml:"enabled"` |
|
| 160 |
+} |
|
| 161 |
+ |
|
| 162 |
+type GatewayFrontendConfig struct {
|
|
| 163 |
+ Enabled *bool `toml:"enabled"` |
|
| 164 |
+ AllowedRepositories []string `toml:"allowedRepositories"` |
|
| 165 |
+} |
| ... | ... |
@@ -7,6 +7,7 @@ import ( |
| 7 | 7 |
"time" |
| 8 | 8 |
|
| 9 | 9 |
"github.com/docker/go-units" |
| 10 |
+ "github.com/moby/buildkit/util/bklog" |
|
| 10 | 11 |
"github.com/pkg/errors" |
| 11 | 12 |
) |
| 12 | 13 |
|
| ... | ... |
@@ -104,3 +105,25 @@ func stripQuotes(s string) string {
|
| 104 | 104 |
} |
| 105 | 105 |
return s |
| 106 | 106 |
} |
| 107 |
+ |
|
| 108 |
+func DetectDefaultGCCap() DiskSpace {
|
|
| 109 |
+ return DiskSpace{Percentage: DiskSpacePercentage}
|
|
| 110 |
+} |
|
| 111 |
+ |
|
| 112 |
+func (d DiskSpace) AsBytes(root string) int64 {
|
|
| 113 |
+ if d.Bytes != 0 {
|
|
| 114 |
+ return d.Bytes |
|
| 115 |
+ } |
|
| 116 |
+ if d.Percentage == 0 {
|
|
| 117 |
+ return 0 |
|
| 118 |
+ } |
|
| 119 |
+ |
|
| 120 |
+ diskSize, err := getDiskSize(root) |
|
| 121 |
+ if err != nil {
|
|
| 122 |
+ bklog.L.Warnf("failed to get disk size: %v", err)
|
|
| 123 |
+ return defaultCap |
|
| 124 |
+ } |
|
| 125 |
+ avail := diskSize * d.Percentage / 100 |
|
| 126 |
+ rounded := (avail/(1<<30) + 1) * 1e9 // round up |
|
| 127 |
+ return rounded |
|
| 128 |
+} |
| 107 | 129 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,19 @@ |
| 0 |
+//go:build openbsd |
|
| 1 |
+// +build openbsd |
|
| 2 |
+ |
|
| 3 |
+package config |
|
| 4 |
+ |
|
| 5 |
+import ( |
|
| 6 |
+ "syscall" |
|
| 7 |
+) |
|
| 8 |
+ |
|
| 9 |
+var DiskSpacePercentage int64 = 10 |
|
| 10 |
+ |
|
| 11 |
+func getDiskSize(root string) (int64, error) {
|
|
| 12 |
+ var st syscall.Statfs_t |
|
| 13 |
+ if err := syscall.Statfs(root, &st); err != nil {
|
|
| 14 |
+ return 0, err |
|
| 15 |
+ } |
|
| 16 |
+ diskSize := int64(st.F_bsize) * int64(st.F_blocks) |
|
| 17 |
+ return diskSize, nil |
|
| 18 |
+} |
| ... | ... |
@@ -1,5 +1,5 @@ |
| 1 |
-//go:build !windows |
|
| 2 |
-// +build !windows |
|
| 1 |
+//go:build !windows && !openbsd |
|
| 2 |
+// +build !windows,!openbsd |
|
| 3 | 3 |
|
| 4 | 4 |
package config |
| 5 | 5 |
|
| ... | ... |
@@ -7,23 +7,13 @@ import ( |
| 7 | 7 |
"syscall" |
| 8 | 8 |
) |
| 9 | 9 |
|
| 10 |
-func DetectDefaultGCCap() DiskSpace {
|
|
| 11 |
- return DiskSpace{Percentage: 10}
|
|
| 12 |
-} |
|
| 13 |
- |
|
| 14 |
-func (d DiskSpace) AsBytes(root string) int64 {
|
|
| 15 |
- if d.Bytes != 0 {
|
|
| 16 |
- return d.Bytes |
|
| 17 |
- } |
|
| 18 |
- if d.Percentage == 0 {
|
|
| 19 |
- return 0 |
|
| 20 |
- } |
|
| 10 |
+var DiskSpacePercentage int64 = 10 |
|
| 21 | 11 |
|
| 12 |
+func getDiskSize(root string) (int64, error) {
|
|
| 22 | 13 |
var st syscall.Statfs_t |
| 23 | 14 |
if err := syscall.Statfs(root, &st); err != nil {
|
| 24 |
- return defaultCap |
|
| 15 |
+ return 0, err |
|
| 25 | 16 |
} |
| 26 | 17 |
diskSize := int64(st.Bsize) * int64(st.Blocks) |
| 27 |
- avail := diskSize * d.Percentage / 100 |
|
| 28 |
- return (avail/(1<<30) + 1) * 1e9 // round up |
|
| 18 |
+ return diskSize, nil |
|
| 29 | 19 |
} |
| ... | ... |
@@ -3,10 +3,29 @@ |
| 3 | 3 |
|
| 4 | 4 |
package config |
| 5 | 5 |
|
| 6 |
-func DetectDefaultGCCap() DiskSpace {
|
|
| 7 |
- return DiskSpace{Bytes: defaultCap}
|
|
| 8 |
-} |
|
| 6 |
+import ( |
|
| 7 |
+ "golang.org/x/sys/windows" |
|
| 8 |
+) |
|
| 9 |
+ |
|
| 10 |
+// set as double that for Linux since |
|
| 11 |
+// Windows images are generally larger. |
|
| 12 |
+var DiskSpacePercentage int64 = 20 |
|
| 13 |
+ |
|
| 14 |
+func getDiskSize(root string) (int64, error) {
|
|
| 15 |
+ rootUTF16, err := windows.UTF16FromString(root) |
|
| 16 |
+ if err != nil {
|
|
| 17 |
+ return 0, err |
|
| 18 |
+ } |
|
| 19 |
+ var freeAvailableBytes uint64 |
|
| 20 |
+ var totalBytes uint64 |
|
| 21 |
+ var totalFreeBytes uint64 |
|
| 9 | 22 |
|
| 10 |
-func (d DiskSpace) AsBytes(root string) int64 {
|
|
| 11 |
- return d.Bytes |
|
| 23 |
+ if err := windows.GetDiskFreeSpaceEx( |
|
| 24 |
+ &rootUTF16[0], |
|
| 25 |
+ &freeAvailableBytes, |
|
| 26 |
+ &totalBytes, |
|
| 27 |
+ &totalFreeBytes); err != nil {
|
|
| 28 |
+ return 0, err |
|
| 29 |
+ } |
|
| 30 |
+ return int64(totalBytes), nil |
|
| 12 | 31 |
} |
| ... | ... |
@@ -102,7 +102,7 @@ func (w *containerdExecutor) prepareExecutionEnv(ctx context.Context, rootMount |
| 102 | 102 |
return resolvConf, hostsFile, releaseAll, nil |
| 103 | 103 |
} |
| 104 | 104 |
|
| 105 |
-func (w *containerdExecutor) ensureCWD(ctx context.Context, details *containerState, meta executor.Meta) error {
|
|
| 105 |
+func (w *containerdExecutor) ensureCWD(_ context.Context, details *containerState, meta executor.Meta) error {
|
|
| 106 | 106 |
newp, err := fs.RootPath(details.rootfsPath, meta.Cwd) |
| 107 | 107 |
if err != nil {
|
| 108 | 108 |
return errors.Wrapf(err, "working dir %s points to invalid target", newp) |
| ... | ... |
@@ -19,13 +19,13 @@ import ( |
| 19 | 19 |
"github.com/pkg/errors" |
| 20 | 20 |
) |
| 21 | 21 |
|
| 22 |
-func getUserSpec(user, rootfsPath string) (specs.User, error) {
|
|
| 22 |
+func getUserSpec(user, _ string) (specs.User, error) {
|
|
| 23 | 23 |
return specs.User{
|
| 24 | 24 |
Username: user, |
| 25 | 25 |
}, nil |
| 26 | 26 |
} |
| 27 | 27 |
|
| 28 |
-func (w *containerdExecutor) prepareExecutionEnv(ctx context.Context, rootMount executor.Mount, mounts []executor.Mount, meta executor.Meta, details *containerState, netMode pb.NetMode) (string, string, func(), error) {
|
|
| 28 |
+func (w *containerdExecutor) prepareExecutionEnv(ctx context.Context, rootMount executor.Mount, _ []executor.Mount, _ executor.Meta, details *containerState, _ pb.NetMode) (string, string, func(), error) {
|
|
| 29 | 29 |
var releasers []func() error |
| 30 | 30 |
releaseAll := func() {
|
| 31 | 31 |
for _, release := range releasers {
|
| ... | ... |
@@ -75,7 +75,7 @@ func (w *containerdExecutor) ensureCWD(ctx context.Context, details *containerSt |
| 75 | 75 |
return nil |
| 76 | 76 |
} |
| 77 | 77 |
|
| 78 |
-func (w *containerdExecutor) createOCISpec(ctx context.Context, id, resolvConf, hostsFile string, namespace network.Namespace, mounts []executor.Mount, meta executor.Meta, details *containerState) (*specs.Spec, func(), error) {
|
|
| 78 |
+func (w *containerdExecutor) createOCISpec(ctx context.Context, id, _, _ string, namespace network.Namespace, mounts []executor.Mount, meta executor.Meta, _ *containerState) (*specs.Spec, func(), error) {
|
|
| 79 | 79 |
var releasers []func() |
| 80 | 80 |
releaseAll := func() {
|
| 81 | 81 |
for _, release := range releasers {
|
| ... | ... |
@@ -53,10 +53,8 @@ func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.Identity |
| 53 | 53 |
generate = true |
| 54 | 54 |
lastNotEmpty = false |
| 55 | 55 |
} |
| 56 |
- } else {
|
|
| 57 |
- if fi.ModTime().Before(fiMain.ModTime()) {
|
|
| 58 |
- generate = true |
|
| 59 |
- } |
|
| 56 |
+ } else if fi.ModTime().Before(fiMain.ModTime()) {
|
|
| 57 |
+ generate = true |
|
| 60 | 58 |
} |
| 61 | 59 |
} |
| 62 | 60 |
} |
| ... | ... |
@@ -183,6 +183,16 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou |
| 183 | 183 |
} |
| 184 | 184 |
releasers = append(releasers, release) |
| 185 | 185 |
for _, mount := range mounts {
|
| 186 |
+ mount, release, err := compactLongOverlayMount(mount, m.Readonly) |
|
| 187 |
+ if err != nil {
|
|
| 188 |
+ releaseAll() |
|
| 189 |
+ return nil, nil, err |
|
| 190 |
+ } |
|
| 191 |
+ |
|
| 192 |
+ if release != nil {
|
|
| 193 |
+ releasers = append(releasers, release) |
|
| 194 |
+ } |
|
| 195 |
+ |
|
| 186 | 196 |
mount, err = sm.subMount(mount, m.Selector) |
| 187 | 197 |
if err != nil {
|
| 188 | 198 |
releaseAll() |
| ... | ... |
@@ -211,6 +221,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou |
| 211 | 211 |
if userns.RunningInUserNS() {
|
| 212 | 212 |
s.Mounts, err = rootlessmountopts.FixUpOCI(s.Mounts) |
| 213 | 213 |
if err != nil {
|
| 214 |
+ releaseAll() |
|
| 214 | 215 |
return nil, nil, err |
| 215 | 216 |
} |
| 216 | 217 |
} |
| ... | ... |
@@ -261,26 +272,8 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error) |
| 261 | 261 |
return mount.Mount{}, err
|
| 262 | 262 |
} |
| 263 | 263 |
|
| 264 |
- var mntType string |
|
| 265 |
- opts := []string{}
|
|
| 266 |
- if m.ReadOnly() {
|
|
| 267 |
- opts = append(opts, "ro") |
|
| 268 |
- } |
|
| 269 |
- |
|
| 270 |
- if runtime.GOOS != "windows" {
|
|
| 271 |
- // Windows uses a mechanism similar to bind mounts, but will err out if we request |
|
| 272 |
- // a mount type it does not understand. Leaving the mount type empty on Windows will |
|
| 273 |
- // yield the same result. |
|
| 274 |
- mntType = "bind" |
|
| 275 |
- opts = append(opts, "rbind") |
|
| 276 |
- } |
|
| 277 |
- |
|
| 278 | 264 |
s.m[h] = mountRef{
|
| 279 |
- mount: mount.Mount{
|
|
| 280 |
- Source: mp, |
|
| 281 |
- Type: mntType, |
|
| 282 |
- Options: opts, |
|
| 283 |
- }, |
|
| 265 |
+ mount: bind(mp, m.ReadOnly()), |
|
| 284 | 266 |
unmount: lm.Unmount, |
| 285 | 267 |
subRefs: map[string]mountRef{},
|
| 286 | 268 |
} |
| ... | ... |
@@ -312,3 +305,45 @@ func (s *submounts) cleanup() {
|
| 312 | 312 |
} |
| 313 | 313 |
wg.Wait() |
| 314 | 314 |
} |
| 315 |
+ |
|
| 316 |
+func bind(p string, ro bool) mount.Mount {
|
|
| 317 |
+ m := mount.Mount{
|
|
| 318 |
+ Source: p, |
|
| 319 |
+ } |
|
| 320 |
+ if runtime.GOOS != "windows" {
|
|
| 321 |
+ // Windows uses a mechanism similar to bind mounts, but will err out if we request |
|
| 322 |
+ // a mount type it does not understand. Leaving the mount type empty on Windows will |
|
| 323 |
+ // yield the same result. |
|
| 324 |
+ m.Type = "bind" |
|
| 325 |
+ m.Options = []string{"rbind"}
|
|
| 326 |
+ } |
|
| 327 |
+ if ro {
|
|
| 328 |
+ m.Options = append(m.Options, "ro") |
|
| 329 |
+ } |
|
| 330 |
+ return m |
|
| 331 |
+} |
|
| 332 |
+ |
|
| 333 |
+func compactLongOverlayMount(m mount.Mount, ro bool) (mount.Mount, func() error, error) {
|
|
| 334 |
+ if m.Type != "overlay" {
|
|
| 335 |
+ return m, nil, nil |
|
| 336 |
+ } |
|
| 337 |
+ |
|
| 338 |
+ sz := 0 |
|
| 339 |
+ for _, opt := range m.Options {
|
|
| 340 |
+ sz += len(opt) + 1 |
|
| 341 |
+ } |
|
| 342 |
+ |
|
| 343 |
+ // can fit to single page, no need to compact |
|
| 344 |
+ if sz < 4096-512 {
|
|
| 345 |
+ return m, nil, nil |
|
| 346 |
+ } |
|
| 347 |
+ |
|
| 348 |
+ lm := snapshot.LocalMounterWithMounts([]mount.Mount{m})
|
|
| 349 |
+ |
|
| 350 |
+ mp, err := lm.Mount() |
|
| 351 |
+ if err != nil {
|
|
| 352 |
+ return mount.Mount{}, nil, err
|
|
| 353 |
+ } |
|
| 354 |
+ |
|
| 355 |
+ return bind(mp, ro), lm.Unmount, nil |
|
| 356 |
+} |
| ... | ... |
@@ -14,12 +14,12 @@ func withProcessArgs(args ...string) oci.SpecOpts {
|
| 14 | 14 |
return oci.WithProcessArgs(args...) |
| 15 | 15 |
} |
| 16 | 16 |
|
| 17 |
-func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) {
|
|
| 17 |
+func generateMountOpts(_, _ string) ([]oci.SpecOpts, error) {
|
|
| 18 | 18 |
return nil, nil |
| 19 | 19 |
} |
| 20 | 20 |
|
| 21 | 21 |
// generateSecurityOpts may affect mounts, so must be called after generateMountOpts |
| 22 |
-func generateSecurityOpts(mode pb.SecurityMode, apparmorProfile string, selinuxB bool) ([]oci.SpecOpts, error) {
|
|
| 22 |
+func generateSecurityOpts(mode pb.SecurityMode, _ string, _ bool) ([]oci.SpecOpts, error) {
|
|
| 23 | 23 |
if mode == pb.SecurityMode_INSECURE {
|
| 24 | 24 |
return nil, errors.New("no support for running in insecure mode on FreeBSD")
|
| 25 | 25 |
} |
| ... | ... |
@@ -49,7 +49,7 @@ func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) {
|
| 49 | 49 |
} |
| 50 | 50 |
|
| 51 | 51 |
// tracing is not implemented on FreeBSD |
| 52 |
-func getTracingSocketMount(socket string) *specs.Mount {
|
|
| 52 |
+func getTracingSocketMount(_ string) *specs.Mount {
|
|
| 53 | 53 |
return nil |
| 54 | 54 |
} |
| 55 | 55 |
|
| ... | ... |
@@ -51,14 +51,14 @@ func withGetUserInfoMount() oci.SpecOpts {
|
| 51 | 51 |
} |
| 52 | 52 |
} |
| 53 | 53 |
|
| 54 |
-func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) {
|
|
| 54 |
+func generateMountOpts(_, _ string) ([]oci.SpecOpts, error) {
|
|
| 55 | 55 |
return []oci.SpecOpts{
|
| 56 | 56 |
withGetUserInfoMount(), |
| 57 | 57 |
}, nil |
| 58 | 58 |
} |
| 59 | 59 |
|
| 60 | 60 |
// generateSecurityOpts may affect mounts, so must be called after generateMountOpts |
| 61 |
-func generateSecurityOpts(mode pb.SecurityMode, apparmorProfile string, selinuxB bool) ([]oci.SpecOpts, error) {
|
|
| 61 |
+func generateSecurityOpts(mode pb.SecurityMode, _ string, _ bool) ([]oci.SpecOpts, error) {
|
|
| 62 | 62 |
if mode == pb.SecurityMode_INSECURE {
|
| 63 | 63 |
return nil, errors.New("no support for running in insecure mode on Windows")
|
| 64 | 64 |
} |
| ... | ... |
@@ -1,3 +1,6 @@ |
| 1 |
+//go:build linux |
|
| 2 |
+// +build linux |
|
| 3 |
+ |
|
| 1 | 4 |
package runcexecutor |
| 2 | 5 |
|
| 3 | 6 |
import ( |
| ... | ... |
@@ -445,7 +448,7 @@ func (w *runcExecutor) Exec(ctx context.Context, id string, process executor.Pro |
| 445 | 445 |
spec.Process.Env = process.Meta.Env |
| 446 | 446 |
} |
| 447 | 447 |
|
| 448 |
- err = w.exec(ctx, id, state.Bundle, spec.Process, process, nil) |
|
| 448 |
+ err = w.exec(ctx, id, spec.Process, process, nil) |
|
| 449 | 449 |
return exitError(ctx, err) |
| 450 | 450 |
} |
| 451 | 451 |
|
| ... | ... |
@@ -476,7 +479,7 @@ func (s *forwardIO) Stderr() io.ReadCloser {
|
| 476 | 476 |
return nil |
| 477 | 477 |
} |
| 478 | 478 |
|
| 479 |
-// newRuncProcKiller returns an abstraction for sending SIGKILL to the |
|
| 479 |
+// newRunProcKiller returns an abstraction for sending SIGKILL to the |
|
| 480 | 480 |
// process inside the container initiated from `runc run`. |
| 481 | 481 |
func newRunProcKiller(runC *runc.Runc, id string) procKiller {
|
| 482 | 482 |
return procKiller{runC: runC, id: id}
|
| ... | ... |
@@ -1,88 +1 @@ |
| 1 |
-//go:build !linux |
|
| 2 |
-// +build !linux |
|
| 3 |
- |
|
| 4 | 1 |
package runcexecutor |
| 5 |
- |
|
| 6 |
-import ( |
|
| 7 |
- "context" |
|
| 8 |
- |
|
| 9 |
- runc "github.com/containerd/go-runc" |
|
| 10 |
- "github.com/moby/buildkit/executor" |
|
| 11 |
- "github.com/moby/buildkit/util/bklog" |
|
| 12 |
- "github.com/opencontainers/runtime-spec/specs-go" |
|
| 13 |
- "github.com/pkg/errors" |
|
| 14 |
- "golang.org/x/sync/errgroup" |
|
| 15 |
-) |
|
| 16 |
- |
|
| 17 |
-var errUnsupportedConsole = errors.New("tty for runc is only supported on linux")
|
|
| 18 |
- |
|
| 19 |
-func updateRuncFieldsForHostOS(runtime *runc.Runc) {}
|
|
| 20 |
- |
|
| 21 |
-func (w *runcExecutor) run(ctx context.Context, id, bundle string, process executor.ProcessInfo, started func(), keep bool) error {
|
|
| 22 |
- if process.Meta.Tty {
|
|
| 23 |
- return errUnsupportedConsole |
|
| 24 |
- } |
|
| 25 |
- extraArgs := []string{}
|
|
| 26 |
- if keep {
|
|
| 27 |
- extraArgs = append(extraArgs, "--keep") |
|
| 28 |
- } |
|
| 29 |
- killer := newRunProcKiller(w.runc, id) |
|
| 30 |
- return w.commonCall(ctx, id, bundle, process, started, killer, func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error {
|
|
| 31 |
- _, err := w.runc.Run(ctx, id, bundle, &runc.CreateOpts{
|
|
| 32 |
- NoPivot: w.noPivot, |
|
| 33 |
- Started: started, |
|
| 34 |
- IO: io, |
|
| 35 |
- ExtraArgs: extraArgs, |
|
| 36 |
- }) |
|
| 37 |
- return err |
|
| 38 |
- }) |
|
| 39 |
-} |
|
| 40 |
- |
|
| 41 |
-func (w *runcExecutor) exec(ctx context.Context, id, bundle string, specsProcess *specs.Process, process executor.ProcessInfo, started func()) error {
|
|
| 42 |
- if process.Meta.Tty {
|
|
| 43 |
- return errUnsupportedConsole |
|
| 44 |
- } |
|
| 45 |
- |
|
| 46 |
- killer, err := newExecProcKiller(w.runc, id) |
|
| 47 |
- if err != nil {
|
|
| 48 |
- return errors.Wrap(err, "failed to initialize process killer") |
|
| 49 |
- } |
|
| 50 |
- defer killer.Cleanup() |
|
| 51 |
- |
|
| 52 |
- return w.commonCall(ctx, id, bundle, process, started, killer, func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error {
|
|
| 53 |
- return w.runc.Exec(ctx, id, *specsProcess, &runc.ExecOpts{
|
|
| 54 |
- Started: started, |
|
| 55 |
- IO: io, |
|
| 56 |
- PidFile: pidfile, |
|
| 57 |
- }) |
|
| 58 |
- }) |
|
| 59 |
-} |
|
| 60 |
- |
|
| 61 |
-type runcCall func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error |
|
| 62 |
- |
|
| 63 |
-// commonCall is the common run/exec logic used for non-linux runtimes. A tty |
|
| 64 |
-// is only supported for linux, so this really just handles signal propagation |
|
| 65 |
-// to the started runc process. |
|
| 66 |
-func (w *runcExecutor) commonCall(ctx context.Context, id, bundle string, process executor.ProcessInfo, started func(), killer procKiller, call runcCall) error {
|
|
| 67 |
- runcProcess, ctx := runcProcessHandle(ctx, killer) |
|
| 68 |
- defer runcProcess.Release() |
|
| 69 |
- |
|
| 70 |
- eg, ctx := errgroup.WithContext(ctx) |
|
| 71 |
- defer func() {
|
|
| 72 |
- if err := eg.Wait(); err != nil && !errors.Is(err, context.Canceled) {
|
|
| 73 |
- bklog.G(ctx).Errorf("runc process monitoring error: %s", err)
|
|
| 74 |
- } |
|
| 75 |
- }() |
|
| 76 |
- defer runcProcess.Shutdown() |
|
| 77 |
- |
|
| 78 |
- startedCh := make(chan int, 1) |
|
| 79 |
- eg.Go(func() error {
|
|
| 80 |
- return runcProcess.WaitForStart(ctx, startedCh, started) |
|
| 81 |
- }) |
|
| 82 |
- |
|
| 83 |
- eg.Go(func() error {
|
|
| 84 |
- return handleSignals(ctx, runcProcess, process.Signal) |
|
| 85 |
- }) |
|
| 86 |
- |
|
| 87 |
- return call(ctx, startedCh, &forwardIO{stdin: process.Stdin, stdout: process.Stdout, stderr: process.Stderr}, killer.pidfile)
|
|
| 88 |
-} |
| ... | ... |
@@ -23,7 +23,7 @@ func updateRuncFieldsForHostOS(runtime *runc.Runc) {
|
| 23 | 23 |
|
| 24 | 24 |
func (w *runcExecutor) run(ctx context.Context, id, bundle string, process executor.ProcessInfo, started func(), keep bool) error {
|
| 25 | 25 |
killer := newRunProcKiller(w.runc, id) |
| 26 |
- return w.callWithIO(ctx, id, bundle, process, started, killer, func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error {
|
|
| 26 |
+ return w.callWithIO(ctx, process, started, killer, func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error {
|
|
| 27 | 27 |
extraArgs := []string{}
|
| 28 | 28 |
if keep {
|
| 29 | 29 |
extraArgs = append(extraArgs, "--keep") |
| ... | ... |
@@ -38,14 +38,14 @@ func (w *runcExecutor) run(ctx context.Context, id, bundle string, process execu |
| 38 | 38 |
}) |
| 39 | 39 |
} |
| 40 | 40 |
|
| 41 |
-func (w *runcExecutor) exec(ctx context.Context, id, bundle string, specsProcess *specs.Process, process executor.ProcessInfo, started func()) error {
|
|
| 41 |
+func (w *runcExecutor) exec(ctx context.Context, id string, specsProcess *specs.Process, process executor.ProcessInfo, started func()) error {
|
|
| 42 | 42 |
killer, err := newExecProcKiller(w.runc, id) |
| 43 | 43 |
if err != nil {
|
| 44 | 44 |
return errors.Wrap(err, "failed to initialize process killer") |
| 45 | 45 |
} |
| 46 | 46 |
defer killer.Cleanup() |
| 47 | 47 |
|
| 48 |
- return w.callWithIO(ctx, id, bundle, process, started, killer, func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error {
|
|
| 48 |
+ return w.callWithIO(ctx, process, started, killer, func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error {
|
|
| 49 | 49 |
return w.runc.Exec(ctx, id, *specsProcess, &runc.ExecOpts{
|
| 50 | 50 |
Started: started, |
| 51 | 51 |
IO: io, |
| ... | ... |
@@ -56,7 +56,7 @@ func (w *runcExecutor) exec(ctx context.Context, id, bundle string, specsProcess |
| 56 | 56 |
|
| 57 | 57 |
type runcCall func(ctx context.Context, started chan<- int, io runc.IO, pidfile string) error |
| 58 | 58 |
|
| 59 |
-func (w *runcExecutor) callWithIO(ctx context.Context, id, bundle string, process executor.ProcessInfo, started func(), killer procKiller, call runcCall) error {
|
|
| 59 |
+func (w *runcExecutor) callWithIO(ctx context.Context, process executor.ProcessInfo, started func(), killer procKiller, call runcCall) error {
|
|
| 60 | 60 |
runcProcess, ctx := runcProcessHandle(ctx, killer) |
| 61 | 61 |
defer runcProcess.Release() |
| 62 | 62 |
|
| ... | ... |
@@ -70,7 +70,7 @@ func MakeInTotoStatements(ctx context.Context, s session.Group, attestations []e |
| 70 | 70 |
|
| 71 | 71 |
switch att.Kind {
|
| 72 | 72 |
case gatewaypb.AttestationKindInToto: |
| 73 |
- stmt, err := makeInTotoStatement(ctx, content, att, defaultSubjects) |
|
| 73 |
+ stmt, err := makeInTotoStatement(content, att, defaultSubjects) |
|
| 74 | 74 |
if err != nil {
|
| 75 | 75 |
return err |
| 76 | 76 |
} |
| ... | ... |
@@ -87,7 +87,7 @@ func MakeInTotoStatements(ctx context.Context, s session.Group, attestations []e |
| 87 | 87 |
return statements, nil |
| 88 | 88 |
} |
| 89 | 89 |
|
| 90 |
-func makeInTotoStatement(ctx context.Context, content []byte, attestation exporter.Attestation, defaultSubjects []intoto.Subject) (*intoto.Statement, error) {
|
|
| 90 |
+func makeInTotoStatement(content []byte, attestation exporter.Attestation, defaultSubjects []intoto.Subject) (*intoto.Statement, error) {
|
|
| 91 | 91 |
if len(attestation.InToto.Subjects) == 0 {
|
| 92 | 92 |
attestation.InToto.Subjects = []result.InTotoSubject{{
|
| 93 | 93 |
Kind: gatewaypb.InTotoSubjectKindSelf, |
| ... | ... |
@@ -59,7 +59,7 @@ func Unbundle(ctx context.Context, s session.Group, bundled []exporter.Attestati |
| 59 | 59 |
} |
| 60 | 60 |
defer lm.Unmount() |
| 61 | 61 |
|
| 62 |
- atts, err := unbundle(ctx, src, att) |
|
| 62 |
+ atts, err := unbundle(src, att) |
|
| 63 | 63 |
if err != nil {
|
| 64 | 64 |
return err |
| 65 | 65 |
} |
| ... | ... |
@@ -116,7 +116,7 @@ func sort(atts []exporter.Attestation) []exporter.Attestation {
|
| 116 | 116 |
return result |
| 117 | 117 |
} |
| 118 | 118 |
|
| 119 |
-func unbundle(ctx context.Context, root string, bundle exporter.Attestation) ([]exporter.Attestation, error) {
|
|
| 119 |
+func unbundle(root string, bundle exporter.Attestation) ([]exporter.Attestation, error) {
|
|
| 120 | 120 |
dir, err := fs.RootPath(root, bundle.Path) |
| 121 | 121 |
if err != nil {
|
| 122 | 122 |
return nil, err |
| ... | ... |
@@ -10,7 +10,6 @@ import ( |
| 10 | 10 |
"strings" |
| 11 | 11 |
|
| 12 | 12 |
"github.com/containerd/containerd/content" |
| 13 |
- "github.com/containerd/containerd/errdefs" |
|
| 14 | 13 |
"github.com/containerd/containerd/images" |
| 15 | 14 |
"github.com/containerd/containerd/labels" |
| 16 | 15 |
"github.com/containerd/containerd/leases" |
| ... | ... |
@@ -18,6 +17,7 @@ import ( |
| 18 | 18 |
"github.com/containerd/containerd/platforms" |
| 19 | 19 |
"github.com/containerd/containerd/remotes/docker" |
| 20 | 20 |
"github.com/containerd/containerd/rootfs" |
| 21 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 21 | 22 |
"github.com/moby/buildkit/cache" |
| 22 | 23 |
cacheconfig "github.com/moby/buildkit/cache/config" |
| 23 | 24 |
"github.com/moby/buildkit/exporter" |
| ... | ... |
@@ -270,7 +270,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source |
| 270 | 270 |
for _, sfx := range sfx {
|
| 271 | 271 |
img.Name = targetName + sfx |
| 272 | 272 |
if _, err := e.opt.Images.Update(imageClientCtx, img); err != nil {
|
| 273 |
- if !errors.Is(err, errdefs.ErrNotFound) {
|
|
| 273 |
+ if !errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 274 | 274 |
return nil, nil, tagDone(err) |
| 275 | 275 |
} |
| 276 | 276 |
|
| ... | ... |
@@ -284,7 +284,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source |
| 284 | 284 |
if e.unpack {
|
| 285 | 285 |
if opts.RewriteTimestamp {
|
| 286 | 286 |
// e.unpackImage cannot be used because src ref does not point to the rewritten image |
| 287 |
- /// |
|
| 287 |
+ // / |
|
| 288 | 288 |
// TODO: change e.unpackImage so that it takes Result[Remote] as parameter. |
| 289 | 289 |
// https://github.com/moby/buildkit/pull/4057#discussion_r1324106088 |
| 290 | 290 |
return nil, nil, errors.New("exporter option \"rewrite-timestamp\" conflicts with \"unpack\"")
|
| ... | ... |
@@ -448,7 +448,7 @@ func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Imag |
| 448 | 448 |
} |
| 449 | 449 |
} |
| 450 | 450 |
|
| 451 |
- layers, err := getLayers(ctx, remote.Descriptors, manifest) |
|
| 451 |
+ layers, err := getLayers(remote.Descriptors, manifest) |
|
| 452 | 452 |
if err != nil {
|
| 453 | 453 |
return err |
| 454 | 454 |
} |
| ... | ... |
@@ -478,7 +478,7 @@ func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Imag |
| 478 | 478 |
return err |
| 479 | 479 |
} |
| 480 | 480 |
|
| 481 |
-func getLayers(ctx context.Context, descs []ocispecs.Descriptor, manifest ocispecs.Manifest) ([]rootfs.Layer, error) {
|
|
| 481 |
+func getLayers(descs []ocispecs.Descriptor, manifest ocispecs.Manifest) ([]rootfs.Layer, error) {
|
|
| 482 | 482 |
if len(descs) != len(manifest.Layers) {
|
| 483 | 483 |
return nil, errors.Errorf("mismatched image rootfs and manifest layers")
|
| 484 | 484 |
} |
| ... | ... |
@@ -12,7 +12,7 @@ import ( |
| 12 | 12 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| 13 | 13 |
) |
| 14 | 14 |
|
| 15 |
-func patchImageLayers(ctx context.Context, remote *solver.Remote, history []ocispecs.History, ref cache.ImmutableRef, opts *ImageCommitOpts, sg session.Group) (*solver.Remote, []ocispecs.History, error) {
|
|
| 15 |
+func patchImageLayers(ctx context.Context, remote *solver.Remote, history []ocispecs.History, ref cache.ImmutableRef, opts *ImageCommitOpts, _ session.Group) (*solver.Remote, []ocispecs.History, error) {
|
|
| 16 | 16 |
remote, history = normalizeLayersAndHistory(ctx, remote, history, ref, opts.OCITypes) |
| 17 | 17 |
return remote, history, nil |
| 18 | 18 |
} |
| ... | ... |
@@ -312,7 +312,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session |
| 312 | 312 |
return nil, err |
| 313 | 313 |
} |
| 314 | 314 |
|
| 315 |
- desc, err := ic.commitAttestationsManifest(ctx, opts, p, desc.Digest.String(), stmts) |
|
| 315 |
+ desc, err := ic.commitAttestationsManifest(ctx, opts, desc.Digest.String(), stmts) |
|
| 316 | 316 |
if err != nil {
|
| 317 | 317 |
return nil, err |
| 318 | 318 |
} |
| ... | ... |
@@ -419,11 +419,16 @@ func (ic *ImageWriter) rewriteRemoteWithEpoch(ctx context.Context, opts *ImageCo |
| 419 | 419 |
var divergedFromBase bool |
| 420 | 420 |
for i, desc := range remoteDescriptors {
|
| 421 | 421 |
i, desc := i, desc |
| 422 |
- info, err := cs.Info(ctx, desc.Digest) |
|
| 423 |
- if err != nil {
|
|
| 424 |
- return nil, err |
|
| 422 |
+ // Usually we get non-empty diffID here, but if the content was ingested via a third-party containerd client, |
|
| 423 |
+ // diffID here can be empty, and will be computed by the converter. |
|
| 424 |
+ diffID := digest.Digest(desc.Annotations[labels.LabelUncompressed]) |
|
| 425 |
+ if diffID == "" {
|
|
| 426 |
+ info, err := cs.Info(ctx, desc.Digest) |
|
| 427 |
+ if err != nil {
|
|
| 428 |
+ return nil, err |
|
| 429 |
+ } |
|
| 430 |
+ diffID = digest.Digest(info.Labels[labels.LabelUncompressed]) // can be still empty |
|
| 425 | 431 |
} |
| 426 |
- diffID := digest.Digest(info.Labels[labels.LabelUncompressed]) // can be empty |
|
| 427 | 432 |
var immDiffID digest.Digest |
| 428 | 433 |
if !divergedFromBase && baseImg != nil && i < len(baseImg.RootFS.DiffIDs) {
|
| 429 | 434 |
immDiffID = baseImg.RootFS.DiffIDs[i] |
| ... | ... |
@@ -548,7 +553,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *Ima |
| 548 | 548 |
}, &configDesc, nil |
| 549 | 549 |
} |
| 550 | 550 |
|
| 551 |
-func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *ImageCommitOpts, p exptypes.Platform, target string, statements []intoto.Statement) (*ocispecs.Descriptor, error) {
|
|
| 551 |
+func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *ImageCommitOpts, target string, statements []intoto.Statement) (*ocispecs.Descriptor, error) {
|
|
| 552 | 552 |
var ( |
| 553 | 553 |
manifestType = ocispecs.MediaTypeImageManifest |
| 554 | 554 |
configType = ocispecs.MediaTypeImageConfig |
| ... | ... |
@@ -142,7 +142,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source |
| 142 | 142 |
if e.opts.PlatformSplit {
|
| 143 | 143 |
st := fstypes.Stat{
|
| 144 | 144 |
Mode: uint32(os.ModeDir | 0755), |
| 145 |
- Path: strings.Replace(k, "/", "_", -1), |
|
| 145 |
+ Path: strings.ReplaceAll(k, "/", "_"), |
|
| 146 | 146 |
} |
| 147 | 147 |
if e.opts.Epoch != nil {
|
| 148 | 148 |
st.ModTime = e.opts.Epoch.UnixNano() |
| ... | ... |
@@ -189,7 +189,7 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab |
| 189 | 189 |
if !opt.PlatformSplit {
|
| 190 | 190 |
nameExt := path.Ext(name) |
| 191 | 191 |
namBase := strings.TrimSuffix(name, nameExt) |
| 192 |
- name = fmt.Sprintf("%s.%s%s", namBase, strings.Replace(k, "/", "_", -1), nameExt)
|
|
| 192 |
+ name = fmt.Sprintf("%s.%s%s", namBase, strings.ReplaceAll(k, "/", "_"), nameExt)
|
|
| 193 | 193 |
} |
| 194 | 194 |
if _, ok := names[name]; ok {
|
| 195 | 195 |
return nil, nil, errors.Errorf("duplicate attestation path name %s", name)
|
| ... | ... |
@@ -96,7 +96,7 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source |
| 96 | 96 |
|
| 97 | 97 |
st := fstypes.Stat{
|
| 98 | 98 |
Mode: uint32(os.ModeDir | 0755), |
| 99 |
- Path: strings.Replace(k, "/", "_", -1), |
|
| 99 |
+ Path: strings.ReplaceAll(k, "/", "_"), |
|
| 100 | 100 |
} |
| 101 | 101 |
if e.opts.Epoch != nil {
|
| 102 | 102 |
st.ModTime = e.opts.Epoch.UnixNano() |
| 103 | 103 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,59 @@ |
| 0 |
+package verifier |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "encoding/json" |
|
| 4 |
+ "strings" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/containerd/containerd/platforms" |
|
| 7 |
+ "github.com/moby/buildkit/solver/result" |
|
| 8 |
+) |
|
| 9 |
+ |
|
| 10 |
+const requestOptsKeys = "verifier.requestopts" |
|
| 11 |
+ |
|
| 12 |
+const ( |
|
| 13 |
+ platformsKey = "platform" |
|
| 14 |
+ labelsPrefix = "label:" |
|
| 15 |
+ keyRequestID = "requestid" |
|
| 16 |
+) |
|
| 17 |
+ |
|
| 18 |
+type RequestOpts struct {
|
|
| 19 |
+ Platforms []string |
|
| 20 |
+ Labels map[string]string |
|
| 21 |
+ Request string |
|
| 22 |
+} |
|
| 23 |
+ |
|
| 24 |
+func CaptureFrontendOpts[T comparable](m map[string]string, res *result.Result[T]) error {
|
|
| 25 |
+ req := &RequestOpts{}
|
|
| 26 |
+ if v, ok := m[platformsKey]; ok {
|
|
| 27 |
+ req.Platforms = strings.Split(v, ",") |
|
| 28 |
+ } else {
|
|
| 29 |
+ req.Platforms = []string{platforms.Format(platforms.Normalize(platforms.DefaultSpec()))}
|
|
| 30 |
+ } |
|
| 31 |
+ |
|
| 32 |
+ req.Labels = map[string]string{}
|
|
| 33 |
+ for k, v := range m {
|
|
| 34 |
+ if strings.HasPrefix(k, labelsPrefix) {
|
|
| 35 |
+ req.Labels[strings.TrimPrefix(k, labelsPrefix)] = v |
|
| 36 |
+ } |
|
| 37 |
+ } |
|
| 38 |
+ req.Request = m[keyRequestID] |
|
| 39 |
+ |
|
| 40 |
+ dt, err := json.Marshal(req) |
|
| 41 |
+ if err != nil {
|
|
| 42 |
+ return err |
|
| 43 |
+ } |
|
| 44 |
+ res.AddMeta(requestOptsKeys, dt) |
|
| 45 |
+ return nil |
|
| 46 |
+} |
|
| 47 |
+ |
|
| 48 |
+func getRequestOpts[T comparable](res *result.Result[T]) (*RequestOpts, error) {
|
|
| 49 |
+ dt, ok := res.Metadata[requestOptsKeys] |
|
| 50 |
+ if !ok {
|
|
| 51 |
+ return nil, nil |
|
| 52 |
+ } |
|
| 53 |
+ req := &RequestOpts{}
|
|
| 54 |
+ if err := json.Unmarshal(dt, req); err != nil {
|
|
| 55 |
+ return nil, err |
|
| 56 |
+ } |
|
| 57 |
+ return req, nil |
|
| 58 |
+} |
| 0 | 59 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,107 @@ |
| 0 |
+package verifier |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "fmt" |
|
| 5 |
+ "sort" |
|
| 6 |
+ "strings" |
|
| 7 |
+ |
|
| 8 |
+ "github.com/containerd/containerd/platforms" |
|
| 9 |
+ "github.com/moby/buildkit/client" |
|
| 10 |
+ "github.com/moby/buildkit/exporter/containerimage/exptypes" |
|
| 11 |
+ "github.com/moby/buildkit/solver/result" |
|
| 12 |
+ "github.com/pkg/errors" |
|
| 13 |
+) |
|
| 14 |
+ |
|
| 15 |
+func CheckInvalidPlatforms[T comparable](ctx context.Context, res *result.Result[T]) ([]client.VertexWarning, error) {
|
|
| 16 |
+ req, err := getRequestOpts(res) |
|
| 17 |
+ if err != nil {
|
|
| 18 |
+ return nil, err |
|
| 19 |
+ } |
|
| 20 |
+ |
|
| 21 |
+ if req.Request != "" {
|
|
| 22 |
+ return nil, nil |
|
| 23 |
+ } |
|
| 24 |
+ |
|
| 25 |
+ if _, ok := res.Metadata[exptypes.ExporterPlatformsKey]; len(res.Refs) > 0 && !ok {
|
|
| 26 |
+ return nil, errors.Errorf("build result contains multiple refs without platforms mapping")
|
|
| 27 |
+ } |
|
| 28 |
+ |
|
| 29 |
+ isMap := len(res.Refs) > 0 |
|
| 30 |
+ |
|
| 31 |
+ ps, err := exptypes.ParsePlatforms(res.Metadata) |
|
| 32 |
+ if err != nil {
|
|
| 33 |
+ return nil, err |
|
| 34 |
+ } |
|
| 35 |
+ |
|
| 36 |
+ warnings := []client.VertexWarning{}
|
|
| 37 |
+ reqMap := map[string]struct{}{}
|
|
| 38 |
+ reqList := []exptypes.Platform{}
|
|
| 39 |
+ |
|
| 40 |
+ for _, v := range req.Platforms {
|
|
| 41 |
+ p, err := platforms.Parse(v) |
|
| 42 |
+ if err != nil {
|
|
| 43 |
+ warnings = append(warnings, client.VertexWarning{
|
|
| 44 |
+ Short: []byte(fmt.Sprintf("Invalid platform result requested %q: %s", v, err.Error())),
|
|
| 45 |
+ }) |
|
| 46 |
+ } |
|
| 47 |
+ p = platforms.Normalize(p) |
|
| 48 |
+ _, ok := reqMap[platforms.Format(p)] |
|
| 49 |
+ if ok {
|
|
| 50 |
+ warnings = append(warnings, client.VertexWarning{
|
|
| 51 |
+ Short: []byte(fmt.Sprintf("Duplicate platform result requested %q", v)),
|
|
| 52 |
+ }) |
|
| 53 |
+ } |
|
| 54 |
+ reqMap[platforms.Format(p)] = struct{}{}
|
|
| 55 |
+ reqList = append(reqList, exptypes.Platform{Platform: p})
|
|
| 56 |
+ } |
|
| 57 |
+ |
|
| 58 |
+ if len(warnings) > 0 {
|
|
| 59 |
+ return warnings, nil |
|
| 60 |
+ } |
|
| 61 |
+ |
|
| 62 |
+ if len(reqMap) == 1 && len(ps.Platforms) == 1 {
|
|
| 63 |
+ pp := platforms.Normalize(ps.Platforms[0].Platform) |
|
| 64 |
+ if _, ok := reqMap[platforms.Format(pp)]; !ok {
|
|
| 65 |
+ return []client.VertexWarning{{
|
|
| 66 |
+ Short: []byte(fmt.Sprintf("Requested platform %q does not match result platform %q", req.Platforms[0], platforms.Format(pp))),
|
|
| 67 |
+ }}, nil |
|
| 68 |
+ } |
|
| 69 |
+ return nil, nil |
|
| 70 |
+ } |
|
| 71 |
+ |
|
| 72 |
+ if !isMap && len(reqMap) > 1 {
|
|
| 73 |
+ return []client.VertexWarning{{
|
|
| 74 |
+ Short: []byte("Multiple platforms requested but result is not multi-platform"),
|
|
| 75 |
+ }}, nil |
|
| 76 |
+ } |
|
| 77 |
+ |
|
| 78 |
+ mismatch := len(reqMap) != len(ps.Platforms) |
|
| 79 |
+ |
|
| 80 |
+ if !mismatch {
|
|
| 81 |
+ for _, p := range ps.Platforms {
|
|
| 82 |
+ pp := platforms.Normalize(p.Platform) |
|
| 83 |
+ if _, ok := reqMap[platforms.Format(pp)]; !ok {
|
|
| 84 |
+ mismatch = true |
|
| 85 |
+ break |
|
| 86 |
+ } |
|
| 87 |
+ } |
|
| 88 |
+ } |
|
| 89 |
+ |
|
| 90 |
+ if mismatch {
|
|
| 91 |
+ return []client.VertexWarning{{
|
|
| 92 |
+ Short: []byte(fmt.Sprintf("Requested platforms %s do not match result platforms %s", platformsString(reqList), platformsString(ps.Platforms))),
|
|
| 93 |
+ }}, nil |
|
| 94 |
+ } |
|
| 95 |
+ |
|
| 96 |
+ return nil, nil |
|
| 97 |
+} |
|
| 98 |
+ |
|
| 99 |
+func platformsString(ps []exptypes.Platform) string {
|
|
| 100 |
+ var ss []string |
|
| 101 |
+ for _, p := range ps {
|
|
| 102 |
+ ss = append(ss, platforms.Format(platforms.Normalize(p.Platform))) |
|
| 103 |
+ } |
|
| 104 |
+ sort.Strings(ss) |
|
| 105 |
+ return strings.Join(ss, ",") |
|
| 106 |
+} |
| ... | ... |
@@ -11,10 +11,12 @@ import ( |
| 11 | 11 |
"github.com/moby/buildkit/frontend" |
| 12 | 12 |
"github.com/moby/buildkit/frontend/attestations/sbom" |
| 13 | 13 |
"github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" |
| 14 |
+ "github.com/moby/buildkit/frontend/dockerfile/linter" |
|
| 14 | 15 |
"github.com/moby/buildkit/frontend/dockerfile/parser" |
| 15 | 16 |
"github.com/moby/buildkit/frontend/dockerui" |
| 16 | 17 |
"github.com/moby/buildkit/frontend/gateway/client" |
| 17 | 18 |
gwpb "github.com/moby/buildkit/frontend/gateway/pb" |
| 19 |
+ "github.com/moby/buildkit/frontend/subrequests/lint" |
|
| 18 | 20 |
"github.com/moby/buildkit/frontend/subrequests/outline" |
| 19 | 21 |
"github.com/moby/buildkit/frontend/subrequests/targets" |
| 20 | 22 |
"github.com/moby/buildkit/solver/errdefs" |
| ... | ... |
@@ -73,8 +75,13 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
|
| 73 | 73 |
Client: bc, |
| 74 | 74 |
SourceMap: src.SourceMap, |
| 75 | 75 |
MetaResolver: c, |
| 76 |
- Warn: func(msg, url string, detail [][]byte, location *parser.Range) {
|
|
| 77 |
- src.Warn(ctx, msg, warnOpts(location, detail, url)) |
|
| 76 |
+ Warn: func(rulename, description, url, msg string, location []parser.Range) {
|
|
| 77 |
+ startLine := 0 |
|
| 78 |
+ if len(location) > 0 {
|
|
| 79 |
+ startLine = location[0].Start.Line |
|
| 80 |
+ } |
|
| 81 |
+ msg = linter.LintFormatShort(rulename, msg, startLine) |
|
| 82 |
+ src.Warn(ctx, msg, warnOpts(location, [][]byte{[]byte(description)}, url))
|
|
| 78 | 83 |
}, |
| 79 | 84 |
} |
| 80 | 85 |
|
| ... | ... |
@@ -85,6 +92,9 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
|
| 85 | 85 |
ListTargets: func(ctx context.Context) (*targets.List, error) {
|
| 86 | 86 |
return dockerfile2llb.ListTargets(ctx, src.Data) |
| 87 | 87 |
}, |
| 88 |
+ Lint: func(ctx context.Context) (*lint.LintResults, error) {
|
|
| 89 |
+ return dockerfile2llb.DockerfileLint(ctx, src.Data, convertOpt) |
|
| 90 |
+ }, |
|
| 88 | 91 |
}); err != nil {
|
| 89 | 92 |
return nil, err |
| 90 | 93 |
} else if ok {
|
| ... | ... |
@@ -236,21 +246,24 @@ func forwardGateway(ctx context.Context, c client.Client, ref string, cmdline st |
| 236 | 236 |
}) |
| 237 | 237 |
} |
| 238 | 238 |
|
| 239 |
-func warnOpts(r *parser.Range, detail [][]byte, url string) client.WarnOpts {
|
|
| 239 |
+func warnOpts(r []parser.Range, detail [][]byte, url string) client.WarnOpts {
|
|
| 240 | 240 |
opts := client.WarnOpts{Level: 1, Detail: detail, URL: url}
|
| 241 | 241 |
if r == nil {
|
| 242 | 242 |
return opts |
| 243 | 243 |
} |
| 244 |
- opts.Range = []*pb.Range{{
|
|
| 245 |
- Start: pb.Position{
|
|
| 246 |
- Line: int32(r.Start.Line), |
|
| 247 |
- Character: int32(r.Start.Character), |
|
| 248 |
- }, |
|
| 249 |
- End: pb.Position{
|
|
| 250 |
- Line: int32(r.End.Line), |
|
| 251 |
- Character: int32(r.End.Character), |
|
| 252 |
- }, |
|
| 253 |
- }} |
|
| 244 |
+ opts.Range = []*pb.Range{}
|
|
| 245 |
+ for _, r := range r {
|
|
| 246 |
+ opts.Range = append(opts.Range, &pb.Range{
|
|
| 247 |
+ Start: pb.Position{
|
|
| 248 |
+ Line: int32(r.Start.Line), |
|
| 249 |
+ Character: int32(r.Start.Character), |
|
| 250 |
+ }, |
|
| 251 |
+ End: pb.Position{
|
|
| 252 |
+ Line: int32(r.End.Line), |
|
| 253 |
+ Character: int32(r.End.Character), |
|
| 254 |
+ }, |
|
| 255 |
+ }) |
|
| 256 |
+ } |
|
| 254 | 257 |
return opts |
| 255 | 258 |
} |
| 256 | 259 |
|
| ... | ... |
@@ -22,9 +22,11 @@ import ( |
| 22 | 22 |
"github.com/moby/buildkit/client/llb/imagemetaresolver" |
| 23 | 23 |
"github.com/moby/buildkit/client/llb/sourceresolver" |
| 24 | 24 |
"github.com/moby/buildkit/frontend/dockerfile/instructions" |
| 25 |
+ "github.com/moby/buildkit/frontend/dockerfile/linter" |
|
| 25 | 26 |
"github.com/moby/buildkit/frontend/dockerfile/parser" |
| 26 | 27 |
"github.com/moby/buildkit/frontend/dockerfile/shell" |
| 27 | 28 |
"github.com/moby/buildkit/frontend/dockerui" |
| 29 |
+ "github.com/moby/buildkit/frontend/subrequests/lint" |
|
| 28 | 30 |
"github.com/moby/buildkit/frontend/subrequests/outline" |
| 29 | 31 |
"github.com/moby/buildkit/frontend/subrequests/targets" |
| 30 | 32 |
"github.com/moby/buildkit/identity" |
| ... | ... |
@@ -62,7 +64,7 @@ type ConvertOpt struct {
|
| 62 | 62 |
TargetPlatform *ocispecs.Platform |
| 63 | 63 |
MetaResolver llb.ImageMetaResolver |
| 64 | 64 |
LLBCaps *apicaps.CapSet |
| 65 |
- Warn func(short, url string, detail [][]byte, location *parser.Range) |
|
| 65 |
+ Warn linter.LintWarnFunc |
|
| 66 | 66 |
} |
| 67 | 67 |
|
| 68 | 68 |
type SBOMTargets struct {
|
| ... | ... |
@@ -88,7 +90,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (st *llb.Sta |
| 88 | 88 |
if ds.ignoreCache {
|
| 89 | 89 |
sbom.IgnoreCache = true |
| 90 | 90 |
} |
| 91 |
- for _, dsi := range findReachable(ds) {
|
|
| 91 |
+ for dsi := range allReachableStages(ds) {
|
|
| 92 | 92 |
if ds != dsi && dsi.scanStage {
|
| 93 | 93 |
sbom.Extras[dsi.stageName] = dsi.state |
| 94 | 94 |
if dsi.ignoreCache {
|
| ... | ... |
@@ -109,12 +111,35 @@ func Dockefile2Outline(ctx context.Context, dt []byte, opt ConvertOpt) (*outline |
| 109 | 109 |
return &o, nil |
| 110 | 110 |
} |
| 111 | 111 |
|
| 112 |
+func DockerfileLint(ctx context.Context, dt []byte, opt ConvertOpt) (*lint.LintResults, error) {
|
|
| 113 |
+ results := &lint.LintResults{}
|
|
| 114 |
+ sourceIndex := results.AddSource(opt.SourceMap) |
|
| 115 |
+ opt.Warn = func(rulename, description, url, fmtmsg string, location []parser.Range) {
|
|
| 116 |
+ results.AddWarning(rulename, description, url, fmtmsg, sourceIndex, location) |
|
| 117 |
+ } |
|
| 118 |
+ _, err := toDispatchState(ctx, dt, opt) |
|
| 119 |
+ |
|
| 120 |
+ var errLoc *parser.ErrorLocation |
|
| 121 |
+ if err != nil {
|
|
| 122 |
+ buildErr := &lint.BuildError{
|
|
| 123 |
+ Message: err.Error(), |
|
| 124 |
+ } |
|
| 125 |
+ if errors.As(err, &errLoc) {
|
|
| 126 |
+ ranges := mergeLocations(errLoc.Locations...) |
|
| 127 |
+ buildErr.Location = toPBLocation(sourceIndex, ranges) |
|
| 128 |
+ } |
|
| 129 |
+ results.Error = buildErr |
|
| 130 |
+ } |
|
| 131 |
+ return results, nil |
|
| 132 |
+} |
|
| 133 |
+ |
|
| 112 | 134 |
func ListTargets(ctx context.Context, dt []byte) (*targets.List, error) {
|
| 113 | 135 |
dockerfile, err := parser.Parse(bytes.NewReader(dt)) |
| 114 | 136 |
if err != nil {
|
| 115 | 137 |
return nil, err |
| 116 | 138 |
} |
| 117 |
- stages, _, err := instructions.Parse(dockerfile.AST) |
|
| 139 |
+ |
|
| 140 |
+ stages, _, err := instructions.Parse(dockerfile.AST, nil) |
|
| 118 | 141 |
if err != nil {
|
| 119 | 142 |
return nil, err |
| 120 | 143 |
} |
| ... | ... |
@@ -137,6 +162,64 @@ func ListTargets(ctx context.Context, dt []byte) (*targets.List, error) {
|
| 137 | 137 |
return l, nil |
| 138 | 138 |
} |
| 139 | 139 |
|
| 140 |
+func parseLintOptions(checkStr string) (*linter.Config, error) {
|
|
| 141 |
+ checkStr = strings.TrimSpace(checkStr) |
|
| 142 |
+ if checkStr == "" {
|
|
| 143 |
+ return &linter.Config{}, nil
|
|
| 144 |
+ } |
|
| 145 |
+ |
|
| 146 |
+ parts := strings.SplitN(checkStr, ";", 2) |
|
| 147 |
+ var skipSet []string |
|
| 148 |
+ var errorOnWarn, skipAll bool |
|
| 149 |
+ for _, p := range parts {
|
|
| 150 |
+ k, v, ok := strings.Cut(p, "=") |
|
| 151 |
+ if !ok {
|
|
| 152 |
+ return nil, errors.Errorf("invalid check option %q", p)
|
|
| 153 |
+ } |
|
| 154 |
+ k = strings.TrimSpace(k) |
|
| 155 |
+ switch k {
|
|
| 156 |
+ case "skip": |
|
| 157 |
+ v = strings.TrimSpace(v) |
|
| 158 |
+ if v == "all" {
|
|
| 159 |
+ skipAll = true |
|
| 160 |
+ } else {
|
|
| 161 |
+ skipSet = strings.Split(v, ",") |
|
| 162 |
+ for i, rule := range skipSet {
|
|
| 163 |
+ skipSet[i] = strings.TrimSpace(rule) |
|
| 164 |
+ } |
|
| 165 |
+ } |
|
| 166 |
+ case "error": |
|
| 167 |
+ v, err := strconv.ParseBool(strings.TrimSpace(v)) |
|
| 168 |
+ if err != nil {
|
|
| 169 |
+ return nil, errors.Wrapf(err, "failed to parse check option %q", p) |
|
| 170 |
+ } |
|
| 171 |
+ errorOnWarn = v |
|
| 172 |
+ default: |
|
| 173 |
+ return nil, errors.Errorf("invalid check option %q", k)
|
|
| 174 |
+ } |
|
| 175 |
+ } |
|
| 176 |
+ return &linter.Config{
|
|
| 177 |
+ SkipRules: skipSet, |
|
| 178 |
+ SkipAll: skipAll, |
|
| 179 |
+ ReturnAsError: errorOnWarn, |
|
| 180 |
+ }, nil |
|
| 181 |
+} |
|
| 182 |
+ |
|
| 183 |
+func newRuleLinter(dt []byte, opt *ConvertOpt) (*linter.Linter, error) {
|
|
| 184 |
+ var lintOptionStr string |
|
| 185 |
+ if opt.Client != nil && opt.Client.LinterConfig != nil {
|
|
| 186 |
+ lintOptionStr = *opt.Client.LinterConfig |
|
| 187 |
+ } else {
|
|
| 188 |
+ lintOptionStr, _, _, _ = parser.ParseDirective("check", dt)
|
|
| 189 |
+ } |
|
| 190 |
+ lintConfig, err := parseLintOptions(lintOptionStr) |
|
| 191 |
+ if err != nil {
|
|
| 192 |
+ return nil, errors.Wrapf(err, "failed to parse check options") |
|
| 193 |
+ } |
|
| 194 |
+ lintConfig.Warn = opt.Warn |
|
| 195 |
+ return linter.New(lintConfig), nil |
|
| 196 |
+} |
|
| 197 |
+ |
|
| 140 | 198 |
func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchState, error) {
|
| 141 | 199 |
if len(dt) == 0 {
|
| 142 | 200 |
return nil, errors.Errorf("the Dockerfile cannot be empty")
|
| ... | ... |
@@ -159,8 +242,9 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 159 | 159 |
return nil, nil, nil |
| 160 | 160 |
} |
| 161 | 161 |
|
| 162 |
- if opt.Warn == nil {
|
|
| 163 |
- opt.Warn = func(string, string, [][]byte, *parser.Range) {}
|
|
| 162 |
+ lint, err := newRuleLinter(dt, &opt) |
|
| 163 |
+ if err != nil {
|
|
| 164 |
+ return nil, err |
|
| 164 | 165 |
} |
| 165 | 166 |
|
| 166 | 167 |
if opt.Client != nil && opt.LLBCaps == nil {
|
| ... | ... |
@@ -180,16 +264,28 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 180 | 180 |
return nil, err |
| 181 | 181 |
} |
| 182 | 182 |
|
| 183 |
- for _, w := range dockerfile.Warnings {
|
|
| 184 |
- opt.Warn(w.Short, w.URL, w.Detail, w.Location) |
|
| 183 |
+ // Moby still uses the `dockerfile.PrintWarnings` method to print non-empty |
|
| 184 |
+ // continuation line warnings. We iterate over those warnings here. |
|
| 185 |
+ for _, warning := range dockerfile.Warnings {
|
|
| 186 |
+ // The `dockerfile.Warnings` *should* only contain warnings about empty continuation |
|
| 187 |
+ // lines, but we'll check the warning message to be sure, so that we don't accidentally |
|
| 188 |
+ // process warnings that are not related to empty continuation lines twice. |
|
| 189 |
+ if warning.URL == linter.RuleNoEmptyContinuations.URL {
|
|
| 190 |
+ location := []parser.Range{*warning.Location}
|
|
| 191 |
+ msg := linter.RuleNoEmptyContinuations.Format() |
|
| 192 |
+ lint.Run(&linter.RuleNoEmptyContinuations, location, msg) |
|
| 193 |
+ } |
|
| 185 | 194 |
} |
| 186 | 195 |
|
| 196 |
+ validateCommandCasing(dockerfile, lint) |
|
| 197 |
+ |
|
| 187 | 198 |
proxyEnv := proxyEnvFromBuildArgs(opt.BuildArgs) |
| 188 | 199 |
|
| 189 |
- stages, metaArgs, err := instructions.Parse(dockerfile.AST) |
|
| 200 |
+ stages, metaArgs, err := instructions.Parse(dockerfile.AST, lint) |
|
| 190 | 201 |
if err != nil {
|
| 191 | 202 |
return nil, err |
| 192 | 203 |
} |
| 204 |
+ validateStageNames(stages, lint) |
|
| 193 | 205 |
|
| 194 | 206 |
shlex := shell.NewLex(dockerfile.EscapeToken) |
| 195 | 207 |
outline := newOutlineCapture() |
| ... | ... |
@@ -199,7 +295,12 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 199 | 199 |
info := argInfo{definition: metaArg, location: cmd.Location()}
|
| 200 | 200 |
if v, ok := opt.BuildArgs[metaArg.Key]; !ok {
|
| 201 | 201 |
if metaArg.Value != nil {
|
| 202 |
- *metaArg.Value, info.deps, _ = shlex.ProcessWordWithMatches(*metaArg.Value, metaArgsToMap(optMetaArgs)) |
|
| 202 |
+ result, err := shlex.ProcessWordWithMatches(*metaArg.Value, metaArgsToMap(optMetaArgs)) |
|
| 203 |
+ if err != nil {
|
|
| 204 |
+ return nil, parser.WithLocation(err, cmd.Location()) |
|
| 205 |
+ } |
|
| 206 |
+ *metaArg.Value = result.Result |
|
| 207 |
+ info.deps = result.Matched |
|
| 203 | 208 |
} |
| 204 | 209 |
} else {
|
| 205 | 210 |
metaArg.Value = &v |
| ... | ... |
@@ -221,14 +322,17 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 221 | 221 |
|
| 222 | 222 |
// set base state for every image |
| 223 | 223 |
for i, st := range stages {
|
| 224 |
- name, used, err := shlex.ProcessWordWithMatches(st.BaseName, metaArgsToMap(optMetaArgs)) |
|
| 224 |
+ nameMatch, err := shlex.ProcessWordWithMatches(st.BaseName, metaArgsToMap(optMetaArgs)) |
|
| 225 |
+ reportUnusedFromArgs(metaArgsKeys(optMetaArgs), nameMatch.Unmatched, st.Location, lint) |
|
| 226 |
+ used := nameMatch.Matched |
|
| 227 |
+ |
|
| 225 | 228 |
if err != nil {
|
| 226 | 229 |
return nil, parser.WithLocation(err, st.Location) |
| 227 | 230 |
} |
| 228 |
- if name == "" {
|
|
| 231 |
+ if nameMatch.Result == "" {
|
|
| 229 | 232 |
return nil, parser.WithLocation(errors.Errorf("base name (%s) should not be blank", st.BaseName), st.Location)
|
| 230 | 233 |
} |
| 231 |
- st.BaseName = name |
|
| 234 |
+ st.BaseName = nameMatch.Result |
|
| 232 | 235 |
|
| 233 | 236 |
ds := &dispatchState{
|
| 234 | 237 |
stage: st, |
| ... | ... |
@@ -242,18 +346,31 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 242 | 242 |
} |
| 243 | 243 |
|
| 244 | 244 |
if v := st.Platform; v != "" {
|
| 245 |
- v, u, err := shlex.ProcessWordWithMatches(v, metaArgsToMap(optMetaArgs)) |
|
| 245 |
+ platMatch, err := shlex.ProcessWordWithMatches(v, metaArgsToMap(optMetaArgs)) |
|
| 246 |
+ reportUnusedFromArgs(metaArgsKeys(optMetaArgs), platMatch.Unmatched, st.Location, lint) |
|
| 247 |
+ |
|
| 246 | 248 |
if err != nil {
|
| 247 |
- return nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location) |
|
| 249 |
+ return nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", platMatch.Result), st.Location) |
|
| 250 |
+ } |
|
| 251 |
+ |
|
| 252 |
+ if platMatch.Result == "" {
|
|
| 253 |
+ err := errors.Errorf("empty platform value from expression %s", v)
|
|
| 254 |
+ err = parser.WithLocation(err, st.Location) |
|
| 255 |
+ err = wrapSuggestAny(err, platMatch.Unmatched, metaArgsKeys(optMetaArgs)) |
|
| 256 |
+ return nil, err |
|
| 248 | 257 |
} |
| 249 | 258 |
|
| 250 |
- p, err := platforms.Parse(v) |
|
| 259 |
+ p, err := platforms.Parse(platMatch.Result) |
|
| 251 | 260 |
if err != nil {
|
| 261 |
+ err = parser.WithLocation(err, st.Location) |
|
| 262 |
+ err = wrapSuggestAny(err, platMatch.Unmatched, metaArgsKeys(optMetaArgs)) |
|
| 252 | 263 |
return nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location) |
| 253 | 264 |
} |
| 254 |
- for k := range u {
|
|
| 265 |
+ |
|
| 266 |
+ for k := range platMatch.Matched {
|
|
| 255 | 267 |
used[k] = struct{}{}
|
| 256 | 268 |
} |
| 269 |
+ |
|
| 257 | 270 |
ds.platform = &p |
| 258 | 271 |
} |
| 259 | 272 |
|
| ... | ... |
@@ -363,11 +480,12 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 363 | 363 |
allStageNames = append(allStageNames, s.stageName) |
| 364 | 364 |
} |
| 365 | 365 |
} |
| 366 |
+ allReachable := allReachableStages(target) |
|
| 366 | 367 |
|
| 367 | 368 |
baseCtx := ctx |
| 368 | 369 |
eg, ctx := errgroup.WithContext(ctx) |
| 369 | 370 |
for i, d := range allDispatchStates.states {
|
| 370 |
- reachable := isReachable(target, d) |
|
| 371 |
+ _, reachable := allReachable[d] |
|
| 371 | 372 |
// resolve image config for every stage |
| 372 | 373 |
if d.base == nil && !d.noinit {
|
| 373 | 374 |
if d.stage.BaseName == emptyImageName {
|
| ... | ... |
@@ -483,6 +601,9 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 483 | 483 |
llb.WithCustomName(prefixCommand(d, "FROM "+d.stage.BaseName, opt.MultiPlatformRequested, platform, nil)), |
| 484 | 484 |
location(opt.SourceMap, d.stage.Location), |
| 485 | 485 |
) |
| 486 |
+ if reachable {
|
|
| 487 |
+ validateBaseImagePlatform(origName, *platform, d.image.Platform, d.stage.Location, lint) |
|
| 488 |
+ } |
|
| 486 | 489 |
} |
| 487 | 490 |
d.platform = platform |
| 488 | 491 |
return nil |
| ... | ... |
@@ -500,21 +621,10 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 500 | 500 |
ctxPaths := map[string]struct{}{}
|
| 501 | 501 |
|
| 502 | 502 |
for _, d := range allDispatchStates.states {
|
| 503 |
- if !isReachable(target, d) || d.noinit {
|
|
| 503 |
+ if _, ok := allReachable[d]; !ok || d.noinit {
|
|
| 504 | 504 |
continue |
| 505 | 505 |
} |
| 506 |
- // mark as initialized, used to determine states that have not been dispatched yet |
|
| 507 |
- d.noinit = true |
|
| 508 |
- |
|
| 509 |
- if d.base != nil {
|
|
| 510 |
- d.state = d.base.state |
|
| 511 |
- d.platform = d.base.platform |
|
| 512 |
- d.image = clone(d.base.image) |
|
| 513 |
- d.baseImg = cloneX(d.base.baseImg) |
|
| 514 |
- // Utilize the same path index as our base image so we propagate |
|
| 515 |
- // the paths we use back to the base image. |
|
| 516 |
- d.paths = d.base.paths |
|
| 517 |
- } |
|
| 506 |
+ d.init() |
|
| 518 | 507 |
|
| 519 | 508 |
// Ensure platform is set. |
| 520 | 509 |
if d.platform == nil {
|
| ... | ... |
@@ -565,6 +675,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 565 | 565 |
cgroupParent: opt.CgroupParent, |
| 566 | 566 |
llbCaps: opt.LLBCaps, |
| 567 | 567 |
sourceMap: opt.SourceMap, |
| 568 |
+ lint: lint, |
|
| 568 | 569 |
} |
| 569 | 570 |
|
| 570 | 571 |
if err = dispatchOnBuildTriggers(d, d.image.Config.OnBuild, opt); err != nil {
|
| ... | ... |
@@ -608,6 +719,14 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 608 | 608 |
target.image.Config.Labels[k] = v |
| 609 | 609 |
} |
| 610 | 610 |
|
| 611 |
+ // If lint.Error() returns an error, it means that |
|
| 612 |
+ // there were warnings, and that our linter has been |
|
| 613 |
+ // configured to return an error on warnings, |
|
| 614 |
+ // so we appropriately return that error here. |
|
| 615 |
+ if err := lint.Error(); err != nil {
|
|
| 616 |
+ return nil, err |
|
| 617 |
+ } |
|
| 618 |
+ |
|
| 611 | 619 |
opts := filterPaths(ctxPaths) |
| 612 | 620 |
bctx := opt.MainContext |
| 613 | 621 |
if opt.Client != nil {
|
| ... | ... |
@@ -651,6 +770,14 @@ func metaArgsToMap(metaArgs []instructions.KeyValuePairOptional) map[string]stri |
| 651 | 651 |
return m |
| 652 | 652 |
} |
| 653 | 653 |
|
| 654 |
+func metaArgsKeys(metaArgs []instructions.KeyValuePairOptional) []string {
|
|
| 655 |
+ s := make([]string, 0, len(metaArgs)) |
|
| 656 |
+ for _, arg := range metaArgs {
|
|
| 657 |
+ s = append(s, arg.Key) |
|
| 658 |
+ } |
|
| 659 |
+ return s |
|
| 660 |
+} |
|
| 661 |
+ |
|
| 654 | 662 |
func toCommand(ic instructions.Command, allDispatchStates *dispatchStates) (command, error) {
|
| 655 | 663 |
cmd := command{Command: ic}
|
| 656 | 664 |
if c, ok := ic.(*instructions.CopyCommand); ok {
|
| ... | ... |
@@ -700,16 +827,23 @@ type dispatchOpt struct {
|
| 700 | 700 |
cgroupParent string |
| 701 | 701 |
llbCaps *apicaps.CapSet |
| 702 | 702 |
sourceMap *llb.SourceMap |
| 703 |
+ lint *linter.Linter |
|
| 703 | 704 |
} |
| 704 | 705 |
|
| 705 | 706 |
func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
|
| 706 |
- if ex, ok := cmd.Command.(instructions.SupportsSingleWordExpansion); ok {
|
|
| 707 |
+ var err error |
|
| 708 |
+ // ARG command value could be ignored, so defer handling the expansion error |
|
| 709 |
+ _, isArg := cmd.Command.(*instructions.ArgCommand) |
|
| 710 |
+ if ex, ok := cmd.Command.(instructions.SupportsSingleWordExpansion); ok && !isArg {
|
|
| 707 | 711 |
err := ex.Expand(func(word string) (string, error) {
|
| 708 | 712 |
env, err := d.state.Env(context.TODO()) |
| 709 | 713 |
if err != nil {
|
| 710 | 714 |
return "", err |
| 711 | 715 |
} |
| 712 |
- return opt.shlex.ProcessWord(word, env) |
|
| 716 |
+ |
|
| 717 |
+ newword, unmatched, err := opt.shlex.ProcessWord(word, env) |
|
| 718 |
+ reportUnmatchedVariables(cmd, d.buildArgs, env, unmatched, &opt) |
|
| 719 |
+ return newword, err |
|
| 713 | 720 |
}) |
| 714 | 721 |
if err != nil {
|
| 715 | 722 |
return err |
| ... | ... |
@@ -721,17 +855,17 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
|
| 721 | 721 |
if err != nil {
|
| 722 | 722 |
return "", err |
| 723 | 723 |
} |
| 724 |
- |
|
| 725 | 724 |
lex := shell.NewLex('\\')
|
| 726 | 725 |
lex.SkipProcessQuotes = true |
| 727 |
- return lex.ProcessWord(word, env) |
|
| 726 |
+ newword, unmatched, err := lex.ProcessWord(word, env) |
|
| 727 |
+ reportUnmatchedVariables(cmd, d.buildArgs, env, unmatched, &opt) |
|
| 728 |
+ return newword, err |
|
| 728 | 729 |
}) |
| 729 | 730 |
if err != nil {
|
| 730 | 731 |
return err |
| 731 | 732 |
} |
| 732 | 733 |
} |
| 733 | 734 |
|
| 734 |
- var err error |
|
| 735 | 735 |
switch c := cmd.Command.(type) {
|
| 736 | 736 |
case *instructions.MaintainerCommand: |
| 737 | 737 |
err = dispatchMaintainer(d, c) |
| ... | ... |
@@ -774,11 +908,11 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
|
| 774 | 774 |
case *instructions.OnbuildCommand: |
| 775 | 775 |
err = dispatchOnbuild(d, c) |
| 776 | 776 |
case *instructions.CmdCommand: |
| 777 |
- err = dispatchCmd(d, c) |
|
| 777 |
+ err = dispatchCmd(d, c, opt.lint) |
|
| 778 | 778 |
case *instructions.EntrypointCommand: |
| 779 |
- err = dispatchEntrypoint(d, c) |
|
| 779 |
+ err = dispatchEntrypoint(d, c, opt.lint) |
|
| 780 | 780 |
case *instructions.HealthCheckCommand: |
| 781 |
- err = dispatchHealthcheck(d, c) |
|
| 781 |
+ err = dispatchHealthcheck(d, c, opt.lint) |
|
| 782 | 782 |
case *instructions.ExposeCommand: |
| 783 | 783 |
err = dispatchExpose(d, c, opt.shlex) |
| 784 | 784 |
case *instructions.UserCommand: |
| ... | ... |
@@ -790,7 +924,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
|
| 790 | 790 |
case *instructions.ShellCommand: |
| 791 | 791 |
err = dispatchShell(d, c) |
| 792 | 792 |
case *instructions.ArgCommand: |
| 793 |
- err = dispatchArg(d, c, opt.metaArgs, opt.buildArgValues) |
|
| 793 |
+ err = dispatchArg(d, c, &opt) |
|
| 794 | 794 |
case *instructions.CopyCommand: |
| 795 | 795 |
l := opt.buildContext |
| 796 | 796 |
if len(cmd.sources) != 0 {
|
| ... | ... |
@@ -850,7 +984,6 @@ type dispatchState struct {
|
| 850 | 850 |
// paths marks the paths that are used by this dispatchState. |
| 851 | 851 |
paths map[string]struct{}
|
| 852 | 852 |
ignoreCache bool |
| 853 |
- cmdSet bool |
|
| 854 | 853 |
unregistered bool |
| 855 | 854 |
stageName string |
| 856 | 855 |
cmdIndex int |
| ... | ... |
@@ -860,12 +993,39 @@ type dispatchState struct {
|
| 860 | 860 |
epoch *time.Time |
| 861 | 861 |
scanStage bool |
| 862 | 862 |
scanContext bool |
| 863 |
+ // workdirSet is set to true if a workdir has been set |
|
| 864 |
+ // within the current dockerfile. |
|
| 865 |
+ workdirSet bool |
|
| 866 |
+ |
|
| 867 |
+ entrypoint instructionTracker |
|
| 868 |
+ cmd instructionTracker |
|
| 869 |
+ healthcheck instructionTracker |
|
| 863 | 870 |
} |
| 864 | 871 |
|
| 865 | 872 |
func (ds *dispatchState) asyncLocalOpts() []llb.LocalOption {
|
| 866 | 873 |
return filterPaths(ds.paths) |
| 867 | 874 |
} |
| 868 | 875 |
|
| 876 |
+// init is invoked when the dispatch state inherits its attributes |
|
| 877 |
+// from the base image. |
|
| 878 |
+func (ds *dispatchState) init() {
|
|
| 879 |
+ // mark as initialized, used to determine states that have not been dispatched yet |
|
| 880 |
+ ds.noinit = true |
|
| 881 |
+ |
|
| 882 |
+ if ds.base == nil {
|
|
| 883 |
+ return |
|
| 884 |
+ } |
|
| 885 |
+ |
|
| 886 |
+ ds.state = ds.base.state |
|
| 887 |
+ ds.platform = ds.base.platform |
|
| 888 |
+ ds.image = clone(ds.base.image) |
|
| 889 |
+ ds.baseImg = cloneX(ds.base.baseImg) |
|
| 890 |
+ // Utilize the same path index as our base image so we propagate |
|
| 891 |
+ // the paths we use back to the base image. |
|
| 892 |
+ ds.paths = ds.base.paths |
|
| 893 |
+ ds.workdirSet = ds.base.workdirSet |
|
| 894 |
+} |
|
| 895 |
+ |
|
| 869 | 896 |
type dispatchStates struct {
|
| 870 | 897 |
states []*dispatchState |
| 871 | 898 |
statesByName map[string]*dispatchState |
| ... | ... |
@@ -1005,6 +1165,8 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE |
| 1005 | 1005 |
} |
| 1006 | 1006 |
} |
| 1007 | 1007 |
if c.PrependShell {
|
| 1008 |
+ // Don't pass the linter function because we do not report a warning for |
|
| 1009 |
+ // shell usage on run commands. |
|
| 1008 | 1010 |
args = withShell(d.image, args) |
| 1009 | 1011 |
} |
| 1010 | 1012 |
|
| ... | ... |
@@ -1078,6 +1240,23 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE |
| 1078 | 1078 |
} |
| 1079 | 1079 |
|
| 1080 | 1080 |
func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bool, opt *dispatchOpt) error {
|
| 1081 |
+ if commit {
|
|
| 1082 |
+ // This linter rule checks if workdir has been set to an absolute value locally |
|
| 1083 |
+ // within the current dockerfile. Absolute paths in base images are ignored |
|
| 1084 |
+ // because they might change and it is not advised to rely on them. |
|
| 1085 |
+ // |
|
| 1086 |
+ // We only run this check when commit is true. Commit is true when we are performing |
|
| 1087 |
+ // this operation on a local call to workdir rather than one coming from |
|
| 1088 |
+ // the base image. We only check the first instance of workdir being set |
|
| 1089 |
+ // so successive relative paths are ignored because every instance is fixed |
|
| 1090 |
+ // by fixing the first one. |
|
| 1091 |
+ if !d.workdirSet && !system.IsAbs(c.Path, d.platform.OS) {
|
|
| 1092 |
+ msg := linter.RuleWorkdirRelativePath.Format(c.Path) |
|
| 1093 |
+ opt.lint.Run(&linter.RuleWorkdirRelativePath, c.Location(), msg) |
|
| 1094 |
+ } |
|
| 1095 |
+ d.workdirSet = true |
|
| 1096 |
+ } |
|
| 1097 |
+ |
|
| 1081 | 1098 |
wd, err := system.NormalizeWorkdir(d.image.Config.WorkingDir, c.Path, d.platform.OS) |
| 1082 | 1099 |
if err != nil {
|
| 1083 | 1100 |
return errors.Wrap(err, "normalizing workdir") |
| ... | ... |
@@ -1125,10 +1304,6 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
|
| 1125 | 1125 |
return err |
| 1126 | 1126 |
} |
| 1127 | 1127 |
|
| 1128 |
- if cfg.params.DestPath == "." || cfg.params.DestPath == "" || cfg.params.DestPath[len(cfg.params.DestPath)-1] == filepath.Separator {
|
|
| 1129 |
- dest += string(filepath.Separator) |
|
| 1130 |
- } |
|
| 1131 |
- |
|
| 1132 | 1128 |
var copyOpt []llb.CopyOption |
| 1133 | 1129 |
|
| 1134 | 1130 |
if cfg.chown != "" {
|
| ... | ... |
@@ -1340,11 +1515,10 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
|
| 1340 | 1340 |
copyOpts := []llb.ConstraintsOpt{
|
| 1341 | 1341 |
llb.Platform(*d.platform), |
| 1342 | 1342 |
} |
| 1343 |
- copy(copyOpts, fileOpt) |
|
| 1343 |
+ copyOpts = append(copyOpts, fileOpt...) |
|
| 1344 | 1344 |
copyOpts = append(copyOpts, llb.ProgressGroup(pgID, pgName, true)) |
| 1345 | 1345 |
|
| 1346 |
- var mergeOpts []llb.ConstraintsOpt |
|
| 1347 |
- copy(mergeOpts, fileOpt) |
|
| 1346 |
+ mergeOpts := append([]llb.ConstraintsOpt{}, fileOpt...)
|
|
| 1348 | 1347 |
d.cmdIndex-- |
| 1349 | 1348 |
mergeOpts = append(mergeOpts, llb.ProgressGroup(pgID, pgName, false), llb.WithCustomName(prefixCommand(d, "LINK "+name, d.prefixPlatform, &platform, env))) |
| 1350 | 1349 |
|
| ... | ... |
@@ -1394,30 +1568,42 @@ func dispatchOnbuild(d *dispatchState, c *instructions.OnbuildCommand) error {
|
| 1394 | 1394 |
return nil |
| 1395 | 1395 |
} |
| 1396 | 1396 |
|
| 1397 |
-func dispatchCmd(d *dispatchState, c *instructions.CmdCommand) error {
|
|
| 1397 |
+func dispatchCmd(d *dispatchState, c *instructions.CmdCommand, lint *linter.Linter) error {
|
|
| 1398 |
+ validateUsedOnce(c, &d.cmd, lint) |
|
| 1399 |
+ |
|
| 1398 | 1400 |
var args []string = c.CmdLine |
| 1399 | 1401 |
if c.PrependShell {
|
| 1402 |
+ if len(d.image.Config.Shell) == 0 {
|
|
| 1403 |
+ msg := linter.RuleJSONArgsRecommended.Format(c.Name()) |
|
| 1404 |
+ lint.Run(&linter.RuleJSONArgsRecommended, c.Location(), msg) |
|
| 1405 |
+ } |
|
| 1400 | 1406 |
args = withShell(d.image, args) |
| 1401 | 1407 |
} |
| 1402 | 1408 |
d.image.Config.Cmd = args |
| 1403 | 1409 |
d.image.Config.ArgsEscaped = true //nolint:staticcheck // ignore SA1019: field is deprecated in OCI Image spec, but used for backward-compatibility with Docker image spec. |
| 1404 |
- d.cmdSet = true |
|
| 1405 | 1410 |
return commitToHistory(&d.image, fmt.Sprintf("CMD %q", args), false, nil, d.epoch)
|
| 1406 | 1411 |
} |
| 1407 | 1412 |
|
| 1408 |
-func dispatchEntrypoint(d *dispatchState, c *instructions.EntrypointCommand) error {
|
|
| 1413 |
+func dispatchEntrypoint(d *dispatchState, c *instructions.EntrypointCommand, lint *linter.Linter) error {
|
|
| 1414 |
+ validateUsedOnce(c, &d.entrypoint, lint) |
|
| 1415 |
+ |
|
| 1409 | 1416 |
var args []string = c.CmdLine |
| 1410 | 1417 |
if c.PrependShell {
|
| 1418 |
+ if len(d.image.Config.Shell) == 0 {
|
|
| 1419 |
+ msg := linter.RuleJSONArgsRecommended.Format(c.Name()) |
|
| 1420 |
+ lint.Run(&linter.RuleJSONArgsRecommended, c.Location(), msg) |
|
| 1421 |
+ } |
|
| 1411 | 1422 |
args = withShell(d.image, args) |
| 1412 | 1423 |
} |
| 1413 | 1424 |
d.image.Config.Entrypoint = args |
| 1414 |
- if !d.cmdSet {
|
|
| 1425 |
+ if !d.cmd.IsSet {
|
|
| 1415 | 1426 |
d.image.Config.Cmd = nil |
| 1416 | 1427 |
} |
| 1417 | 1428 |
return commitToHistory(&d.image, fmt.Sprintf("ENTRYPOINT %q", args), false, nil, d.epoch)
|
| 1418 | 1429 |
} |
| 1419 | 1430 |
|
| 1420 |
-func dispatchHealthcheck(d *dispatchState, c *instructions.HealthCheckCommand) error {
|
|
| 1431 |
+func dispatchHealthcheck(d *dispatchState, c *instructions.HealthCheckCommand, lint *linter.Linter) error {
|
|
| 1432 |
+ validateUsedOnce(c, &d.healthcheck, lint) |
|
| 1421 | 1433 |
d.image.Config.Healthcheck = &dockerspec.HealthcheckConfig{
|
| 1422 | 1434 |
Test: c.Health.Test, |
| 1423 | 1435 |
Interval: c.Health.Interval, |
| ... | ... |
@@ -1494,34 +1680,46 @@ func dispatchShell(d *dispatchState, c *instructions.ShellCommand) error {
|
| 1494 | 1494 |
return commitToHistory(&d.image, fmt.Sprintf("SHELL %v", c.Shell), false, nil, d.epoch)
|
| 1495 | 1495 |
} |
| 1496 | 1496 |
|
| 1497 |
-func dispatchArg(d *dispatchState, c *instructions.ArgCommand, metaArgs []instructions.KeyValuePairOptional, buildArgValues map[string]string) error {
|
|
| 1497 |
+func dispatchArg(d *dispatchState, c *instructions.ArgCommand, opt *dispatchOpt) error {
|
|
| 1498 | 1498 |
commitStrs := make([]string, 0, len(c.Args)) |
| 1499 | 1499 |
for _, arg := range c.Args {
|
| 1500 |
- buildArg := setKVValue(arg, buildArgValues) |
|
| 1501 |
- |
|
| 1502 |
- commitStr := arg.Key |
|
| 1503 |
- if arg.Value != nil {
|
|
| 1504 |
- commitStr += "=" + *arg.Value |
|
| 1505 |
- } |
|
| 1506 |
- commitStrs = append(commitStrs, commitStr) |
|
| 1500 |
+ _, hasValue := opt.buildArgValues[arg.Key] |
|
| 1501 |
+ hasDefault := arg.Value != nil |
|
| 1507 | 1502 |
|
| 1508 | 1503 |
skipArgInfo := false // skip the arg info if the arg is inherited from global scope |
| 1509 |
- if buildArg.Value == nil {
|
|
| 1510 |
- for _, ma := range metaArgs {
|
|
| 1511 |
- if ma.Key == buildArg.Key {
|
|
| 1512 |
- buildArg.Value = ma.Value |
|
| 1504 |
+ if !hasDefault && !hasValue {
|
|
| 1505 |
+ for _, ma := range opt.metaArgs {
|
|
| 1506 |
+ if ma.Key == arg.Key {
|
|
| 1507 |
+ arg.Value = ma.Value |
|
| 1513 | 1508 |
skipArgInfo = true |
| 1509 |
+ hasDefault = false |
|
| 1514 | 1510 |
} |
| 1515 | 1511 |
} |
| 1516 | 1512 |
} |
| 1517 | 1513 |
|
| 1514 |
+ if hasValue {
|
|
| 1515 |
+ v := opt.buildArgValues[arg.Key] |
|
| 1516 |
+ arg.Value = &v |
|
| 1517 |
+ } else if hasDefault {
|
|
| 1518 |
+ env, err := d.state.Env(context.TODO()) |
|
| 1519 |
+ if err != nil {
|
|
| 1520 |
+ return err |
|
| 1521 |
+ } |
|
| 1522 |
+ v, unmatched, err := opt.shlex.ProcessWord(*arg.Value, env) |
|
| 1523 |
+ reportUnmatchedVariables(c, d.buildArgs, env, unmatched, opt) |
|
| 1524 |
+ if err != nil {
|
|
| 1525 |
+ return err |
|
| 1526 |
+ } |
|
| 1527 |
+ arg.Value = &v |
|
| 1528 |
+ } |
|
| 1529 |
+ |
|
| 1518 | 1530 |
ai := argInfo{definition: arg, location: c.Location()}
|
| 1519 | 1531 |
|
| 1520 |
- if buildArg.Value != nil {
|
|
| 1521 |
- if _, ok := nonEnvArgs[buildArg.Key]; !ok {
|
|
| 1522 |
- d.state = d.state.AddEnv(buildArg.Key, *buildArg.Value) |
|
| 1532 |
+ if arg.Value != nil {
|
|
| 1533 |
+ if _, ok := nonEnvArgs[arg.Key]; !ok {
|
|
| 1534 |
+ d.state = d.state.AddEnv(arg.Key, *arg.Value) |
|
| 1523 | 1535 |
} |
| 1524 |
- ai.value = *buildArg.Value |
|
| 1536 |
+ ai.value = *arg.Value |
|
| 1525 | 1537 |
} |
| 1526 | 1538 |
|
| 1527 | 1539 |
if !skipArgInfo {
|
| ... | ... |
@@ -1529,7 +1727,13 @@ func dispatchArg(d *dispatchState, c *instructions.ArgCommand, metaArgs []instru |
| 1529 | 1529 |
} |
| 1530 | 1530 |
d.outline.usedArgs[arg.Key] = struct{}{}
|
| 1531 | 1531 |
|
| 1532 |
- d.buildArgs = append(d.buildArgs, buildArg) |
|
| 1532 |
+ d.buildArgs = append(d.buildArgs, arg) |
|
| 1533 |
+ |
|
| 1534 |
+ commitStr := arg.Key |
|
| 1535 |
+ if arg.Value != nil {
|
|
| 1536 |
+ commitStr += "=" + *arg.Value |
|
| 1537 |
+ } |
|
| 1538 |
+ commitStrs = append(commitStrs, commitStr) |
|
| 1533 | 1539 |
} |
| 1534 | 1540 |
return commitToHistory(&d.image, "ARG "+strings.Join(commitStrs, " "), false, nil, d.epoch) |
| 1535 | 1541 |
} |
| ... | ... |
@@ -1549,9 +1753,9 @@ func pathRelativeToWorkingDir(s llb.State, p string, platform ocispecs.Platform) |
| 1549 | 1549 |
} |
| 1550 | 1550 |
|
| 1551 | 1551 |
if system.IsAbs(p, platform.OS) {
|
| 1552 |
- return system.NormalizePath("/", p, platform.OS, false)
|
|
| 1552 |
+ return system.NormalizePath("/", p, platform.OS, true)
|
|
| 1553 | 1553 |
} |
| 1554 |
- return system.NormalizePath(dir, p, platform.OS, false) |
|
| 1554 |
+ return system.NormalizePath(dir, p, platform.OS, true) |
|
| 1555 | 1555 |
} |
| 1556 | 1556 |
|
| 1557 | 1557 |
func addEnv(env []string, k, v string) []string {
|
| ... | ... |
@@ -1631,33 +1835,23 @@ func commitToHistory(img *dockerspec.DockerOCIImage, msg string, withLayer bool, |
| 1631 | 1631 |
return nil |
| 1632 | 1632 |
} |
| 1633 | 1633 |
|
| 1634 |
-func isReachable(from, to *dispatchState) (ret bool) {
|
|
| 1635 |
- if from == nil {
|
|
| 1636 |
- return false |
|
| 1637 |
- } |
|
| 1638 |
- if from == to || isReachable(from.base, to) {
|
|
| 1639 |
- return true |
|
| 1640 |
- } |
|
| 1641 |
- for d := range from.deps {
|
|
| 1642 |
- if isReachable(d, to) {
|
|
| 1643 |
- return true |
|
| 1644 |
- } |
|
| 1645 |
- } |
|
| 1646 |
- return false |
|
| 1634 |
+func allReachableStages(s *dispatchState) map[*dispatchState]struct{} {
|
|
| 1635 |
+ stages := make(map[*dispatchState]struct{})
|
|
| 1636 |
+ addReachableStages(s, stages) |
|
| 1637 |
+ return stages |
|
| 1647 | 1638 |
} |
| 1648 | 1639 |
|
| 1649 |
-func findReachable(from *dispatchState) (ret []*dispatchState) {
|
|
| 1650 |
- if from == nil {
|
|
| 1651 |
- return nil |
|
| 1640 |
+func addReachableStages(s *dispatchState, stages map[*dispatchState]struct{}) {
|
|
| 1641 |
+ if _, ok := stages[s]; ok {
|
|
| 1642 |
+ return |
|
| 1652 | 1643 |
} |
| 1653 |
- ret = append(ret, from) |
|
| 1654 |
- if from.base != nil {
|
|
| 1655 |
- ret = append(ret, findReachable(from.base)...) |
|
| 1644 |
+ stages[s] = struct{}{}
|
|
| 1645 |
+ if s.base != nil {
|
|
| 1646 |
+ addReachableStages(s.base, stages) |
|
| 1656 | 1647 |
} |
| 1657 |
- for d := range from.deps {
|
|
| 1658 |
- ret = append(ret, findReachable(d)...) |
|
| 1648 |
+ for d := range s.deps {
|
|
| 1649 |
+ addReachableStages(d, stages) |
|
| 1659 | 1650 |
} |
| 1660 |
- return ret |
|
| 1661 | 1651 |
} |
| 1662 | 1652 |
|
| 1663 | 1653 |
func validateCircularDependency(states []*dispatchState) error {
|
| ... | ... |
@@ -1794,7 +1988,7 @@ func uppercaseCmd(str string) string {
|
| 1794 | 1794 |
} |
| 1795 | 1795 |
|
| 1796 | 1796 |
func processCmdEnv(shlex *shell.Lex, cmd string, env []string) string {
|
| 1797 |
- w, err := shlex.ProcessWord(cmd, env) |
|
| 1797 |
+ w, _, err := shlex.ProcessWord(cmd, env) |
|
| 1798 | 1798 |
if err != nil {
|
| 1799 | 1799 |
return cmd |
| 1800 | 1800 |
} |
| ... | ... |
@@ -1928,3 +2122,190 @@ func isEnabledForStage(stage string, value string) bool {
|
| 1928 | 1928 |
} |
| 1929 | 1929 |
return false |
| 1930 | 1930 |
} |
| 1931 |
+ |
|
| 1932 |
+func isSelfConsistentCasing(s string) bool {
|
|
| 1933 |
+ return s == strings.ToLower(s) || s == strings.ToUpper(s) |
|
| 1934 |
+} |
|
| 1935 |
+ |
|
| 1936 |
+func validateCommandCasing(dockerfile *parser.Result, lint *linter.Linter) {
|
|
| 1937 |
+ var lowerCount, upperCount int |
|
| 1938 |
+ for _, node := range dockerfile.AST.Children {
|
|
| 1939 |
+ if isSelfConsistentCasing(node.Value) {
|
|
| 1940 |
+ if strings.ToLower(node.Value) == node.Value {
|
|
| 1941 |
+ lowerCount++ |
|
| 1942 |
+ } else {
|
|
| 1943 |
+ upperCount++ |
|
| 1944 |
+ } |
|
| 1945 |
+ } |
|
| 1946 |
+ } |
|
| 1947 |
+ |
|
| 1948 |
+ isMajorityLower := lowerCount > upperCount |
|
| 1949 |
+ for _, node := range dockerfile.AST.Children {
|
|
| 1950 |
+ // Here, we check both if the command is consistent per command (ie, "CMD" or "cmd", not "Cmd") |
|
| 1951 |
+ // as well as ensuring that the casing is consistent throughout the dockerfile by comparing the |
|
| 1952 |
+ // command to the casing of the majority of commands. |
|
| 1953 |
+ if !isSelfConsistentCasing(node.Value) {
|
|
| 1954 |
+ msg := linter.RuleSelfConsistentCommandCasing.Format(node.Value) |
|
| 1955 |
+ lint.Run(&linter.RuleSelfConsistentCommandCasing, node.Location(), msg) |
|
| 1956 |
+ } else {
|
|
| 1957 |
+ var msg string |
|
| 1958 |
+ var needsLintWarn bool |
|
| 1959 |
+ if isMajorityLower && strings.ToUpper(node.Value) == node.Value {
|
|
| 1960 |
+ msg = linter.RuleFileConsistentCommandCasing.Format(node.Value, "lowercase") |
|
| 1961 |
+ needsLintWarn = true |
|
| 1962 |
+ } else if !isMajorityLower && strings.ToLower(node.Value) == node.Value {
|
|
| 1963 |
+ msg = linter.RuleFileConsistentCommandCasing.Format(node.Value, "uppercase") |
|
| 1964 |
+ needsLintWarn = true |
|
| 1965 |
+ } |
|
| 1966 |
+ if needsLintWarn {
|
|
| 1967 |
+ lint.Run(&linter.RuleFileConsistentCommandCasing, node.Location(), msg) |
|
| 1968 |
+ } |
|
| 1969 |
+ } |
|
| 1970 |
+ } |
|
| 1971 |
+} |
|
| 1972 |
+ |
|
| 1973 |
+var reservedStageNames = map[string]struct{}{
|
|
| 1974 |
+ "context": {},
|
|
| 1975 |
+ "scratch": {},
|
|
| 1976 |
+} |
|
| 1977 |
+ |
|
| 1978 |
+func validateStageNames(stages []instructions.Stage, lint *linter.Linter) {
|
|
| 1979 |
+ stageNames := make(map[string]struct{})
|
|
| 1980 |
+ for _, stage := range stages {
|
|
| 1981 |
+ if stage.Name != "" {
|
|
| 1982 |
+ if _, ok := reservedStageNames[stage.Name]; ok {
|
|
| 1983 |
+ msg := linter.RuleReservedStageName.Format(stage.Name) |
|
| 1984 |
+ lint.Run(&linter.RuleReservedStageName, stage.Location, msg) |
|
| 1985 |
+ } |
|
| 1986 |
+ |
|
| 1987 |
+ if _, ok := stageNames[stage.Name]; ok {
|
|
| 1988 |
+ msg := linter.RuleDuplicateStageName.Format(stage.Name) |
|
| 1989 |
+ lint.Run(&linter.RuleDuplicateStageName, stage.Location, msg) |
|
| 1990 |
+ } |
|
| 1991 |
+ stageNames[stage.Name] = struct{}{}
|
|
| 1992 |
+ } |
|
| 1993 |
+ } |
|
| 1994 |
+} |
|
| 1995 |
+ |
|
| 1996 |
+func reportUnmatchedVariables(cmd instructions.Command, buildArgs []instructions.KeyValuePairOptional, env []string, unmatched map[string]struct{}, opt *dispatchOpt) {
|
|
| 1997 |
+ if len(unmatched) == 0 {
|
|
| 1998 |
+ return |
|
| 1999 |
+ } |
|
| 2000 |
+ for _, buildArg := range buildArgs {
|
|
| 2001 |
+ delete(unmatched, buildArg.Key) |
|
| 2002 |
+ } |
|
| 2003 |
+ if len(unmatched) == 0 {
|
|
| 2004 |
+ return |
|
| 2005 |
+ } |
|
| 2006 |
+ options := metaArgsKeys(opt.metaArgs) |
|
| 2007 |
+ for _, envVar := range env {
|
|
| 2008 |
+ key, _ := parseKeyValue(envVar) |
|
| 2009 |
+ options = append(options, key) |
|
| 2010 |
+ } |
|
| 2011 |
+ for cmdVar := range unmatched {
|
|
| 2012 |
+ if _, nonEnvOk := nonEnvArgs[cmdVar]; nonEnvOk {
|
|
| 2013 |
+ continue |
|
| 2014 |
+ } |
|
| 2015 |
+ match, _ := suggest.Search(cmdVar, options, true) |
|
| 2016 |
+ msg := linter.RuleUndefinedVar.Format(cmdVar, match) |
|
| 2017 |
+ opt.lint.Run(&linter.RuleUndefinedVar, cmd.Location(), msg) |
|
| 2018 |
+ } |
|
| 2019 |
+} |
|
| 2020 |
+ |
|
| 2021 |
+func mergeLocations(locations ...[]parser.Range) []parser.Range {
|
|
| 2022 |
+ allRanges := []parser.Range{}
|
|
| 2023 |
+ for _, ranges := range locations {
|
|
| 2024 |
+ allRanges = append(allRanges, ranges...) |
|
| 2025 |
+ } |
|
| 2026 |
+ if len(allRanges) == 0 {
|
|
| 2027 |
+ return []parser.Range{}
|
|
| 2028 |
+ } |
|
| 2029 |
+ if len(allRanges) == 1 {
|
|
| 2030 |
+ return allRanges |
|
| 2031 |
+ } |
|
| 2032 |
+ |
|
| 2033 |
+ sort.Slice(allRanges, func(i, j int) bool {
|
|
| 2034 |
+ return allRanges[i].Start.Line < allRanges[j].Start.Line |
|
| 2035 |
+ }) |
|
| 2036 |
+ |
|
| 2037 |
+ location := []parser.Range{}
|
|
| 2038 |
+ currentRange := allRanges[0] |
|
| 2039 |
+ for _, r := range allRanges[1:] {
|
|
| 2040 |
+ if r.Start.Line <= currentRange.End.Line {
|
|
| 2041 |
+ currentRange.End.Line = max(currentRange.End.Line, r.End.Line) |
|
| 2042 |
+ } else {
|
|
| 2043 |
+ location = append(location, currentRange) |
|
| 2044 |
+ currentRange = r |
|
| 2045 |
+ } |
|
| 2046 |
+ } |
|
| 2047 |
+ location = append(location, currentRange) |
|
| 2048 |
+ return location |
|
| 2049 |
+} |
|
| 2050 |
+ |
|
| 2051 |
+func toPBLocation(sourceIndex int, location []parser.Range) pb.Location {
|
|
| 2052 |
+ loc := make([]*pb.Range, 0, len(location)) |
|
| 2053 |
+ for _, l := range location {
|
|
| 2054 |
+ loc = append(loc, &pb.Range{
|
|
| 2055 |
+ Start: pb.Position{
|
|
| 2056 |
+ Line: int32(l.Start.Line), |
|
| 2057 |
+ Character: int32(l.Start.Character), |
|
| 2058 |
+ }, |
|
| 2059 |
+ End: pb.Position{
|
|
| 2060 |
+ Line: int32(l.End.Line), |
|
| 2061 |
+ Character: int32(l.End.Character), |
|
| 2062 |
+ }, |
|
| 2063 |
+ }) |
|
| 2064 |
+ } |
|
| 2065 |
+ return pb.Location{
|
|
| 2066 |
+ SourceIndex: int32(sourceIndex), |
|
| 2067 |
+ Ranges: loc, |
|
| 2068 |
+ } |
|
| 2069 |
+} |
|
| 2070 |
+ |
|
| 2071 |
+func reportUnusedFromArgs(values []string, unmatched map[string]struct{}, location []parser.Range, lint *linter.Linter) {
|
|
| 2072 |
+ for arg := range unmatched {
|
|
| 2073 |
+ suggest, _ := suggest.Search(arg, values, true) |
|
| 2074 |
+ msg := linter.RuleUndeclaredArgInFrom.Format(arg, suggest) |
|
| 2075 |
+ lint.Run(&linter.RuleUndeclaredArgInFrom, location, msg) |
|
| 2076 |
+ } |
|
| 2077 |
+} |
|
| 2078 |
+ |
|
| 2079 |
+type instructionTracker struct {
|
|
| 2080 |
+ Loc []parser.Range |
|
| 2081 |
+ IsSet bool |
|
| 2082 |
+} |
|
| 2083 |
+ |
|
| 2084 |
+func (v *instructionTracker) MarkUsed(loc []parser.Range) {
|
|
| 2085 |
+ v.Loc = loc |
|
| 2086 |
+ v.IsSet = true |
|
| 2087 |
+} |
|
| 2088 |
+ |
|
| 2089 |
+func validateUsedOnce(c instructions.Command, loc *instructionTracker, lint *linter.Linter) {
|
|
| 2090 |
+ if loc.IsSet {
|
|
| 2091 |
+ msg := linter.RuleMultipleInstructionsDisallowed.Format(c.Name()) |
|
| 2092 |
+ // Report the location of the previous invocation because it is the one |
|
| 2093 |
+ // that will be ignored. |
|
| 2094 |
+ lint.Run(&linter.RuleMultipleInstructionsDisallowed, loc.Loc, msg) |
|
| 2095 |
+ } |
|
| 2096 |
+ loc.MarkUsed(c.Location()) |
|
| 2097 |
+} |
|
| 2098 |
+ |
|
| 2099 |
+func wrapSuggestAny(err error, keys map[string]struct{}, options []string) error {
|
|
| 2100 |
+ for k := range keys {
|
|
| 2101 |
+ var ok bool |
|
| 2102 |
+ ok, err = suggest.WrapErrorMaybe(err, k, options, true) |
|
| 2103 |
+ if ok {
|
|
| 2104 |
+ break |
|
| 2105 |
+ } |
|
| 2106 |
+ } |
|
| 2107 |
+ return err |
|
| 2108 |
+} |
|
| 2109 |
+ |
|
| 2110 |
+func validateBaseImagePlatform(name string, expected, actual ocispecs.Platform, location []parser.Range, lint *linter.Linter) {
|
|
| 2111 |
+ if expected.OS != actual.OS || expected.Architecture != actual.Architecture {
|
|
| 2112 |
+ expectedStr := platforms.Format(platforms.Normalize(expected)) |
|
| 2113 |
+ actualStr := platforms.Format(platforms.Normalize(actual)) |
|
| 2114 |
+ msg := linter.RuleInvalidBaseImagePlatform.Format(name, expectedStr, actualStr) |
|
| 2115 |
+ lint.Run(&linter.RuleInvalidBaseImagePlatform, location, msg) |
|
| 2116 |
+ } |
|
| 2117 |
+} |
| ... | ... |
@@ -8,6 +8,6 @@ import ( |
| 8 | 8 |
"github.com/moby/buildkit/frontend/dockerfile/instructions" |
| 9 | 9 |
) |
| 10 | 10 |
|
| 11 |
-func dispatchRunSecurity(c *instructions.RunCommand) (llb.RunOption, error) {
|
|
| 11 |
+func dispatchRunSecurity(_ *instructions.RunCommand) (llb.RunOption, error) {
|
|
| 12 | 12 |
return nil, nil |
| 13 | 13 |
} |
| ... | ... |
@@ -130,11 +130,9 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []* |
| 130 | 130 |
} |
| 131 | 131 |
if src := path.Join("/", mount.Source); src != "/" {
|
| 132 | 132 |
mountOpts = append(mountOpts, llb.SourcePath(src)) |
| 133 |
- } else {
|
|
| 134 |
- if mount.UID != nil || mount.GID != nil || mount.Mode != nil {
|
|
| 135 |
- st = setCacheUIDGID(mount, st) |
|
| 136 |
- mountOpts = append(mountOpts, llb.SourcePath("/cache"))
|
|
| 137 |
- } |
|
| 133 |
+ } else if mount.UID != nil || mount.GID != nil || mount.Mode != nil {
|
|
| 134 |
+ st = setCacheUIDGID(mount, st) |
|
| 135 |
+ mountOpts = append(mountOpts, llb.SourcePath("/cache"))
|
|
| 138 | 136 |
} |
| 139 | 137 |
|
| 140 | 138 |
out = append(out, llb.AddMount(target, st, mountOpts...)) |
| ... | ... |
@@ -3,9 +3,8 @@ package instructions |
| 3 | 3 |
import ( |
| 4 | 4 |
"strings" |
| 5 | 5 |
|
| 6 |
- "github.com/docker/docker/api/types/container" |
|
| 7 |
- "github.com/docker/docker/api/types/strslice" |
|
| 8 | 6 |
"github.com/moby/buildkit/frontend/dockerfile/parser" |
| 7 |
+ dockerspec "github.com/moby/docker-image-spec/specs-go/v1" |
|
| 9 | 8 |
"github.com/pkg/errors" |
| 10 | 9 |
) |
| 11 | 10 |
|
| ... | ... |
@@ -155,7 +154,7 @@ type MaintainerCommand struct {
|
| 155 | 155 |
} |
| 156 | 156 |
|
| 157 | 157 |
// NewLabelCommand creates a new 'LABEL' command |
| 158 |
-func NewLabelCommand(k string, v string, NoExp bool) *LabelCommand {
|
|
| 158 |
+func NewLabelCommand(k string, v string, noExp bool) *LabelCommand {
|
|
| 159 | 159 |
kvp := KeyValuePair{Key: k, Value: v}
|
| 160 | 160 |
c := "LABEL " |
| 161 | 161 |
c += kvp.String() |
| ... | ... |
@@ -165,7 +164,7 @@ func NewLabelCommand(k string, v string, NoExp bool) *LabelCommand {
|
| 165 | 165 |
Labels: KeyValuePairs{
|
| 166 | 166 |
kvp, |
| 167 | 167 |
}, |
| 168 |
- noExpand: NoExp, |
|
| 168 |
+ noExpand: noExp, |
|
| 169 | 169 |
} |
| 170 | 170 |
return cmd |
| 171 | 171 |
} |
| ... | ... |
@@ -325,7 +324,7 @@ type ShellInlineFile struct {
|
| 325 | 325 |
|
| 326 | 326 |
// ShellDependantCmdLine represents a cmdline optionally prepended with the shell |
| 327 | 327 |
type ShellDependantCmdLine struct {
|
| 328 |
- CmdLine strslice.StrSlice |
|
| 328 |
+ CmdLine []string |
|
| 329 | 329 |
Files []ShellInlineFile |
| 330 | 330 |
PrependShell bool |
| 331 | 331 |
} |
| ... | ... |
@@ -368,7 +367,7 @@ type CmdCommand struct {
|
| 368 | 368 |
// HEALTHCHECK <health-config> |
| 369 | 369 |
type HealthCheckCommand struct {
|
| 370 | 370 |
withNameAndCode |
| 371 |
- Health *container.HealthConfig |
|
| 371 |
+ Health *dockerspec.HealthcheckConfig |
|
| 372 | 372 |
} |
| 373 | 373 |
|
| 374 | 374 |
// EntrypointCommand sets the default entrypoint of the container to use the |
| ... | ... |
@@ -479,7 +478,7 @@ func (c *ArgCommand) Expand(expander SingleWordExpander) error {
|
| 479 | 479 |
// SHELL bash -e -c |
| 480 | 480 |
type ShellCommand struct {
|
| 481 | 481 |
withNameAndCode |
| 482 |
- Shell strslice.StrSlice |
|
| 482 |
+ Shell []string |
|
| 483 | 483 |
} |
| 484 | 484 |
|
| 485 | 485 |
// Stage represents a bundled collection of commands. |
| ... | ... |
@@ -18,7 +18,7 @@ func errNotJSON(command, original string) error {
|
| 18 | 18 |
// double backslash and a [] pair. No, this is not perfect, but it doesn't |
| 19 | 19 |
// have to be. It's simply a hint to make life a little easier. |
| 20 | 20 |
extra := "" |
| 21 |
- original = filepath.FromSlash(strings.ToLower(strings.Replace(strings.ToLower(original), strings.ToLower(command)+" ", "", -1))) |
|
| 21 |
+ original = filepath.FromSlash(strings.ToLower(strings.ReplaceAll(strings.ToLower(original), strings.ToLower(command)+" ", ""))) |
|
| 22 | 22 |
if len(regexp.MustCompile(`"[a-z]:\\.*`).FindStringSubmatch(original)) > 0 && |
| 23 | 23 |
!strings.Contains(original, `\\`) && |
| 24 | 24 |
strings.Contains(original, "[") && |
| ... | ... |
@@ -12,11 +12,11 @@ import ( |
| 12 | 12 |
"strings" |
| 13 | 13 |
"time" |
| 14 | 14 |
|
| 15 |
- "github.com/docker/docker/api/types/container" |
|
| 16 |
- "github.com/docker/docker/api/types/strslice" |
|
| 17 | 15 |
"github.com/moby/buildkit/frontend/dockerfile/command" |
| 16 |
+ "github.com/moby/buildkit/frontend/dockerfile/linter" |
|
| 18 | 17 |
"github.com/moby/buildkit/frontend/dockerfile/parser" |
| 19 | 18 |
"github.com/moby/buildkit/util/suggest" |
| 19 |
+ dockerspec "github.com/moby/docker-image-spec/specs-go/v1" |
|
| 20 | 20 |
"github.com/pkg/errors" |
| 21 | 21 |
) |
| 22 | 22 |
|
| ... | ... |
@@ -66,24 +66,38 @@ func newParseRequestFromNode(node *parser.Node) parseRequest {
|
| 66 | 66 |
} |
| 67 | 67 |
} |
| 68 | 68 |
|
| 69 |
-// ParseInstruction converts an AST to a typed instruction (either a command or a build stage beginning when encountering a `FROM` statement) |
|
| 70 | 69 |
func ParseInstruction(node *parser.Node) (v interface{}, err error) {
|
| 70 |
+ return ParseInstructionWithLinter(node, nil) |
|
| 71 |
+} |
|
| 72 |
+ |
|
| 73 |
+// ParseInstruction converts an AST to a typed instruction (either a command or a build stage beginning when encountering a `FROM` statement) |
|
| 74 |
+func ParseInstructionWithLinter(node *parser.Node, lint *linter.Linter) (v interface{}, err error) {
|
|
| 71 | 75 |
defer func() {
|
| 72 | 76 |
err = parser.WithLocation(err, node.Location()) |
| 73 | 77 |
}() |
| 74 | 78 |
req := newParseRequestFromNode(node) |
| 75 | 79 |
switch strings.ToLower(node.Value) {
|
| 76 | 80 |
case command.Env: |
| 77 |
- return parseEnv(req) |
|
| 81 |
+ return parseEnv(req, lint) |
|
| 78 | 82 |
case command.Maintainer: |
| 83 |
+ msg := linter.RuleMaintainerDeprecated.Format() |
|
| 84 |
+ lint.Run(&linter.RuleMaintainerDeprecated, node.Location(), msg) |
|
| 79 | 85 |
return parseMaintainer(req) |
| 80 | 86 |
case command.Label: |
| 81 |
- return parseLabel(req) |
|
| 87 |
+ return parseLabel(req, lint) |
|
| 82 | 88 |
case command.Add: |
| 83 | 89 |
return parseAdd(req) |
| 84 | 90 |
case command.Copy: |
| 85 | 91 |
return parseCopy(req) |
| 86 | 92 |
case command.From: |
| 93 |
+ if !isLowerCaseStageName(req.args) {
|
|
| 94 |
+ msg := linter.RuleStageNameCasing.Format(req.args[2]) |
|
| 95 |
+ lint.Run(&linter.RuleStageNameCasing, node.Location(), msg) |
|
| 96 |
+ } |
|
| 97 |
+ if !doesFromCaseMatchAsCase(req) {
|
|
| 98 |
+ msg := linter.RuleFromAsCasing.Format(req.command, req.args[1]) |
|
| 99 |
+ lint.Run(&linter.RuleFromAsCasing, node.Location(), msg) |
|
| 100 |
+ } |
|
| 87 | 101 |
return parseFrom(req) |
| 88 | 102 |
case command.Onbuild: |
| 89 | 103 |
return parseOnBuild(req) |
| ... | ... |
@@ -150,9 +164,9 @@ func (e *parseError) Unwrap() error {
|
| 150 | 150 |
|
| 151 | 151 |
// Parse a Dockerfile into a collection of buildable stages. |
| 152 | 152 |
// metaArgs is a collection of ARG instructions that occur before the first FROM. |
| 153 |
-func Parse(ast *parser.Node) (stages []Stage, metaArgs []ArgCommand, err error) {
|
|
| 153 |
+func Parse(ast *parser.Node, lint *linter.Linter) (stages []Stage, metaArgs []ArgCommand, err error) {
|
|
| 154 | 154 |
for _, n := range ast.Children {
|
| 155 |
- cmd, err := ParseInstruction(n) |
|
| 155 |
+ cmd, err := ParseInstructionWithLinter(n, lint) |
|
| 156 | 156 |
if err != nil {
|
| 157 | 157 |
return nil, nil, &parseError{inner: err, node: n}
|
| 158 | 158 |
} |
| ... | ... |
@@ -179,31 +193,34 @@ func Parse(ast *parser.Node) (stages []Stage, metaArgs []ArgCommand, err error) |
| 179 | 179 |
return stages, metaArgs, nil |
| 180 | 180 |
} |
| 181 | 181 |
|
| 182 |
-func parseKvps(args []string, cmdName string) (KeyValuePairs, error) {
|
|
| 182 |
+func parseKvps(args []string, cmdName string, location []parser.Range, lint *linter.Linter) (KeyValuePairs, error) {
|
|
| 183 | 183 |
if len(args) == 0 {
|
| 184 | 184 |
return nil, errAtLeastOneArgument(cmdName) |
| 185 | 185 |
} |
| 186 |
- if len(args)%2 != 0 {
|
|
| 186 |
+ if len(args)%3 != 0 {
|
|
| 187 | 187 |
// should never get here, but just in case |
| 188 | 188 |
return nil, errTooManyArguments(cmdName) |
| 189 | 189 |
} |
| 190 | 190 |
var res KeyValuePairs |
| 191 |
- for j := 0; j < len(args); j += 2 {
|
|
| 191 |
+ for j := 0; j < len(args); j += 3 {
|
|
| 192 | 192 |
if len(args[j]) == 0 {
|
| 193 | 193 |
return nil, errBlankCommandNames(cmdName) |
| 194 | 194 |
} |
| 195 |
- name := args[j] |
|
| 196 |
- value := args[j+1] |
|
| 195 |
+ name, value, sep := args[j], args[j+1], args[j+2] |
|
| 196 |
+ if sep == "" {
|
|
| 197 |
+ msg := linter.RuleLegacyKeyValueFormat.Format(cmdName) |
|
| 198 |
+ lint.Run(&linter.RuleLegacyKeyValueFormat, location, msg) |
|
| 199 |
+ } |
|
| 197 | 200 |
res = append(res, KeyValuePair{Key: name, Value: value})
|
| 198 | 201 |
} |
| 199 | 202 |
return res, nil |
| 200 | 203 |
} |
| 201 | 204 |
|
| 202 |
-func parseEnv(req parseRequest) (*EnvCommand, error) {
|
|
| 205 |
+func parseEnv(req parseRequest, lint *linter.Linter) (*EnvCommand, error) {
|
|
| 203 | 206 |
if err := req.flags.Parse(); err != nil {
|
| 204 | 207 |
return nil, err |
| 205 | 208 |
} |
| 206 |
- envs, err := parseKvps(req.args, "ENV") |
|
| 209 |
+ envs, err := parseKvps(req.args, "ENV", req.location, lint) |
|
| 207 | 210 |
if err != nil {
|
| 208 | 211 |
return nil, err |
| 209 | 212 |
} |
| ... | ... |
@@ -227,12 +244,12 @@ func parseMaintainer(req parseRequest) (*MaintainerCommand, error) {
|
| 227 | 227 |
}, nil |
| 228 | 228 |
} |
| 229 | 229 |
|
| 230 |
-func parseLabel(req parseRequest) (*LabelCommand, error) {
|
|
| 230 |
+func parseLabel(req parseRequest, lint *linter.Linter) (*LabelCommand, error) {
|
|
| 231 | 231 |
if err := req.flags.Parse(); err != nil {
|
| 232 | 232 |
return nil, err |
| 233 | 233 |
} |
| 234 | 234 |
|
| 235 |
- labels, err := parseKvps(req.args, "LABEL") |
|
| 235 |
+ labels, err := parseKvps(req.args, "LABEL", req.location, lint) |
|
| 236 | 236 |
if err != nil {
|
| 237 | 237 |
return nil, err |
| 238 | 238 |
} |
| ... | ... |
@@ -450,7 +467,7 @@ func parseWorkdir(req parseRequest) (*WorkdirCommand, error) {
|
| 450 | 450 |
}, nil |
| 451 | 451 |
} |
| 452 | 452 |
|
| 453 |
-func parseShellDependentCommand(req parseRequest, command string, emptyAsNil bool) (ShellDependantCmdLine, error) {
|
|
| 453 |
+func parseShellDependentCommand(req parseRequest, emptyAsNil bool) (ShellDependantCmdLine, error) {
|
|
| 454 | 454 |
var files []ShellInlineFile |
| 455 | 455 |
for _, heredoc := range req.heredocs {
|
| 456 | 456 |
file := ShellInlineFile{
|
| ... | ... |
@@ -462,12 +479,11 @@ func parseShellDependentCommand(req parseRequest, command string, emptyAsNil boo |
| 462 | 462 |
} |
| 463 | 463 |
|
| 464 | 464 |
args := handleJSONArgs(req.args, req.attributes) |
| 465 |
- cmd := strslice.StrSlice(args) |
|
| 466 |
- if emptyAsNil && len(cmd) == 0 {
|
|
| 467 |
- cmd = nil |
|
| 465 |
+ if emptyAsNil && len(args) == 0 {
|
|
| 466 |
+ args = nil |
|
| 468 | 467 |
} |
| 469 | 468 |
return ShellDependantCmdLine{
|
| 470 |
- CmdLine: cmd, |
|
| 469 |
+ CmdLine: args, |
|
| 471 | 470 |
Files: files, |
| 472 | 471 |
PrependShell: !req.attributes["json"], |
| 473 | 472 |
}, nil |
| ... | ... |
@@ -487,7 +503,7 @@ func parseRun(req parseRequest) (*RunCommand, error) {
|
| 487 | 487 |
} |
| 488 | 488 |
cmd.FlagsUsed = req.flags.Used() |
| 489 | 489 |
|
| 490 |
- cmdline, err := parseShellDependentCommand(req, "RUN", false) |
|
| 490 |
+ cmdline, err := parseShellDependentCommand(req, false) |
|
| 491 | 491 |
if err != nil {
|
| 492 | 492 |
return nil, err |
| 493 | 493 |
} |
| ... | ... |
@@ -509,7 +525,7 @@ func parseCmd(req parseRequest) (*CmdCommand, error) {
|
| 509 | 509 |
return nil, err |
| 510 | 510 |
} |
| 511 | 511 |
|
| 512 |
- cmdline, err := parseShellDependentCommand(req, "CMD", false) |
|
| 512 |
+ cmdline, err := parseShellDependentCommand(req, false) |
|
| 513 | 513 |
if err != nil {
|
| 514 | 514 |
return nil, err |
| 515 | 515 |
} |
| ... | ... |
@@ -525,7 +541,7 @@ func parseEntrypoint(req parseRequest) (*EntrypointCommand, error) {
|
| 525 | 525 |
return nil, err |
| 526 | 526 |
} |
| 527 | 527 |
|
| 528 |
- cmdline, err := parseShellDependentCommand(req, "ENTRYPOINT", true) |
|
| 528 |
+ cmdline, err := parseShellDependentCommand(req, true) |
|
| 529 | 529 |
if err != nil {
|
| 530 | 530 |
return nil, err |
| 531 | 531 |
} |
| ... | ... |
@@ -550,8 +566,10 @@ func parseOptInterval(f *Flag) (time.Duration, error) {
|
| 550 | 550 |
if d == 0 {
|
| 551 | 551 |
return 0, nil |
| 552 | 552 |
} |
| 553 |
- if d < container.MinimumDuration {
|
|
| 554 |
- return 0, errors.Errorf("Interval %#v cannot be less than %s", f.name, container.MinimumDuration)
|
|
| 553 |
+ |
|
| 554 |
+ const minimumDuration = time.Millisecond |
|
| 555 |
+ if d < minimumDuration {
|
|
| 556 |
+ return 0, errors.Errorf("Interval %#v cannot be less than %s", f.name, minimumDuration)
|
|
| 555 | 557 |
} |
| 556 | 558 |
return d, nil |
| 557 | 559 |
} |
| ... | ... |
@@ -569,12 +587,11 @@ func parseHealthcheck(req parseRequest) (*HealthCheckCommand, error) {
|
| 569 | 569 |
if len(args) != 0 {
|
| 570 | 570 |
return nil, errors.New("HEALTHCHECK NONE takes no arguments")
|
| 571 | 571 |
} |
| 572 |
- test := strslice.StrSlice{typ}
|
|
| 573 |
- cmd.Health = &container.HealthConfig{
|
|
| 574 |
- Test: test, |
|
| 572 |
+ cmd.Health = &dockerspec.HealthcheckConfig{
|
|
| 573 |
+ Test: []string{typ},
|
|
| 575 | 574 |
} |
| 576 | 575 |
} else {
|
| 577 |
- healthcheck := container.HealthConfig{}
|
|
| 576 |
+ healthcheck := dockerspec.HealthcheckConfig{}
|
|
| 578 | 577 |
|
| 579 | 578 |
flInterval := req.flags.AddString("interval", "")
|
| 580 | 579 |
flTimeout := req.flags.AddString("timeout", "")
|
| ... | ... |
@@ -597,7 +614,7 @@ func parseHealthcheck(req parseRequest) (*HealthCheckCommand, error) {
|
| 597 | 597 |
typ = "CMD-SHELL" |
| 598 | 598 |
} |
| 599 | 599 |
|
| 600 |
- healthcheck.Test = strslice.StrSlice(append([]string{typ}, cmdSlice...))
|
|
| 600 |
+ healthcheck.Test = append([]string{typ}, cmdSlice...)
|
|
| 601 | 601 |
default: |
| 602 | 602 |
return nil, errors.Errorf("Unknown type %#v in HEALTHCHECK (try CMD)", typ)
|
| 603 | 603 |
} |
| ... | ... |
@@ -761,7 +778,7 @@ func parseShell(req parseRequest) (*ShellCommand, error) {
|
| 761 | 761 |
// SHELL ["powershell", "-command"] |
| 762 | 762 |
|
| 763 | 763 |
return &ShellCommand{
|
| 764 |
- Shell: strslice.StrSlice(shellSlice), |
|
| 764 |
+ Shell: shellSlice, |
|
| 765 | 765 |
withNameAndCode: newWithNameAndCode(req), |
| 766 | 766 |
}, nil |
| 767 | 767 |
default: |
| ... | ... |
@@ -815,3 +832,30 @@ func allInstructionNames() []string {
|
| 815 | 815 |
} |
| 816 | 816 |
return out |
| 817 | 817 |
} |
| 818 |
+ |
|
| 819 |
+func isLowerCaseStageName(cmdArgs []string) bool {
|
|
| 820 |
+ if len(cmdArgs) != 3 {
|
|
| 821 |
+ return true |
|
| 822 |
+ } |
|
| 823 |
+ stageName := cmdArgs[2] |
|
| 824 |
+ return stageName == strings.ToLower(stageName) |
|
| 825 |
+} |
|
| 826 |
+ |
|
| 827 |
+func doesFromCaseMatchAsCase(req parseRequest) bool {
|
|
| 828 |
+ if len(req.args) < 3 {
|
|
| 829 |
+ return true |
|
| 830 |
+ } |
|
| 831 |
+ // consistent casing for the command is handled elsewhere. |
|
| 832 |
+ // If the command is not consistent, there's no need to |
|
| 833 |
+ // add an additional lint warning for the `as` argument. |
|
| 834 |
+ fromHasLowerCasing := req.command == strings.ToLower(req.command) |
|
| 835 |
+ fromHasUpperCasing := req.command == strings.ToUpper(req.command) |
|
| 836 |
+ if !fromHasLowerCasing && !fromHasUpperCasing {
|
|
| 837 |
+ return true |
|
| 838 |
+ } |
|
| 839 |
+ |
|
| 840 |
+ if fromHasLowerCasing {
|
|
| 841 |
+ return req.args[1] == strings.ToLower(req.args[1]) |
|
| 842 |
+ } |
|
| 843 |
+ return req.args[1] == strings.ToUpper(req.args[1]) |
|
| 844 |
+} |
| 818 | 845 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,98 @@ |
| 0 |
+package linter |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "fmt" |
|
| 4 |
+ "strings" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/moby/buildkit/frontend/dockerfile/parser" |
|
| 7 |
+ "github.com/pkg/errors" |
|
| 8 |
+) |
|
| 9 |
+ |
|
| 10 |
+type Config struct {
|
|
| 11 |
+ Warn LintWarnFunc |
|
| 12 |
+ SkipRules []string |
|
| 13 |
+ SkipAll bool |
|
| 14 |
+ ReturnAsError bool |
|
| 15 |
+} |
|
| 16 |
+ |
|
| 17 |
+type Linter struct {
|
|
| 18 |
+ SkippedRules map[string]struct{}
|
|
| 19 |
+ CalledRules []string |
|
| 20 |
+ SkipAll bool |
|
| 21 |
+ ReturnAsError bool |
|
| 22 |
+ Warn LintWarnFunc |
|
| 23 |
+} |
|
| 24 |
+ |
|
| 25 |
+func New(config *Config) *Linter {
|
|
| 26 |
+ toret := &Linter{
|
|
| 27 |
+ SkippedRules: map[string]struct{}{},
|
|
| 28 |
+ CalledRules: []string{},
|
|
| 29 |
+ Warn: config.Warn, |
|
| 30 |
+ } |
|
| 31 |
+ toret.SkipAll = config.SkipAll |
|
| 32 |
+ toret.ReturnAsError = config.ReturnAsError |
|
| 33 |
+ for _, rule := range config.SkipRules {
|
|
| 34 |
+ toret.SkippedRules[rule] = struct{}{}
|
|
| 35 |
+ } |
|
| 36 |
+ return toret |
|
| 37 |
+} |
|
| 38 |
+ |
|
| 39 |
+func (lc *Linter) Run(rule LinterRuleI, location []parser.Range, txt ...string) {
|
|
| 40 |
+ if lc == nil || lc.Warn == nil || lc.SkipAll {
|
|
| 41 |
+ return |
|
| 42 |
+ } |
|
| 43 |
+ rulename := rule.RuleName() |
|
| 44 |
+ if _, ok := lc.SkippedRules[rulename]; ok {
|
|
| 45 |
+ return |
|
| 46 |
+ } |
|
| 47 |
+ lc.CalledRules = append(lc.CalledRules, rulename) |
|
| 48 |
+ rule.Run(lc.Warn, location, txt...) |
|
| 49 |
+} |
|
| 50 |
+ |
|
| 51 |
+func (lc *Linter) Error() error {
|
|
| 52 |
+ if lc == nil || !lc.ReturnAsError {
|
|
| 53 |
+ return nil |
|
| 54 |
+ } |
|
| 55 |
+ if len(lc.CalledRules) == 0 {
|
|
| 56 |
+ return nil |
|
| 57 |
+ } |
|
| 58 |
+ var rules []string |
|
| 59 |
+ uniqueRules := map[string]struct{}{}
|
|
| 60 |
+ for _, r := range lc.CalledRules {
|
|
| 61 |
+ uniqueRules[r] = struct{}{}
|
|
| 62 |
+ } |
|
| 63 |
+ for r := range uniqueRules {
|
|
| 64 |
+ rules = append(rules, r) |
|
| 65 |
+ } |
|
| 66 |
+ return errors.Errorf("lint violation found for rules: %s", strings.Join(rules, ", "))
|
|
| 67 |
+} |
|
| 68 |
+ |
|
| 69 |
+type LinterRuleI interface {
|
|
| 70 |
+ RuleName() string |
|
| 71 |
+ Run(warn LintWarnFunc, location []parser.Range, txt ...string) |
|
| 72 |
+} |
|
| 73 |
+ |
|
| 74 |
+type LinterRule[F any] struct {
|
|
| 75 |
+ Name string |
|
| 76 |
+ Description string |
|
| 77 |
+ URL string |
|
| 78 |
+ Format F |
|
| 79 |
+} |
|
| 80 |
+ |
|
| 81 |
+func (rule *LinterRule[F]) RuleName() string {
|
|
| 82 |
+ return rule.Name |
|
| 83 |
+} |
|
| 84 |
+ |
|
| 85 |
+func (rule *LinterRule[F]) Run(warn LintWarnFunc, location []parser.Range, txt ...string) {
|
|
| 86 |
+ if len(txt) == 0 {
|
|
| 87 |
+ txt = []string{rule.Description}
|
|
| 88 |
+ } |
|
| 89 |
+ short := strings.Join(txt, " ") |
|
| 90 |
+ warn(rule.Name, rule.Description, rule.URL, short, location) |
|
| 91 |
+} |
|
| 92 |
+ |
|
| 93 |
+func LintFormatShort(rulename, msg string, startLine int) string {
|
|
| 94 |
+ return fmt.Sprintf("Lint Rule '%s': %s (line %d)", rulename, msg, startLine)
|
|
| 95 |
+} |
|
| 96 |
+ |
|
| 97 |
+type LintWarnFunc func(rulename, description, url, fmtmsg string, location []parser.Range) |
| 0 | 98 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,130 @@ |
| 0 |
+package linter |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "fmt" |
|
| 4 |
+) |
|
| 5 |
+ |
|
| 6 |
+var ( |
|
| 7 |
+ RuleStageNameCasing = LinterRule[func(string) string]{
|
|
| 8 |
+ Name: "StageNameCasing", |
|
| 9 |
+ Description: "Stage names should be lowercase", |
|
| 10 |
+ Format: func(stageName string) string {
|
|
| 11 |
+ return fmt.Sprintf("Stage name '%s' should be lowercase", stageName)
|
|
| 12 |
+ }, |
|
| 13 |
+ } |
|
| 14 |
+ RuleFromAsCasing = LinterRule[func(string, string) string]{
|
|
| 15 |
+ Name: "FromAsCasing", |
|
| 16 |
+ Description: "The 'as' keyword should match the case of the 'from' keyword", |
|
| 17 |
+ Format: func(from, as string) string {
|
|
| 18 |
+ return fmt.Sprintf("'%s' and '%s' keywords' casing do not match", as, from)
|
|
| 19 |
+ }, |
|
| 20 |
+ } |
|
| 21 |
+ RuleNoEmptyContinuations = LinterRule[func() string]{
|
|
| 22 |
+ Name: "NoEmptyContinuations", |
|
| 23 |
+ Description: "Empty continuation lines will become errors in a future release", |
|
| 24 |
+ URL: "https://github.com/moby/moby/pull/33719", |
|
| 25 |
+ Format: func() string {
|
|
| 26 |
+ return "Empty continuation line" |
|
| 27 |
+ }, |
|
| 28 |
+ } |
|
| 29 |
+ RuleSelfConsistentCommandCasing = LinterRule[func(string) string]{
|
|
| 30 |
+ Name: "SelfConsistentCommandCasing", |
|
| 31 |
+ Description: "Commands should be in consistent casing (all lower or all upper)", |
|
| 32 |
+ Format: func(command string) string {
|
|
| 33 |
+ return fmt.Sprintf("Command '%s' should be consistently cased", command)
|
|
| 34 |
+ }, |
|
| 35 |
+ } |
|
| 36 |
+ RuleFileConsistentCommandCasing = LinterRule[func(string, string) string]{
|
|
| 37 |
+ Name: "FileConsistentCommandCasing", |
|
| 38 |
+ Description: "All commands within the Dockerfile should use the same casing (either upper or lower)", |
|
| 39 |
+ Format: func(violatingCommand, correctCasing string) string {
|
|
| 40 |
+ return fmt.Sprintf("Command '%s' should match the case of the command majority (%s)", violatingCommand, correctCasing)
|
|
| 41 |
+ }, |
|
| 42 |
+ } |
|
| 43 |
+ RuleDuplicateStageName = LinterRule[func(string) string]{
|
|
| 44 |
+ Name: "DuplicateStageName", |
|
| 45 |
+ Description: "Stage names should be unique", |
|
| 46 |
+ Format: func(stageName string) string {
|
|
| 47 |
+ return fmt.Sprintf("Duplicate stage name %q, stage names should be unique", stageName)
|
|
| 48 |
+ }, |
|
| 49 |
+ } |
|
| 50 |
+ RuleReservedStageName = LinterRule[func(string) string]{
|
|
| 51 |
+ Name: "ReservedStageName", |
|
| 52 |
+ Description: "Reserved stage names should not be used to name a stage", |
|
| 53 |
+ Format: func(reservedStageName string) string {
|
|
| 54 |
+ return fmt.Sprintf("Stage name should not use the same name as reserved stage %q", reservedStageName)
|
|
| 55 |
+ }, |
|
| 56 |
+ } |
|
| 57 |
+ RuleJSONArgsRecommended = LinterRule[func(instructionName string) string]{
|
|
| 58 |
+ Name: "JSONArgsRecommended", |
|
| 59 |
+ Description: "JSON arguments recommended for ENTRYPOINT/CMD to prevent unintended behavior related to OS signals", |
|
| 60 |
+ Format: func(instructionName string) string {
|
|
| 61 |
+ return fmt.Sprintf("JSON arguments recommended for %s to prevent unintended behavior related to OS signals", instructionName)
|
|
| 62 |
+ }, |
|
| 63 |
+ } |
|
| 64 |
+ RuleMaintainerDeprecated = LinterRule[func() string]{
|
|
| 65 |
+ Name: "MaintainerDeprecated", |
|
| 66 |
+ Description: "The maintainer instruction is deprecated, use a label instead to define an image author", |
|
| 67 |
+ URL: "https://docs.docker.com/reference/dockerfile/#maintainer-deprecated", |
|
| 68 |
+ Format: func() string {
|
|
| 69 |
+ return "Maintainer instruction is deprecated in favor of using label" |
|
| 70 |
+ }, |
|
| 71 |
+ } |
|
| 72 |
+ RuleUndeclaredArgInFrom = LinterRule[func(string, string) string]{
|
|
| 73 |
+ Name: "UndeclaredArgInFrom", |
|
| 74 |
+ Description: "FROM command must use declared ARGs", |
|
| 75 |
+ Format: func(baseArg, suggest string) string {
|
|
| 76 |
+ out := fmt.Sprintf("FROM argument '%s' is not declared", baseArg)
|
|
| 77 |
+ if suggest != "" {
|
|
| 78 |
+ out += fmt.Sprintf(" (did you mean %s?)", suggest)
|
|
| 79 |
+ } |
|
| 80 |
+ return out |
|
| 81 |
+ }, |
|
| 82 |
+ } |
|
| 83 |
+ RuleWorkdirRelativePath = LinterRule[func(workdir string) string]{
|
|
| 84 |
+ Name: "WorkdirRelativePath", |
|
| 85 |
+ Description: "Relative workdir without an absolute workdir declared within the build can have unexpected results if the base image changes", |
|
| 86 |
+ Format: func(workdir string) string {
|
|
| 87 |
+ return fmt.Sprintf("Relative workdir %q can have unexpected results if the base image changes", workdir)
|
|
| 88 |
+ }, |
|
| 89 |
+ } |
|
| 90 |
+ RuleUndefinedArg = LinterRule[func(string) string]{
|
|
| 91 |
+ Name: "UndefinedArg", |
|
| 92 |
+ Description: "ARGs should be defined before their use", |
|
| 93 |
+ Format: func(arg string) string {
|
|
| 94 |
+ return fmt.Sprintf("Usage of undefined variable '$%s'", arg)
|
|
| 95 |
+ }, |
|
| 96 |
+ } |
|
| 97 |
+ RuleUndefinedVar = LinterRule[func(string, string) string]{
|
|
| 98 |
+ Name: "UndefinedVar", |
|
| 99 |
+ Description: "Variables should be defined before their use", |
|
| 100 |
+ Format: func(arg, suggest string) string {
|
|
| 101 |
+ out := fmt.Sprintf("Usage of undefined variable '$%s'", arg)
|
|
| 102 |
+ if suggest != "" {
|
|
| 103 |
+ out += fmt.Sprintf(" (did you mean $%s?)", suggest)
|
|
| 104 |
+ } |
|
| 105 |
+ return out |
|
| 106 |
+ }, |
|
| 107 |
+ } |
|
| 108 |
+ RuleMultipleInstructionsDisallowed = LinterRule[func(instructionName string) string]{
|
|
| 109 |
+ Name: "MultipleInstructionsDisallowed", |
|
| 110 |
+ Description: "Multiple instructions of the same type should not be used in the same stage", |
|
| 111 |
+ Format: func(instructionName string) string {
|
|
| 112 |
+ return fmt.Sprintf("Multiple %s instructions should not be used in the same stage because only the last one will be used", instructionName)
|
|
| 113 |
+ }, |
|
| 114 |
+ } |
|
| 115 |
+ RuleLegacyKeyValueFormat = LinterRule[func(cmdName string) string]{
|
|
| 116 |
+ Name: "LegacyKeyValueFormat", |
|
| 117 |
+ Description: "Legacy key/value format with whitespace separator should not be used", |
|
| 118 |
+ Format: func(cmdName string) string {
|
|
| 119 |
+ return fmt.Sprintf("\"%s key=value\" should be used instead of legacy \"%s key value\" format", cmdName, cmdName)
|
|
| 120 |
+ }, |
|
| 121 |
+ } |
|
| 122 |
+ RuleInvalidBaseImagePlatform = LinterRule[func(string, string, string) string]{
|
|
| 123 |
+ Name: "InvalidBaseImagePlatform", |
|
| 124 |
+ Description: "Base image platform does not match expected target platform", |
|
| 125 |
+ Format: func(image, expected, actual string) string {
|
|
| 126 |
+ return fmt.Sprintf("Base image %s was pulled with platform %q, expected %q for current build", image, actual, expected)
|
|
| 127 |
+ }, |
|
| 128 |
+ } |
|
| 129 |
+) |
| ... | ... |
@@ -13,12 +13,14 @@ import ( |
| 13 | 13 |
|
| 14 | 14 |
const ( |
| 15 | 15 |
keySyntax = "syntax" |
| 16 |
+ keyCheck = "check" |
|
| 16 | 17 |
keyEscape = "escape" |
| 17 | 18 |
) |
| 18 | 19 |
|
| 19 | 20 |
var validDirectives = map[string]struct{}{
|
| 20 | 21 |
keySyntax: {},
|
| 21 | 22 |
keyEscape: {},
|
| 23 |
+ keyCheck: {},
|
|
| 22 | 24 |
} |
| 23 | 25 |
|
| 24 | 26 |
type Directive struct {
|
| ... | ... |
@@ -110,6 +112,10 @@ func (d *DirectiveParser) ParseAll(data []byte) ([]*Directive, error) {
|
| 110 | 110 |
// This allows for a flexible range of input formats, and appropriate syntax |
| 111 | 111 |
// selection. |
| 112 | 112 |
func DetectSyntax(dt []byte) (string, string, []Range, bool) {
|
| 113 |
+ return ParseDirective(keySyntax, dt) |
|
| 114 |
+} |
|
| 115 |
+ |
|
| 116 |
+func ParseDirective(key string, dt []byte) (string, string, []Range, bool) {
|
|
| 113 | 117 |
dt, hadShebang, err := discardShebang(dt) |
| 114 | 118 |
if err != nil {
|
| 115 | 119 |
return "", "", nil, false |
| ... | ... |
@@ -119,42 +125,38 @@ func DetectSyntax(dt []byte) (string, string, []Range, bool) {
|
| 119 | 119 |
line++ |
| 120 | 120 |
} |
| 121 | 121 |
|
| 122 |
- // use default directive parser, and search for #syntax= |
|
| 122 |
+ // use default directive parser, and search for #key= |
|
| 123 | 123 |
directiveParser := DirectiveParser{line: line}
|
| 124 |
- if syntax, cmdline, loc, ok := detectSyntaxFromParser(dt, directiveParser); ok {
|
|
| 124 |
+ if syntax, cmdline, loc, ok := detectDirectiveFromParser(key, dt, directiveParser); ok {
|
|
| 125 | 125 |
return syntax, cmdline, loc, true |
| 126 | 126 |
} |
| 127 | 127 |
|
| 128 |
- // use directive with different comment prefix, and search for //syntax= |
|
| 128 |
+ // use directive with different comment prefix, and search for //key= |
|
| 129 | 129 |
directiveParser = DirectiveParser{line: line}
|
| 130 | 130 |
directiveParser.setComment("//")
|
| 131 |
- if syntax, cmdline, loc, ok := detectSyntaxFromParser(dt, directiveParser); ok {
|
|
| 131 |
+ if syntax, cmdline, loc, ok := detectDirectiveFromParser(key, dt, directiveParser); ok {
|
|
| 132 | 132 |
return syntax, cmdline, loc, true |
| 133 | 133 |
} |
| 134 | 134 |
|
| 135 |
- // search for possible json directives |
|
| 136 |
- var directive struct {
|
|
| 137 |
- Syntax string `json:"syntax"` |
|
| 138 |
- } |
|
| 139 |
- if err := json.Unmarshal(dt, &directive); err == nil {
|
|
| 140 |
- if directive.Syntax != "" {
|
|
| 135 |
+ // use json directive, and search for { "key": "..." }
|
|
| 136 |
+ jsonDirective := map[string]string{}
|
|
| 137 |
+ if err := json.Unmarshal(dt, &jsonDirective); err == nil {
|
|
| 138 |
+ if v, ok := jsonDirective[key]; ok {
|
|
| 141 | 139 |
loc := []Range{{
|
| 142 | 140 |
Start: Position{Line: line},
|
| 143 | 141 |
End: Position{Line: line},
|
| 144 | 142 |
}} |
| 145 |
- return directive.Syntax, directive.Syntax, loc, true |
|
| 143 |
+ return v, v, loc, true |
|
| 146 | 144 |
} |
| 147 | 145 |
} |
| 148 | 146 |
|
| 149 | 147 |
return "", "", nil, false |
| 150 | 148 |
} |
| 151 | 149 |
|
| 152 |
-func detectSyntaxFromParser(dt []byte, parser DirectiveParser) (string, string, []Range, bool) {
|
|
| 150 |
+func detectDirectiveFromParser(key string, dt []byte, parser DirectiveParser) (string, string, []Range, bool) {
|
|
| 153 | 151 |
directives, _ := parser.ParseAll(dt) |
| 154 | 152 |
for _, d := range directives {
|
| 155 |
- // check for syntax directive before erroring out, since the error |
|
| 156 |
- // might have occurred *after* the syntax directive |
|
| 157 |
- if d.Name == keySyntax {
|
|
| 153 |
+ if d.Name == key {
|
|
| 158 | 154 |
p, _, _ := strings.Cut(d.Value, " ") |
| 159 | 155 |
return p, d.Value, d.Location, true |
| 160 | 156 |
} |
| ... | ... |
@@ -154,7 +154,7 @@ func parseNameVal(rest string, key string, d *directives) (*Node, error) {
|
| 154 | 154 |
if len(parts) < 2 {
|
| 155 | 155 |
return nil, errors.Errorf("%s must have two arguments", key)
|
| 156 | 156 |
} |
| 157 |
- return newKeyValueNode(parts[0], parts[1]), nil |
|
| 157 |
+ return newKeyValueNode(parts[0], parts[1], ""), nil |
|
| 158 | 158 |
} |
| 159 | 159 |
|
| 160 | 160 |
var rootNode *Node |
| ... | ... |
@@ -165,17 +165,20 @@ func parseNameVal(rest string, key string, d *directives) (*Node, error) {
|
| 165 | 165 |
} |
| 166 | 166 |
|
| 167 | 167 |
parts := strings.SplitN(word, "=", 2) |
| 168 |
- node := newKeyValueNode(parts[0], parts[1]) |
|
| 168 |
+ node := newKeyValueNode(parts[0], parts[1], "=") |
|
| 169 | 169 |
rootNode, prevNode = appendKeyValueNode(node, rootNode, prevNode) |
| 170 | 170 |
} |
| 171 | 171 |
|
| 172 | 172 |
return rootNode, nil |
| 173 | 173 |
} |
| 174 | 174 |
|
| 175 |
-func newKeyValueNode(key, value string) *Node {
|
|
| 175 |
+func newKeyValueNode(key, value, sep string) *Node {
|
|
| 176 | 176 |
return &Node{
|
| 177 | 177 |
Value: key, |
| 178 |
- Next: &Node{Value: value},
|
|
| 178 |
+ Next: &Node{
|
|
| 179 |
+ Value: value, |
|
| 180 |
+ Next: &Node{Value: sep},
|
|
| 181 |
+ }, |
|
| 179 | 182 |
} |
| 180 | 183 |
} |
| 181 | 184 |
|
| ... | ... |
@@ -187,7 +190,9 @@ func appendKeyValueNode(node, rootNode, prevNode *Node) (*Node, *Node) {
|
| 187 | 187 |
prevNode.Next = node |
| 188 | 188 |
} |
| 189 | 189 |
|
| 190 |
- prevNode = node.Next |
|
| 190 |
+ for prevNode = node.Next; prevNode.Next != nil; {
|
|
| 191 |
+ prevNode = prevNode.Next |
|
| 192 |
+ } |
|
| 191 | 193 |
return rootNode, prevNode |
| 192 | 194 |
} |
| 193 | 195 |
|
| ... | ... |
@@ -269,7 +274,7 @@ func parseString(rest string, d *directives) (*Node, map[string]bool, error) {
|
| 269 | 269 |
} |
| 270 | 270 |
|
| 271 | 271 |
// parseJSON converts JSON arrays to an AST. |
| 272 |
-func parseJSON(rest string, d *directives) (*Node, map[string]bool, error) {
|
|
| 272 |
+func parseJSON(rest string) (*Node, map[string]bool, error) {
|
|
| 273 | 273 |
rest = strings.TrimLeftFunc(rest, unicode.IsSpace) |
| 274 | 274 |
if !strings.HasPrefix(rest, "[") {
|
| 275 | 275 |
return nil, nil, errors.Errorf("Error parsing %q as a JSON array", rest)
|
| ... | ... |
@@ -307,7 +312,7 @@ func parseMaybeJSON(rest string, d *directives) (*Node, map[string]bool, error) |
| 307 | 307 |
return nil, nil, nil |
| 308 | 308 |
} |
| 309 | 309 |
|
| 310 |
- node, attrs, err := parseJSON(rest, d) |
|
| 310 |
+ node, attrs, err := parseJSON(rest) |
|
| 311 | 311 |
|
| 312 | 312 |
if err == nil {
|
| 313 | 313 |
return node, attrs, nil |
| ... | ... |
@@ -325,7 +330,7 @@ func parseMaybeJSON(rest string, d *directives) (*Node, map[string]bool, error) |
| 325 | 325 |
// so, passes to parseJSON; if not, attempts to parse it as a whitespace |
| 326 | 326 |
// delimited string. |
| 327 | 327 |
func parseMaybeJSONToList(rest string, d *directives) (*Node, map[string]bool, error) {
|
| 328 |
- node, attrs, err := parseJSON(rest, d) |
|
| 328 |
+ node, attrs, err := parseJSON(rest) |
|
| 329 | 329 |
|
| 330 | 330 |
if err == nil {
|
| 331 | 331 |
return node, attrs, nil |
| ... | ... |
@@ -32,10 +32,12 @@ func NewLex(escapeToken rune) *Lex {
|
| 32 | 32 |
} |
| 33 | 33 |
|
| 34 | 34 |
// ProcessWord will use the 'env' list of environment variables, |
| 35 |
-// and replace any env var references in 'word'. |
|
| 36 |
-func (s *Lex) ProcessWord(word string, env []string) (string, error) {
|
|
| 37 |
- word, _, err := s.process(word, BuildEnvs(env)) |
|
| 38 |
- return word, err |
|
| 35 |
+// and replace any env var references in 'word'. It will also |
|
| 36 |
+// return variables in word which were not found in the 'env' list, |
|
| 37 |
+// which is useful in later linting. |
|
| 38 |
+func (s *Lex) ProcessWord(word string, env []string) (string, map[string]struct{}, error) {
|
|
| 39 |
+ result, err := s.process(word, BuildEnvs(env)) |
|
| 40 |
+ return result.Result, result.Unmatched, err |
|
| 39 | 41 |
} |
| 40 | 42 |
|
| 41 | 43 |
// ProcessWords will use the 'env' list of environment variables, |
| ... | ... |
@@ -46,28 +48,40 @@ func (s *Lex) ProcessWord(word string, env []string) (string, error) {
|
| 46 | 46 |
// Note, each one is trimmed to remove leading and trailing spaces (unless |
| 47 | 47 |
// they are quoted", but ProcessWord retains spaces between words. |
| 48 | 48 |
func (s *Lex) ProcessWords(word string, env []string) ([]string, error) {
|
| 49 |
- _, words, err := s.process(word, BuildEnvs(env)) |
|
| 50 |
- return words, err |
|
| 49 |
+ result, err := s.process(word, BuildEnvs(env)) |
|
| 50 |
+ return result.Words, err |
|
| 51 | 51 |
} |
| 52 | 52 |
|
| 53 | 53 |
// ProcessWordWithMap will use the 'env' list of environment variables, |
| 54 | 54 |
// and replace any env var references in 'word'. |
| 55 | 55 |
func (s *Lex) ProcessWordWithMap(word string, env map[string]string) (string, error) {
|
| 56 |
- word, _, err := s.process(word, env) |
|
| 57 |
- return word, err |
|
| 56 |
+ result, err := s.process(word, env) |
|
| 57 |
+ return result.Result, err |
|
| 58 |
+} |
|
| 59 |
+ |
|
| 60 |
+type ProcessWordResult struct {
|
|
| 61 |
+ Result string |
|
| 62 |
+ Words []string |
|
| 63 |
+ Matched map[string]struct{}
|
|
| 64 |
+ Unmatched map[string]struct{}
|
|
| 58 | 65 |
} |
| 59 | 66 |
|
| 60 | 67 |
// ProcessWordWithMatches will use the 'env' list of environment variables, |
| 61 | 68 |
// replace any env var references in 'word' and return the env that were used. |
| 62 |
-func (s *Lex) ProcessWordWithMatches(word string, env map[string]string) (string, map[string]struct{}, error) {
|
|
| 69 |
+func (s *Lex) ProcessWordWithMatches(word string, env map[string]string) (ProcessWordResult, error) {
|
|
| 63 | 70 |
sw := s.init(word, env) |
| 64 |
- word, _, err := sw.process(word) |
|
| 65 |
- return word, sw.matches, err |
|
| 71 |
+ word, words, err := sw.process(word) |
|
| 72 |
+ return ProcessWordResult{
|
|
| 73 |
+ Result: word, |
|
| 74 |
+ Words: words, |
|
| 75 |
+ Matched: sw.matches, |
|
| 76 |
+ Unmatched: sw.nonmatches, |
|
| 77 |
+ }, err |
|
| 66 | 78 |
} |
| 67 | 79 |
|
| 68 | 80 |
func (s *Lex) ProcessWordsWithMap(word string, env map[string]string) ([]string, error) {
|
| 69 |
- _, words, err := s.process(word, env) |
|
| 70 |
- return words, err |
|
| 81 |
+ result, err := s.process(word, env) |
|
| 82 |
+ return result.Words, err |
|
| 71 | 83 |
} |
| 72 | 84 |
|
| 73 | 85 |
func (s *Lex) init(word string, env map[string]string) *shellWord {
|
| ... | ... |
@@ -79,14 +93,21 @@ func (s *Lex) init(word string, env map[string]string) *shellWord {
|
| 79 | 79 |
rawQuotes: s.RawQuotes, |
| 80 | 80 |
rawEscapes: s.RawEscapes, |
| 81 | 81 |
matches: make(map[string]struct{}),
|
| 82 |
+ nonmatches: make(map[string]struct{}),
|
|
| 82 | 83 |
} |
| 83 | 84 |
sw.scanner.Init(strings.NewReader(word)) |
| 84 | 85 |
return sw |
| 85 | 86 |
} |
| 86 | 87 |
|
| 87 |
-func (s *Lex) process(word string, env map[string]string) (string, []string, error) {
|
|
| 88 |
+func (s *Lex) process(word string, env map[string]string) (*ProcessWordResult, error) {
|
|
| 88 | 89 |
sw := s.init(word, env) |
| 89 |
- return sw.process(word) |
|
| 90 |
+ word, words, err := sw.process(word) |
|
| 91 |
+ return &ProcessWordResult{
|
|
| 92 |
+ Result: word, |
|
| 93 |
+ Words: words, |
|
| 94 |
+ Matched: sw.matches, |
|
| 95 |
+ Unmatched: sw.nonmatches, |
|
| 96 |
+ }, err |
|
| 90 | 97 |
} |
| 91 | 98 |
|
| 92 | 99 |
type shellWord struct {
|
| ... | ... |
@@ -98,6 +119,7 @@ type shellWord struct {
|
| 98 | 98 |
skipUnsetEnv bool |
| 99 | 99 |
skipProcessQuotes bool |
| 100 | 100 |
matches map[string]struct{}
|
| 101 |
+ nonmatches map[string]struct{}
|
|
| 101 | 102 |
} |
| 102 | 103 |
|
| 103 | 104 |
func (sw *shellWord) process(source string) (string, []string, error) {
|
| ... | ... |
@@ -511,6 +533,7 @@ func (sw *shellWord) getEnv(name string) (string, bool) {
|
| 511 | 511 |
return value, true |
| 512 | 512 |
} |
| 513 | 513 |
} |
| 514 |
+ sw.nonmatches[name] = struct{}{}
|
|
| 514 | 515 |
return "", false |
| 515 | 516 |
} |
| 516 | 517 |
|
| ... | ... |
@@ -47,6 +47,7 @@ const ( |
| 47 | 47 |
keyCacheNSArg = "build-arg:BUILDKIT_CACHE_MOUNT_NS" |
| 48 | 48 |
keyMultiPlatformArg = "build-arg:BUILDKIT_MULTI_PLATFORM" |
| 49 | 49 |
keyHostnameArg = "build-arg:BUILDKIT_SANDBOX_HOSTNAME" |
| 50 |
+ keyDockerfileLintArg = "build-arg:BUILDKIT_DOCKERFILE_CHECK" |
|
| 50 | 51 |
keyContextKeepGitDirArg = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR" |
| 51 | 52 |
keySourceDateEpoch = "build-arg:SOURCE_DATE_EPOCH" |
| 52 | 53 |
) |
| ... | ... |
@@ -64,6 +65,7 @@ type Config struct {
|
| 64 | 64 |
ShmSize int64 |
| 65 | 65 |
Target string |
| 66 | 66 |
Ulimits []pb.Ulimit |
| 67 |
+ LinterConfig *string |
|
| 67 | 68 |
|
| 68 | 69 |
CacheImports []client.CacheOptionsEntry |
| 69 | 70 |
TargetPlatforms []ocispecs.Platform // nil means default |
| ... | ... |
@@ -277,6 +279,10 @@ func (bc *Client) init() error {
|
| 277 | 277 |
opts[keyHostname] = v |
| 278 | 278 |
} |
| 279 | 279 |
bc.Hostname = opts[keyHostname] |
| 280 |
+ |
|
| 281 |
+ if v, ok := opts[keyDockerfileLintArg]; ok {
|
|
| 282 |
+ bc.LinterConfig = &v |
|
| 283 |
+ } |
|
| 280 | 284 |
return nil |
| 281 | 285 |
} |
| 282 | 286 |
|
| ... | ... |
@@ -7,6 +7,7 @@ import ( |
| 7 | 7 |
|
| 8 | 8 |
"github.com/moby/buildkit/frontend/gateway/client" |
| 9 | 9 |
"github.com/moby/buildkit/frontend/subrequests" |
| 10 |
+ "github.com/moby/buildkit/frontend/subrequests/lint" |
|
| 10 | 11 |
"github.com/moby/buildkit/frontend/subrequests/outline" |
| 11 | 12 |
"github.com/moby/buildkit/frontend/subrequests/targets" |
| 12 | 13 |
"github.com/moby/buildkit/solver/errdefs" |
| ... | ... |
@@ -19,6 +20,7 @@ const ( |
| 19 | 19 |
type RequestHandler struct {
|
| 20 | 20 |
Outline func(context.Context) (*outline.Outline, error) |
| 21 | 21 |
ListTargets func(context.Context) (*targets.List, error) |
| 22 |
+ Lint func(context.Context) (*lint.LintResults, error) |
|
| 22 | 23 |
AllowOther bool |
| 23 | 24 |
} |
| 24 | 25 |
|
| ... | ... |
@@ -55,6 +57,18 @@ func (bc *Client) HandleSubrequest(ctx context.Context, h RequestHandler) (*clie |
| 55 | 55 |
res, err := targets.ToResult() |
| 56 | 56 |
return res, true, err |
| 57 | 57 |
} |
| 58 |
+ case lint.SubrequestLintDefinition.Name: |
|
| 59 |
+ if f := h.Lint; f != nil {
|
|
| 60 |
+ warnings, err := f(ctx) |
|
| 61 |
+ if err != nil {
|
|
| 62 |
+ return nil, false, err |
|
| 63 |
+ } |
|
| 64 |
+ if warnings == nil {
|
|
| 65 |
+ return nil, true, nil |
|
| 66 |
+ } |
|
| 67 |
+ res, err := warnings.ToResult() |
|
| 68 |
+ return res, true, err |
|
| 69 |
+ } |
|
| 58 | 70 |
} |
| 59 | 71 |
if h.AllowOther {
|
| 60 | 72 |
return nil, false, nil |
| ... | ... |
@@ -14,6 +14,7 @@ import ( |
| 14 | 14 |
"syscall" |
| 15 | 15 |
"time" |
| 16 | 16 |
|
| 17 |
+ "github.com/containerd/containerd/defaults" |
|
| 17 | 18 |
"github.com/containerd/containerd/mount" |
| 18 | 19 |
"github.com/distribution/reference" |
| 19 | 20 |
"github.com/docker/docker/pkg/idtools" |
| ... | ... |
@@ -67,14 +68,26 @@ const ( |
| 67 | 67 |
keyDevel = "gateway-devel" |
| 68 | 68 |
) |
| 69 | 69 |
|
| 70 |
-func NewGatewayFrontend(w worker.Infos) frontend.Frontend {
|
|
| 71 |
- return &gatewayFrontend{
|
|
| 72 |
- workers: w, |
|
| 70 |
+func NewGatewayFrontend(workers worker.Infos, allowedRepositories []string) (frontend.Frontend, error) {
|
|
| 71 |
+ var parsedAllowedRepositories []string |
|
| 72 |
+ |
|
| 73 |
+ for _, allowedRepository := range allowedRepositories {
|
|
| 74 |
+ sourceRef, err := reference.ParseNormalizedNamed(allowedRepository) |
|
| 75 |
+ if err != nil {
|
|
| 76 |
+ return nil, err |
|
| 77 |
+ } |
|
| 78 |
+ parsedAllowedRepositories = append(parsedAllowedRepositories, reference.TrimNamed(sourceRef).Name()) |
|
| 73 | 79 |
} |
| 80 |
+ |
|
| 81 |
+ return &gatewayFrontend{
|
|
| 82 |
+ workers: workers, |
|
| 83 |
+ allowedRepositories: parsedAllowedRepositories, |
|
| 84 |
+ }, nil |
|
| 74 | 85 |
} |
| 75 | 86 |
|
| 76 | 87 |
type gatewayFrontend struct {
|
| 77 |
- workers worker.Infos |
|
| 88 |
+ workers worker.Infos |
|
| 89 |
+ allowedRepositories []string |
|
| 78 | 90 |
} |
| 79 | 91 |
|
| 80 | 92 |
func filterPrefix(opts map[string]string, pfx string) map[string]string {
|
| ... | ... |
@@ -87,6 +100,30 @@ func filterPrefix(opts map[string]string, pfx string) map[string]string {
|
| 87 | 87 |
return m |
| 88 | 88 |
} |
| 89 | 89 |
|
| 90 |
+func (gf *gatewayFrontend) checkSourceIsAllowed(source string) error {
|
|
| 91 |
+ // Returns nil if the source is allowed. |
|
| 92 |
+ // Returns an error if the source is not allowed. |
|
| 93 |
+ if len(gf.allowedRepositories) == 0 {
|
|
| 94 |
+ // No source restrictions in place |
|
| 95 |
+ return nil |
|
| 96 |
+ } |
|
| 97 |
+ |
|
| 98 |
+ sourceRef, err := reference.ParseNormalizedNamed(source) |
|
| 99 |
+ if err != nil {
|
|
| 100 |
+ return err |
|
| 101 |
+ } |
|
| 102 |
+ |
|
| 103 |
+ taglessSource := reference.TrimNamed(sourceRef).Name() |
|
| 104 |
+ |
|
| 105 |
+ for _, allowedRepository := range gf.allowedRepositories {
|
|
| 106 |
+ if taglessSource == allowedRepository {
|
|
| 107 |
+ // Allowed |
|
| 108 |
+ return nil |
|
| 109 |
+ } |
|
| 110 |
+ } |
|
| 111 |
+ return errors.Errorf("'%s' is not an allowed gateway source", source)
|
|
| 112 |
+} |
|
| 113 |
+ |
|
| 90 | 114 |
func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, opts map[string]string, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*frontend.Result, error) {
|
| 91 | 115 |
source, ok := opts[keySource] |
| 92 | 116 |
if !ok {
|
| ... | ... |
@@ -101,6 +138,11 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten |
| 101 | 101 |
|
| 102 | 102 |
var frontendDef *opspb.Definition |
| 103 | 103 |
|
| 104 |
+ err := gf.checkSourceIsAllowed(source) |
|
| 105 |
+ if err != nil {
|
|
| 106 |
+ return nil, err |
|
| 107 |
+ } |
|
| 108 |
+ |
|
| 104 | 109 |
if isDevel {
|
| 105 | 110 |
devRes, err := llbBridge.Solve(ctx, |
| 106 | 111 |
frontend.SolveRequest{
|
| ... | ... |
@@ -456,7 +498,13 @@ func newBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg |
| 456 | 456 |
func serveLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, exec executor.Executor, workers worker.Infos, inputs map[string]*opspb.Definition, sid string, sm *session.Manager) (*llbBridgeForwarder, context.Context, error) {
|
| 457 | 457 |
ctx, cancel := context.WithCancelCause(ctx) |
| 458 | 458 |
lbf := newBridgeForwarder(ctx, llbBridge, exec, workers, inputs, sid, sm) |
| 459 |
- server := grpc.NewServer(grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor)) |
|
| 459 |
+ serverOpt := []grpc.ServerOption{
|
|
| 460 |
+ grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), |
|
| 461 |
+ grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor), |
|
| 462 |
+ grpc.MaxRecvMsgSize(defaults.DefaultMaxRecvMsgSize), |
|
| 463 |
+ grpc.MaxSendMsgSize(defaults.DefaultMaxSendMsgSize), |
|
| 464 |
+ } |
|
| 465 |
+ server := grpc.NewServer(serverOpt...) |
|
| 460 | 466 |
grpc_health_v1.RegisterHealthServer(server, health.NewServer()) |
| 461 | 467 |
pb.RegisterLLBBridgeServer(server, lbf) |
| 462 | 468 |
|
| ... | ... |
@@ -1249,11 +1249,18 @@ func (r *reference) StatFile(ctx context.Context, req client.StatRequest) (*fsty |
| 1249 | 1249 |
} |
| 1250 | 1250 |
|
| 1251 | 1251 |
func grpcClientConn(ctx context.Context) (context.Context, *grpc.ClientConn, error) {
|
| 1252 |
- dialOpt := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
|
| 1253 |
- return stdioConn(), nil |
|
| 1254 |
- }) |
|
| 1255 |
- |
|
| 1256 |
- cc, err := grpc.DialContext(ctx, "localhost", dialOpt, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor)) |
|
| 1252 |
+ dialOpts := []grpc.DialOption{
|
|
| 1253 |
+ grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
|
| 1254 |
+ return stdioConn(), nil |
|
| 1255 |
+ }), |
|
| 1256 |
+ grpc.WithTransportCredentials(insecure.NewCredentials()), |
|
| 1257 |
+ grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), |
|
| 1258 |
+ grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor), |
|
| 1259 |
+ grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(16 << 20)), |
|
| 1260 |
+ grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(16 << 20)), |
|
| 1261 |
+ } |
|
| 1262 |
+ |
|
| 1263 |
+ cc, err := grpc.DialContext(ctx, "localhost", dialOpts...) |
|
| 1257 | 1264 |
if err != nil {
|
| 1258 | 1265 |
return nil, nil, errors.Wrap(err, "failed to create grpc client") |
| 1259 | 1266 |
} |
| 1260 | 1267 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,200 @@ |
| 0 |
+package lint |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "bytes" |
|
| 4 |
+ "encoding/json" |
|
| 5 |
+ "fmt" |
|
| 6 |
+ "io" |
|
| 7 |
+ "sort" |
|
| 8 |
+ |
|
| 9 |
+ "github.com/moby/buildkit/client/llb" |
|
| 10 |
+ "github.com/moby/buildkit/frontend/dockerfile/parser" |
|
| 11 |
+ "github.com/moby/buildkit/frontend/gateway/client" |
|
| 12 |
+ "github.com/moby/buildkit/frontend/subrequests" |
|
| 13 |
+ "github.com/moby/buildkit/solver/errdefs" |
|
| 14 |
+ "github.com/moby/buildkit/solver/pb" |
|
| 15 |
+ "github.com/pkg/errors" |
|
| 16 |
+) |
|
| 17 |
+ |
|
| 18 |
+const RequestLint = "frontend.lint" |
|
| 19 |
+ |
|
| 20 |
+var SubrequestLintDefinition = subrequests.Request{
|
|
| 21 |
+ Name: RequestLint, |
|
| 22 |
+ Version: "1.0.0", |
|
| 23 |
+ Type: subrequests.TypeRPC, |
|
| 24 |
+ Description: "Lint a Dockerfile", |
|
| 25 |
+ Opts: []subrequests.Named{},
|
|
| 26 |
+ Metadata: []subrequests.Named{
|
|
| 27 |
+ {Name: "result.json"},
|
|
| 28 |
+ {Name: "result.txt"},
|
|
| 29 |
+ {Name: "result.statuscode"},
|
|
| 30 |
+ }, |
|
| 31 |
+} |
|
| 32 |
+ |
|
| 33 |
+type Warning struct {
|
|
| 34 |
+ RuleName string `json:"ruleName"` |
|
| 35 |
+ Description string `json:"description,omitempty"` |
|
| 36 |
+ URL string `json:"url,omitempty"` |
|
| 37 |
+ Detail string `json:"detail,omitempty"` |
|
| 38 |
+ Location pb.Location `json:"location,omitempty"` |
|
| 39 |
+} |
|
| 40 |
+ |
|
| 41 |
+type BuildError struct {
|
|
| 42 |
+ Message string `json:"message"` |
|
| 43 |
+ Location pb.Location `json:"location"` |
|
| 44 |
+} |
|
| 45 |
+ |
|
| 46 |
+type LintResults struct {
|
|
| 47 |
+ Warnings []Warning `json:"warnings"` |
|
| 48 |
+ Sources []*pb.SourceInfo `json:"sources"` |
|
| 49 |
+ Error *BuildError `json:"buildError,omitempty"` |
|
| 50 |
+} |
|
| 51 |
+ |
|
| 52 |
+func (results *LintResults) AddSource(sourceMap *llb.SourceMap) int {
|
|
| 53 |
+ newSource := &pb.SourceInfo{
|
|
| 54 |
+ Filename: sourceMap.Filename, |
|
| 55 |
+ Language: sourceMap.Language, |
|
| 56 |
+ Definition: sourceMap.Definition.ToPB(), |
|
| 57 |
+ Data: sourceMap.Data, |
|
| 58 |
+ } |
|
| 59 |
+ for i, source := range results.Sources {
|
|
| 60 |
+ if sourceInfoEqual(source, newSource) {
|
|
| 61 |
+ return i |
|
| 62 |
+ } |
|
| 63 |
+ } |
|
| 64 |
+ results.Sources = append(results.Sources, newSource) |
|
| 65 |
+ return len(results.Sources) - 1 |
|
| 66 |
+} |
|
| 67 |
+ |
|
| 68 |
+func (results *LintResults) AddWarning(rulename, description, url, fmtmsg string, sourceIndex int, location []parser.Range) {
|
|
| 69 |
+ sourceLocation := []*pb.Range{}
|
|
| 70 |
+ for _, loc := range location {
|
|
| 71 |
+ sourceLocation = append(sourceLocation, &pb.Range{
|
|
| 72 |
+ Start: pb.Position{
|
|
| 73 |
+ Line: int32(loc.Start.Line), |
|
| 74 |
+ Character: int32(loc.Start.Character), |
|
| 75 |
+ }, |
|
| 76 |
+ End: pb.Position{
|
|
| 77 |
+ Line: int32(loc.End.Line), |
|
| 78 |
+ Character: int32(loc.End.Character), |
|
| 79 |
+ }, |
|
| 80 |
+ }) |
|
| 81 |
+ } |
|
| 82 |
+ pbLocation := pb.Location{
|
|
| 83 |
+ SourceIndex: int32(sourceIndex), |
|
| 84 |
+ Ranges: sourceLocation, |
|
| 85 |
+ } |
|
| 86 |
+ results.Warnings = append(results.Warnings, Warning{
|
|
| 87 |
+ RuleName: rulename, |
|
| 88 |
+ Description: description, |
|
| 89 |
+ URL: url, |
|
| 90 |
+ Detail: fmtmsg, |
|
| 91 |
+ Location: pbLocation, |
|
| 92 |
+ }) |
|
| 93 |
+} |
|
| 94 |
+ |
|
| 95 |
+func (results *LintResults) ToResult() (*client.Result, error) {
|
|
| 96 |
+ res := client.NewResult() |
|
| 97 |
+ dt, err := json.MarshalIndent(results, "", " ") |
|
| 98 |
+ if err != nil {
|
|
| 99 |
+ return nil, err |
|
| 100 |
+ } |
|
| 101 |
+ res.AddMeta("result.json", dt)
|
|
| 102 |
+ |
|
| 103 |
+ b := bytes.NewBuffer(nil) |
|
| 104 |
+ if err := PrintLintViolations(dt, b); err != nil {
|
|
| 105 |
+ return nil, err |
|
| 106 |
+ } |
|
| 107 |
+ res.AddMeta("result.txt", b.Bytes())
|
|
| 108 |
+ |
|
| 109 |
+ status := 0 |
|
| 110 |
+ if len(results.Warnings) > 0 {
|
|
| 111 |
+ status = 1 |
|
| 112 |
+ } |
|
| 113 |
+ res.AddMeta("result.statuscode", []byte(fmt.Sprintf("%d", status)))
|
|
| 114 |
+ |
|
| 115 |
+ res.AddMeta("version", []byte(SubrequestLintDefinition.Version))
|
|
| 116 |
+ return res, nil |
|
| 117 |
+} |
|
| 118 |
+ |
|
| 119 |
+func (results *LintResults) validateWarnings() error {
|
|
| 120 |
+ for _, warning := range results.Warnings {
|
|
| 121 |
+ if int(warning.Location.SourceIndex) >= len(results.Sources) {
|
|
| 122 |
+ return errors.Errorf("sourceIndex is out of range")
|
|
| 123 |
+ } |
|
| 124 |
+ if warning.Location.SourceIndex > 0 {
|
|
| 125 |
+ warningSource := results.Sources[warning.Location.SourceIndex] |
|
| 126 |
+ if warningSource == nil {
|
|
| 127 |
+ return errors.Errorf("sourceIndex points to nil source")
|
|
| 128 |
+ } |
|
| 129 |
+ } |
|
| 130 |
+ } |
|
| 131 |
+ return nil |
|
| 132 |
+} |
|
| 133 |
+ |
|
| 134 |
+func PrintLintViolations(dt []byte, w io.Writer) error {
|
|
| 135 |
+ var results LintResults |
|
| 136 |
+ |
|
| 137 |
+ if err := json.Unmarshal(dt, &results); err != nil {
|
|
| 138 |
+ return err |
|
| 139 |
+ } |
|
| 140 |
+ |
|
| 141 |
+ if err := results.validateWarnings(); err != nil {
|
|
| 142 |
+ return err |
|
| 143 |
+ } |
|
| 144 |
+ |
|
| 145 |
+ sort.Slice(results.Warnings, func(i, j int) bool {
|
|
| 146 |
+ warningI := results.Warnings[i] |
|
| 147 |
+ warningJ := results.Warnings[j] |
|
| 148 |
+ sourceIndexI := warningI.Location.SourceIndex |
|
| 149 |
+ sourceIndexJ := warningJ.Location.SourceIndex |
|
| 150 |
+ if sourceIndexI < 0 && sourceIndexJ < 0 {
|
|
| 151 |
+ return warningI.RuleName < warningJ.RuleName |
|
| 152 |
+ } else if sourceIndexI < 0 || sourceIndexJ < 0 {
|
|
| 153 |
+ return sourceIndexI < 0 |
|
| 154 |
+ } |
|
| 155 |
+ |
|
| 156 |
+ sourceInfoI := results.Sources[warningI.Location.SourceIndex] |
|
| 157 |
+ sourceInfoJ := results.Sources[warningJ.Location.SourceIndex] |
|
| 158 |
+ if sourceInfoI.Filename != sourceInfoJ.Filename {
|
|
| 159 |
+ return sourceInfoI.Filename < sourceInfoJ.Filename |
|
| 160 |
+ } |
|
| 161 |
+ if len(warningI.Location.Ranges) == 0 && len(warningJ.Location.Ranges) == 0 {
|
|
| 162 |
+ return warningI.RuleName < warningJ.RuleName |
|
| 163 |
+ } else if len(warningI.Location.Ranges) == 0 || len(warningJ.Location.Ranges) == 0 {
|
|
| 164 |
+ return len(warningI.Location.Ranges) == 0 |
|
| 165 |
+ } |
|
| 166 |
+ |
|
| 167 |
+ return warningI.Location.Ranges[0].Start.Line < warningJ.Location.Ranges[0].Start.Line |
|
| 168 |
+ }) |
|
| 169 |
+ |
|
| 170 |
+ for _, warning := range results.Warnings {
|
|
| 171 |
+ fmt.Fprintf(w, "%s", warning.RuleName) |
|
| 172 |
+ if warning.URL != "" {
|
|
| 173 |
+ fmt.Fprintf(w, " - %s", warning.URL) |
|
| 174 |
+ } |
|
| 175 |
+ fmt.Fprintf(w, "\n%s\n", warning.Description) |
|
| 176 |
+ |
|
| 177 |
+ if warning.Location.SourceIndex < 0 {
|
|
| 178 |
+ continue |
|
| 179 |
+ } |
|
| 180 |
+ sourceInfo := results.Sources[warning.Location.SourceIndex] |
|
| 181 |
+ source := errdefs.Source{
|
|
| 182 |
+ Info: sourceInfo, |
|
| 183 |
+ Ranges: warning.Location.Ranges, |
|
| 184 |
+ } |
|
| 185 |
+ err := source.Print(w) |
|
| 186 |
+ if err != nil {
|
|
| 187 |
+ return err |
|
| 188 |
+ } |
|
| 189 |
+ fmt.Fprintln(w) |
|
| 190 |
+ } |
|
| 191 |
+ return nil |
|
| 192 |
+} |
|
| 193 |
+ |
|
| 194 |
+func sourceInfoEqual(a, b *pb.SourceInfo) bool {
|
|
| 195 |
+ if a.Filename != b.Filename || a.Language != b.Language {
|
|
| 196 |
+ return false |
|
| 197 |
+ } |
|
| 198 |
+ return bytes.Equal(a.Data, b.Data) |
|
| 199 |
+} |
| ... | ... |
@@ -72,10 +72,8 @@ func PrintTargets(dt []byte, w io.Writer) error {
|
| 72 | 72 |
name := t.Name |
| 73 | 73 |
if name == "" && t.Default {
|
| 74 | 74 |
name = "(default)" |
| 75 |
- } else {
|
|
| 76 |
- if t.Default {
|
|
| 77 |
- name = fmt.Sprintf("%s (default)", name)
|
|
| 78 |
- } |
|
| 75 |
+ } else if t.Default {
|
|
| 76 |
+ name = fmt.Sprintf("%s (default)", name)
|
|
| 79 | 77 |
} |
| 80 | 78 |
fmt.Fprintf(tw, "%s\t%s\n", name, t.Description) |
| 81 | 79 |
} |
| ... | ... |
@@ -5,8 +5,8 @@ import ( |
| 5 | 5 |
|
| 6 | 6 |
api "github.com/containerd/containerd/api/services/content/v1" |
| 7 | 7 |
"github.com/containerd/containerd/content" |
| 8 |
- "github.com/containerd/containerd/errdefs" |
|
| 9 | 8 |
"github.com/containerd/containerd/services/content/contentserver" |
| 9 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 10 | 10 |
"github.com/moby/buildkit/session" |
| 11 | 11 |
digest "github.com/opencontainers/go-digest" |
| 12 | 12 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -25,17 +25,17 @@ type attachableContentStore struct {
|
| 25 | 25 |
func (cs *attachableContentStore) choose(ctx context.Context) (content.Store, error) {
|
| 26 | 26 |
md, ok := metadata.FromIncomingContext(ctx) |
| 27 | 27 |
if !ok {
|
| 28 |
- return nil, errors.Wrap(errdefs.ErrInvalidArgument, "request lacks metadata") |
|
| 28 |
+ return nil, errors.Wrap(cerrdefs.ErrInvalidArgument, "request lacks metadata") |
|
| 29 | 29 |
} |
| 30 | 30 |
|
| 31 | 31 |
values := md[GRPCHeaderID] |
| 32 | 32 |
if len(values) == 0 {
|
| 33 |
- return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "request lacks metadata %q", GRPCHeaderID) |
|
| 33 |
+ return nil, errors.Wrapf(cerrdefs.ErrInvalidArgument, "request lacks metadata %q", GRPCHeaderID) |
|
| 34 | 34 |
} |
| 35 | 35 |
id := values[0] |
| 36 | 36 |
store, ok := cs.stores[id] |
| 37 | 37 |
if !ok {
|
| 38 |
- return nil, errors.Wrapf(errdefs.ErrNotFound, "unknown store %s", id) |
|
| 38 |
+ return nil, errors.Wrapf(cerrdefs.ErrNotFound, "unknown store %s", id) |
|
| 39 | 39 |
} |
| 40 | 40 |
return store, nil |
| 41 | 41 |
} |
| ... | ... |
@@ -8,9 +8,9 @@ import ( |
| 8 | 8 |
"time" |
| 9 | 9 |
|
| 10 | 10 |
"github.com/containerd/containerd/defaults" |
| 11 |
- grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" |
|
| 12 | 11 |
"github.com/moby/buildkit/util/bklog" |
| 13 | 12 |
"github.com/moby/buildkit/util/grpcerrors" |
| 13 |
+ "github.com/moby/buildkit/util/tracing" |
|
| 14 | 14 |
"github.com/pkg/errors" |
| 15 | 15 |
"github.com/sirupsen/logrus" |
| 16 | 16 |
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" |
| ... | ... |
@@ -31,9 +31,6 @@ func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) {
|
| 31 | 31 |
} |
| 32 | 32 |
|
| 33 | 33 |
func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc.ClientConn, error) {
|
| 34 |
- var unary []grpc.UnaryClientInterceptor |
|
| 35 |
- var stream []grpc.StreamClientInterceptor |
|
| 36 |
- |
|
| 37 | 34 |
var dialCount int64 |
| 38 | 35 |
dialer := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
| 39 | 36 |
if c := atomic.AddInt64(&dialCount, 1); c > 1 {
|
| ... | ... |
@@ -47,26 +44,16 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc. |
| 47 | 47 |
grpc.WithTransportCredentials(insecure.NewCredentials()), |
| 48 | 48 |
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), |
| 49 | 49 |
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), |
| 50 |
+ grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), |
|
| 51 |
+ grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor), |
|
| 50 | 52 |
} |
| 51 | 53 |
|
| 52 | 54 |
if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
|
| 53 |
- unary = append(unary, filterClient(otelgrpc.UnaryClientInterceptor(otelgrpc.WithTracerProvider(span.TracerProvider()), otelgrpc.WithPropagators(propagators)))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681 |
|
| 54 |
- stream = append(stream, otelgrpc.StreamClientInterceptor(otelgrpc.WithTracerProvider(span.TracerProvider()), otelgrpc.WithPropagators(propagators))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681 |
|
| 55 |
- } |
|
| 56 |
- |
|
| 57 |
- unary = append(unary, grpcerrors.UnaryClientInterceptor) |
|
| 58 |
- stream = append(stream, grpcerrors.StreamClientInterceptor) |
|
| 59 |
- |
|
| 60 |
- if len(unary) == 1 {
|
|
| 61 |
- dialOpts = append(dialOpts, grpc.WithUnaryInterceptor(unary[0])) |
|
| 62 |
- } else if len(unary) > 1 {
|
|
| 63 |
- dialOpts = append(dialOpts, grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unary...))) |
|
| 64 |
- } |
|
| 65 |
- |
|
| 66 |
- if len(stream) == 1 {
|
|
| 67 |
- dialOpts = append(dialOpts, grpc.WithStreamInterceptor(stream[0])) |
|
| 68 |
- } else if len(stream) > 1 {
|
|
| 69 |
- dialOpts = append(dialOpts, grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(stream...))) |
|
| 55 |
+ statsHandler := tracing.ClientStatsHandler( |
|
| 56 |
+ otelgrpc.WithTracerProvider(span.TracerProvider()), |
|
| 57 |
+ otelgrpc.WithPropagators(propagators), |
|
| 58 |
+ ) |
|
| 59 |
+ dialOpts = append(dialOpts, grpc.WithStatsHandler(statsHandler)) |
|
| 70 | 60 |
} |
| 71 | 61 |
|
| 72 | 62 |
cc, err := grpc.DialContext(ctx, "localhost", dialOpts...) |
| ... | ... |
@@ -3,12 +3,11 @@ package session |
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 | 5 |
"net" |
| 6 |
- "strings" |
|
| 7 | 6 |
"sync" |
| 8 | 7 |
|
| 9 |
- grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" |
|
| 10 | 8 |
"github.com/moby/buildkit/identity" |
| 11 | 9 |
"github.com/moby/buildkit/util/grpcerrors" |
| 10 |
+ "github.com/moby/buildkit/util/tracing" |
|
| 12 | 11 |
"github.com/pkg/errors" |
| 13 | 12 |
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" |
| 14 | 13 |
"go.opentelemetry.io/otel/propagation" |
| ... | ... |
@@ -53,29 +52,17 @@ type Session struct {
|
| 53 | 53 |
func NewSession(ctx context.Context, name, sharedKey string) (*Session, error) {
|
| 54 | 54 |
id := identity.NewID() |
| 55 | 55 |
|
| 56 |
- var unary []grpc.UnaryServerInterceptor |
|
| 57 |
- var stream []grpc.StreamServerInterceptor |
|
| 58 |
- |
|
| 59 |
- serverOpts := []grpc.ServerOption{}
|
|
| 60 |
- |
|
| 61 |
- if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
|
|
| 62 |
- unary = append(unary, filterServer(otelgrpc.UnaryServerInterceptor(otelgrpc.WithTracerProvider(span.TracerProvider()), otelgrpc.WithPropagators(propagators)))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681 |
|
| 63 |
- stream = append(stream, otelgrpc.StreamServerInterceptor(otelgrpc.WithTracerProvider(span.TracerProvider()), otelgrpc.WithPropagators(propagators))) //nolint:staticcheck // TODO(thaJeztah): ignore SA1019 for deprecated options: see https://github.com/moby/buildkit/issues/4681 |
|
| 64 |
- } |
|
| 65 |
- |
|
| 66 |
- unary = append(unary, grpcerrors.UnaryServerInterceptor) |
|
| 67 |
- stream = append(stream, grpcerrors.StreamServerInterceptor) |
|
| 68 |
- |
|
| 69 |
- if len(unary) == 1 {
|
|
| 70 |
- serverOpts = append(serverOpts, grpc.UnaryInterceptor(unary[0])) |
|
| 71 |
- } else if len(unary) > 1 {
|
|
| 72 |
- serverOpts = append(serverOpts, grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary...))) |
|
| 56 |
+ serverOpts := []grpc.ServerOption{
|
|
| 57 |
+ grpc.UnaryInterceptor(grpcerrors.UnaryServerInterceptor), |
|
| 58 |
+ grpc.StreamInterceptor(grpcerrors.StreamServerInterceptor), |
|
| 73 | 59 |
} |
| 74 | 60 |
|
| 75 |
- if len(stream) == 1 {
|
|
| 76 |
- serverOpts = append(serverOpts, grpc.StreamInterceptor(stream[0])) |
|
| 77 |
- } else if len(stream) > 1 {
|
|
| 78 |
- serverOpts = append(serverOpts, grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream...))) |
|
| 61 |
+ if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {
|
|
| 62 |
+ statsHandler := tracing.ServerStatsHandler( |
|
| 63 |
+ otelgrpc.WithTracerProvider(span.TracerProvider()), |
|
| 64 |
+ otelgrpc.WithPropagators(propagators), |
|
| 65 |
+ ) |
|
| 66 |
+ serverOpts = append(serverOpts, grpc.StatsHandler(statsHandler)) |
|
| 79 | 67 |
} |
| 80 | 68 |
|
| 81 | 69 |
s := &Session{
|
| ... | ... |
@@ -167,22 +154,3 @@ func (s *Session) closed() bool {
|
| 167 | 167 |
func MethodURL(s, m string) string {
|
| 168 | 168 |
return "/" + s + "/" + m |
| 169 | 169 |
} |
| 170 |
- |
|
| 171 |
-// updates needed in opentelemetry-contrib to avoid this |
|
| 172 |
-func filterServer(intercept grpc.UnaryServerInterceptor) grpc.UnaryServerInterceptor {
|
|
| 173 |
- return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
|
|
| 174 |
- if strings.HasSuffix(info.FullMethod, "Health/Check") {
|
|
| 175 |
- return handler(ctx, req) |
|
| 176 |
- } |
|
| 177 |
- return intercept(ctx, req, info, handler) |
|
| 178 |
- } |
|
| 179 |
-} |
|
| 180 |
- |
|
| 181 |
-func filterClient(intercept grpc.UnaryClientInterceptor) grpc.UnaryClientInterceptor {
|
|
| 182 |
- return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
|
| 183 |
- if strings.HasSuffix(method, "Health/Check") {
|
|
| 184 |
- return invoker(ctx, method, req, reply, cc, opts...) |
|
| 185 |
- } |
|
| 186 |
- return intercept(ctx, method, req, reply, cc, invoker, opts...) |
|
| 187 |
- } |
|
| 188 |
-} |
| ... | ... |
@@ -37,8 +37,8 @@ type Upload struct {
|
| 37 | 37 |
cc Upload_PullClient |
| 38 | 38 |
} |
| 39 | 39 |
|
| 40 |
-func (u *Upload) WriteTo(w io.Writer) (int, error) {
|
|
| 41 |
- n := 0 |
|
| 40 |
+func (u *Upload) WriteTo(w io.Writer) (int64, error) {
|
|
| 41 |
+ var n int64 |
|
| 42 | 42 |
for {
|
| 43 | 43 |
var bm BytesMessage |
| 44 | 44 |
if err := u.cc.RecvMsg(&bm); err != nil {
|
| ... | ... |
@@ -48,7 +48,7 @@ func (u *Upload) WriteTo(w io.Writer) (int, error) {
|
| 48 | 48 |
return n, errors.WithStack(err) |
| 49 | 49 |
} |
| 50 | 50 |
nn, err := w.Write(bm.Data) |
| 51 |
- n += nn |
|
| 51 |
+ n += int64(nn) |
|
| 52 | 52 |
if err != nil {
|
| 53 | 53 |
return n, errors.WithStack(err) |
| 54 | 54 |
} |
| ... | ... |
@@ -68,7 +68,12 @@ func (c *Store) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content |
| 68 | 68 |
} |
| 69 | 69 |
|
| 70 | 70 |
func (c *Store) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
|
| 71 |
- return c.writer(ctx, 3, opts...) |
|
| 71 |
+ ctx = namespaces.WithNamespace(ctx, c.ns) |
|
| 72 |
+ w, err := c.Store.Writer(ctx, opts...) |
|
| 73 |
+ if err != nil {
|
|
| 74 |
+ return nil, err |
|
| 75 |
+ } |
|
| 76 |
+ return &nsWriter{Writer: w, ns: c.ns}, nil
|
|
| 72 | 77 |
} |
| 73 | 78 |
|
| 74 | 79 |
func (c *Store) WithFallbackNS(ns string) content.Store {
|
| ... | ... |
@@ -78,15 +83,6 @@ func (c *Store) WithFallbackNS(ns string) content.Store {
|
| 78 | 78 |
} |
| 79 | 79 |
} |
| 80 | 80 |
|
| 81 |
-func (c *Store) writer(ctx context.Context, retries int, opts ...content.WriterOpt) (content.Writer, error) {
|
|
| 82 |
- ctx = namespaces.WithNamespace(ctx, c.ns) |
|
| 83 |
- w, err := c.Store.Writer(ctx, opts...) |
|
| 84 |
- if err != nil {
|
|
| 85 |
- return nil, err |
|
| 86 |
- } |
|
| 87 |
- return &nsWriter{Writer: w, ns: c.ns}, nil
|
|
| 88 |
-} |
|
| 89 |
- |
|
| 90 | 81 |
type nsWriter struct {
|
| 91 | 82 |
content.Writer |
| 92 | 83 |
ns string |
| ... | ... |
@@ -8,10 +8,10 @@ import ( |
| 8 | 8 |
"github.com/pkg/errors" |
| 9 | 9 |
) |
| 10 | 10 |
|
| 11 |
-func (sn *mergeSnapshotter) diffApply(ctx context.Context, dest Mountable, diffs ...Diff) (_ snapshots.Usage, rerr error) {
|
|
| 11 |
+func (sn *mergeSnapshotter) diffApply(_ context.Context, _ Mountable, _ ...Diff) (_ snapshots.Usage, rerr error) {
|
|
| 12 | 12 |
return snapshots.Usage{}, errors.New("diffApply not yet supported on FreeBSD")
|
| 13 | 13 |
} |
| 14 | 14 |
|
| 15 |
-func needsUserXAttr(ctx context.Context, sn Snapshotter, lm leases.Manager) (bool, error) {
|
|
| 15 |
+func needsUserXAttr(_ context.Context, _ Snapshotter, _ leases.Manager) (bool, error) {
|
|
| 16 | 16 |
return false, errors.New("needs userxattr not supported on FreeBSD")
|
| 17 | 17 |
} |
| ... | ... |
@@ -235,7 +235,7 @@ func (a *applier) Apply(ctx context.Context, c *change) error {
|
| 235 | 235 |
dstStat: dstStat, |
| 236 | 236 |
} |
| 237 | 237 |
|
| 238 |
- if done, err := a.applyDelete(ctx, ca); err != nil {
|
|
| 238 |
+ if done, err := a.applyDelete(ca); err != nil {
|
|
| 239 | 239 |
return errors.Wrap(err, "failed to delete during apply") |
| 240 | 240 |
} else if done {
|
| 241 | 241 |
return nil |
| ... | ... |
@@ -253,7 +253,7 @@ func (a *applier) Apply(ctx context.Context, c *change) error {
|
| 253 | 253 |
return nil |
| 254 | 254 |
} |
| 255 | 255 |
|
| 256 |
-func (a *applier) applyDelete(ctx context.Context, ca *changeApply) (bool, error) {
|
|
| 256 |
+func (a *applier) applyDelete(ca *changeApply) (bool, error) {
|
|
| 257 | 257 |
// Even when not deleting, we may be overwriting a file, in which case we should |
| 258 | 258 |
// delete the existing file at the path, if any. Don't delete when both are dirs |
| 259 | 259 |
// in this case though because they should get merged, not overwritten. |
| ... | ... |
@@ -11,10 +11,10 @@ import ( |
| 11 | 11 |
"github.com/pkg/errors" |
| 12 | 12 |
) |
| 13 | 13 |
|
| 14 |
-func (sn *mergeSnapshotter) diffApply(ctx context.Context, dest Mountable, diffs ...Diff) (_ snapshots.Usage, rerr error) {
|
|
| 14 |
+func (sn *mergeSnapshotter) diffApply(_ context.Context, _ Mountable, _ ...Diff) (_ snapshots.Usage, rerr error) {
|
|
| 15 | 15 |
return snapshots.Usage{}, errors.New("diffApply not yet supported on windows")
|
| 16 | 16 |
} |
| 17 | 17 |
|
| 18 |
-func needsUserXAttr(ctx context.Context, sn Snapshotter, lm leases.Manager) (bool, error) {
|
|
| 18 |
+func needsUserXAttr(_ context.Context, _ Snapshotter, _ leases.Manager) (bool, error) {
|
|
| 19 | 19 |
return false, errors.New("needs userxattr not supported on windows")
|
| 20 | 20 |
} |
| ... | ... |
@@ -4,8 +4,8 @@ import ( |
| 4 | 4 |
"os" |
| 5 | 5 |
|
| 6 | 6 |
"github.com/Microsoft/go-winio/pkg/bindfilter" |
| 7 |
- "github.com/containerd/containerd/errdefs" |
|
| 8 | 7 |
"github.com/containerd/containerd/mount" |
| 8 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 9 | 9 |
"github.com/pkg/errors" |
| 10 | 10 |
"golang.org/x/sys/windows" |
| 11 | 11 |
) |
| ... | ... |
@@ -26,7 +26,7 @@ func (lm *localMounter) Mount() (string, error) {
|
| 26 | 26 |
// Windows can only mount a single mount at a given location. |
| 27 | 27 |
// Parent layers are carried in Options, opaquely to localMounter. |
| 28 | 28 |
if len(lm.mounts) != 1 {
|
| 29 |
- return "", errors.Wrapf(errdefs.ErrNotImplemented, "request to mount %d layers, only 1 is supported", len(lm.mounts)) |
|
| 29 |
+ return "", errors.Wrapf(cerrdefs.ErrNotImplemented, "request to mount %d layers, only 1 is supported", len(lm.mounts)) |
|
| 30 | 30 |
} |
| 31 | 31 |
|
| 32 | 32 |
m := lm.mounts[0] |
| ... | ... |
@@ -66,7 +66,7 @@ func (lm *localMounter) Unmount() error {
|
| 66 | 66 |
// Calling Mount() would fail on an instance of the localMounter where mounts contains |
| 67 | 67 |
// anything other than 1 mount. |
| 68 | 68 |
if len(lm.mounts) != 1 {
|
| 69 |
- return errors.Wrapf(errdefs.ErrNotImplemented, "request to mount %d layers, only 1 is supported", len(lm.mounts)) |
|
| 69 |
+ return errors.Wrapf(cerrdefs.ErrNotImplemented, "request to mount %d layers, only 1 is supported", len(lm.mounts)) |
|
| 70 | 70 |
} |
| 71 | 71 |
m := lm.mounts[0] |
| 72 | 72 |
|
| ... | ... |
@@ -4,8 +4,11 @@ import ( |
| 4 | 4 |
"bytes" |
| 5 | 5 |
"encoding/json" |
| 6 | 6 |
"fmt" |
| 7 |
+ "os" |
|
| 7 | 8 |
|
| 9 |
+ "github.com/moby/buildkit/identity" |
|
| 8 | 10 |
"github.com/moby/buildkit/solver" |
| 11 |
+ "github.com/moby/buildkit/util/bklog" |
|
| 9 | 12 |
digest "github.com/opencontainers/go-digest" |
| 10 | 13 |
"github.com/pkg/errors" |
| 11 | 14 |
bolt "go.etcd.io/bbolt" |
| ... | ... |
@@ -23,10 +26,12 @@ type Store struct {
|
| 23 | 23 |
} |
| 24 | 24 |
|
| 25 | 25 |
func NewStore(dbPath string) (*Store, error) {
|
| 26 |
- db, err := bolt.Open(dbPath, 0600, nil) |
|
| 26 |
+ db, err := safeOpenDB(dbPath) |
|
| 27 | 27 |
if err != nil {
|
| 28 |
- return nil, errors.Wrapf(err, "failed to open database file %s", dbPath) |
|
| 28 |
+ return nil, err |
|
| 29 | 29 |
} |
| 30 |
+ |
|
| 31 |
+ // Initialize the database with the needed buckets if they do not exist. |
|
| 30 | 32 |
if err := db.Update(func(tx *bolt.Tx) error {
|
| 31 | 33 |
for _, b := range []string{resultBucket, linksBucket, byResultBucket, backlinksBucket} {
|
| 32 | 34 |
if _, err := tx.CreateBucketIfNotExists([]byte(b)); err != nil {
|
| ... | ... |
@@ -455,3 +460,51 @@ func isEmptyBucket(b *bolt.Bucket) bool {
|
| 455 | 455 |
k, _ := b.Cursor().First() |
| 456 | 456 |
return k == nil |
| 457 | 457 |
} |
| 458 |
+ |
|
| 459 |
+// safeOpenDB opens a bolt database and recovers from panic that |
|
| 460 |
+// can be caused by a corrupted database file. |
|
| 461 |
+func safeOpenDB(dbPath string) (db *bolt.DB, err error) {
|
|
| 462 |
+ defer func() {
|
|
| 463 |
+ if r := recover(); r != nil {
|
|
| 464 |
+ err = errors.Errorf("%v", r)
|
|
| 465 |
+ } |
|
| 466 |
+ |
|
| 467 |
+ // If we get an error when opening the database, but we have |
|
| 468 |
+ // access to the file and the file looks like it has content, |
|
| 469 |
+ // then fallback to resetting the database since the database |
|
| 470 |
+ // may be corrupt. |
|
| 471 |
+ if err != nil && fileHasContent(dbPath) {
|
|
| 472 |
+ db, err = fallbackOpenDB(dbPath, err) |
|
| 473 |
+ } |
|
| 474 |
+ }() |
|
| 475 |
+ return openDB(dbPath) |
|
| 476 |
+} |
|
| 477 |
+ |
|
| 478 |
+// fallbackOpenDB performs database recovery and opens the new database |
|
| 479 |
+// file when the database fails to open. Called after the first database |
|
| 480 |
+// open fails. |
|
| 481 |
+func fallbackOpenDB(dbPath string, openErr error) (*bolt.DB, error) {
|
|
| 482 |
+ backupPath := dbPath + "." + identity.NewID() + ".bak" |
|
| 483 |
+ bklog.L.Errorf("failed to open database file %s, resetting to empty. Old database is backed up to %s. "+
|
|
| 484 |
+ "This error signifies that buildkitd likely crashed or was sigkilled abrubtly, leaving the database corrupted. "+ |
|
| 485 |
+ "If you see logs from a previous panic then please report in the issue tracker at https://github.com/moby/buildkit . %+v", dbPath, backupPath, openErr) |
|
| 486 |
+ if err := os.Rename(dbPath, backupPath); err != nil {
|
|
| 487 |
+ return nil, errors.Wrapf(err, "failed to rename database file %s to %s", dbPath, backupPath) |
|
| 488 |
+ } |
|
| 489 |
+ |
|
| 490 |
+ // Attempt to open the database again. This should be a new database. |
|
| 491 |
+ // If this fails, it is a permanent error. |
|
| 492 |
+ return openDB(dbPath) |
|
| 493 |
+} |
|
| 494 |
+ |
|
| 495 |
+// openDB opens a bolt database in user-only read/write mode. |
|
| 496 |
+func openDB(dbPath string) (*bolt.DB, error) {
|
|
| 497 |
+ return bolt.Open(dbPath, 0600, nil) |
|
| 498 |
+} |
|
| 499 |
+ |
|
| 500 |
+// fileHasContent checks if we have access to the file with appropriate |
|
| 501 |
+// permissions and the file has a non-zero size. |
|
| 502 |
+func fileHasContent(dbPath string) bool {
|
|
| 503 |
+ st, err := os.Stat(dbPath) |
|
| 504 |
+ return err == nil && st.Size() > 0 |
|
| 505 |
+} |
| ... | ... |
@@ -843,12 +843,44 @@ func (e *edge) createInputRequests(desiredState edgeStatusType, f *pipeFactory, |
| 843 | 843 |
addNew := true |
| 844 | 844 |
if dep.req != nil && !dep.req.Status().Completed {
|
| 845 | 845 |
if dep.req.Request().(*edgeRequest).desiredState != desiredStateDep {
|
| 846 |
+ if e.debug {
|
|
| 847 |
+ bklog.G(context.TODO()). |
|
| 848 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 849 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 850 |
+ WithField("dep_index", dep.index).
|
|
| 851 |
+ WithField("dep_req_desired_state", dep.req.Request().(*edgeRequest).desiredState).
|
|
| 852 |
+ WithField("dep_desired_state", desiredStateDep).
|
|
| 853 |
+ WithField("dep_state", dep.state).
|
|
| 854 |
+ Debug("cancel input request")
|
|
| 855 |
+ } |
|
| 846 | 856 |
dep.req.Cancel() |
| 847 | 857 |
} else {
|
| 858 |
+ if e.debug {
|
|
| 859 |
+ bklog.G(context.TODO()). |
|
| 860 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 861 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 862 |
+ WithField("dep_index", dep.index).
|
|
| 863 |
+ WithField("dep_req_desired_state", dep.req.Request().(*edgeRequest).desiredState).
|
|
| 864 |
+ WithField("dep_desired_state", desiredStateDep).
|
|
| 865 |
+ WithField("dep_state", dep.state).
|
|
| 866 |
+ Debug("skip input request based on existing request")
|
|
| 867 |
+ } |
|
| 848 | 868 |
addNew = false |
| 849 | 869 |
} |
| 850 | 870 |
} |
| 851 | 871 |
if addNew {
|
| 872 |
+ if e.debug {
|
|
| 873 |
+ bklog.G(context.TODO()). |
|
| 874 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 875 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 876 |
+ WithField("dep_index", dep.index).
|
|
| 877 |
+ WithField("dep_desired_state", desiredStateDep).
|
|
| 878 |
+ WithField("dep_state", dep.state).
|
|
| 879 |
+ WithField("dep_vertex_name", e.edge.Vertex.Inputs()[dep.index].Vertex.Name()).
|
|
| 880 |
+ WithField("dep_vertex_digest", e.edge.Vertex.Inputs()[dep.index].Vertex.Digest()).
|
|
| 881 |
+ Debug("add input request")
|
|
| 882 |
+ } |
|
| 883 |
+ |
|
| 852 | 884 |
req := f.NewInputRequest(e.edge.Vertex.Inputs()[int(dep.index)], &edgeRequest{
|
| 853 | 885 |
currentState: dep.edgeState, |
| 854 | 886 |
desiredState: desiredStateDep, |
| ... | ... |
@@ -858,6 +890,16 @@ func (e *edge) createInputRequests(desiredState edgeStatusType, f *pipeFactory, |
| 858 | 858 |
dep.req = req |
| 859 | 859 |
addedNew = true |
| 860 | 860 |
} |
| 861 |
+ } else if e.debug {
|
|
| 862 |
+ bklog.G(context.TODO()). |
|
| 863 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 864 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 865 |
+ WithField("dep_index", dep.index).
|
|
| 866 |
+ WithField("dep_desired_state", desiredStateDep).
|
|
| 867 |
+ WithField("dep_state", dep.state).
|
|
| 868 |
+ WithField("dep_vertex_name", e.edge.Vertex.Inputs()[dep.index].Vertex.Name()).
|
|
| 869 |
+ WithField("dep_vertex_digest", e.edge.Vertex.Inputs()[dep.index].Vertex.Digest()).
|
|
| 870 |
+ Debug("skip input request based on dep state")
|
|
| 861 | 871 |
} |
| 862 | 872 |
// initialize function to compute cache key based on dependency result |
| 863 | 873 |
if dep.state == edgeStatusComplete && dep.slowCacheReq == nil && (e.slowCacheFunc(dep) != nil || e.preprocessFunc(dep) != nil) && e.cacheMap != nil {
|
| ... | ... |
@@ -199,9 +199,11 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach |
| 199 | 199 |
} |
| 200 | 200 |
} |
| 201 | 201 |
|
| 202 |
- for cm, id := range k.ids {
|
|
| 203 |
- if _, err := addBacklinks(t, rec, cm, id, bkm); err != nil {
|
|
| 204 |
- return nil, err |
|
| 202 |
+ if !opt.IgnoreBacklinks {
|
|
| 203 |
+ for cm, id := range k.ids {
|
|
| 204 |
+ if _, err := addBacklinks(t, rec, cm, id, bkm); err != nil {
|
|
| 205 |
+ return nil, err |
|
| 206 |
+ } |
|
| 205 | 207 |
} |
| 206 | 208 |
} |
| 207 | 209 |
} |
| ... | ... |
@@ -32,12 +32,12 @@ func (ei *edgeIndex) Release(e *edge) {
|
| 32 | 32 |
defer ei.mu.Unlock() |
| 33 | 33 |
|
| 34 | 34 |
for id := range ei.backRefs[e] {
|
| 35 |
- ei.releaseEdge(id, e) |
|
| 35 |
+ ei.releaseEdge(id) |
|
| 36 | 36 |
} |
| 37 | 37 |
delete(ei.backRefs, e) |
| 38 | 38 |
} |
| 39 | 39 |
|
| 40 |
-func (ei *edgeIndex) releaseEdge(id string, e *edge) {
|
|
| 40 |
+func (ei *edgeIndex) releaseEdge(id string) {
|
|
| 41 | 41 |
item, ok := ei.items[id] |
| 42 | 42 |
if !ok {
|
| 43 | 43 |
return |
| ... | ... |
@@ -10,6 +10,7 @@ import ( |
| 10 | 10 |
"github.com/moby/buildkit/identity" |
| 11 | 11 |
"github.com/moby/buildkit/session" |
| 12 | 12 |
"github.com/moby/buildkit/solver/errdefs" |
| 13 |
+ "github.com/moby/buildkit/util/bklog" |
|
| 13 | 14 |
"github.com/moby/buildkit/util/flightcontrol" |
| 14 | 15 |
"github.com/moby/buildkit/util/progress" |
| 15 | 16 |
"github.com/moby/buildkit/util/progress/controller" |
| ... | ... |
@@ -149,7 +150,7 @@ func (s *state) getEdge(index Index) *edge {
|
| 149 | 149 |
} |
| 150 | 150 |
|
| 151 | 151 |
if s.op == nil {
|
| 152 |
- s.op = newSharedOp(s.opts.ResolveOpFunc, s.opts.DefaultCache, s) |
|
| 152 |
+ s.op = newSharedOp(s.opts.ResolveOpFunc, s) |
|
| 153 | 153 |
} |
| 154 | 154 |
|
| 155 | 155 |
e := newEdge(Edge{Index: index, Vertex: s.vtx}, s.op, s.index)
|
| ... | ... |
@@ -175,6 +176,8 @@ func (s *state) setEdge(index Index, targetEdge *edge, targetState *state) {
|
| 175 | 175 |
targetEdge.takeOwnership(e) |
| 176 | 176 |
|
| 177 | 177 |
if targetState != nil {
|
| 178 |
+ targetState.addJobs(s, map[*state]struct{}{})
|
|
| 179 |
+ |
|
| 178 | 180 |
if _, ok := targetState.allPw[s.mpw]; !ok {
|
| 179 | 181 |
targetState.mpw.Add(s.mpw) |
| 180 | 182 |
targetState.allPw[s.mpw] = struct{}{}
|
| ... | ... |
@@ -182,6 +185,51 @@ func (s *state) setEdge(index Index, targetEdge *edge, targetState *state) {
|
| 182 | 182 |
} |
| 183 | 183 |
} |
| 184 | 184 |
|
| 185 |
+// addJobs recursively adds jobs to state and all its ancestors. currently |
|
| 186 |
+// only used during edge merges to add jobs from the source of the merge to the |
|
| 187 |
+// target and its ancestors. |
|
| 188 |
+// requires that Solver.mu is read-locked and srcState.mu is locked |
|
| 189 |
+func (s *state) addJobs(srcState *state, memo map[*state]struct{}) {
|
|
| 190 |
+ if _, ok := memo[s]; ok {
|
|
| 191 |
+ return |
|
| 192 |
+ } |
|
| 193 |
+ memo[s] = struct{}{}
|
|
| 194 |
+ |
|
| 195 |
+ s.mu.Lock() |
|
| 196 |
+ defer s.mu.Unlock() |
|
| 197 |
+ |
|
| 198 |
+ for j := range srcState.jobs {
|
|
| 199 |
+ s.jobs[j] = struct{}{}
|
|
| 200 |
+ } |
|
| 201 |
+ |
|
| 202 |
+ for _, inputEdge := range s.vtx.Inputs() {
|
|
| 203 |
+ inputState, ok := s.solver.actives[inputEdge.Vertex.Digest()] |
|
| 204 |
+ if !ok {
|
|
| 205 |
+ bklog.G(context.TODO()). |
|
| 206 |
+ WithField("vertex_digest", inputEdge.Vertex.Digest()).
|
|
| 207 |
+ Error("input vertex not found during addJobs")
|
|
| 208 |
+ continue |
|
| 209 |
+ } |
|
| 210 |
+ inputState.addJobs(srcState, memo) |
|
| 211 |
+ |
|
| 212 |
+ // tricky case: if the inputState's edge was *already* merged we should |
|
| 213 |
+ // also add jobs to the merged edge's state |
|
| 214 |
+ mergedInputEdge := inputState.getEdge(inputEdge.Index) |
|
| 215 |
+ if mergedInputEdge == nil || mergedInputEdge.edge.Vertex.Digest() == inputEdge.Vertex.Digest() {
|
|
| 216 |
+ // not merged |
|
| 217 |
+ continue |
|
| 218 |
+ } |
|
| 219 |
+ mergedInputState, ok := s.solver.actives[mergedInputEdge.edge.Vertex.Digest()] |
|
| 220 |
+ if !ok {
|
|
| 221 |
+ bklog.G(context.TODO()). |
|
| 222 |
+ WithField("vertex_digest", mergedInputEdge.edge.Vertex.Digest()).
|
|
| 223 |
+ Error("merged input vertex not found during addJobs")
|
|
| 224 |
+ continue |
|
| 225 |
+ } |
|
| 226 |
+ mergedInputState.addJobs(srcState, memo) |
|
| 227 |
+ } |
|
| 228 |
+} |
|
| 229 |
+ |
|
| 185 | 230 |
func (s *state) combinedCacheManager() CacheManager {
|
| 186 | 231 |
s.mu.Lock() |
| 187 | 232 |
cms := make([]CacheManager, 0, len(s.cache)+1) |
| ... | ... |
@@ -350,7 +398,23 @@ func (jl *Solver) getState(e Edge) *state {
|
| 350 | 350 |
return st |
| 351 | 351 |
} |
| 352 | 352 |
|
| 353 |
-func (jl *Solver) getEdge(e Edge) *edge {
|
|
| 353 |
+func (jl *Solver) getEdge(e Edge) (redge *edge) {
|
|
| 354 |
+ if debugScheduler {
|
|
| 355 |
+ defer func() {
|
|
| 356 |
+ lg := bklog.G(context.TODO()). |
|
| 357 |
+ WithField("edge_vertex_name", e.Vertex.Name()).
|
|
| 358 |
+ WithField("edge_vertex_digest", e.Vertex.Digest()).
|
|
| 359 |
+ WithField("edge_index", e.Index)
|
|
| 360 |
+ if redge != nil {
|
|
| 361 |
+ lg = lg. |
|
| 362 |
+ WithField("return_edge_vertex_name", redge.edge.Vertex.Name()).
|
|
| 363 |
+ WithField("return_edge_vertex_digest", redge.edge.Vertex.Digest()).
|
|
| 364 |
+ WithField("return_edge_index", redge.edge.Index)
|
|
| 365 |
+ } |
|
| 366 |
+ lg.Debug("getEdge return")
|
|
| 367 |
+ }() |
|
| 368 |
+ } |
|
| 369 |
+ |
|
| 354 | 370 |
jl.mu.RLock() |
| 355 | 371 |
defer jl.mu.RUnlock() |
| 356 | 372 |
|
| ... | ... |
@@ -362,7 +426,7 @@ func (jl *Solver) getEdge(e Edge) *edge {
|
| 362 | 362 |
} |
| 363 | 363 |
|
| 364 | 364 |
func (jl *Solver) subBuild(ctx context.Context, e Edge, parent Vertex) (CachedResult, error) {
|
| 365 |
- v, err := jl.load(e.Vertex, parent, nil) |
|
| 365 |
+ v, err := jl.load(ctx, e.Vertex, parent, nil) |
|
| 366 | 366 |
if err != nil {
|
| 367 | 367 |
return nil, err |
| 368 | 368 |
} |
| ... | ... |
@@ -374,16 +438,17 @@ func (jl *Solver) Close() {
|
| 374 | 374 |
jl.s.Stop() |
| 375 | 375 |
} |
| 376 | 376 |
|
| 377 |
-func (jl *Solver) load(v, parent Vertex, j *Job) (Vertex, error) {
|
|
| 377 |
+func (jl *Solver) load(ctx context.Context, v, parent Vertex, j *Job) (Vertex, error) {
|
|
| 378 | 378 |
jl.mu.Lock() |
| 379 | 379 |
defer jl.mu.Unlock() |
| 380 | 380 |
|
| 381 | 381 |
cache := map[Vertex]Vertex{}
|
| 382 | 382 |
|
| 383 |
- return jl.loadUnlocked(v, parent, j, cache) |
|
| 383 |
+ return jl.loadUnlocked(ctx, v, parent, j, cache) |
|
| 384 | 384 |
} |
| 385 | 385 |
|
| 386 |
-func (jl *Solver) loadUnlocked(v, parent Vertex, j *Job, cache map[Vertex]Vertex) (Vertex, error) {
|
|
| 386 |
+// called with solver lock |
|
| 387 |
+func (jl *Solver) loadUnlocked(ctx context.Context, v, parent Vertex, j *Job, cache map[Vertex]Vertex) (Vertex, error) {
|
|
| 387 | 388 |
if v, ok := cache[v]; ok {
|
| 388 | 389 |
return v, nil |
| 389 | 390 |
} |
| ... | ... |
@@ -391,7 +456,7 @@ func (jl *Solver) loadUnlocked(v, parent Vertex, j *Job, cache map[Vertex]Vertex |
| 391 | 391 |
|
| 392 | 392 |
inputs := make([]Edge, len(v.Inputs())) |
| 393 | 393 |
for i, e := range v.Inputs() {
|
| 394 |
- v, err := jl.loadUnlocked(e.Vertex, parent, j, cache) |
|
| 394 |
+ v, err := jl.loadUnlocked(ctx, e.Vertex, parent, j, cache) |
|
| 395 | 395 |
if err != nil {
|
| 396 | 396 |
return nil, err |
| 397 | 397 |
} |
| ... | ... |
@@ -448,6 +513,33 @@ func (jl *Solver) loadUnlocked(v, parent Vertex, j *Job, cache map[Vertex]Vertex |
| 448 | 448 |
origDigest: origVtx.Digest(), |
| 449 | 449 |
} |
| 450 | 450 |
jl.actives[dgst] = st |
| 451 |
+ |
|
| 452 |
+ if debugScheduler {
|
|
| 453 |
+ lg := bklog.G(ctx). |
|
| 454 |
+ WithField("vertex_name", v.Name()).
|
|
| 455 |
+ WithField("vertex_digest", v.Digest()).
|
|
| 456 |
+ WithField("actives_digest_key", dgst)
|
|
| 457 |
+ if j != nil {
|
|
| 458 |
+ lg = lg.WithField("job", j.id)
|
|
| 459 |
+ } |
|
| 460 |
+ lg.Debug("adding active vertex")
|
|
| 461 |
+ for i, inp := range v.Inputs() {
|
|
| 462 |
+ lg.WithField("input_index", i).
|
|
| 463 |
+ WithField("input_vertex_name", inp.Vertex.Name()).
|
|
| 464 |
+ WithField("input_vertex_digest", inp.Vertex.Digest()).
|
|
| 465 |
+ WithField("input_edge_index", inp.Index).
|
|
| 466 |
+ Debug("new active vertex input")
|
|
| 467 |
+ } |
|
| 468 |
+ } |
|
| 469 |
+ } else if debugScheduler {
|
|
| 470 |
+ lg := bklog.G(ctx). |
|
| 471 |
+ WithField("vertex_name", v.Name()).
|
|
| 472 |
+ WithField("vertex_digest", v.Digest()).
|
|
| 473 |
+ WithField("actives_digest_key", dgst)
|
|
| 474 |
+ if j != nil {
|
|
| 475 |
+ lg = lg.WithField("job", j.id)
|
|
| 476 |
+ } |
|
| 477 |
+ lg.Debug("reusing active vertex")
|
|
| 451 | 478 |
} |
| 452 | 479 |
|
| 453 | 480 |
st.mu.Lock() |
| ... | ... |
@@ -563,6 +655,21 @@ func (jl *Solver) Get(id string) (*Job, error) {
|
| 563 | 563 |
// called with solver lock |
| 564 | 564 |
func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) {
|
| 565 | 565 |
if len(st.jobs) == 0 && len(st.parents) == 0 {
|
| 566 |
+ if debugScheduler {
|
|
| 567 |
+ bklog.G(context.TODO()). |
|
| 568 |
+ WithField("vertex_name", st.vtx.Name()).
|
|
| 569 |
+ WithField("vertex_digest", st.vtx.Digest()).
|
|
| 570 |
+ WithField("actives_key", k).
|
|
| 571 |
+ Debug("deleting unreferenced active vertex")
|
|
| 572 |
+ for _, e := range st.edges {
|
|
| 573 |
+ bklog.G(context.TODO()). |
|
| 574 |
+ WithField("vertex_name", e.edge.Vertex.Name()).
|
|
| 575 |
+ WithField("vertex_digest", e.edge.Vertex.Digest()).
|
|
| 576 |
+ WithField("index", e.edge.Index).
|
|
| 577 |
+ WithField("state", e.state).
|
|
| 578 |
+ Debug("edge in deleted unreferenced state")
|
|
| 579 |
+ } |
|
| 580 |
+ } |
|
| 566 | 581 |
for chKey := range st.childVtx {
|
| 567 | 582 |
chState := jl.actives[chKey] |
| 568 | 583 |
delete(chState.parents, k) |
| ... | ... |
@@ -570,6 +677,17 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) {
|
| 570 | 570 |
} |
| 571 | 571 |
st.Release() |
| 572 | 572 |
delete(jl.actives, k) |
| 573 |
+ } else if debugScheduler {
|
|
| 574 |
+ var jobIDs []string |
|
| 575 |
+ for j := range st.jobs {
|
|
| 576 |
+ jobIDs = append(jobIDs, j.id) |
|
| 577 |
+ } |
|
| 578 |
+ bklog.G(context.TODO()). |
|
| 579 |
+ WithField("vertex_name", st.vtx.Name()).
|
|
| 580 |
+ WithField("vertex_digest", st.vtx.Digest()).
|
|
| 581 |
+ WithField("actives_key", k).
|
|
| 582 |
+ WithField("jobs", jobIDs).
|
|
| 583 |
+ Debug("not deleting referenced active vertex")
|
|
| 573 | 584 |
} |
| 574 | 585 |
} |
| 575 | 586 |
|
| ... | ... |
@@ -578,7 +696,7 @@ func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, er |
| 578 | 578 |
j.span = span |
| 579 | 579 |
} |
| 580 | 580 |
|
| 581 |
- v, err := j.list.load(e.Vertex, nil, j) |
|
| 581 |
+ v, err := j.list.load(ctx, e.Vertex, nil, j) |
|
| 582 | 582 |
if err != nil {
|
| 583 | 583 |
return nil, err |
| 584 | 584 |
} |
| ... | ... |
@@ -589,8 +707,6 @@ func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, er |
| 589 | 589 |
return nil, err |
| 590 | 590 |
} |
| 591 | 591 |
|
| 592 |
- j.list.mu.Lock() |
|
| 593 |
- defer j.list.mu.Unlock() |
|
| 594 | 592 |
return &withProvenance{CachedResult: res, j: j, e: e}, nil
|
| 595 | 593 |
} |
| 596 | 594 |
|
| ... | ... |
@@ -610,6 +726,7 @@ func (wp *withProvenance) WalkProvenance(ctx context.Context, f func(ProvenanceP |
| 610 | 610 |
return wp.j.walkProvenance(ctx, wp.e, f, m) |
| 611 | 611 |
} |
| 612 | 612 |
|
| 613 |
+// called with solver lock |
|
| 613 | 614 |
func (j *Job) walkProvenance(ctx context.Context, e Edge, f func(ProvenanceProvider) error, visited map[digest.Digest]struct{}) error {
|
| 614 | 615 |
if _, ok := visited[e.Vertex.Digest()]; ok {
|
| 615 | 616 |
return nil |
| ... | ... |
@@ -647,6 +764,14 @@ func (j *Job) Discard() error {
|
| 647 | 647 |
for k, st := range j.list.actives {
|
| 648 | 648 |
st.mu.Lock() |
| 649 | 649 |
if _, ok := st.jobs[j]; ok {
|
| 650 |
+ if debugScheduler {
|
|
| 651 |
+ bklog.G(context.TODO()). |
|
| 652 |
+ WithField("job", j.id).
|
|
| 653 |
+ WithField("vertex_name", st.vtx.Name()).
|
|
| 654 |
+ WithField("vertex_digest", st.vtx.Digest()).
|
|
| 655 |
+ WithField("actives_key", k).
|
|
| 656 |
+ Debug("deleting job from state")
|
|
| 657 |
+ } |
|
| 650 | 658 |
delete(st.jobs, j) |
| 651 | 659 |
j.list.deleteIfUnreferenced(k, st) |
| 652 | 660 |
} |
| ... | ... |
@@ -709,7 +834,7 @@ type activeOp interface {
|
| 709 | 709 |
CalcSlowCache(context.Context, Index, PreprocessFunc, ResultBasedCacheFunc, Result) (digest.Digest, error) |
| 710 | 710 |
} |
| 711 | 711 |
|
| 712 |
-func newSharedOp(resolver ResolveOpFunc, cacheManager CacheManager, st *state) *sharedOp {
|
|
| 712 |
+func newSharedOp(resolver ResolveOpFunc, st *state) *sharedOp {
|
|
| 713 | 713 |
so := &sharedOp{
|
| 714 | 714 |
resolver: resolver, |
| 715 | 715 |
st: st, |
| ... | ... |
@@ -97,11 +97,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp |
| 97 | 97 |
if srcPol != nil {
|
| 98 | 98 |
pol = append([]*spb.Policy{srcPol}, pol...)
|
| 99 | 99 |
} |
| 100 |
- |
|
| 101 | 100 |
polEngine = sourcepolicy.NewEngine(pol) |
| 102 |
- if err != nil {
|
|
| 103 |
- return nil, err |
|
| 104 |
- } |
|
| 105 | 101 |
} |
| 106 | 102 |
var cms []solver.CacheManager |
| 107 | 103 |
for _, im := range cacheImports {
|
| ... | ... |
@@ -27,7 +27,7 @@ func timestampToTime(ts int64) *time.Time {
|
| 27 | 27 |
return &tm |
| 28 | 28 |
} |
| 29 | 29 |
|
| 30 |
-func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *copy.User, idmap *idtools.IdentityMapping) error {
|
|
| 30 |
+func mkdir(d string, action pb.FileActionMkDir, user *copy.User, idmap *idtools.IdentityMapping) error {
|
|
| 31 | 31 |
p, err := fs.RootPath(d, action.Path) |
| 32 | 32 |
if err != nil {
|
| 33 | 33 |
return err |
| ... | ... |
@@ -60,7 +60,7 @@ func mkdir(ctx context.Context, d string, action pb.FileActionMkDir, user *copy. |
| 60 | 60 |
return nil |
| 61 | 61 |
} |
| 62 | 62 |
|
| 63 |
-func mkfile(ctx context.Context, d string, action pb.FileActionMkFile, user *copy.User, idmap *idtools.IdentityMapping) error {
|
|
| 63 |
+func mkfile(d string, action pb.FileActionMkFile, user *copy.User, idmap *idtools.IdentityMapping) error {
|
|
| 64 | 64 |
p, err := fs.RootPath(d, filepath.Join("/", action.Path))
|
| 65 | 65 |
if err != nil {
|
| 66 | 66 |
return err |
| ... | ... |
@@ -86,7 +86,7 @@ func mkfile(ctx context.Context, d string, action pb.FileActionMkFile, user *cop |
| 86 | 86 |
return nil |
| 87 | 87 |
} |
| 88 | 88 |
|
| 89 |
-func rm(ctx context.Context, d string, action pb.FileActionRm) error {
|
|
| 89 |
+func rm(d string, action pb.FileActionRm) error {
|
|
| 90 | 90 |
if action.AllowWildcard {
|
| 91 | 91 |
src, err := cleanPath(action.Path) |
| 92 | 92 |
if err != nil {
|
| ... | ... |
@@ -139,7 +139,7 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u * |
| 139 | 139 |
} |
| 140 | 140 |
destPath, err := cleanPath(action.Dest) |
| 141 | 141 |
if err != nil {
|
| 142 |
- return errors.Wrap(err, "cleaning path") |
|
| 142 |
+ return errors.Wrap(err, "cleaning destination path") |
|
| 143 | 143 |
} |
| 144 | 144 |
if !action.CreateDestPath {
|
| 145 | 145 |
p, err := fs.RootPath(dest, filepath.Join("/", action.Dest))
|
| ... | ... |
@@ -172,6 +172,7 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u * |
| 172 | 172 |
} |
| 173 | 173 |
ci.CopyDirContents = action.DirCopyContents |
| 174 | 174 |
ci.FollowLinks = action.FollowSymlink |
| 175 |
+ ci.AlwaysReplaceExistingDestPaths = action.AlwaysReplaceExistingDestPaths |
|
| 175 | 176 |
}, |
| 176 | 177 |
copy.WithXAttrErrorHandler(xattrErrorHandler), |
| 177 | 178 |
} |
| ... | ... |
@@ -245,7 +246,7 @@ func (fb *Backend) Mkdir(ctx context.Context, m, user, group fileoptypes.Mount, |
| 245 | 245 |
return err |
| 246 | 246 |
} |
| 247 | 247 |
|
| 248 |
- return mkdir(ctx, dir, action, u, mnt.m.IdentityMapping()) |
|
| 248 |
+ return mkdir(dir, action, u, mnt.m.IdentityMapping()) |
|
| 249 | 249 |
} |
| 250 | 250 |
|
| 251 | 251 |
func (fb *Backend) Mkfile(ctx context.Context, m, user, group fileoptypes.Mount, action pb.FileActionMkFile) error {
|
| ... | ... |
@@ -266,7 +267,7 @@ func (fb *Backend) Mkfile(ctx context.Context, m, user, group fileoptypes.Mount, |
| 266 | 266 |
return err |
| 267 | 267 |
} |
| 268 | 268 |
|
| 269 |
- return mkfile(ctx, dir, action, u, mnt.m.IdentityMapping()) |
|
| 269 |
+ return mkfile(dir, action, u, mnt.m.IdentityMapping()) |
|
| 270 | 270 |
} |
| 271 | 271 |
|
| 272 | 272 |
func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileActionRm) error {
|
| ... | ... |
@@ -282,7 +283,7 @@ func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileAc |
| 282 | 282 |
} |
| 283 | 283 |
defer lm.Unmount() |
| 284 | 284 |
|
| 285 |
- return rm(ctx, dir, action) |
|
| 285 |
+ return rm(dir, action) |
|
| 286 | 286 |
} |
| 287 | 287 |
|
| 288 | 288 |
func (fb *Backend) Copy(ctx context.Context, m1, m2, user, group fileoptypes.Mount, action pb.FileActionCopy) error {
|
| ... | ... |
@@ -5,7 +5,7 @@ import ( |
| 5 | 5 |
copy "github.com/tonistiigi/fsutil/copy" |
| 6 | 6 |
) |
| 7 | 7 |
|
| 8 |
-func mapUserToChowner(user *copy.User, idmap *idtools.IdentityMapping) (copy.Chowner, error) {
|
|
| 8 |
+func mapUserToChowner(user *copy.User, _ *idtools.IdentityMapping) (copy.Chowner, error) {
|
|
| 9 | 9 |
if user == nil || user.SID == "" {
|
| 10 | 10 |
return func(old *copy.User) (*copy.User, error) {
|
| 11 | 11 |
if old == nil || old.SID == "" {
|
| ... | ... |
@@ -14,13 +14,14 @@ import ( |
| 14 | 14 |
"time" |
| 15 | 15 |
|
| 16 | 16 |
"github.com/containerd/containerd/content" |
| 17 |
- "github.com/containerd/containerd/errdefs" |
|
| 18 | 17 |
"github.com/containerd/containerd/leases" |
| 18 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 19 | 19 |
controlapi "github.com/moby/buildkit/api/services/control" |
| 20 | 20 |
"github.com/moby/buildkit/client" |
| 21 | 21 |
"github.com/moby/buildkit/cmd/buildkitd/config" |
| 22 | 22 |
"github.com/moby/buildkit/identity" |
| 23 | 23 |
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd" |
| 24 |
+ "github.com/moby/buildkit/util/iohelper" |
|
| 24 | 25 |
"github.com/moby/buildkit/util/leaseutil" |
| 25 | 26 |
digest "github.com/opencontainers/go-digest" |
| 26 | 27 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -136,7 +137,7 @@ func (h *HistoryQueue) migrateV2() error {
|
| 136 | 136 |
return b.ForEach(func(key, dt []byte) error {
|
| 137 | 137 |
recs, err := h.opt.LeaseManager.ListResources(ctx, leases.Lease{ID: h.leaseID(string(key))})
|
| 138 | 138 |
if err != nil {
|
| 139 |
- if errdefs.IsNotFound(err) {
|
|
| 139 |
+ if cerrdefs.IsNotFound(err) {
|
|
| 140 | 140 |
return nil |
| 141 | 141 |
} |
| 142 | 142 |
return err |
| ... | ... |
@@ -156,7 +157,7 @@ func (h *HistoryQueue) migrateV2() error {
|
| 156 | 156 |
|
| 157 | 157 |
l, err := h.hLeaseManager.Create(ctx, leases.WithID(h.leaseID(string(key)))) |
| 158 | 158 |
if err != nil {
|
| 159 |
- if !errors.Is(err, errdefs.ErrAlreadyExists) {
|
|
| 159 |
+ if !errors.Is(err, cerrdefs.ErrAlreadyExists) {
|
|
| 160 | 160 |
return err |
| 161 | 161 |
} |
| 162 | 162 |
l = leases.Lease{ID: string(key)}
|
| ... | ... |
@@ -241,7 +242,7 @@ func (h *HistoryQueue) migrateBlobV2(ctx context.Context, id string, detectSkipL |
| 241 | 241 |
Digest: dgst, |
| 242 | 242 |
}), content.WithRef("history-migrate-"+id))
|
| 243 | 243 |
if err != nil {
|
| 244 |
- if errdefs.IsAlreadyExists(err) {
|
|
| 244 |
+ if cerrdefs.IsAlreadyExists(err) {
|
|
| 245 | 245 |
return true, nil |
| 246 | 246 |
} |
| 247 | 247 |
return false, err |
| ... | ... |
@@ -254,7 +255,7 @@ func (h *HistoryQueue) migrateBlobV2(ctx context.Context, id string, detectSkipL |
| 254 | 254 |
return false, nil // allow skipping |
| 255 | 255 |
} |
| 256 | 256 |
defer ra.Close() |
| 257 |
- if err := content.Copy(ctx, w, &reader{ReaderAt: ra}, 0, dgst, content.WithLabels(labels)); err != nil {
|
|
| 257 |
+ if err := content.Copy(ctx, w, iohelper.ReadCloser(ra), 0, dgst, content.WithLabels(labels)); err != nil {
|
|
| 258 | 258 |
return false, err |
| 259 | 259 |
} |
| 260 | 260 |
|
| ... | ... |
@@ -364,12 +365,12 @@ func (h *HistoryQueue) addResource(ctx context.Context, l leases.Lease, desc *co |
| 364 | 364 |
return nil |
| 365 | 365 |
} |
| 366 | 366 |
if _, err := h.hContentStore.Info(ctx, desc.Digest); err != nil {
|
| 367 |
- if errdefs.IsNotFound(err) {
|
|
| 368 |
- ctx, release, err := leaseutil.WithLease(ctx, h.hLeaseManager, leases.WithID("history_migration_"+identity.NewID()), leaseutil.MakeTemporary)
|
|
| 367 |
+ if cerrdefs.IsNotFound(err) {
|
|
| 368 |
+ lr, ctx, err := leaseutil.NewLease(ctx, h.hLeaseManager, leases.WithID("history_migration_"+identity.NewID()), leaseutil.MakeTemporary)
|
|
| 369 | 369 |
if err != nil {
|
| 370 | 370 |
return err |
| 371 | 371 |
} |
| 372 |
- defer release(ctx) |
|
| 372 |
+ defer lr.Discard() |
|
| 373 | 373 |
ok, err := h.migrateBlobV2(ctx, string(desc.Digest), detectSkipLayers) |
| 374 | 374 |
if err != nil {
|
| 375 | 375 |
return err |
| ... | ... |
@@ -460,9 +461,11 @@ func (h *HistoryQueue) Status(ctx context.Context, ref string, st chan<- *client |
| 460 | 460 |
if err != nil {
|
| 461 | 461 |
return err |
| 462 | 462 |
} |
| 463 |
- defer ra.Close() |
|
| 464 | 463 |
|
| 465 |
- brdr := bufio.NewReader(&reader{ReaderAt: ra})
|
|
| 464 |
+ rc := iohelper.ReadCloser(ra) |
|
| 465 |
+ defer rc.Close() |
|
| 466 |
+ |
|
| 467 |
+ brdr := bufio.NewReader(rc) |
|
| 466 | 468 |
|
| 467 | 469 |
buf := make([]byte, 32*1024) |
| 468 | 470 |
|
| ... | ... |
@@ -506,7 +509,7 @@ func (h *HistoryQueue) update(ctx context.Context, rec controlapi.BuildHistoryRe |
| 506 | 506 |
l, err := h.hLeaseManager.Create(ctx, leases.WithID(h.leaseID(rec.Ref))) |
| 507 | 507 |
created := true |
| 508 | 508 |
if err != nil {
|
| 509 |
- if !errors.Is(err, errdefs.ErrAlreadyExists) {
|
|
| 509 |
+ if !errors.Is(err, cerrdefs.ErrAlreadyExists) {
|
|
| 510 | 510 |
return err |
| 511 | 511 |
} |
| 512 | 512 |
l = leases.Lease{ID: h.leaseID(rec.Ref)}
|
| ... | ... |
@@ -642,7 +645,7 @@ func (w *Writer) Commit(ctx context.Context) (*ocispecs.Descriptor, func(), erro |
| 642 | 642 |
dgst := w.dgstr.Digest() |
| 643 | 643 |
sz := int64(w.sz) |
| 644 | 644 |
if err := w.w.Commit(leases.WithLease(ctx, w.l.ID), int64(w.sz), dgst); err != nil {
|
| 645 |
- if !errdefs.IsAlreadyExists(err) {
|
|
| 645 |
+ if !cerrdefs.IsAlreadyExists(err) {
|
|
| 646 | 646 |
w.Discard() |
| 647 | 647 |
return nil, nil, err |
| 648 | 648 |
} |
| ... | ... |
@@ -906,14 +909,3 @@ func (p *channel[T]) close() {
|
| 906 | 906 |
close(p.done) |
| 907 | 907 |
}) |
| 908 | 908 |
} |
| 909 |
- |
|
| 910 |
-type reader struct {
|
|
| 911 |
- io.ReaderAt |
|
| 912 |
- pos int64 |
|
| 913 |
-} |
|
| 914 |
- |
|
| 915 |
-func (r *reader) Read(p []byte) (int, error) {
|
|
| 916 |
- n, err := r.ReaderAt.ReadAt(p, r.pos) |
|
| 917 |
- r.pos += int64(len(p)) |
|
| 918 |
- return n, err |
|
| 919 |
-} |
| ... | ... |
@@ -423,7 +423,7 @@ func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu |
| 423 | 423 |
return nil, err |
| 424 | 424 |
} |
| 425 | 425 |
|
| 426 |
- emu, err := getEmulator(ctx, e.platform, e.cm.IdentityMapping()) |
|
| 426 |
+ emu, err := getEmulator(ctx, e.platform) |
|
| 427 | 427 |
if err != nil {
|
| 428 | 428 |
return nil, err |
| 429 | 429 |
} |
| ... | ... |
@@ -85,7 +85,7 @@ func (m *staticEmulatorMount) IdentityMapping() *idtools.IdentityMapping {
|
| 85 | 85 |
return m.idmap |
| 86 | 86 |
} |
| 87 | 87 |
|
| 88 |
-func getEmulator(ctx context.Context, p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) {
|
|
| 88 |
+func getEmulator(ctx context.Context, p *pb.Platform) (*emulator, error) {
|
|
| 89 | 89 |
all := archutil.SupportedPlatforms(false) |
| 90 | 90 |
pp := platforms.Normalize(ocispecs.Platform{
|
| 91 | 91 |
Architecture: p.Architecture, |
| ... | ... |
@@ -13,7 +13,7 @@ import ( |
| 13 | 13 |
copy "github.com/tonistiigi/fsutil/copy" |
| 14 | 14 |
) |
| 15 | 15 |
|
| 16 |
-func getReadUserFn(worker worker.Worker) func(chopt *pb.ChownOpt, mu, mg snapshot.Mountable) (*copy.User, error) {
|
|
| 16 |
+func getReadUserFn(_ worker.Worker) func(chopt *pb.ChownOpt, mu, mg snapshot.Mountable) (*copy.User, error) {
|
|
| 17 | 17 |
return readUser |
| 18 | 18 |
} |
| 19 | 19 |
|
| ... | ... |
@@ -11,7 +11,7 @@ import ( |
| 11 | 11 |
copy "github.com/tonistiigi/fsutil/copy" |
| 12 | 12 |
) |
| 13 | 13 |
|
| 14 |
-func getReadUserFn(worker worker.Worker) func(chopt *pb.ChownOpt, mu, mg snapshot.Mountable) (*copy.User, error) {
|
|
| 14 |
+func getReadUserFn(_ worker.Worker) func(chopt *pb.ChownOpt, mu, mg snapshot.Mountable) (*copy.User, error) {
|
|
| 15 | 15 |
return readUser |
| 16 | 16 |
} |
| 17 | 17 |
|
| ... | ... |
@@ -18,7 +18,7 @@ func getReadUserFn(worker worker.Worker) func(chopt *pb.ChownOpt, mu, mg snapsho |
| 18 | 18 |
} |
| 19 | 19 |
} |
| 20 | 20 |
|
| 21 |
-func readUser(chopt *pb.ChownOpt, mu, mg snapshot.Mountable, worker worker.Worker) (*copy.User, error) {
|
|
| 21 |
+func readUser(chopt *pb.ChownOpt, mu, _ snapshot.Mountable, worker worker.Worker) (*copy.User, error) {
|
|
| 22 | 22 |
if chopt == nil {
|
| 23 | 23 |
return nil, nil |
| 24 | 24 |
} |
| ... | ... |
@@ -411,9 +411,10 @@ func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solve |
| 411 | 411 |
} |
| 412 | 412 |
|
| 413 | 413 |
if _, err := r.CacheKeys()[0].Exporter.ExportTo(ctx, e, solver.CacheExportOpt{
|
| 414 |
- ResolveRemotes: resolveRemotes, |
|
| 415 |
- Mode: solver.CacheExportModeRemoteOnly, |
|
| 416 |
- ExportRoots: true, |
|
| 414 |
+ ResolveRemotes: resolveRemotes, |
|
| 415 |
+ Mode: solver.CacheExportModeRemoteOnly, |
|
| 416 |
+ ExportRoots: true, |
|
| 417 |
+ IgnoreBacklinks: true, |
|
| 417 | 418 |
}); err != nil {
|
| 418 | 419 |
return err |
| 419 | 420 |
} |
| ... | ... |
@@ -21,7 +21,9 @@ import ( |
| 21 | 21 |
resourcestypes "github.com/moby/buildkit/executor/resources/types" |
| 22 | 22 |
"github.com/moby/buildkit/exporter" |
| 23 | 23 |
"github.com/moby/buildkit/exporter/containerimage/exptypes" |
| 24 |
+ "github.com/moby/buildkit/exporter/verifier" |
|
| 24 | 25 |
"github.com/moby/buildkit/frontend" |
| 26 |
+ "github.com/moby/buildkit/frontend/attestations" |
|
| 25 | 27 |
"github.com/moby/buildkit/frontend/gateway" |
| 26 | 28 |
"github.com/moby/buildkit/identity" |
| 27 | 29 |
"github.com/moby/buildkit/session" |
| ... | ... |
@@ -33,13 +35,12 @@ import ( |
| 33 | 33 |
"github.com/moby/buildkit/util/compression" |
| 34 | 34 |
"github.com/moby/buildkit/util/entitlements" |
| 35 | 35 |
"github.com/moby/buildkit/util/grpcerrors" |
| 36 |
+ "github.com/moby/buildkit/util/leaseutil" |
|
| 36 | 37 |
"github.com/moby/buildkit/util/progress" |
| 37 | 38 |
"github.com/moby/buildkit/util/tracing/detect" |
| 38 | 39 |
"github.com/moby/buildkit/worker" |
| 39 | 40 |
digest "github.com/opencontainers/go-digest" |
| 40 | 41 |
"github.com/pkg/errors" |
| 41 |
- "go.opentelemetry.io/otel/sdk/trace/tracetest" |
|
| 42 |
- "go.opentelemetry.io/otel/trace" |
|
| 43 | 42 |
"golang.org/x/sync/errgroup" |
| 44 | 43 |
"google.golang.org/grpc/codes" |
| 45 | 44 |
"google.golang.org/grpc/status" |
| ... | ... |
@@ -157,15 +158,10 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge {
|
| 157 | 157 |
return s.bridge(b) |
| 158 | 158 |
} |
| 159 | 159 |
|
| 160 |
-func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend.SolveRequest, exp ExporterRequest, j *solver.Job, usage *resources.SysSampler) (func(*Result, []exporter.DescriptorReference, error) error, error) {
|
|
| 161 |
- var stopTrace func() []tracetest.SpanStub |
|
| 162 |
- |
|
| 163 |
- if s := trace.SpanFromContext(ctx); s.SpanContext().IsValid() {
|
|
| 164 |
- if exp, _, err := detect.Exporter(); err == nil {
|
|
| 165 |
- if rec, ok := exp.(*detect.TraceRecorder); ok {
|
|
| 166 |
- stopTrace = rec.Record(s.SpanContext().TraceID()) |
|
| 167 |
- } |
|
| 168 |
- } |
|
| 160 |
+func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend.SolveRequest, exp ExporterRequest, j *solver.Job, usage *resources.SysSampler) (func(context.Context, *Result, []exporter.DescriptorReference, error) error, error) {
|
|
| 161 |
+ stopTrace, err := detect.Recorder.Record(ctx) |
|
| 162 |
+ if err != nil {
|
|
| 163 |
+ return nil, err |
|
| 169 | 164 |
} |
| 170 | 165 |
|
| 171 | 166 |
st := time.Now() |
| ... | ... |
@@ -190,7 +186,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend |
| 190 | 190 |
return nil, err |
| 191 | 191 |
} |
| 192 | 192 |
|
| 193 |
- return func(res *Result, descrefs []exporter.DescriptorReference, err error) error {
|
|
| 193 |
+ return func(ctx context.Context, res *Result, descrefs []exporter.DescriptorReference, err error) error {
|
|
| 194 | 194 |
en := time.Now() |
| 195 | 195 |
rec.CompletedAt = &en |
| 196 | 196 |
|
| ... | ... |
@@ -203,8 +199,8 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend |
| 203 | 203 |
} |
| 204 | 204 |
} |
| 205 | 205 |
|
| 206 |
- ctx, cancel := context.WithCancelCause(context.Background()) |
|
| 207 |
- ctx, _ = context.WithTimeoutCause(ctx, 20*time.Second, errors.WithStack(context.DeadlineExceeded)) |
|
| 206 |
+ ctx, cancel := context.WithCancelCause(ctx) |
|
| 207 |
+ ctx, _ = context.WithTimeoutCause(ctx, 300*time.Second, errors.WithStack(context.DeadlineExceeded)) |
|
| 208 | 208 |
defer cancel(errors.WithStack(context.Canceled)) |
| 209 | 209 |
|
| 210 | 210 |
var mu sync.Mutex |
| ... | ... |
@@ -217,6 +213,15 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend |
| 217 | 217 |
"capture-usage": "true", |
| 218 | 218 |
} |
| 219 | 219 |
|
| 220 |
+ // infer builder-id from user input if available |
|
| 221 |
+ if attests, err := attestations.Parse(rec.FrontendAttrs); err == nil {
|
|
| 222 |
+ if prvAttrs, ok := attests["provenance"]; ok {
|
|
| 223 |
+ if builderID, ok := prvAttrs["builder-id"]; ok {
|
|
| 224 |
+ attrs["builder-id"] = builderID |
|
| 225 |
+ } |
|
| 226 |
+ } |
|
| 227 |
+ } |
|
| 228 |
+ |
|
| 220 | 229 |
makeProvenance := func(res solver.ResultProxy, cap *provenance.Capture) (*controlapi.Descriptor, func(), error) {
|
| 221 | 230 |
prc, err := NewProvenanceCreator(ctx2, cap, res, attrs, j, usage) |
| 222 | 231 |
if err != nil {
|
| ... | ... |
@@ -506,7 +511,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro |
| 506 | 506 |
return nil, err1 |
| 507 | 507 |
} |
| 508 | 508 |
defer func() {
|
| 509 |
- err = rec(resProv, descrefs, err) |
|
| 509 |
+ err = rec(context.WithoutCancel(ctx), resProv, descrefs, err) |
|
| 510 | 510 |
}() |
| 511 | 511 |
} |
| 512 | 512 |
|
| ... | ... |
@@ -532,6 +537,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro |
| 532 | 532 |
res = &frontend.Result{}
|
| 533 | 533 |
} |
| 534 | 534 |
|
| 535 |
+ if err := verifier.CaptureFrontendOpts(req.FrontendOpt, res); err != nil {
|
|
| 536 |
+ return nil, err |
|
| 537 |
+ } |
|
| 538 |
+ |
|
| 535 | 539 |
releasers = append(releasers, func() {
|
| 536 | 540 |
res.EachRef(func(ref solver.ResultProxy) error {
|
| 537 | 541 |
go ref.Release(context.TODO()) |
| ... | ... |
@@ -582,6 +591,22 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro |
| 582 | 582 |
return nil, err |
| 583 | 583 |
} |
| 584 | 584 |
|
| 585 |
+ // Functions that create new objects in containerd (eg. content blobs) need to have a lease to ensure |
|
| 586 |
+ // that the object is not garbage collected immediately. This is protected by the indivual components, |
|
| 587 |
+ // but because creating a lease is not cheap and requires a disk write, we create a single lease here |
|
| 588 |
+ // early and let all the exporters, cache export and provenance creation use the same one. |
|
| 589 |
+ lm, err := s.leaseManager() |
|
| 590 |
+ if err != nil {
|
|
| 591 |
+ return nil, err |
|
| 592 |
+ } |
|
| 593 |
+ ctx, done, err := leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) |
|
| 594 |
+ if err != nil {
|
|
| 595 |
+ return nil, err |
|
| 596 |
+ } |
|
| 597 |
+ releasers = append(releasers, func() {
|
|
| 598 |
+ done(context.WithoutCancel(ctx)) |
|
| 599 |
+ }) |
|
| 600 |
+ |
|
| 585 | 601 |
cacheExporters, inlineCacheExporter := splitCacheExporters(exp.CacheExporters) |
| 586 | 602 |
|
| 587 | 603 |
var exporterResponse map[string]string |
| ... | ... |
@@ -706,6 +731,11 @@ func runInlineCacheExporter(ctx context.Context, e exporter.ExporterInstance, in |
| 706 | 706 |
} |
| 707 | 707 |
|
| 708 | 708 |
func (s *Solver) runExporters(ctx context.Context, exporters []exporter.ExporterInstance, inlineCacheExporter inlineCacheExporter, job *solver.Job, cached *result.Result[solver.CachedResult], inp *result.Result[cache.ImmutableRef]) (exporterResponse map[string]string, descrefs []exporter.DescriptorReference, err error) {
|
| 709 |
+ warnings, err := verifier.CheckInvalidPlatforms(ctx, inp) |
|
| 710 |
+ if err != nil {
|
|
| 711 |
+ return nil, nil, err |
|
| 712 |
+ } |
|
| 713 |
+ |
|
| 709 | 714 |
eg, ctx := errgroup.WithContext(ctx) |
| 710 | 715 |
resps := make([]map[string]string, len(exporters)) |
| 711 | 716 |
descs := make([]exporter.DescriptorReference, len(exporters)) |
| ... | ... |
@@ -714,6 +744,15 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter |
| 714 | 714 |
eg.Go(func() error {
|
| 715 | 715 |
id := fmt.Sprint(job.SessionID, "-export-", i) |
| 716 | 716 |
return inBuilderContext(ctx, job, exp.Name(), id, func(ctx context.Context, _ session.Group) error {
|
| 717 |
+ if i == 0 && len(warnings) > 0 {
|
|
| 718 |
+ pw, _, _ := progress.NewFromContext(ctx) |
|
| 719 |
+ for _, w := range warnings {
|
|
| 720 |
+ pw.Write(identity.NewID(), w) |
|
| 721 |
+ } |
|
| 722 |
+ if err := pw.Close(); err != nil {
|
|
| 723 |
+ return err |
|
| 724 |
+ } |
|
| 725 |
+ } |
|
| 717 | 726 |
inlineCache := exptypes.InlineCache(func(ctx context.Context) (*result.Result[*exptypes.InlineCacheEntry], error) {
|
| 718 | 727 |
return runInlineCacheExporter(ctx, exp, inlineCacheExporter, job, cached) |
| 719 | 728 |
}) |
| ... | ... |
@@ -730,6 +769,19 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter |
| 730 | 730 |
return nil, nil, err |
| 731 | 731 |
} |
| 732 | 732 |
|
| 733 |
+ if len(exporters) == 0 && len(warnings) > 0 {
|
|
| 734 |
+ err := inBuilderContext(ctx, job, "Verifying build result", identity.NewID(), func(ctx context.Context, _ session.Group) error {
|
|
| 735 |
+ pw, _, _ := progress.NewFromContext(ctx) |
|
| 736 |
+ for _, w := range warnings {
|
|
| 737 |
+ pw.Write(identity.NewID(), w) |
|
| 738 |
+ } |
|
| 739 |
+ return pw.Close() |
|
| 740 |
+ }) |
|
| 741 |
+ if err != nil {
|
|
| 742 |
+ return nil, nil, err |
|
| 743 |
+ } |
|
| 744 |
+ } |
|
| 745 |
+ |
|
| 733 | 746 |
// TODO: separate these out, and return multiple exporter responses to the |
| 734 | 747 |
// client |
| 735 | 748 |
for _, resp := range resps {
|
| ... | ... |
@@ -744,6 +796,14 @@ func (s *Solver) runExporters(ctx context.Context, exporters []exporter.Exporter |
| 744 | 744 |
return exporterResponse, descs, nil |
| 745 | 745 |
} |
| 746 | 746 |
|
| 747 |
+func (s *Solver) leaseManager() (*leaseutil.Manager, error) {
|
|
| 748 |
+ w, err := defaultResolver(s.workerController)() |
|
| 749 |
+ if err != nil {
|
|
| 750 |
+ return nil, err |
|
| 751 |
+ } |
|
| 752 |
+ return w.LeaseManager(), nil |
|
| 753 |
+} |
|
| 754 |
+ |
|
| 747 | 755 |
func splitCacheExporters(exporters []RemoteCacheExporter) (rest []RemoteCacheExporter, inline inlineCacheExporter) {
|
| 748 | 756 |
rest = make([]RemoteCacheExporter, 0, len(exporters)) |
| 749 | 757 |
for _, exp := range exporters {
|
| ... | ... |
@@ -61,10 +61,11 @@ const ( |
| 61 | 61 |
CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" |
| 62 | 62 |
CapExecSecretEnv apicaps.CapID = "exec.secretenv" |
| 63 | 63 |
|
| 64 |
- CapFileBase apicaps.CapID = "file.base" |
|
| 65 |
- CapFileRmWildcard apicaps.CapID = "file.rm.wildcard" |
|
| 66 |
- CapFileCopyIncludeExcludePatterns apicaps.CapID = "file.copy.includeexcludepatterns" |
|
| 67 |
- CapFileRmNoFollowSymlink apicaps.CapID = "file.rm.nofollowsymlink" |
|
| 64 |
+ CapFileBase apicaps.CapID = "file.base" |
|
| 65 |
+ CapFileRmWildcard apicaps.CapID = "file.rm.wildcard" |
|
| 66 |
+ CapFileCopyIncludeExcludePatterns apicaps.CapID = "file.copy.includeexcludepatterns" |
|
| 67 |
+ CapFileRmNoFollowSymlink apicaps.CapID = "file.rm.nofollowsymlink" |
|
| 68 |
+ CapFileCopyAlwaysReplaceExistingDestPaths apicaps.CapID = "file.copy.alwaysreplaceexistingdestpaths" |
|
| 68 | 69 |
|
| 69 | 70 |
CapConstraints apicaps.CapID = "constraints" |
| 70 | 71 |
CapPlatform apicaps.CapID = "platform" |
| ... | ... |
@@ -385,6 +386,12 @@ func init() {
|
| 385 | 385 |
}) |
| 386 | 386 |
|
| 387 | 387 |
Caps.Init(apicaps.Cap{
|
| 388 |
+ ID: CapFileCopyAlwaysReplaceExistingDestPaths, |
|
| 389 |
+ Enabled: true, |
|
| 390 |
+ Status: apicaps.CapStatusExperimental, |
|
| 391 |
+ }) |
|
| 392 |
+ |
|
| 393 |
+ Caps.Init(apicaps.Cap{
|
|
| 388 | 394 |
ID: CapConstraints, |
| 389 | 395 |
Enabled: true, |
| 390 | 396 |
Status: apicaps.CapStatusExperimental, |
| ... | ... |
@@ -2137,6 +2137,8 @@ type FileActionCopy struct {
|
| 2137 | 2137 |
IncludePatterns []string `protobuf:"bytes,12,rep,name=include_patterns,json=includePatterns,proto3" json:"include_patterns,omitempty"` |
| 2138 | 2138 |
// exclude files/dir matching any of these patterns (even if they match an include pattern) |
| 2139 | 2139 |
ExcludePatterns []string `protobuf:"bytes,13,rep,name=exclude_patterns,json=excludePatterns,proto3" json:"exclude_patterns,omitempty"` |
| 2140 |
+ // alwaysReplaceExistingDestPaths results in an existing dest path that differs in type from the src path being replaced rather than the default of returning an error |
|
| 2141 |
+ AlwaysReplaceExistingDestPaths bool `protobuf:"varint,14,opt,name=alwaysReplaceExistingDestPaths,proto3" json:"alwaysReplaceExistingDestPaths,omitempty"` |
|
| 2140 | 2142 |
} |
| 2141 | 2143 |
|
| 2142 | 2144 |
func (m *FileActionCopy) Reset() { *m = FileActionCopy{} }
|
| ... | ... |
@@ -2259,6 +2261,13 @@ func (m *FileActionCopy) GetExcludePatterns() []string {
|
| 2259 | 2259 |
return nil |
| 2260 | 2260 |
} |
| 2261 | 2261 |
|
| 2262 |
+func (m *FileActionCopy) GetAlwaysReplaceExistingDestPaths() bool {
|
|
| 2263 |
+ if m != nil {
|
|
| 2264 |
+ return m.AlwaysReplaceExistingDestPaths |
|
| 2265 |
+ } |
|
| 2266 |
+ return false |
|
| 2267 |
+} |
|
| 2268 |
+ |
|
| 2262 | 2269 |
type FileActionMkFile struct {
|
| 2263 | 2270 |
// path for the new file |
| 2264 | 2271 |
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` |
| ... | ... |
@@ -2892,172 +2901,173 @@ func init() {
|
| 2892 | 2892 |
func init() { proto.RegisterFile("ops.proto", fileDescriptor_8de16154b2733812) }
|
| 2893 | 2893 |
|
| 2894 | 2894 |
var fileDescriptor_8de16154b2733812 = []byte{
|
| 2895 |
- // 2627 bytes of a gzipped FileDescriptorProto |
|
| 2896 |
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x59, 0xcf, 0x6f, 0x5b, 0xc7, |
|
| 2897 |
- 0xf1, 0x17, 0x7f, 0x93, 0x43, 0x8a, 0x66, 0xd6, 0x4e, 0xc2, 0xe8, 0xeb, 0xaf, 0xac, 0xbc, 0xe4, |
|
| 2895 |
+ // 2656 bytes of a gzipped FileDescriptorProto |
|
| 2896 |
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x59, 0xcf, 0x6f, 0x1b, 0xc7, |
|
| 2897 |
+ 0xf5, 0x17, 0x7f, 0x93, 0x8f, 0x14, 0xcd, 0x8c, 0x9d, 0x84, 0xd1, 0xd7, 0x5f, 0x59, 0xd9, 0xe4, |
|
| 2898 | 2898 |
0x1b, 0xc8, 0xb2, 0x2d, 0x21, 0x0a, 0x10, 0xe7, 0x6b, 0x04, 0x45, 0x25, 0x91, 0x8a, 0x18, 0xdb, |
| 2899 |
- 0xa2, 0xb0, 0xb4, 0x9c, 0x1e, 0x0a, 0x18, 0x4f, 0x8f, 0x4b, 0xea, 0x41, 0x8f, 0xef, 0x3d, 0xec, |
|
| 2900 |
- 0x5b, 0x5a, 0x62, 0x0f, 0x3d, 0xf4, 0xd4, 0x63, 0x80, 0x02, 0xbd, 0x15, 0xfd, 0x27, 0x7a, 0x6c, |
|
| 2901 |
- 0xef, 0x01, 0x72, 0x09, 0xd0, 0x1e, 0x82, 0x1e, 0xd2, 0xc2, 0xb9, 0xf4, 0x8f, 0x68, 0x81, 0x62, |
|
| 2902 |
- 0x66, 0xf7, 0xfd, 0x20, 0x25, 0xc3, 0x71, 0x5b, 0xf4, 0xc4, 0xd9, 0x99, 0xcf, 0xce, 0xce, 0xcc, |
|
| 2903 |
- 0xce, 0xec, 0xce, 0x5b, 0x42, 0x2d, 0x08, 0xa3, 0xcd, 0x50, 0x06, 0x2a, 0x60, 0xf9, 0xf0, 0x64, |
|
| 2904 |
- 0xe5, 0xde, 0xd8, 0x55, 0xa7, 0xd3, 0x93, 0x4d, 0x27, 0x98, 0x6c, 0x8d, 0x83, 0x71, 0xb0, 0x45, |
|
| 2905 |
- 0xa2, 0x93, 0xe9, 0x88, 0x46, 0x34, 0x20, 0x4a, 0x4f, 0xb1, 0xfe, 0x96, 0x87, 0x7c, 0x3f, 0x64, |
|
| 2906 |
- 0xef, 0x42, 0xd9, 0xf5, 0xc3, 0xa9, 0x8a, 0xda, 0xb9, 0xb5, 0xc2, 0x7a, 0x7d, 0xbb, 0xb6, 0x19, |
|
| 2907 |
- 0x9e, 0x6c, 0xf6, 0x90, 0xc3, 0x8d, 0x80, 0xad, 0x41, 0x51, 0x5c, 0x08, 0xa7, 0x9d, 0x5f, 0xcb, |
|
| 2908 |
- 0xad, 0xd7, 0xb7, 0x01, 0x01, 0xdd, 0x0b, 0xe1, 0xf4, 0xc3, 0x83, 0x25, 0x4e, 0x12, 0xf6, 0x01, |
|
| 2909 |
- 0x94, 0xa3, 0x60, 0x2a, 0x1d, 0xd1, 0x2e, 0x10, 0xa6, 0x81, 0x98, 0x01, 0x71, 0x08, 0x65, 0xa4, |
|
| 2910 |
- 0xa8, 0x69, 0xe4, 0x7a, 0xa2, 0x5d, 0x4c, 0x35, 0xed, 0xbb, 0x9e, 0xc6, 0x90, 0x84, 0xbd, 0x07, |
|
| 2911 |
- 0xa5, 0x93, 0xa9, 0xeb, 0x0d, 0xdb, 0x25, 0x82, 0xd4, 0x11, 0xb2, 0x8b, 0x0c, 0xc2, 0x68, 0x19, |
|
| 2912 |
- 0x82, 0x26, 0x42, 0x8e, 0x45, 0xbb, 0x9c, 0x82, 0x1e, 0x23, 0x43, 0x83, 0x48, 0x86, 0x6b, 0x0d, |
|
| 2913 |
- 0xdd, 0xd1, 0xa8, 0x5d, 0x49, 0xd7, 0xea, 0xb8, 0xa3, 0x91, 0x5e, 0x0b, 0x25, 0x6c, 0x1d, 0xaa, |
|
| 2914 |
- 0xa1, 0x67, 0xab, 0x51, 0x20, 0x27, 0x6d, 0x48, 0xed, 0x3e, 0x32, 0x3c, 0x9e, 0x48, 0xd9, 0x7d, |
|
| 2915 |
- 0xa8, 0x3b, 0x81, 0x1f, 0x29, 0x69, 0xbb, 0xbe, 0x8a, 0xda, 0x75, 0x02, 0xbf, 0x89, 0xe0, 0x2f, |
|
| 2916 |
- 0x02, 0x79, 0x26, 0xe4, 0x5e, 0x2a, 0xe4, 0x59, 0xe4, 0x6e, 0x11, 0xf2, 0x41, 0x68, 0xfd, 0x3a, |
|
| 2917 |
- 0x07, 0xd5, 0x58, 0x2b, 0xb3, 0xa0, 0xb1, 0x23, 0x9d, 0x53, 0x57, 0x09, 0x47, 0x4d, 0xa5, 0x68, |
|
| 2918 |
- 0xe7, 0xd6, 0x72, 0xeb, 0x35, 0x3e, 0xc7, 0x63, 0x4d, 0xc8, 0xf7, 0x07, 0x14, 0xef, 0x1a, 0xcf, |
|
| 2919 |
- 0xf7, 0x07, 0xac, 0x0d, 0x95, 0xa7, 0xb6, 0x74, 0x6d, 0x5f, 0x51, 0x80, 0x6b, 0x3c, 0x1e, 0xb2, |
|
| 2920 |
- 0x9b, 0x50, 0xeb, 0x0f, 0x9e, 0x0a, 0x19, 0xb9, 0x81, 0x4f, 0x61, 0xad, 0xf1, 0x94, 0xc1, 0x56, |
|
| 2921 |
- 0x01, 0xfa, 0x83, 0x7d, 0x61, 0xa3, 0xd2, 0xa8, 0x5d, 0x5a, 0x2b, 0xac, 0xd7, 0x78, 0x86, 0x63, |
|
| 2922 |
- 0xfd, 0x1c, 0x4a, 0xb4, 0xd5, 0xec, 0x73, 0x28, 0x0f, 0xdd, 0xb1, 0x88, 0x94, 0x36, 0x67, 0x77, |
|
| 2923 |
- 0xfb, 0xab, 0xef, 0x6e, 0x2d, 0xfd, 0xf9, 0xbb, 0x5b, 0x1b, 0x99, 0x9c, 0x0a, 0x42, 0xe1, 0x3b, |
|
| 2924 |
- 0x81, 0xaf, 0x6c, 0xd7, 0x17, 0x32, 0xda, 0x1a, 0x07, 0xf7, 0xf4, 0x94, 0xcd, 0x0e, 0xfd, 0x70, |
|
| 2925 |
- 0xa3, 0x81, 0xdd, 0x86, 0x92, 0xeb, 0x0f, 0xc5, 0x05, 0xd9, 0x5f, 0xd8, 0xbd, 0x6e, 0x54, 0xd5, |
|
| 2926 |
- 0xfb, 0x53, 0x15, 0x4e, 0x55, 0x0f, 0x45, 0x5c, 0x23, 0xac, 0xaf, 0x73, 0x50, 0xd6, 0xa9, 0xc4, |
|
| 2927 |
- 0x6e, 0x42, 0x71, 0x22, 0x94, 0x4d, 0xeb, 0xd7, 0xb7, 0xab, 0x7a, 0x4b, 0x95, 0xcd, 0x89, 0x8b, |
|
| 2928 |
- 0x59, 0x3a, 0x09, 0xa6, 0x18, 0xfb, 0x7c, 0x9a, 0xa5, 0x8f, 0x91, 0xc3, 0x8d, 0x80, 0xfd, 0x1f, |
|
| 2929 |
- 0x54, 0x7c, 0xa1, 0xce, 0x03, 0x79, 0x46, 0x31, 0x6a, 0xea, 0xb4, 0x38, 0x14, 0xea, 0x71, 0x30, |
|
| 2930 |
- 0x14, 0x3c, 0x96, 0xb1, 0xbb, 0x50, 0x8d, 0x84, 0x33, 0x95, 0xae, 0x9a, 0x51, 0xbc, 0x9a, 0xdb, |
|
| 2931 |
- 0x2d, 0x4a, 0x56, 0xc3, 0x23, 0x70, 0x82, 0x60, 0x77, 0xa0, 0x16, 0x09, 0x47, 0x0a, 0x25, 0xfc, |
|
| 2932 |
- 0xe7, 0x14, 0xbf, 0xfa, 0xf6, 0xb2, 0x81, 0x4b, 0xa1, 0xba, 0xfe, 0x73, 0x9e, 0xca, 0xad, 0xaf, |
|
| 2933 |
- 0xf3, 0x50, 0x44, 0x9b, 0x19, 0x83, 0xa2, 0x2d, 0xc7, 0xba, 0xa2, 0x6a, 0x9c, 0x68, 0xd6, 0x82, |
|
| 2934 |
- 0x02, 0xea, 0xc8, 0x13, 0x0b, 0x49, 0xe4, 0x38, 0xe7, 0x43, 0xb3, 0xa1, 0x48, 0xe2, 0xbc, 0x69, |
|
| 2935 |
- 0x24, 0xa4, 0xd9, 0x47, 0xa2, 0xd9, 0x6d, 0xa8, 0x85, 0x32, 0xb8, 0x98, 0x3d, 0xd3, 0x16, 0xa4, |
|
| 2936 |
- 0x59, 0x8a, 0x4c, 0x34, 0xa0, 0x1a, 0x1a, 0x8a, 0x6d, 0x00, 0x88, 0x0b, 0x25, 0xed, 0x83, 0x20, |
|
| 2937 |
- 0x52, 0x51, 0xbb, 0x4c, 0xd6, 0x52, 0xde, 0x23, 0xa3, 0x77, 0xc4, 0x33, 0x52, 0xb6, 0x02, 0xd5, |
|
| 2938 |
- 0xd3, 0x20, 0x52, 0xbe, 0x3d, 0x11, 0x54, 0x21, 0x35, 0x9e, 0x8c, 0x99, 0x05, 0xe5, 0xa9, 0xe7, |
|
| 2939 |
- 0x4e, 0x5c, 0xd5, 0xae, 0xa5, 0x3a, 0x8e, 0x89, 0xc3, 0x8d, 0x04, 0xb3, 0xd8, 0x19, 0xcb, 0x60, |
|
| 2940 |
- 0x1a, 0x1e, 0xd9, 0x52, 0xf8, 0x8a, 0xea, 0xa7, 0xc6, 0xe7, 0x78, 0xec, 0x53, 0x78, 0x47, 0x8a, |
|
| 2941 |
- 0x49, 0xf0, 0x5c, 0xd0, 0x46, 0x0d, 0xd4, 0xf4, 0x24, 0xe2, 0x18, 0xd8, 0xc8, 0x7d, 0x2e, 0xa8, |
|
| 2942 |
- 0x86, 0xaa, 0xfc, 0xe5, 0x00, 0xeb, 0x2e, 0x94, 0xb5, 0xdd, 0x18, 0x16, 0xa4, 0x4c, 0xa5, 0x10, |
|
| 2943 |
- 0x8d, 0x15, 0xd2, 0x3b, 0x8a, 0x2b, 0xa4, 0x77, 0x64, 0x75, 0xa0, 0xac, 0x2d, 0x44, 0xf4, 0x21, |
|
| 2944 |
- 0x7a, 0x65, 0xd0, 0x48, 0x23, 0x6f, 0x10, 0x8c, 0x94, 0xce, 0x48, 0x4e, 0x34, 0x69, 0xb5, 0xa5, |
|
| 2945 |
- 0x8e, 0x7f, 0x81, 0x13, 0x6d, 0x3d, 0x84, 0x5a, 0xb2, 0xb3, 0xb4, 0x44, 0xc7, 0xa8, 0xc9, 0xf7, |
|
| 2946 |
- 0x3a, 0x38, 0x81, 0xc2, 0xa5, 0x17, 0x25, 0x1a, 0xc3, 0x18, 0x84, 0xca, 0x0d, 0x7c, 0xdb, 0x23, |
|
| 2947 |
- 0x45, 0x55, 0x9e, 0x8c, 0xad, 0x3f, 0x16, 0xa0, 0x44, 0x8e, 0xb1, 0x75, 0xac, 0x88, 0x70, 0xaa, |
|
| 2948 |
- 0x3d, 0x28, 0xec, 0x32, 0x53, 0x11, 0x40, 0xb5, 0x97, 0x14, 0x04, 0xd6, 0xe1, 0x0a, 0x66, 0xa7, |
|
| 2949 |
- 0x27, 0x1c, 0x15, 0x48, 0xb3, 0x4e, 0x32, 0xc6, 0xf5, 0x87, 0x58, 0xa1, 0x3a, 0x61, 0x88, 0x66, |
|
| 2950 |
- 0x77, 0xa0, 0x1c, 0x50, 0x59, 0x51, 0xce, 0xbc, 0xa4, 0xd8, 0x0c, 0x04, 0x95, 0x4b, 0x61, 0x0f, |
|
| 2951 |
- 0x03, 0xdf, 0x9b, 0x51, 0x26, 0x55, 0x79, 0x32, 0xc6, 0x44, 0xa7, 0x3a, 0x7a, 0x32, 0x0b, 0xf5, |
|
| 2952 |
- 0xb1, 0xda, 0xd4, 0x89, 0xfe, 0x38, 0x66, 0xf2, 0x54, 0x8e, 0x07, 0xe7, 0x93, 0x49, 0x38, 0x8a, |
|
| 2953 |
- 0xfa, 0xa1, 0x6a, 0x5f, 0x4f, 0x53, 0x32, 0xe6, 0xf1, 0x44, 0x8a, 0x48, 0xc7, 0x76, 0x4e, 0x05, |
|
| 2954 |
- 0x22, 0x6f, 0xa4, 0xc8, 0x3d, 0xc3, 0xe3, 0x89, 0x34, 0xad, 0x34, 0x84, 0xbe, 0x49, 0xd0, 0x4c, |
|
| 2955 |
- 0xa5, 0x21, 0x36, 0x95, 0x63, 0x86, 0x0e, 0x06, 0x07, 0x88, 0x7c, 0x2b, 0x3d, 0xdd, 0x35, 0x87, |
|
| 2956 |
- 0x1b, 0x89, 0xf6, 0x36, 0x9a, 0x7a, 0xaa, 0xd7, 0x69, 0xbf, 0xad, 0x43, 0x19, 0x8f, 0xd9, 0xff, |
|
| 2957 |
- 0x43, 0x03, 0x4f, 0x32, 0xe1, 0x2b, 0xb2, 0xa4, 0xdd, 0x26, 0x87, 0xdf, 0x4c, 0x1c, 0xde, 0xcb, |
|
| 2958 |
- 0x08, 0xf9, 0x1c, 0xd4, 0x5a, 0x4d, 0x7d, 0xc7, 0x1d, 0x89, 0xdc, 0x9f, 0xe9, 0x54, 0x2b, 0x70, |
|
| 2959 |
- 0xa2, 0xad, 0x1e, 0x54, 0x63, 0xef, 0x2e, 0x65, 0xd0, 0x3d, 0xa8, 0x44, 0xa7, 0xb6, 0x74, 0xfd, |
|
| 2960 |
- 0x31, 0x6d, 0x6e, 0x73, 0xfb, 0x7a, 0x12, 0x8c, 0x81, 0xe6, 0xa3, 0x03, 0x31, 0xc6, 0x0a, 0xe2, |
|
| 2961 |
- 0x6c, 0xbc, 0x4a, 0x57, 0x0b, 0x0a, 0x53, 0x77, 0x48, 0x7a, 0x96, 0x39, 0x92, 0xc8, 0x19, 0xbb, |
|
| 2962 |
- 0x3a, 0x9f, 0x97, 0x39, 0x92, 0x68, 0xdf, 0x24, 0x18, 0xea, 0xeb, 0x76, 0x99, 0x13, 0x3d, 0x97, |
|
| 2963 |
- 0xb1, 0xa5, 0x85, 0x8c, 0xf5, 0xe2, 0xb0, 0xfe, 0x57, 0x56, 0xfb, 0x55, 0x0e, 0xaa, 0x71, 0x8f, |
|
| 2964 |
- 0x80, 0x37, 0x95, 0x3b, 0x14, 0xbe, 0x72, 0x47, 0xae, 0x90, 0x66, 0xe1, 0x0c, 0x87, 0xdd, 0x83, |
|
| 2965 |
- 0x92, 0xad, 0x94, 0x8c, 0xcf, 0xff, 0xb7, 0xb3, 0x0d, 0xc6, 0xe6, 0x0e, 0x4a, 0xba, 0xbe, 0x92, |
|
| 2966 |
- 0x33, 0xae, 0x51, 0x2b, 0x9f, 0x00, 0xa4, 0x4c, 0xb4, 0xf5, 0x4c, 0xcc, 0x8c, 0x56, 0x24, 0xd9, |
|
| 2967 |
- 0x0d, 0x28, 0x3d, 0xb7, 0xbd, 0x69, 0x5c, 0xcc, 0x7a, 0xf0, 0x20, 0xff, 0x49, 0xce, 0xfa, 0x43, |
|
| 2968 |
- 0x1e, 0x2a, 0xa6, 0xe1, 0x60, 0x77, 0xa1, 0x42, 0x0d, 0x87, 0xb1, 0xe8, 0xea, 0xca, 0x8d, 0x21, |
|
| 2969 |
- 0x6c, 0x2b, 0xe9, 0xa4, 0x32, 0x36, 0x1a, 0x55, 0xba, 0xa3, 0x32, 0x36, 0xa6, 0x7d, 0x55, 0x61, |
|
| 2970 |
- 0x28, 0x46, 0xa6, 0x65, 0x6a, 0x52, 0x83, 0x22, 0x46, 0xae, 0xef, 0x62, 0x7c, 0x38, 0x8a, 0xd8, |
|
| 2971 |
- 0xdd, 0xd8, 0xeb, 0x22, 0x69, 0x7c, 0x2b, 0xab, 0xf1, 0xb2, 0xd3, 0x3d, 0xa8, 0x67, 0x96, 0xb9, |
|
| 2972 |
- 0xc2, 0xeb, 0xf7, 0xb3, 0x5e, 0x9b, 0x25, 0x49, 0x9d, 0xee, 0xf7, 0xd2, 0x28, 0xfc, 0x1b, 0xf1, |
|
| 2973 |
- 0xfb, 0x18, 0x20, 0x55, 0xf9, 0xc3, 0x4f, 0x3e, 0xeb, 0xf7, 0x05, 0x80, 0x7e, 0x88, 0xd7, 0xe7, |
|
| 2974 |
- 0xd0, 0xa6, 0x0b, 0xbf, 0xe1, 0x8e, 0xfd, 0x40, 0x8a, 0x67, 0x74, 0x42, 0xd0, 0xfc, 0x2a, 0xaf, |
|
| 2975 |
- 0x6b, 0x1e, 0x55, 0x0c, 0xdb, 0x81, 0xfa, 0x50, 0x44, 0x8e, 0x74, 0x29, 0xa1, 0x4c, 0xd0, 0x6f, |
|
| 2976 |
- 0xa1, 0x4f, 0xa9, 0x9e, 0xcd, 0x4e, 0x8a, 0xd0, 0xb1, 0xca, 0xce, 0x61, 0xdb, 0xd0, 0x10, 0x17, |
|
| 2977 |
- 0x61, 0x20, 0x95, 0x59, 0x45, 0xf7, 0xa5, 0xd7, 0x74, 0x87, 0x8b, 0x7c, 0x7d, 0x02, 0xd4, 0x45, |
|
| 2978 |
- 0x3a, 0x60, 0x36, 0x14, 0x1d, 0x3b, 0x8c, 0x4c, 0x37, 0xd0, 0x5e, 0x58, 0x6f, 0xcf, 0x0e, 0x75, |
|
| 2979 |
- 0xd0, 0x76, 0x3f, 0x42, 0x5f, 0x7f, 0xf1, 0x97, 0x5b, 0x77, 0x32, 0x2d, 0xd4, 0x24, 0x38, 0x99, |
|
| 2980 |
- 0x6d, 0x51, 0xbe, 0x9c, 0xb9, 0x6a, 0x6b, 0xaa, 0x5c, 0x6f, 0xcb, 0x0e, 0x5d, 0x54, 0x87, 0x13, |
|
| 2981 |
- 0x7b, 0x1d, 0x4e, 0xaa, 0xd9, 0x27, 0xd0, 0x0c, 0x65, 0x30, 0x96, 0x22, 0x8a, 0x9e, 0xd1, 0x85, |
|
| 2982 |
- 0x6a, 0x1a, 0xdd, 0x37, 0xcc, 0xc5, 0x4f, 0x92, 0xcf, 0x50, 0xc0, 0x97, 0xc3, 0xec, 0x70, 0xe5, |
|
| 2983 |
- 0x47, 0xd0, 0x5a, 0xf4, 0xf8, 0x75, 0x76, 0x6f, 0xe5, 0x3e, 0xd4, 0x12, 0x0f, 0x5e, 0x35, 0xb1, |
|
| 2984 |
- 0x9a, 0xdd, 0xf6, 0xdf, 0xe5, 0xa0, 0xac, 0xeb, 0x91, 0xdd, 0x87, 0x9a, 0x17, 0x38, 0x36, 0x1a, |
|
| 2985 |
- 0x10, 0x7f, 0x54, 0xbc, 0x93, 0x96, 0xeb, 0xe6, 0xa3, 0x58, 0xa6, 0xf7, 0x23, 0xc5, 0x62, 0x7a, |
|
| 2986 |
- 0xba, 0xfe, 0x28, 0x88, 0xeb, 0xa7, 0x99, 0x4e, 0xea, 0xf9, 0xa3, 0x80, 0x6b, 0xe1, 0xca, 0x43, |
|
| 2987 |
- 0x68, 0xce, 0xab, 0xb8, 0xc2, 0xce, 0xf7, 0xe6, 0x13, 0x9d, 0x2e, 0x92, 0x64, 0x52, 0xd6, 0xec, |
|
| 2988 |
- 0xfb, 0x50, 0x4b, 0xf8, 0x6c, 0xe3, 0xb2, 0xe1, 0x8d, 0xec, 0xcc, 0x8c, 0xad, 0xd6, 0x2f, 0x73, |
|
| 2989 |
- 0x00, 0xa9, 0x6d, 0x78, 0xce, 0xe1, 0xe7, 0x8b, 0x9f, 0x36, 0x1e, 0xc9, 0x98, 0xee, 0x6d, 0x5b, |
|
| 2990 |
- 0xd9, 0x64, 0x4b, 0x83, 0x13, 0xcd, 0x36, 0x01, 0x86, 0x49, 0xad, 0xbf, 0xe4, 0x04, 0xc8, 0x20, |
|
| 2991 |
- 0x50, 0xbf, 0x67, 0xfb, 0xe3, 0xa9, 0x3d, 0x16, 0xa6, 0x3b, 0x4c, 0xc6, 0x56, 0x1f, 0xaa, 0xb1, |
|
| 2992 |
- 0x85, 0x6c, 0x0d, 0xea, 0x91, 0xb1, 0x0a, 0x3b, 0x70, 0x34, 0xa5, 0xc4, 0xb3, 0x2c, 0xec, 0xa4, |
|
| 2993 |
- 0xa5, 0xed, 0x8f, 0xc5, 0x5c, 0x27, 0xcd, 0x91, 0xc3, 0x8d, 0xc0, 0xfa, 0x02, 0x4a, 0xc4, 0xc0, |
|
| 2994 |
- 0xea, 0x8d, 0x94, 0x2d, 0x95, 0x69, 0xca, 0x75, 0xdf, 0x19, 0x44, 0x64, 0xd2, 0x6e, 0x11, 0xf3, |
|
| 2995 |
- 0x9b, 0x6b, 0x00, 0x7b, 0x1f, 0xbb, 0xdb, 0xa1, 0x09, 0xf7, 0x55, 0x38, 0x14, 0x5b, 0x9f, 0x42, |
|
| 2996 |
- 0x35, 0x66, 0x63, 0x54, 0x3c, 0xd7, 0x17, 0xc6, 0x44, 0xa2, 0xf1, 0x63, 0xc6, 0x39, 0xb5, 0xa5, |
|
| 2997 |
- 0xed, 0x28, 0xa1, 0xdb, 0x9f, 0x12, 0x4f, 0x19, 0xd6, 0x7b, 0x50, 0xcf, 0x14, 0x25, 0xe6, 0xe2, |
|
| 2998 |
- 0x53, 0xda, 0x63, 0x7d, 0x34, 0xe8, 0x81, 0xf5, 0x19, 0x2c, 0xcf, 0x15, 0x08, 0xde, 0x64, 0xee, |
|
| 2999 |
- 0x30, 0xbe, 0xc9, 0xf4, 0x2d, 0x75, 0xa9, 0x8b, 0x63, 0x50, 0x3c, 0x17, 0xf6, 0x99, 0xe9, 0xe0, |
|
| 3000 |
- 0x88, 0xb6, 0x7e, 0x8b, 0xdf, 0x6c, 0x71, 0x67, 0xfd, 0xbf, 0x00, 0xa7, 0x4a, 0x85, 0xcf, 0xa8, |
|
| 3001 |
- 0xd5, 0x36, 0xca, 0x6a, 0xc8, 0x21, 0x04, 0xbb, 0x05, 0x75, 0x1c, 0x44, 0x46, 0xae, 0x55, 0xd3, |
|
| 3002 |
- 0x8c, 0x48, 0x03, 0xfe, 0x07, 0x6a, 0xa3, 0x64, 0x7a, 0xc1, 0xe4, 0x47, 0x3c, 0xfb, 0x1d, 0xa8, |
|
| 3003 |
- 0xfa, 0x81, 0x91, 0xe9, 0xbd, 0xad, 0xf8, 0x41, 0x32, 0xcf, 0xf6, 0x3c, 0x23, 0x2b, 0xe9, 0x79, |
|
| 3004 |
- 0xb6, 0xe7, 0x91, 0xd0, 0xba, 0x03, 0x6f, 0x5c, 0xfa, 0xfa, 0x64, 0x6f, 0x41, 0x79, 0xe4, 0x7a, |
|
| 3005 |
- 0x8a, 0x6e, 0x2c, 0xfc, 0xd2, 0x30, 0x23, 0xeb, 0x1f, 0x39, 0x80, 0x34, 0xb7, 0xb0, 0x64, 0xf0, |
|
| 3006 |
- 0xea, 0x41, 0x4c, 0x43, 0x5f, 0x35, 0x1e, 0x54, 0x27, 0xe6, 0x10, 0x33, 0x99, 0x71, 0x73, 0x3e, |
|
| 3007 |
- 0x1f, 0x37, 0xe3, 0x33, 0x4e, 0x1f, 0x6f, 0xdb, 0xe6, 0x78, 0x7b, 0x9d, 0x2f, 0xc4, 0x64, 0x05, |
|
| 3008 |
- 0x6a, 0xe0, 0xb2, 0x0f, 0x06, 0x90, 0xd6, 0x3a, 0x37, 0x92, 0x95, 0x87, 0xb0, 0x3c, 0xb7, 0xe4, |
|
| 3009 |
- 0x0f, 0xbc, 0xd0, 0xd2, 0xc3, 0x38, 0x5b, 0xe8, 0xdb, 0x50, 0xd6, 0x2f, 0x0d, 0x6c, 0x1d, 0x2a, |
|
| 3010 |
- 0xb6, 0xa3, 0x6b, 0x3c, 0x73, 0xce, 0xa0, 0x70, 0x87, 0xd8, 0x3c, 0x16, 0x5b, 0x7f, 0xca, 0x03, |
|
| 3011 |
- 0xa4, 0xfc, 0xd7, 0xe8, 0xe2, 0x1f, 0x40, 0x33, 0x12, 0x4e, 0xe0, 0x0f, 0x6d, 0x39, 0x23, 0xa9, |
|
| 3012 |
- 0xf9, 0x14, 0xbe, 0x6a, 0xca, 0x02, 0x32, 0xd3, 0xd1, 0x17, 0x5e, 0xdd, 0xd1, 0xaf, 0x43, 0xd1, |
|
| 3013 |
- 0x09, 0xc2, 0x99, 0xb9, 0xb7, 0xd8, 0xbc, 0x23, 0x7b, 0x41, 0x38, 0x3b, 0x58, 0xe2, 0x84, 0x60, |
|
| 3014 |
- 0x9b, 0x50, 0x9e, 0x9c, 0xd1, 0xdb, 0x8b, 0xfe, 0x86, 0xbc, 0x31, 0x8f, 0x7d, 0x7c, 0x86, 0xf4, |
|
| 3015 |
- 0xc1, 0x12, 0x37, 0x28, 0x76, 0x07, 0x4a, 0x93, 0xb3, 0xa1, 0x2b, 0xcd, 0xcd, 0x73, 0x7d, 0x11, |
|
| 3016 |
- 0xde, 0x71, 0x25, 0x3d, 0xb5, 0x20, 0x86, 0x59, 0x90, 0x97, 0x13, 0xf3, 0xd0, 0xd2, 0x5a, 0x88, |
|
| 3017 |
- 0xe6, 0xe4, 0x60, 0x89, 0xe7, 0xe5, 0x64, 0xb7, 0x0a, 0x65, 0x1d, 0x57, 0xeb, 0xef, 0x05, 0x68, |
|
| 3018 |
- 0xce, 0x5b, 0x89, 0x3b, 0x1b, 0x49, 0x27, 0xde, 0xd9, 0x48, 0x3a, 0xc9, 0xc7, 0x4e, 0x3e, 0xf3, |
|
| 3019 |
- 0xb1, 0x63, 0x41, 0x29, 0x38, 0xf7, 0x85, 0xcc, 0x3e, 0x32, 0xed, 0x9d, 0x06, 0xe7, 0x3e, 0x76, |
|
| 3020 |
- 0xcd, 0x5a, 0x34, 0xd7, 0x84, 0x96, 0x4c, 0x13, 0xfa, 0x3e, 0x2c, 0x8f, 0x02, 0xcf, 0x0b, 0xce, |
|
| 3021 |
- 0x07, 0xb3, 0x89, 0xe7, 0xfa, 0x67, 0xa6, 0x13, 0x9d, 0x67, 0xb2, 0x75, 0xb8, 0x36, 0x74, 0x25, |
|
| 3022 |
- 0x9a, 0x63, 0xba, 0xff, 0x88, 0x7c, 0xaf, 0xf2, 0x45, 0x36, 0xfb, 0x1c, 0xd6, 0x6c, 0xa5, 0xc4, |
|
| 3023 |
- 0x24, 0x54, 0xc7, 0x7e, 0x68, 0x3b, 0x67, 0x9d, 0xc0, 0xa1, 0x2a, 0x9c, 0x84, 0xb6, 0x72, 0x4f, |
|
| 3024 |
- 0x5c, 0xcf, 0x55, 0x33, 0x0a, 0x46, 0x95, 0xbf, 0x12, 0xc7, 0x3e, 0x80, 0xa6, 0x23, 0x85, 0xad, |
|
| 3025 |
- 0x44, 0x47, 0x44, 0xea, 0xc8, 0x56, 0xa7, 0xed, 0x2a, 0xcd, 0x5c, 0xe0, 0xa2, 0x0f, 0x36, 0x5a, |
|
| 3026 |
- 0xfb, 0x85, 0xeb, 0x0d, 0x1d, 0xfc, 0x6c, 0xad, 0x69, 0x1f, 0xe6, 0x98, 0x6c, 0x13, 0x18, 0x31, |
|
| 3027 |
- 0xba, 0x93, 0x50, 0xcd, 0x12, 0x28, 0x10, 0xf4, 0x0a, 0x09, 0x1e, 0xb8, 0xca, 0x9d, 0x88, 0x48, |
|
| 3028 |
- 0xd9, 0x93, 0x90, 0xbe, 0xc8, 0x0b, 0x3c, 0x65, 0xb0, 0xdb, 0xd0, 0x72, 0x7d, 0xc7, 0x9b, 0x0e, |
|
| 3029 |
- 0xc5, 0xb3, 0x10, 0x1d, 0x91, 0x7e, 0xd4, 0x6e, 0xd0, 0xa9, 0x72, 0xcd, 0xf0, 0x8f, 0x0c, 0x1b, |
|
| 3030 |
- 0xa1, 0xe2, 0x62, 0x01, 0xba, 0xac, 0xa1, 0x86, 0x1f, 0x43, 0xad, 0x2f, 0x73, 0xd0, 0x5a, 0x4c, |
|
| 3031 |
- 0x3c, 0xdc, 0xb6, 0x10, 0x9d, 0x37, 0x1f, 0xed, 0x48, 0x27, 0x5b, 0x99, 0xcf, 0x6c, 0x65, 0x7c, |
|
| 3032 |
- 0x97, 0x16, 0x32, 0x77, 0x69, 0x92, 0x16, 0xc5, 0x97, 0xa7, 0xc5, 0x9c, 0xa3, 0xa5, 0x05, 0x47, |
|
| 3033 |
- 0xad, 0xdf, 0xe4, 0xe0, 0xda, 0x42, 0x72, 0xff, 0x60, 0x8b, 0xd6, 0xa0, 0x3e, 0xb1, 0xcf, 0x84, |
|
| 3034 |
- 0x7e, 0xf2, 0x88, 0xcc, 0x15, 0x92, 0x65, 0xfd, 0x07, 0xec, 0xf3, 0xa1, 0x91, 0xad, 0xa8, 0x2b, |
|
| 3035 |
- 0x6d, 0x8b, 0x13, 0xe4, 0x30, 0x50, 0xfb, 0xc1, 0xd4, 0xdc, 0xc5, 0x71, 0x82, 0xc4, 0xcc, 0xcb, |
|
| 3036 |
- 0x69, 0x54, 0xb8, 0x22, 0x8d, 0xac, 0x43, 0xa8, 0xc6, 0x06, 0xb2, 0x5b, 0xe6, 0x4d, 0x2a, 0x97, |
|
| 3037 |
- 0x3e, 0xb5, 0x1e, 0x47, 0x42, 0xa2, 0xed, 0xfa, 0x81, 0xea, 0x5d, 0x28, 0xe9, 0x1e, 0x35, 0x7f, |
|
| 3038 |
- 0x19, 0xa1, 0x25, 0xd6, 0x00, 0x2a, 0x86, 0xc3, 0x36, 0xa0, 0x7c, 0x32, 0x4b, 0xde, 0x67, 0xcc, |
|
| 3039 |
- 0x71, 0x81, 0xe3, 0xa1, 0x41, 0xe0, 0x19, 0xa4, 0x11, 0xec, 0x06, 0x14, 0x4f, 0x66, 0xbd, 0x8e, |
|
| 3040 |
- 0xfe, 0xea, 0xc4, 0x93, 0x0c, 0x47, 0xbb, 0x65, 0x6d, 0x90, 0xf5, 0x08, 0x1a, 0xd9, 0x79, 0xc9, |
|
| 3041 |
- 0xc5, 0x9e, 0xcb, 0x5c, 0xec, 0xc9, 0x91, 0x9d, 0x7f, 0xd5, 0xe7, 0xc7, 0xc7, 0x00, 0xf4, 0x82, |
|
| 3042 |
- 0xfc, 0xba, 0x9f, 0x2d, 0x1f, 0x42, 0xc5, 0xbc, 0x3c, 0xb3, 0x0f, 0x16, 0x5e, 0xd2, 0x9b, 0xc9, |
|
| 3043 |
- 0xb3, 0xf4, 0xdc, 0x73, 0xba, 0xf5, 0x00, 0x1b, 0xd8, 0x73, 0x21, 0x3b, 0xee, 0x68, 0xf4, 0xba, |
|
| 3044 |
- 0xcb, 0x3d, 0x80, 0xe6, 0x71, 0x18, 0xfe, 0x6b, 0x73, 0x7f, 0x0a, 0x65, 0xfd, 0x00, 0x8e, 0x73, |
|
| 3045 |
- 0x3c, 0xb4, 0xc0, 0xec, 0x01, 0xd3, 0x4d, 0x6e, 0xd6, 0x24, 0xae, 0x01, 0x88, 0x9c, 0xe2, 0x7a, |
|
| 3046 |
- 0x66, 0x73, 0x09, 0x39, 0x6f, 0x00, 0xd7, 0x80, 0x8d, 0x75, 0xa8, 0x98, 0xb7, 0x56, 0x56, 0x83, |
|
| 3047 |
- 0xd2, 0xf1, 0xe1, 0xa0, 0xfb, 0xa4, 0xb5, 0xc4, 0xaa, 0x50, 0x3c, 0xe8, 0x0f, 0x9e, 0xb4, 0x72, |
|
| 3048 |
- 0x48, 0x1d, 0xf6, 0x0f, 0xbb, 0xad, 0xfc, 0xc6, 0x6d, 0x68, 0x64, 0x5f, 0x5b, 0x59, 0x1d, 0x2a, |
|
| 3049 |
- 0x83, 0x9d, 0xc3, 0xce, 0x6e, 0xff, 0x27, 0xad, 0x25, 0xd6, 0x80, 0x6a, 0xef, 0x70, 0xd0, 0xdd, |
|
| 3050 |
- 0x3b, 0xe6, 0xdd, 0x56, 0x6e, 0xe3, 0xc7, 0x50, 0x4b, 0x1e, 0xa0, 0x50, 0xc3, 0x6e, 0xef, 0xb0, |
|
| 3051 |
- 0xd3, 0x5a, 0x62, 0x00, 0xe5, 0x41, 0x77, 0x8f, 0x77, 0x51, 0x6f, 0x05, 0x0a, 0x83, 0xc1, 0x41, |
|
| 3052 |
- 0x2b, 0x8f, 0xab, 0xee, 0xed, 0xec, 0x1d, 0x74, 0x5b, 0x05, 0x24, 0x9f, 0x3c, 0x3e, 0xda, 0x1f, |
|
| 3053 |
- 0xb4, 0x8a, 0x1b, 0x1f, 0xc2, 0x1b, 0x97, 0x5e, 0x74, 0x70, 0xc5, 0x4e, 0x77, 0x7f, 0xe7, 0xf8, |
|
| 3054 |
- 0x11, 0x9a, 0x58, 0x86, 0x7c, 0xff, 0x50, 0x2b, 0xea, 0xef, 0xef, 0xb7, 0xf2, 0x1b, 0x1f, 0xc3, |
|
| 3055 |
- 0xb5, 0x85, 0x27, 0x19, 0x5a, 0xf0, 0x60, 0x87, 0x77, 0x71, 0xf1, 0x3a, 0x54, 0x8e, 0x78, 0xef, |
|
| 3056 |
- 0xe9, 0xce, 0x93, 0x6e, 0x2b, 0x87, 0x82, 0x47, 0xfd, 0xbd, 0x87, 0xdd, 0x4e, 0x2b, 0xbf, 0x7b, |
|
| 3057 |
- 0xf3, 0xab, 0x17, 0xab, 0xb9, 0x6f, 0x5e, 0xac, 0xe6, 0xbe, 0x7d, 0xb1, 0x9a, 0xfb, 0xeb, 0x8b, |
|
| 3058 |
- 0xd5, 0xdc, 0x97, 0xdf, 0xaf, 0x2e, 0x7d, 0xf3, 0xfd, 0xea, 0xd2, 0xb7, 0xdf, 0xaf, 0x2e, 0x9d, |
|
| 3059 |
- 0x94, 0xe9, 0x5f, 0x97, 0x8f, 0xfe, 0x19, 0x00, 0x00, 0xff, 0xff, 0xe8, 0xd7, 0x00, 0x96, 0xb5, |
|
| 3060 |
- 0x19, 0x00, 0x00, |
|
| 2899 |
+ 0xa2, 0x30, 0xb4, 0x9c, 0x1e, 0x0a, 0x18, 0xab, 0xe5, 0x90, 0x5a, 0x68, 0xb9, 0xbb, 0x98, 0x1d, |
|
| 2900 |
+ 0x5a, 0x62, 0x0f, 0x3d, 0xf4, 0xd4, 0x63, 0x80, 0x02, 0xbd, 0x15, 0xfd, 0x27, 0x7a, 0x6c, 0x6f, |
|
| 2901 |
+ 0x3d, 0x04, 0xc8, 0x25, 0x40, 0x7b, 0x08, 0x7a, 0x48, 0x0b, 0xe7, 0xd2, 0x7f, 0xa2, 0x40, 0xf1, |
|
| 2902 |
+ 0xde, 0xcc, 0xfe, 0x20, 0x25, 0xc3, 0x71, 0x5b, 0xf4, 0xc4, 0x99, 0xcf, 0xfb, 0xcc, 0x9b, 0x37, |
|
| 2903 |
+ 0x6f, 0xde, 0x9b, 0x79, 0x3b, 0x84, 0x5a, 0x10, 0x46, 0x9b, 0xa1, 0x0c, 0x54, 0xc0, 0xf2, 0xe1, |
|
| 2904 |
+ 0xc9, 0xca, 0xbd, 0xb1, 0xab, 0x4e, 0xa7, 0x27, 0x9b, 0x4e, 0x30, 0xd9, 0x1a, 0x07, 0xe3, 0x60, |
|
| 2905 |
+ 0x8b, 0x44, 0x27, 0xd3, 0x11, 0xf5, 0xa8, 0x43, 0x2d, 0x3d, 0xc4, 0xfa, 0x7b, 0x1e, 0xf2, 0xfd, |
|
| 2906 |
+ 0x90, 0xbd, 0x0b, 0x65, 0xd7, 0x0f, 0xa7, 0x2a, 0x6a, 0xe7, 0xd6, 0x0a, 0xeb, 0xf5, 0xed, 0xda, |
|
| 2907 |
+ 0x66, 0x78, 0xb2, 0xd9, 0x43, 0x84, 0x1b, 0x01, 0x5b, 0x83, 0xa2, 0xb8, 0x10, 0x4e, 0x3b, 0xbf, |
|
| 2908 |
+ 0x96, 0x5b, 0xaf, 0x6f, 0x03, 0x12, 0xba, 0x17, 0xc2, 0xe9, 0x87, 0x07, 0x4b, 0x9c, 0x24, 0xec, |
|
| 2909 |
+ 0x03, 0x28, 0x47, 0xc1, 0x54, 0x3a, 0xa2, 0x5d, 0x20, 0x4e, 0x03, 0x39, 0x03, 0x42, 0x88, 0x65, |
|
| 2910 |
+ 0xa4, 0xa8, 0x69, 0xe4, 0x7a, 0xa2, 0x5d, 0x4c, 0x35, 0xed, 0xbb, 0x9e, 0xe6, 0x90, 0x84, 0xbd, |
|
| 2911 |
+ 0x07, 0xa5, 0x93, 0xa9, 0xeb, 0x0d, 0xdb, 0x25, 0xa2, 0xd4, 0x91, 0xb2, 0x8b, 0x00, 0x71, 0xb4, |
|
| 2912 |
+ 0x0c, 0x49, 0x13, 0x21, 0xc7, 0xa2, 0x5d, 0x4e, 0x49, 0x8f, 0x11, 0xd0, 0x24, 0x92, 0xe1, 0x5c, |
|
| 2913 |
+ 0x43, 0x77, 0x34, 0x6a, 0x57, 0xd2, 0xb9, 0x3a, 0xee, 0x68, 0xa4, 0xe7, 0x42, 0x09, 0x5b, 0x87, |
|
| 2914 |
+ 0x6a, 0xe8, 0xd9, 0x6a, 0x14, 0xc8, 0x49, 0x1b, 0x52, 0xbb, 0x8f, 0x0c, 0xc6, 0x13, 0x29, 0xbb, |
|
| 2915 |
+ 0x0f, 0x75, 0x27, 0xf0, 0x23, 0x25, 0x6d, 0xd7, 0x57, 0x51, 0xbb, 0x4e, 0xe4, 0x37, 0x91, 0xfc, |
|
| 2916 |
+ 0x45, 0x20, 0xcf, 0x84, 0xdc, 0x4b, 0x85, 0x3c, 0xcb, 0xdc, 0x2d, 0x42, 0x3e, 0x08, 0xad, 0x5f, |
|
| 2917 |
+ 0xe7, 0xa0, 0x1a, 0x6b, 0x65, 0x16, 0x34, 0x76, 0xa4, 0x73, 0xea, 0x2a, 0xe1, 0xa8, 0xa9, 0x14, |
|
| 2918 |
+ 0xed, 0xdc, 0x5a, 0x6e, 0xbd, 0xc6, 0xe7, 0x30, 0xd6, 0x84, 0x7c, 0x7f, 0x40, 0xfe, 0xae, 0xf1, |
|
| 2919 |
+ 0x7c, 0x7f, 0xc0, 0xda, 0x50, 0x79, 0x6a, 0x4b, 0xd7, 0xf6, 0x15, 0x39, 0xb8, 0xc6, 0xe3, 0x2e, |
|
| 2920 |
+ 0xbb, 0x09, 0xb5, 0xfe, 0xe0, 0xa9, 0x90, 0x91, 0x1b, 0xf8, 0xe4, 0xd6, 0x1a, 0x4f, 0x01, 0xb6, |
|
| 2921 |
+ 0x0a, 0xd0, 0x1f, 0xec, 0x0b, 0x1b, 0x95, 0x46, 0xed, 0xd2, 0x5a, 0x61, 0xbd, 0xc6, 0x33, 0x88, |
|
| 2922 |
+ 0xf5, 0x73, 0x28, 0xd1, 0x56, 0xb3, 0xcf, 0xa1, 0x3c, 0x74, 0xc7, 0x22, 0x52, 0xda, 0x9c, 0xdd, |
|
| 2923 |
+ 0xed, 0xaf, 0xbe, 0xbb, 0xb5, 0xf4, 0x97, 0xef, 0x6e, 0x6d, 0x64, 0x62, 0x2a, 0x08, 0x85, 0xef, |
|
| 2924 |
+ 0x04, 0xbe, 0xb2, 0x5d, 0x5f, 0xc8, 0x68, 0x6b, 0x1c, 0xdc, 0xd3, 0x43, 0x36, 0x3b, 0xf4, 0xc3, |
|
| 2925 |
+ 0x8d, 0x06, 0x76, 0x1b, 0x4a, 0xae, 0x3f, 0x14, 0x17, 0x64, 0x7f, 0x61, 0xf7, 0xba, 0x51, 0x55, |
|
| 2926 |
+ 0xef, 0x4f, 0x55, 0x38, 0x55, 0x3d, 0x14, 0x71, 0xcd, 0xb0, 0xbe, 0xce, 0x41, 0x59, 0x87, 0x12, |
|
| 2927 |
+ 0xbb, 0x09, 0xc5, 0x89, 0x50, 0x36, 0xcd, 0x5f, 0xdf, 0xae, 0xea, 0x2d, 0x55, 0x36, 0x27, 0x14, |
|
| 2928 |
+ 0xa3, 0x74, 0x12, 0x4c, 0xd1, 0xf7, 0xf9, 0x34, 0x4a, 0x1f, 0x23, 0xc2, 0x8d, 0x80, 0xfd, 0x1f, |
|
| 2929 |
+ 0x54, 0x7c, 0xa1, 0xce, 0x03, 0x79, 0x46, 0x3e, 0x6a, 0xea, 0xb0, 0x38, 0x14, 0xea, 0x71, 0x30, |
|
| 2930 |
+ 0x14, 0x3c, 0x96, 0xb1, 0xbb, 0x50, 0x8d, 0x84, 0x33, 0x95, 0xae, 0x9a, 0x91, 0xbf, 0x9a, 0xdb, |
|
| 2931 |
+ 0x2d, 0x0a, 0x56, 0x83, 0x11, 0x39, 0x61, 0xb0, 0x3b, 0x50, 0x8b, 0x84, 0x23, 0x85, 0x12, 0xfe, |
|
| 2932 |
+ 0x73, 0xf2, 0x5f, 0x7d, 0x7b, 0xd9, 0xd0, 0xa5, 0x50, 0x5d, 0xff, 0x39, 0x4f, 0xe5, 0xd6, 0xd7, |
|
| 2933 |
+ 0x79, 0x28, 0xa2, 0xcd, 0x8c, 0x41, 0xd1, 0x96, 0x63, 0x9d, 0x51, 0x35, 0x4e, 0x6d, 0xd6, 0x82, |
|
| 2934 |
+ 0x02, 0xea, 0xc8, 0x13, 0x84, 0x4d, 0x44, 0x9c, 0xf3, 0xa1, 0xd9, 0x50, 0x6c, 0xe2, 0xb8, 0x69, |
|
| 2935 |
+ 0x24, 0xa4, 0xd9, 0x47, 0x6a, 0xb3, 0xdb, 0x50, 0x0b, 0x65, 0x70, 0x31, 0x7b, 0xa6, 0x2d, 0x48, |
|
| 2936 |
+ 0xa3, 0x14, 0x41, 0x34, 0xa0, 0x1a, 0x9a, 0x16, 0xdb, 0x00, 0x10, 0x17, 0x4a, 0xda, 0x07, 0x41, |
|
| 2937 |
+ 0xa4, 0xa2, 0x76, 0x99, 0xac, 0xa5, 0xb8, 0x47, 0xa0, 0x77, 0xc4, 0x33, 0x52, 0xb6, 0x02, 0xd5, |
|
| 2938 |
+ 0xd3, 0x20, 0x52, 0xbe, 0x3d, 0x11, 0x94, 0x21, 0x35, 0x9e, 0xf4, 0x99, 0x05, 0xe5, 0xa9, 0xe7, |
|
| 2939 |
+ 0x4e, 0x5c, 0xd5, 0xae, 0xa5, 0x3a, 0x8e, 0x09, 0xe1, 0x46, 0x82, 0x51, 0xec, 0x8c, 0x65, 0x30, |
|
| 2940 |
+ 0x0d, 0x8f, 0x6c, 0x29, 0x7c, 0x45, 0xf9, 0x53, 0xe3, 0x73, 0x18, 0xfb, 0x14, 0xde, 0x91, 0x62, |
|
| 2941 |
+ 0x12, 0x3c, 0x17, 0xb4, 0x51, 0x03, 0x35, 0x3d, 0x89, 0x38, 0x3a, 0x36, 0x72, 0x9f, 0x0b, 0xca, |
|
| 2942 |
+ 0xa1, 0x2a, 0x7f, 0x39, 0xc1, 0xba, 0x0b, 0x65, 0x6d, 0x37, 0xba, 0x05, 0x5b, 0x26, 0x53, 0xa8, |
|
| 2943 |
+ 0x8d, 0x19, 0xd2, 0x3b, 0x8a, 0x33, 0xa4, 0x77, 0x64, 0x75, 0xa0, 0xac, 0x2d, 0x44, 0xf6, 0x21, |
|
| 2944 |
+ 0xae, 0xca, 0xb0, 0xb1, 0x8d, 0xd8, 0x20, 0x18, 0x29, 0x1d, 0x91, 0x9c, 0xda, 0xa4, 0xd5, 0x96, |
|
| 2945 |
+ 0xda, 0xff, 0x05, 0x4e, 0x6d, 0xeb, 0x21, 0xd4, 0x92, 0x9d, 0xa5, 0x29, 0x3a, 0x46, 0x4d, 0xbe, |
|
| 2946 |
+ 0xd7, 0xc1, 0x01, 0xe4, 0x2e, 0x3d, 0x29, 0xb5, 0xd1, 0x8d, 0x41, 0xa8, 0xdc, 0xc0, 0xb7, 0x3d, |
|
| 2947 |
+ 0x52, 0x54, 0xe5, 0x49, 0xdf, 0xfa, 0x53, 0x01, 0x4a, 0xb4, 0x30, 0xb6, 0x8e, 0x19, 0x11, 0x4e, |
|
| 2948 |
+ 0xf5, 0x0a, 0x0a, 0xbb, 0xcc, 0x64, 0x04, 0x50, 0xee, 0x25, 0x09, 0x81, 0x79, 0xb8, 0x82, 0xd1, |
|
| 2949 |
+ 0xe9, 0x09, 0x47, 0x05, 0xd2, 0xcc, 0x93, 0xf4, 0x71, 0xfe, 0x21, 0x66, 0xa8, 0x0e, 0x18, 0x6a, |
|
| 2950 |
+ 0xb3, 0x3b, 0x50, 0x0e, 0x28, 0xad, 0x28, 0x66, 0x5e, 0x92, 0x6c, 0x86, 0x82, 0xca, 0xa5, 0xb0, |
|
| 2951 |
+ 0x87, 0x81, 0xef, 0xcd, 0x28, 0x92, 0xaa, 0x3c, 0xe9, 0x63, 0xa0, 0x53, 0x1e, 0x3d, 0x99, 0x85, |
|
| 2952 |
+ 0xfa, 0x58, 0x6d, 0xea, 0x40, 0x7f, 0x1c, 0x83, 0x3c, 0x95, 0xe3, 0xc1, 0xf9, 0x64, 0x12, 0x8e, |
|
| 2953 |
+ 0xa2, 0x7e, 0xa8, 0xda, 0xd7, 0xd3, 0x90, 0x8c, 0x31, 0x9e, 0x48, 0x91, 0xe9, 0xd8, 0xce, 0xa9, |
|
| 2954 |
+ 0x40, 0xe6, 0x8d, 0x94, 0xb9, 0x67, 0x30, 0x9e, 0x48, 0xd3, 0x4c, 0x43, 0xea, 0x9b, 0x44, 0xcd, |
|
| 2955 |
+ 0x64, 0x1a, 0x72, 0x53, 0x39, 0x46, 0xe8, 0x60, 0x70, 0x80, 0xcc, 0xb7, 0xd2, 0xd3, 0x5d, 0x23, |
|
| 2956 |
+ 0xdc, 0x48, 0xf4, 0x6a, 0xa3, 0xa9, 0xa7, 0x7a, 0x9d, 0xf6, 0xdb, 0xda, 0x95, 0x71, 0x9f, 0xfd, |
|
| 2957 |
+ 0x3f, 0x34, 0xf0, 0x24, 0x13, 0xbe, 0x22, 0x4b, 0xda, 0x6d, 0x5a, 0xf0, 0x9b, 0xc9, 0x82, 0xf7, |
|
| 2958 |
+ 0x32, 0x42, 0x3e, 0x47, 0xb5, 0x56, 0xd3, 0xb5, 0xe3, 0x8e, 0x44, 0xee, 0xcf, 0x74, 0xa8, 0x15, |
|
| 2959 |
+ 0x38, 0xb5, 0xad, 0x1e, 0x54, 0xe3, 0xd5, 0x5d, 0x8a, 0xa0, 0x7b, 0x50, 0x89, 0x4e, 0x6d, 0xe9, |
|
| 2960 |
+ 0xfa, 0x63, 0xda, 0xdc, 0xe6, 0xf6, 0xf5, 0xc4, 0x19, 0x03, 0x8d, 0xe3, 0x02, 0x62, 0x8e, 0x15, |
|
| 2961 |
+ 0xc4, 0xd1, 0x78, 0x95, 0xae, 0x16, 0x14, 0xa6, 0xee, 0x90, 0xf4, 0x2c, 0x73, 0x6c, 0x22, 0x32, |
|
| 2962 |
+ 0x76, 0x75, 0x3c, 0x2f, 0x73, 0x6c, 0xa2, 0x7d, 0x93, 0x60, 0xa8, 0xaf, 0xdb, 0x65, 0x4e, 0xed, |
|
| 2963 |
+ 0xb9, 0x88, 0x2d, 0x2d, 0x44, 0xac, 0x17, 0xbb, 0xf5, 0xbf, 0x32, 0xdb, 0xaf, 0x72, 0x50, 0x8d, |
|
| 2964 |
+ 0x6b, 0x04, 0xbc, 0xa9, 0xdc, 0xa1, 0xf0, 0x95, 0x3b, 0x72, 0x85, 0x34, 0x13, 0x67, 0x10, 0x76, |
|
| 2965 |
+ 0x0f, 0x4a, 0xb6, 0x52, 0x32, 0x3e, 0xff, 0xdf, 0xce, 0x16, 0x18, 0x9b, 0x3b, 0x28, 0xe9, 0xfa, |
|
| 2966 |
+ 0x4a, 0xce, 0xb8, 0x66, 0xad, 0x7c, 0x02, 0x90, 0x82, 0x68, 0xeb, 0x99, 0x98, 0x19, 0xad, 0xd8, |
|
| 2967 |
+ 0x64, 0x37, 0xa0, 0xf4, 0xdc, 0xf6, 0xa6, 0x71, 0x32, 0xeb, 0xce, 0x83, 0xfc, 0x27, 0x39, 0xeb, |
|
| 2968 |
+ 0x0f, 0x79, 0xa8, 0x98, 0x82, 0x83, 0xdd, 0x85, 0x0a, 0x15, 0x1c, 0xc6, 0xa2, 0xab, 0x33, 0x37, |
|
| 2969 |
+ 0xa6, 0xb0, 0xad, 0xa4, 0x92, 0xca, 0xd8, 0x68, 0x54, 0xe9, 0x8a, 0xca, 0xd8, 0x98, 0xd6, 0x55, |
|
| 2970 |
+ 0x85, 0xa1, 0x18, 0x99, 0x92, 0xa9, 0x49, 0x05, 0x8a, 0x18, 0xb9, 0xbe, 0x8b, 0xfe, 0xe1, 0x28, |
|
| 2971 |
+ 0x62, 0x77, 0xe3, 0x55, 0x17, 0x49, 0xe3, 0x5b, 0x59, 0x8d, 0x97, 0x17, 0xdd, 0x83, 0x7a, 0x66, |
|
| 2972 |
+ 0x9a, 0x2b, 0x56, 0xfd, 0x7e, 0x76, 0xd5, 0x66, 0x4a, 0x52, 0xa7, 0xeb, 0xbd, 0xd4, 0x0b, 0xff, |
|
| 2973 |
+ 0x86, 0xff, 0x3e, 0x06, 0x48, 0x55, 0xfe, 0xf0, 0x93, 0xcf, 0xfa, 0x7d, 0x01, 0xa0, 0x1f, 0xe2, |
|
| 2974 |
+ 0xf5, 0x39, 0xb4, 0xe9, 0xc2, 0x6f, 0xb8, 0x63, 0x3f, 0x90, 0xe2, 0x19, 0x9d, 0x10, 0x34, 0xbe, |
|
| 2975 |
+ 0xca, 0xeb, 0x1a, 0xa3, 0x8c, 0x61, 0x3b, 0x50, 0x1f, 0x8a, 0xc8, 0x91, 0x2e, 0x05, 0x94, 0x71, |
|
| 2976 |
+ 0xfa, 0x2d, 0x5c, 0x53, 0xaa, 0x67, 0xb3, 0x93, 0x32, 0xb4, 0xaf, 0xb2, 0x63, 0xd8, 0x36, 0x34, |
|
| 2977 |
+ 0xc4, 0x45, 0x18, 0x48, 0x65, 0x66, 0xd1, 0x75, 0xe9, 0x35, 0x5d, 0xe1, 0x22, 0xae, 0x4f, 0x80, |
|
| 2978 |
+ 0xba, 0x48, 0x3b, 0xcc, 0x86, 0xa2, 0x63, 0x87, 0x91, 0xa9, 0x06, 0xda, 0x0b, 0xf3, 0xed, 0xd9, |
|
| 2979 |
+ 0xa1, 0x76, 0xda, 0xee, 0x47, 0xb8, 0xd6, 0x5f, 0xfc, 0xf5, 0xd6, 0x9d, 0x4c, 0x09, 0x35, 0x09, |
|
| 2980 |
+ 0x4e, 0x66, 0x5b, 0x14, 0x2f, 0x67, 0xae, 0xda, 0x9a, 0x2a, 0xd7, 0xdb, 0xb2, 0x43, 0x17, 0xd5, |
|
| 2981 |
+ 0xe1, 0xc0, 0x5e, 0x87, 0x93, 0x6a, 0xf6, 0x09, 0x34, 0x43, 0x19, 0x8c, 0xa5, 0x88, 0xa2, 0x67, |
|
| 2982 |
+ 0x74, 0xa1, 0x9a, 0x42, 0xf7, 0x0d, 0x73, 0xf1, 0x93, 0xe4, 0x33, 0x14, 0xf0, 0xe5, 0x30, 0xdb, |
|
| 2983 |
+ 0x5d, 0xf9, 0x11, 0xb4, 0x16, 0x57, 0xfc, 0x3a, 0xbb, 0xb7, 0x72, 0x1f, 0x6a, 0xc9, 0x0a, 0x5e, |
|
| 2984 |
+ 0x35, 0xb0, 0x9a, 0xdd, 0xf6, 0xdf, 0xe5, 0xa0, 0xac, 0xf3, 0x91, 0xdd, 0x87, 0x9a, 0x17, 0x38, |
|
| 2985 |
+ 0x36, 0x1a, 0x10, 0x7f, 0x54, 0xbc, 0x93, 0xa6, 0xeb, 0xe6, 0xa3, 0x58, 0xa6, 0xf7, 0x23, 0xe5, |
|
| 2986 |
+ 0x62, 0x78, 0xba, 0xfe, 0x28, 0x88, 0xf3, 0xa7, 0x99, 0x0e, 0xea, 0xf9, 0xa3, 0x80, 0x6b, 0xe1, |
|
| 2987 |
+ 0xca, 0x43, 0x68, 0xce, 0xab, 0xb8, 0xc2, 0xce, 0xf7, 0xe6, 0x03, 0x9d, 0x2e, 0x92, 0x64, 0x50, |
|
| 2988 |
+ 0xd6, 0xec, 0xfb, 0x50, 0x4b, 0x70, 0xb6, 0x71, 0xd9, 0xf0, 0x46, 0x76, 0x64, 0xc6, 0x56, 0xeb, |
|
| 2989 |
+ 0x97, 0x39, 0x80, 0xd4, 0x36, 0x3c, 0xe7, 0xf0, 0xf3, 0xc5, 0x4f, 0x0b, 0x8f, 0xa4, 0x4f, 0xf7, |
|
| 2990 |
+ 0xb6, 0xad, 0x6c, 0xb2, 0xa5, 0xc1, 0xa9, 0xcd, 0x36, 0x01, 0x86, 0x49, 0xae, 0xbf, 0xe4, 0x04, |
|
| 2991 |
+ 0xc8, 0x30, 0x50, 0xbf, 0x67, 0xfb, 0xe3, 0xa9, 0x3d, 0x16, 0xa6, 0x3a, 0x4c, 0xfa, 0x56, 0x1f, |
|
| 2992 |
+ 0xaa, 0xb1, 0x85, 0x6c, 0x0d, 0xea, 0x91, 0xb1, 0x0a, 0x2b, 0x70, 0x34, 0xa5, 0xc4, 0xb3, 0x10, |
|
| 2993 |
+ 0x56, 0xd2, 0xd2, 0xf6, 0xc7, 0x62, 0xae, 0x92, 0xe6, 0x88, 0x70, 0x23, 0xb0, 0xbe, 0x80, 0x12, |
|
| 2994 |
+ 0x01, 0x98, 0xbd, 0x91, 0xb2, 0xa5, 0x32, 0x45, 0xb9, 0xae, 0x3b, 0x83, 0x88, 0x4c, 0xda, 0x2d, |
|
| 2995 |
+ 0x62, 0x7c, 0x73, 0x4d, 0x60, 0xef, 0x63, 0x75, 0x3b, 0x34, 0xee, 0xbe, 0x8a, 0x87, 0x62, 0xeb, |
|
| 2996 |
+ 0x53, 0xa8, 0xc6, 0x30, 0x7a, 0xc5, 0x73, 0x7d, 0x61, 0x4c, 0xa4, 0x36, 0x7e, 0xcc, 0x38, 0xa7, |
|
| 2997 |
+ 0xb6, 0xb4, 0x1d, 0x25, 0x74, 0xf9, 0x53, 0xe2, 0x29, 0x60, 0xbd, 0x07, 0xf5, 0x4c, 0x52, 0x62, |
|
| 2998 |
+ 0x2c, 0x3e, 0xa5, 0x3d, 0xd6, 0x47, 0x83, 0xee, 0x58, 0x9f, 0xc1, 0xf2, 0x5c, 0x82, 0xe0, 0x4d, |
|
| 2999 |
+ 0xe6, 0x0e, 0xe3, 0x9b, 0x4c, 0xdf, 0x52, 0x97, 0xaa, 0x38, 0x06, 0xc5, 0x73, 0x61, 0x9f, 0x99, |
|
| 3000 |
+ 0x0a, 0x8e, 0xda, 0xd6, 0x6f, 0xf1, 0x9b, 0x2d, 0xae, 0xac, 0xff, 0x17, 0xe0, 0x54, 0xa9, 0xf0, |
|
| 3001 |
+ 0x19, 0x95, 0xda, 0x46, 0x59, 0x0d, 0x11, 0x62, 0xb0, 0x5b, 0x50, 0xc7, 0x4e, 0x64, 0xe4, 0x5a, |
|
| 3002 |
+ 0x35, 0x8d, 0x88, 0x34, 0xe1, 0x7f, 0xa0, 0x36, 0x4a, 0x86, 0x17, 0x4c, 0x7c, 0xc4, 0xa3, 0xdf, |
|
| 3003 |
+ 0x81, 0xaa, 0x1f, 0x18, 0x99, 0xde, 0xdb, 0x8a, 0x1f, 0x24, 0xe3, 0x6c, 0xcf, 0x33, 0xb2, 0x92, |
|
| 3004 |
+ 0x1e, 0x67, 0x7b, 0x1e, 0x09, 0xad, 0x3b, 0xf0, 0xc6, 0xa5, 0xaf, 0x4f, 0xf6, 0x16, 0x94, 0x47, |
|
| 3005 |
+ 0xae, 0xa7, 0xe8, 0xc6, 0xc2, 0x2f, 0x0d, 0xd3, 0xb3, 0xfe, 0x91, 0x03, 0x48, 0x63, 0x0b, 0x53, |
|
| 3006 |
+ 0x06, 0xaf, 0x1e, 0xe4, 0x34, 0xf4, 0x55, 0xe3, 0x41, 0x75, 0x62, 0x0e, 0x31, 0x13, 0x19, 0x37, |
|
| 3007 |
+ 0xe7, 0xe3, 0x71, 0x33, 0x3e, 0xe3, 0xf4, 0xf1, 0xb6, 0x6d, 0x8e, 0xb7, 0xd7, 0xf9, 0x42, 0x4c, |
|
| 3008 |
+ 0x66, 0xa0, 0x02, 0x2e, 0xfb, 0x60, 0x00, 0x69, 0xae, 0x73, 0x23, 0x59, 0x79, 0x08, 0xcb, 0x73, |
|
| 3009 |
+ 0x53, 0xfe, 0xc0, 0x0b, 0x2d, 0x3d, 0x8c, 0xb3, 0x89, 0xbe, 0x0d, 0x65, 0xfd, 0xd2, 0xc0, 0xd6, |
|
| 3010 |
+ 0xa1, 0x62, 0x3b, 0x3a, 0xc7, 0x33, 0xe7, 0x0c, 0x0a, 0x77, 0x08, 0xe6, 0xb1, 0xd8, 0xfa, 0x73, |
|
| 3011 |
+ 0x1e, 0x20, 0xc5, 0x5f, 0xa3, 0x8a, 0x7f, 0x00, 0xcd, 0x48, 0x38, 0x81, 0x3f, 0xb4, 0xe5, 0x8c, |
|
| 3012 |
+ 0xa4, 0xe6, 0x53, 0xf8, 0xaa, 0x21, 0x0b, 0xcc, 0x4c, 0x45, 0x5f, 0x78, 0x75, 0x45, 0xbf, 0x0e, |
|
| 3013 |
+ 0x45, 0x27, 0x08, 0x67, 0xe6, 0xde, 0x62, 0xf3, 0x0b, 0xd9, 0x0b, 0xc2, 0xd9, 0xc1, 0x12, 0x27, |
|
| 3014 |
+ 0x06, 0xdb, 0x84, 0xf2, 0xe4, 0x8c, 0xde, 0x5e, 0xf4, 0x37, 0xe4, 0x8d, 0x79, 0xee, 0xe3, 0x33, |
|
| 3015 |
+ 0x6c, 0x1f, 0x2c, 0x71, 0xc3, 0x62, 0x77, 0xa0, 0x34, 0x39, 0x1b, 0xba, 0xd2, 0xdc, 0x3c, 0xd7, |
|
| 3016 |
+ 0x17, 0xe9, 0x1d, 0x57, 0xd2, 0x53, 0x0b, 0x72, 0x98, 0x05, 0x79, 0x39, 0x31, 0x0f, 0x2d, 0xad, |
|
| 3017 |
+ 0x05, 0x6f, 0x4e, 0x0e, 0x96, 0x78, 0x5e, 0x4e, 0x76, 0xab, 0x50, 0xd6, 0x7e, 0xb5, 0xfe, 0x58, |
|
| 3018 |
+ 0x84, 0xe6, 0xbc, 0x95, 0xb8, 0xb3, 0x91, 0x74, 0xe2, 0x9d, 0x8d, 0xa4, 0x93, 0x7c, 0xec, 0xe4, |
|
| 3019 |
+ 0x33, 0x1f, 0x3b, 0x16, 0x94, 0x82, 0x73, 0x5f, 0xc8, 0xec, 0x23, 0xd3, 0xde, 0x69, 0x70, 0xee, |
|
| 3020 |
+ 0x63, 0xd5, 0xac, 0x45, 0x73, 0x45, 0x68, 0xc9, 0x14, 0xa1, 0xef, 0xc3, 0xf2, 0x28, 0xf0, 0xbc, |
|
| 3021 |
+ 0xe0, 0x7c, 0x30, 0x9b, 0x78, 0xae, 0x7f, 0x66, 0x2a, 0xd1, 0x79, 0x90, 0xad, 0xc3, 0xb5, 0xa1, |
|
| 3022 |
+ 0x2b, 0xd1, 0x1c, 0x53, 0xfd, 0x47, 0xb4, 0xf6, 0x2a, 0x5f, 0x84, 0xd9, 0xe7, 0xb0, 0x66, 0x2b, |
|
| 3023 |
+ 0x25, 0x26, 0xa1, 0x3a, 0xf6, 0x43, 0xdb, 0x39, 0xeb, 0x04, 0x0e, 0x65, 0xe1, 0x24, 0xb4, 0x95, |
|
| 3024 |
+ 0x7b, 0xe2, 0x7a, 0xae, 0x9a, 0x91, 0x33, 0xaa, 0xfc, 0x95, 0x3c, 0xf6, 0x01, 0x34, 0x1d, 0x29, |
|
| 3025 |
+ 0x6c, 0x25, 0x3a, 0x22, 0x52, 0x47, 0xb6, 0x3a, 0x6d, 0x57, 0x69, 0xe4, 0x02, 0x8a, 0x6b, 0xb0, |
|
| 3026 |
+ 0xd1, 0xda, 0x2f, 0x5c, 0x6f, 0xe8, 0xe0, 0x67, 0x6b, 0x4d, 0xaf, 0x61, 0x0e, 0x64, 0x9b, 0xc0, |
|
| 3027 |
+ 0x08, 0xe8, 0x4e, 0x42, 0x35, 0x4b, 0xa8, 0x40, 0xd4, 0x2b, 0x24, 0x78, 0xe0, 0x2a, 0x77, 0x22, |
|
| 3028 |
+ 0x22, 0x65, 0x4f, 0x42, 0xfa, 0x22, 0x2f, 0xf0, 0x14, 0x60, 0xb7, 0xa1, 0xe5, 0xfa, 0x8e, 0x37, |
|
| 3029 |
+ 0x1d, 0x8a, 0x67, 0x21, 0x2e, 0x44, 0xfa, 0x51, 0xbb, 0x41, 0xa7, 0xca, 0x35, 0x83, 0x1f, 0x19, |
|
| 3030 |
+ 0x18, 0xa9, 0xe2, 0x62, 0x81, 0xba, 0xac, 0xa9, 0x06, 0x4f, 0xa8, 0xfb, 0xb0, 0x6a, 0x7b, 0xe7, |
|
| 3031 |
+ 0xf6, 0x2c, 0xe2, 0x22, 0xf4, 0x6c, 0x47, 0x74, 0x2f, 0xdc, 0x48, 0xb9, 0xfe, 0x38, 0x5e, 0x6a, |
|
| 3032 |
+ 0xd4, 0x6e, 0x92, 0xbd, 0xaf, 0x60, 0x59, 0x5f, 0xe6, 0xa0, 0xb5, 0x18, 0xc0, 0xb8, 0xfd, 0x21, |
|
| 3033 |
+ 0x3a, 0xd1, 0x7c, 0xfc, 0x63, 0x3b, 0x09, 0x89, 0x7c, 0x26, 0x24, 0xe2, 0x3b, 0xb9, 0x90, 0xb9, |
|
| 3034 |
+ 0x93, 0x93, 0xf0, 0x2a, 0xbe, 0x3c, 0xbc, 0xe6, 0x1c, 0x56, 0x5a, 0x70, 0x98, 0xf5, 0x9b, 0x1c, |
|
| 3035 |
+ 0x5c, 0x5b, 0x48, 0x92, 0x1f, 0x6c, 0xd1, 0x1a, 0xd4, 0x27, 0xf6, 0x99, 0xd0, 0x4f, 0x27, 0x91, |
|
| 3036 |
+ 0xb9, 0x8a, 0xb2, 0xd0, 0x7f, 0xc0, 0x3e, 0x1f, 0x1a, 0xd9, 0xcc, 0xbc, 0xd2, 0xb6, 0x38, 0xd0, |
|
| 3037 |
+ 0x0e, 0x03, 0xb5, 0x1f, 0x4c, 0xcd, 0x9d, 0x1e, 0x07, 0x5a, 0x0c, 0x5e, 0x0e, 0xc7, 0xc2, 0x15, |
|
| 3038 |
+ 0xe1, 0x68, 0x1d, 0x42, 0x35, 0x36, 0x90, 0xdd, 0x32, 0x6f, 0x5b, 0xb9, 0xf4, 0xc9, 0xf6, 0x38, |
|
| 3039 |
+ 0x12, 0x12, 0x6d, 0xd7, 0x0f, 0x5d, 0xef, 0x42, 0x49, 0xd7, 0xba, 0xf9, 0xcb, 0x0c, 0x2d, 0xb1, |
|
| 3040 |
+ 0x06, 0x50, 0x31, 0x08, 0xdb, 0x80, 0xf2, 0xc9, 0x2c, 0x79, 0xe7, 0x31, 0xc7, 0x0e, 0xf6, 0x87, |
|
| 3041 |
+ 0x86, 0x81, 0x67, 0x99, 0x66, 0xb0, 0x1b, 0x50, 0x3c, 0x99, 0xf5, 0x3a, 0xfa, 0xeb, 0x15, 0x4f, |
|
| 3042 |
+ 0x44, 0xec, 0xed, 0x96, 0xb5, 0x41, 0xd6, 0x23, 0x68, 0x64, 0xc7, 0x25, 0x05, 0x42, 0x2e, 0x53, |
|
| 3043 |
+ 0x20, 0x24, 0x47, 0x7f, 0xfe, 0x55, 0x9f, 0x31, 0x1f, 0x03, 0xd0, 0x4b, 0xf4, 0xeb, 0x7e, 0xfe, |
|
| 3044 |
+ 0x7c, 0x08, 0x15, 0xf3, 0x82, 0xcd, 0x3e, 0x58, 0x78, 0x91, 0x6f, 0x26, 0xcf, 0xdb, 0x73, 0xcf, |
|
| 3045 |
+ 0xf2, 0xd6, 0x03, 0x2c, 0x84, 0xcf, 0x85, 0xec, 0xb8, 0xa3, 0xd1, 0xeb, 0x4e, 0xf7, 0x00, 0x9a, |
|
| 3046 |
+ 0xc7, 0x61, 0xf8, 0xaf, 0x8d, 0xfd, 0x29, 0x94, 0xf5, 0x43, 0x3a, 0x8e, 0xf1, 0xd0, 0x02, 0xb3, |
|
| 3047 |
+ 0x07, 0x4c, 0x17, 0xcb, 0x59, 0x93, 0xb8, 0x26, 0x20, 0x73, 0x8a, 0xf3, 0x99, 0xcd, 0x25, 0xe6, |
|
| 3048 |
+ 0xbc, 0x01, 0x5c, 0x13, 0x36, 0xd6, 0xa1, 0x62, 0xde, 0x6c, 0x59, 0x0d, 0x4a, 0xc7, 0x87, 0x83, |
|
| 3049 |
+ 0xee, 0x93, 0xd6, 0x12, 0xab, 0x42, 0xf1, 0xa0, 0x3f, 0x78, 0xd2, 0xca, 0x61, 0xeb, 0xb0, 0x7f, |
|
| 3050 |
+ 0xd8, 0x6d, 0xe5, 0x37, 0x6e, 0x43, 0x23, 0xfb, 0x6a, 0xcb, 0xea, 0x50, 0x19, 0xec, 0x1c, 0x76, |
|
| 3051 |
+ 0x76, 0xfb, 0x3f, 0x69, 0x2d, 0xb1, 0x06, 0x54, 0x7b, 0x87, 0x83, 0xee, 0xde, 0x31, 0xef, 0xb6, |
|
| 3052 |
+ 0x72, 0x1b, 0x3f, 0x86, 0x5a, 0xf2, 0x90, 0x85, 0x1a, 0x76, 0x7b, 0x87, 0x9d, 0xd6, 0x12, 0x03, |
|
| 3053 |
+ 0x28, 0x0f, 0xba, 0x7b, 0xbc, 0x8b, 0x7a, 0x2b, 0x50, 0x18, 0x0c, 0x0e, 0x5a, 0x79, 0x9c, 0x75, |
|
| 3054 |
+ 0x6f, 0x67, 0xef, 0xa0, 0xdb, 0x2a, 0x60, 0xf3, 0xc9, 0xe3, 0xa3, 0xfd, 0x41, 0xab, 0xb8, 0xf1, |
|
| 3055 |
+ 0x21, 0xbc, 0x71, 0xe9, 0x65, 0x08, 0x67, 0xec, 0x74, 0xf7, 0x77, 0x8e, 0x1f, 0xa1, 0x89, 0x65, |
|
| 3056 |
+ 0xc8, 0xf7, 0x0f, 0xb5, 0xa2, 0xfe, 0xfe, 0x7e, 0x2b, 0xbf, 0xf1, 0x31, 0x5c, 0x5b, 0x78, 0xda, |
|
| 3057 |
+ 0xa1, 0x09, 0x0f, 0x76, 0x78, 0x17, 0x27, 0xaf, 0x43, 0xe5, 0x88, 0xf7, 0x9e, 0xee, 0x3c, 0xe9, |
|
| 3058 |
+ 0xb6, 0x72, 0x28, 0x78, 0xd4, 0xdf, 0x7b, 0xd8, 0xed, 0xb4, 0xf2, 0xbb, 0x37, 0xbf, 0x7a, 0xb1, |
|
| 3059 |
+ 0x9a, 0xfb, 0xe6, 0xc5, 0x6a, 0xee, 0xdb, 0x17, 0xab, 0xb9, 0xbf, 0xbd, 0x58, 0xcd, 0x7d, 0xf9, |
|
| 3060 |
+ 0xfd, 0xea, 0xd2, 0x37, 0xdf, 0xaf, 0x2e, 0x7d, 0xfb, 0xfd, 0xea, 0xd2, 0x49, 0x99, 0xfe, 0xbd, |
|
| 3061 |
+ 0xf9, 0xe8, 0x9f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xef, 0xce, 0x68, 0x38, 0xfd, 0x19, 0x00, 0x00, |
|
| 3061 | 3062 |
} |
| 3062 | 3063 |
|
| 3063 | 3064 |
func (m *Op) Marshal() (dAtA []byte, err error) {
|
| ... | ... |
@@ -4969,6 +4979,16 @@ func (m *FileActionCopy) MarshalToSizedBuffer(dAtA []byte) (int, error) {
|
| 4969 | 4969 |
_ = i |
| 4970 | 4970 |
var l int |
| 4971 | 4971 |
_ = l |
| 4972 |
+ if m.AlwaysReplaceExistingDestPaths {
|
|
| 4973 |
+ i-- |
|
| 4974 |
+ if m.AlwaysReplaceExistingDestPaths {
|
|
| 4975 |
+ dAtA[i] = 1 |
|
| 4976 |
+ } else {
|
|
| 4977 |
+ dAtA[i] = 0 |
|
| 4978 |
+ } |
|
| 4979 |
+ i-- |
|
| 4980 |
+ dAtA[i] = 0x70 |
|
| 4981 |
+ } |
|
| 4972 | 4982 |
if len(m.ExcludePatterns) > 0 {
|
| 4973 | 4983 |
for iNdEx := len(m.ExcludePatterns) - 1; iNdEx >= 0; iNdEx-- {
|
| 4974 | 4984 |
i -= len(m.ExcludePatterns[iNdEx]) |
| ... | ... |
@@ -6463,6 +6483,9 @@ func (m *FileActionCopy) Size() (n int) {
|
| 6463 | 6463 |
n += 1 + l + sovOps(uint64(l)) |
| 6464 | 6464 |
} |
| 6465 | 6465 |
} |
| 6466 |
+ if m.AlwaysReplaceExistingDestPaths {
|
|
| 6467 |
+ n += 2 |
|
| 6468 |
+ } |
|
| 6466 | 6469 |
return n |
| 6467 | 6470 |
} |
| 6468 | 6471 |
|
| ... | ... |
@@ -12391,6 +12414,26 @@ func (m *FileActionCopy) Unmarshal(dAtA []byte) error {
|
| 12391 | 12391 |
} |
| 12392 | 12392 |
m.ExcludePatterns = append(m.ExcludePatterns, string(dAtA[iNdEx:postIndex])) |
| 12393 | 12393 |
iNdEx = postIndex |
| 12394 |
+ case 14: |
|
| 12395 |
+ if wireType != 0 {
|
|
| 12396 |
+ return fmt.Errorf("proto: wrong wireType = %d for field AlwaysReplaceExistingDestPaths", wireType)
|
|
| 12397 |
+ } |
|
| 12398 |
+ var v int |
|
| 12399 |
+ for shift := uint(0); ; shift += 7 {
|
|
| 12400 |
+ if shift >= 64 {
|
|
| 12401 |
+ return ErrIntOverflowOps |
|
| 12402 |
+ } |
|
| 12403 |
+ if iNdEx >= l {
|
|
| 12404 |
+ return io.ErrUnexpectedEOF |
|
| 12405 |
+ } |
|
| 12406 |
+ b := dAtA[iNdEx] |
|
| 12407 |
+ iNdEx++ |
|
| 12408 |
+ v |= int(b&0x7F) << shift |
|
| 12409 |
+ if b < 0x80 {
|
|
| 12410 |
+ break |
|
| 12411 |
+ } |
|
| 12412 |
+ } |
|
| 12413 |
+ m.AlwaysReplaceExistingDestPaths = bool(v != 0) |
|
| 12394 | 12414 |
default: |
| 12395 | 12415 |
iNdEx = preIndex |
| 12396 | 12416 |
skippy, err := skipOps(dAtA[iNdEx:]) |
| ... | ... |
@@ -340,6 +340,8 @@ message FileActionCopy {
|
| 340 | 340 |
repeated string include_patterns = 12; |
| 341 | 341 |
// exclude files/dir matching any of these patterns (even if they match an include pattern) |
| 342 | 342 |
repeated string exclude_patterns = 13; |
| 343 |
+ // alwaysReplaceExistingDestPaths results in an existing dest path that differs in type from the src path being replaced rather than the default of returning an error |
|
| 344 |
+ bool alwaysReplaceExistingDestPaths = 14; |
|
| 343 | 345 |
} |
| 344 | 346 |
|
| 345 | 347 |
message FileActionMkFile {
|
| ... | ... |
@@ -184,16 +184,47 @@ func (s *scheduler) dispatch(e *edge) {
|
| 184 | 184 |
origEdge := e.index.LoadOrStore(k, e) |
| 185 | 185 |
if origEdge != nil {
|
| 186 | 186 |
if e.isDep(origEdge) || origEdge.isDep(e) {
|
| 187 |
- bklog.G(context.TODO()).Debugf("skip merge due to dependency")
|
|
| 187 |
+ bklog.G(context.TODO()). |
|
| 188 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 189 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 190 |
+ WithField("edge_index", e.edge.Index).
|
|
| 191 |
+ WithField("origEdge_vertex_name", origEdge.edge.Vertex.Name()).
|
|
| 192 |
+ WithField("origEdge_vertex_digest", origEdge.edge.Vertex.Digest()).
|
|
| 193 |
+ WithField("origEdge_index", origEdge.edge.Index).
|
|
| 194 |
+ Debug("skip merge due to dependency")
|
|
| 188 | 195 |
} else {
|
| 189 | 196 |
dest, src := origEdge, e |
| 190 | 197 |
if s.ef.hasOwner(origEdge.edge, e.edge) {
|
| 198 |
+ bklog.G(context.TODO()). |
|
| 199 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 200 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 201 |
+ WithField("edge_index", e.edge.Index).
|
|
| 202 |
+ WithField("origEdge_vertex_name", origEdge.edge.Vertex.Name()).
|
|
| 203 |
+ WithField("origEdge_vertex_digest", origEdge.edge.Vertex.Digest()).
|
|
| 204 |
+ WithField("origEdge_index", origEdge.edge.Index).
|
|
| 205 |
+ Debug("swap merge due to owner")
|
|
| 191 | 206 |
dest, src = src, dest |
| 192 | 207 |
} |
| 193 | 208 |
|
| 194 |
- bklog.G(context.TODO()).Debugf("merging edge %s[%d] to %s[%d]\n", src.edge.Vertex.Name(), src.edge.Index, dest.edge.Vertex.Name(), dest.edge.Index)
|
|
| 209 |
+ bklog.G(context.TODO()). |
|
| 210 |
+ WithField("source_edge_vertex_name", src.edge.Vertex.Name()).
|
|
| 211 |
+ WithField("source_edge_vertex_digest", src.edge.Vertex.Digest()).
|
|
| 212 |
+ WithField("source_edge_index", src.edge.Index).
|
|
| 213 |
+ WithField("dest_vertex_name", dest.edge.Vertex.Name()).
|
|
| 214 |
+ WithField("dest_vertex_digest", dest.edge.Vertex.Digest()).
|
|
| 215 |
+ WithField("dest_index", dest.edge.Index).
|
|
| 216 |
+ Debug("merging edges")
|
|
| 195 | 217 |
if s.mergeTo(dest, src) {
|
| 196 | 218 |
s.ef.setEdge(src.edge, dest) |
| 219 |
+ } else {
|
|
| 220 |
+ bklog.G(context.TODO()). |
|
| 221 |
+ WithField("source_edge_vertex_name", src.edge.Vertex.Name()).
|
|
| 222 |
+ WithField("source_edge_vertex_digest", src.edge.Vertex.Digest()).
|
|
| 223 |
+ WithField("source_edge_index", src.edge.Index).
|
|
| 224 |
+ WithField("dest_vertex_name", dest.edge.Vertex.Name()).
|
|
| 225 |
+ WithField("dest_vertex_digest", dest.edge.Vertex.Digest()).
|
|
| 226 |
+ WithField("dest_index", dest.edge.Index).
|
|
| 227 |
+ Debug("merging edges skipped")
|
|
| 197 | 228 |
} |
| 198 | 229 |
} |
| 199 | 230 |
} |
| ... | ... |
@@ -367,8 +398,13 @@ type pipeFactory struct {
|
| 367 | 367 |
func (pf *pipeFactory) NewInputRequest(ee Edge, req *edgeRequest) pipe.Receiver {
|
| 368 | 368 |
target := pf.s.ef.getEdge(ee) |
| 369 | 369 |
if target == nil {
|
| 370 |
+ bklog.G(context.TODO()). |
|
| 371 |
+ WithField("edge_vertex_name", ee.Vertex.Name()).
|
|
| 372 |
+ WithField("edge_vertex_digest", ee.Vertex.Digest()).
|
|
| 373 |
+ WithField("edge_index", ee.Index).
|
|
| 374 |
+ Error("failed to get edge: inconsistent graph state")
|
|
| 370 | 375 |
return pf.NewFuncRequest(func(_ context.Context) (interface{}, error) {
|
| 371 |
- return nil, errors.Errorf("failed to get edge: inconsistent graph state")
|
|
| 376 |
+ return nil, errors.Errorf("failed to get edge: inconsistent graph state in edge %s %s %d", ee.Vertex.Name(), ee.Vertex.Digest(), ee.Index)
|
|
| 372 | 377 |
}) |
| 373 | 378 |
} |
| 374 | 379 |
p := pf.s.newPipe(target, pf.e, pipe.Request{Payload: req})
|
| ... | ... |
@@ -387,28 +423,58 @@ func (pf *pipeFactory) NewFuncRequest(f func(context.Context) (interface{}, erro
|
| 387 | 387 |
} |
| 388 | 388 |
|
| 389 | 389 |
func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pipe.Receiver) {
|
| 390 |
- log := bklog.G(context.TODO()) |
|
| 391 |
- |
|
| 392 |
- log.Debugf(">> unpark %s req=%d upt=%d out=%d state=%s %s", e.edge.Vertex.Name(), len(inc), len(updates), len(allPipes), e.state, e.edge.Vertex.Digest())
|
|
| 390 |
+ log := bklog.G(context.TODO()). |
|
| 391 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 392 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 393 |
+ WithField("edge_index", e.edge.Index)
|
|
| 394 |
+ |
|
| 395 |
+ log. |
|
| 396 |
+ WithField("edge_state", e.state).
|
|
| 397 |
+ WithField("req", len(inc)).
|
|
| 398 |
+ WithField("upt", len(updates)).
|
|
| 399 |
+ WithField("out", len(allPipes)).
|
|
| 400 |
+ Debug(">> unpark")
|
|
| 393 | 401 |
|
| 394 | 402 |
for i, dep := range e.deps {
|
| 395 | 403 |
des := edgeStatusInitial |
| 396 | 404 |
if dep.req != nil {
|
| 397 | 405 |
des = dep.req.Request().(*edgeRequest).desiredState |
| 398 | 406 |
} |
| 399 |
- log.Debugf(":: dep%d %s state=%s des=%s keys=%d hasslowcache=%v preprocessfunc=%v", i, e.edge.Vertex.Inputs()[i].Vertex.Name(), dep.state, des, len(dep.keys), e.slowCacheFunc(dep) != nil, e.preprocessFunc(dep) != nil)
|
|
| 407 |
+ log. |
|
| 408 |
+ WithField("dep_index", i).
|
|
| 409 |
+ WithField("dep_vertex_name", e.edge.Vertex.Inputs()[i].Vertex.Name()).
|
|
| 410 |
+ WithField("dep_vertex_digest", e.edge.Vertex.Inputs()[i].Vertex.Digest()).
|
|
| 411 |
+ WithField("dep_state", dep.state).
|
|
| 412 |
+ WithField("dep_desired_state", des).
|
|
| 413 |
+ WithField("dep_keys", len(dep.keys)).
|
|
| 414 |
+ WithField("dep_has_slow_cache", e.slowCacheFunc(dep) != nil).
|
|
| 415 |
+ WithField("dep_preprocess_func", e.preprocessFunc(dep) != nil).
|
|
| 416 |
+ Debug(":: dep")
|
|
| 400 | 417 |
} |
| 401 | 418 |
|
| 402 | 419 |
for i, in := range inc {
|
| 403 | 420 |
req := in.Request() |
| 404 |
- log.Debugf("> incoming-%d: %p dstate=%s canceled=%v", i, in, req.Payload.(*edgeRequest).desiredState, req.Canceled)
|
|
| 421 |
+ log. |
|
| 422 |
+ WithField("incoming_index", i).
|
|
| 423 |
+ WithField("incoming_pointer", in).
|
|
| 424 |
+ WithField("incoming_desired_state", req.Payload.(*edgeRequest).desiredState).
|
|
| 425 |
+ WithField("incoming_canceled", req.Canceled).
|
|
| 426 |
+ Debug("> incoming")
|
|
| 405 | 427 |
} |
| 406 | 428 |
|
| 407 | 429 |
for i, up := range updates {
|
| 408 | 430 |
if up == e.cacheMapReq {
|
| 409 |
- log.Debugf("> update-%d: %p cacheMapReq complete=%v", i, up, up.Status().Completed)
|
|
| 431 |
+ log. |
|
| 432 |
+ WithField("update_index", i).
|
|
| 433 |
+ WithField("update_pointer", up).
|
|
| 434 |
+ WithField("update_complete", up.Status().Completed).
|
|
| 435 |
+ Debug("> update cacheMapReq")
|
|
| 410 | 436 |
} else if up == e.execReq {
|
| 411 |
- log.Debugf("> update-%d: %p execReq complete=%v", i, up, up.Status().Completed)
|
|
| 437 |
+ log. |
|
| 438 |
+ WithField("update_index", i).
|
|
| 439 |
+ WithField("update_pointer", up).
|
|
| 440 |
+ WithField("update_complete", up.Status().Completed).
|
|
| 441 |
+ Debug("> update execReq")
|
|
| 412 | 442 |
} else {
|
| 413 | 443 |
st, ok := up.Status().Value.(*edgeState) |
| 414 | 444 |
if ok {
|
| ... | ... |
@@ -416,9 +482,18 @@ func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pip |
| 416 | 416 |
if dep, ok := e.depRequests[up]; ok {
|
| 417 | 417 |
index = int(dep.index) |
| 418 | 418 |
} |
| 419 |
- log.Debugf("> update-%d: %p input-%d keys=%d state=%s", i, up, index, len(st.keys), st.state)
|
|
| 419 |
+ log. |
|
| 420 |
+ WithField("update_index", i).
|
|
| 421 |
+ WithField("update_pointer", up).
|
|
| 422 |
+ WithField("update_complete", up.Status().Completed).
|
|
| 423 |
+ WithField("update_input_index", index).
|
|
| 424 |
+ WithField("update_keys", len(st.keys)).
|
|
| 425 |
+ WithField("update_state", st.state).
|
|
| 426 |
+ Debugf("> update edgeState")
|
|
| 420 | 427 |
} else {
|
| 421 |
- log.Debugf("> update-%d: unknown", i)
|
|
| 428 |
+ log. |
|
| 429 |
+ WithField("update_index", i).
|
|
| 430 |
+ Debug("> update unknown")
|
|
| 422 | 431 |
} |
| 423 | 432 |
} |
| 424 | 433 |
} |
| ... | ... |
@@ -427,7 +502,16 @@ func debugSchedulerPreUnpark(e *edge, inc []pipe.Sender, updates, allPipes []pip |
| 427 | 427 |
func debugSchedulerPostUnpark(e *edge, inc []pipe.Sender) {
|
| 428 | 428 |
log := bklog.G(context.TODO()) |
| 429 | 429 |
for i, in := range inc {
|
| 430 |
- log.Debugf("< incoming-%d: %p completed=%v", i, in, in.Status().Completed)
|
|
| 431 |
- } |
|
| 432 |
- log.Debugf("<< unpark %s\n", e.edge.Vertex.Name())
|
|
| 430 |
+ log. |
|
| 431 |
+ WithField("incoming_index", i).
|
|
| 432 |
+ WithField("incoming_pointer", in).
|
|
| 433 |
+ WithField("incoming_complete", in.Status().Completed).
|
|
| 434 |
+ Debug("< incoming")
|
|
| 435 |
+ } |
|
| 436 |
+ log. |
|
| 437 |
+ WithField("edge_vertex_name", e.edge.Vertex.Name()).
|
|
| 438 |
+ WithField("edge_vertex_digest", e.edge.Vertex.Digest()).
|
|
| 439 |
+ WithField("edge_index", e.edge.Index).
|
|
| 440 |
+ WithField("edge_state", e.state).
|
|
| 441 |
+ Debug("<< unpark")
|
|
| 433 | 442 |
} |
| ... | ... |
@@ -112,6 +112,9 @@ type CacheExportOpt struct {
|
| 112 | 112 |
CompressionOpt *compression.Config |
| 113 | 113 |
// ExportRoots defines if records for root vertexes should be exported. |
| 114 | 114 |
ExportRoots bool |
| 115 |
+ // IgnoreBacklinks defines if other cache chains for same result that did not |
|
| 116 |
+ // participate in the current build should be exported. |
|
| 117 |
+ IgnoreBacklinks bool |
|
| 115 | 118 |
} |
| 116 | 119 |
|
| 117 | 120 |
// CacheExporter can export the artifacts of the build chain |
| ... | ... |
@@ -12,6 +12,7 @@ import ( |
| 12 | 12 |
"github.com/moby/buildkit/session" |
| 13 | 13 |
sessioncontent "github.com/moby/buildkit/session/content" |
| 14 | 14 |
"github.com/moby/buildkit/util/imageutil" |
| 15 |
+ "github.com/moby/buildkit/util/iohelper" |
|
| 15 | 16 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| 16 | 17 |
"github.com/pkg/errors" |
| 17 | 18 |
) |
| ... | ... |
@@ -51,7 +52,7 @@ func (r *ociLayoutResolver) Fetch(ctx context.Context, desc ocispecs.Descriptor) |
| 51 | 51 |
if err != nil {
|
| 52 | 52 |
return err |
| 53 | 53 |
} |
| 54 |
- rc = &readerAtWrapper{readerAt: readerAt}
|
|
| 54 |
+ rc = iohelper.ReadCloser(readerAt) |
|
| 55 | 55 |
return nil |
| 56 | 56 |
}) |
| 57 | 57 |
return rc, err |
| ... | ... |
@@ -137,18 +138,3 @@ func (r *ociLayoutResolver) withCaller(ctx context.Context, f func(context.Conte |
| 137 | 137 |
return f(ctx, caller) |
| 138 | 138 |
}) |
| 139 | 139 |
} |
| 140 |
- |
|
| 141 |
-// readerAtWrapper wraps a ReaderAt to give a Reader |
|
| 142 |
-type readerAtWrapper struct {
|
|
| 143 |
- offset int64 |
|
| 144 |
- readerAt content.ReaderAt |
|
| 145 |
-} |
|
| 146 |
- |
|
| 147 |
-func (r *readerAtWrapper) Read(p []byte) (n int, err error) {
|
|
| 148 |
- n, err = r.readerAt.ReadAt(p, r.offset) |
|
| 149 |
- r.offset += int64(n) |
|
| 150 |
- return |
|
| 151 |
-} |
|
| 152 |
-func (r *readerAtWrapper) Close() error {
|
|
| 153 |
- return r.readerAt.Close() |
|
| 154 |
-} |
| ... | ... |
@@ -7,12 +7,12 @@ import ( |
| 7 | 7 |
"time" |
| 8 | 8 |
|
| 9 | 9 |
"github.com/containerd/containerd/content" |
| 10 |
- containerderrdefs "github.com/containerd/containerd/errdefs" |
|
| 11 | 10 |
"github.com/containerd/containerd/images" |
| 12 | 11 |
"github.com/containerd/containerd/leases" |
| 13 | 12 |
"github.com/containerd/containerd/remotes" |
| 14 | 13 |
"github.com/containerd/containerd/remotes/docker" |
| 15 | 14 |
"github.com/containerd/containerd/snapshots" |
| 15 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 16 | 16 |
"github.com/moby/buildkit/cache" |
| 17 | 17 |
"github.com/moby/buildkit/client" |
| 18 | 18 |
"github.com/moby/buildkit/client/llb/sourceresolver" |
| ... | ... |
@@ -58,7 +58,7 @@ type puller struct {
|
| 58 | 58 |
*pull.Puller |
| 59 | 59 |
} |
| 60 | 60 |
|
| 61 |
-func mainManifestKey(ctx context.Context, desc ocispecs.Descriptor, platform ocispecs.Platform, layerLimit *int) (digest.Digest, error) {
|
|
| 61 |
+func mainManifestKey(desc ocispecs.Descriptor, platform ocispecs.Platform, layerLimit *int) (digest.Digest, error) {
|
|
| 62 | 62 |
dt, err := json.Marshal(struct {
|
| 63 | 63 |
Digest digest.Digest |
| 64 | 64 |
OS string |
| ... | ... |
@@ -164,7 +164,7 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach |
| 164 | 164 |
} |
| 165 | 165 |
|
| 166 | 166 |
desc := p.manifest.MainManifestDesc |
| 167 |
- k, err := mainManifestKey(ctx, desc, p.Platform, p.layerLimit) |
|
| 167 |
+ k, err := mainManifestKey(desc, p.Platform, p.layerLimit) |
|
| 168 | 168 |
if err != nil {
|
| 169 | 169 |
return struct{}{}, err
|
| 170 | 170 |
} |
| ... | ... |
@@ -249,7 +249,7 @@ func (p *puller) Snapshot(ctx context.Context, g session.Group) (ir cache.Immuta |
| 249 | 249 |
} |
| 250 | 250 |
|
| 251 | 251 |
for _, desc := range p.manifest.Nonlayers {
|
| 252 |
- if _, err := p.ContentStore.Info(ctx, desc.Digest); containerderrdefs.IsNotFound(err) {
|
|
| 252 |
+ if _, err := p.ContentStore.Info(ctx, desc.Digest); cerrdefs.IsNotFound(err) {
|
|
| 253 | 253 |
// manifest or config must have gotten gc'd after CacheKey, re-pull them |
| 254 | 254 |
ctx, done, err := leaseutil.WithLease(ctx, p.LeaseManager, leaseutil.MakeTemporary) |
| 255 | 255 |
if err != nil {
|
| ... | ... |
@@ -312,7 +312,7 @@ func (gs *gitSourceHandler) mountSSHAuthSock(ctx context.Context, sshID string, |
| 312 | 312 |
return sock, cleanup, nil |
| 313 | 313 |
} |
| 314 | 314 |
|
| 315 |
-func (gs *gitSourceHandler) mountKnownHosts(ctx context.Context) (string, func() error, error) {
|
|
| 315 |
+func (gs *gitSourceHandler) mountKnownHosts() (string, func() error, error) {
|
|
| 316 | 316 |
if gs.src.KnownSSHHosts == "" {
|
| 317 | 317 |
return "", nil, errors.Errorf("no configured known hosts forwarded from the client")
|
| 318 | 318 |
} |
| ... | ... |
@@ -440,9 +440,6 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out |
| 440 | 440 |
if err != nil {
|
| 441 | 441 |
return nil, err |
| 442 | 442 |
} |
| 443 |
- if err != nil {
|
|
| 444 |
- return nil, err |
|
| 445 |
- } |
|
| 446 | 443 |
defer cleanup() |
| 447 | 444 |
gitDir, err := git.GitDir(ctx) |
| 448 | 445 |
if err != nil {
|
| ... | ... |
@@ -695,7 +692,7 @@ func (gs *gitSourceHandler) gitCli(ctx context.Context, g session.Group, opts .. |
| 695 | 695 |
var knownHosts string |
| 696 | 696 |
if gs.src.KnownSSHHosts != "" {
|
| 697 | 697 |
var unmountKnownHosts func() error |
| 698 |
- knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx) |
|
| 698 |
+ knownHosts, unmountKnownHosts, err = gs.mountKnownHosts() |
|
| 699 | 699 |
if err != nil {
|
| 700 | 700 |
cleanup() |
| 701 | 701 |
return nil, nil, err |
| ... | ... |
@@ -129,7 +129,7 @@ func (e *Engine) evaluatePolicy(ctx context.Context, pol *spb.Policy, srcOp *pb. |
| 129 | 129 |
var deny bool |
| 130 | 130 |
for _, rule := range pol.Rules {
|
| 131 | 131 |
selector := e.selectorCache(rule.Selector) |
| 132 |
- matched, err := match(ctx, selector, ident, srcOp.Attrs) |
|
| 132 |
+ matched, err := match(selector, ident, srcOp.Attrs) |
|
| 133 | 133 |
if err != nil {
|
| 134 | 134 |
return false, errors.Wrap(err, "error matching source policy") |
| 135 | 135 |
} |
| ... | ... |
@@ -8,7 +8,7 @@ import ( |
| 8 | 8 |
"github.com/pkg/errors" |
| 9 | 9 |
) |
| 10 | 10 |
|
| 11 |
-// Source wraps a a protobuf source in order to store cached state such as the compiled regexes. |
|
| 11 |
+// Source wraps a protobuf source in order to store cached state such as the compiled regexes. |
|
| 12 | 12 |
type selectorCache struct {
|
| 13 | 13 |
*spb.Selector |
| 14 | 14 |
|
| ... | ... |
@@ -1,14 +1,13 @@ |
| 1 | 1 |
package sourcepolicy |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
- "context" |
|
| 5 | 4 |
"regexp" |
| 6 | 5 |
|
| 7 | 6 |
spb "github.com/moby/buildkit/sourcepolicy/pb" |
| 8 | 7 |
"github.com/pkg/errors" |
| 9 | 8 |
) |
| 10 | 9 |
|
| 11 |
-func match(ctx context.Context, src *selectorCache, ref string, attrs map[string]string) (bool, error) {
|
|
| 10 |
+func match(src *selectorCache, ref string, attrs map[string]string) (bool, error) {
|
|
| 12 | 11 |
for _, c := range src.Constraints {
|
| 13 | 12 |
if c == nil {
|
| 14 | 13 |
return false, errors.Errorf("invalid nil constraint for %v", src)
|
| ... | ... |
@@ -4,21 +4,28 @@ import ( |
| 4 | 4 |
"sort" |
| 5 | 5 |
"strings" |
| 6 | 6 |
"sync" |
| 7 |
+ "time" |
|
| 7 | 8 |
|
| 8 | 9 |
"github.com/containerd/containerd/platforms" |
| 9 | 10 |
"github.com/moby/buildkit/util/bklog" |
| 10 | 11 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| 11 | 12 |
) |
| 12 | 13 |
|
| 13 |
-var mu sync.Mutex |
|
| 14 |
-var arr []ocispecs.Platform |
|
| 14 |
+var CacheMaxAge = 20 * time.Second |
|
| 15 |
+ |
|
| 16 |
+var ( |
|
| 17 |
+ mu sync.Mutex |
|
| 18 |
+ arr []ocispecs.Platform |
|
| 19 |
+ lastRefresh time.Time |
|
| 20 |
+) |
|
| 15 | 21 |
|
| 16 | 22 |
func SupportedPlatforms(noCache bool) []ocispecs.Platform {
|
| 17 | 23 |
mu.Lock() |
| 18 | 24 |
defer mu.Unlock() |
| 19 |
- if !noCache && arr != nil {
|
|
| 25 |
+ if arr != nil && (!noCache || CacheMaxAge < 0 || time.Since(lastRefresh) < CacheMaxAge) {
|
|
| 20 | 26 |
return arr |
| 21 | 27 |
} |
| 28 |
+ defer func() { lastRefresh = time.Now() }()
|
|
| 22 | 29 |
def := nativePlatform() |
| 23 | 30 |
arr = append([]ocispecs.Platform{}, def)
|
| 24 | 31 |
|
| ... | ... |
@@ -6,7 +6,6 @@ import ( |
| 6 | 6 |
|
| 7 | 7 |
"github.com/containerd/containerd/content" |
| 8 | 8 |
"github.com/containerd/containerd/images" |
| 9 |
- "github.com/docker/docker/pkg/ioutils" |
|
| 10 | 9 |
"github.com/moby/buildkit/util/iohelper" |
| 11 | 10 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| 12 | 11 |
) |
| ... | ... |
@@ -22,8 +21,7 @@ func (c uncompressedType) Decompress(ctx context.Context, cs content.Store, desc |
| 22 | 22 |
if err != nil {
|
| 23 | 23 |
return nil, err |
| 24 | 24 |
} |
| 25 |
- rdr := io.NewSectionReader(ra, 0, ra.Size()) |
|
| 26 |
- return ioutils.NewReadCloserWrapper(rdr, ra.Close), nil |
|
| 25 |
+ return iohelper.ReadCloser(ra), nil |
|
| 27 | 26 |
} |
| 28 | 27 |
|
| 29 | 28 |
func (c uncompressedType) NeedsConversion(ctx context.Context, cs content.Store, desc ocispecs.Descriptor) (bool, error) {
|
| ... | ... |
@@ -9,7 +9,7 @@ import ( |
| 9 | 9 |
"time" |
| 10 | 10 |
|
| 11 | 11 |
"github.com/containerd/containerd/content" |
| 12 |
- "github.com/containerd/containerd/errdefs" |
|
| 12 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 13 | 13 |
digest "github.com/opencontainers/go-digest" |
| 14 | 14 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| 15 | 15 |
"github.com/pkg/errors" |
| ... | ... |
@@ -43,7 +43,7 @@ func (b *buffer) Info(ctx context.Context, dgst digest.Digest) (content.Info, er |
| 43 | 43 |
v, ok := b.infos[dgst] |
| 44 | 44 |
b.mu.Unlock() |
| 45 | 45 |
if !ok {
|
| 46 |
- return content.Info{}, errdefs.ErrNotFound
|
|
| 46 |
+ return content.Info{}, cerrdefs.ErrNotFound
|
|
| 47 | 47 |
} |
| 48 | 48 |
return v, nil |
| 49 | 49 |
} |
| ... | ... |
@@ -54,7 +54,7 @@ func (b *buffer) Update(ctx context.Context, new content.Info, fieldpaths ...str |
| 54 | 54 |
|
| 55 | 55 |
updated, ok := b.infos[new.Digest] |
| 56 | 56 |
if !ok {
|
| 57 |
- return content.Info{}, errdefs.ErrNotFound
|
|
| 57 |
+ return content.Info{}, cerrdefs.ErrNotFound
|
|
| 58 | 58 |
} |
| 59 | 59 |
|
| 60 | 60 |
if len(fieldpaths) == 0 {
|
| ... | ... |
@@ -96,7 +96,7 @@ func (b *buffer) Writer(ctx context.Context, opts ...content.WriterOpt) (content |
| 96 | 96 |
} |
| 97 | 97 |
b.mu.Lock() |
| 98 | 98 |
if _, ok := b.refs[wOpts.Ref]; ok {
|
| 99 |
- return nil, errors.Wrapf(errdefs.ErrUnavailable, "ref %s locked", wOpts.Ref) |
|
| 99 |
+ return nil, errors.Wrapf(cerrdefs.ErrUnavailable, "ref %s locked", wOpts.Ref) |
|
| 100 | 100 |
} |
| 101 | 101 |
b.mu.Unlock() |
| 102 | 102 |
return &bufferedWriter{
|
| ... | ... |
@@ -113,14 +113,14 @@ func (b *buffer) Writer(ctx context.Context, opts ...content.WriterOpt) (content |
| 113 | 113 |
} |
| 114 | 114 |
|
| 115 | 115 |
func (b *buffer) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
|
| 116 |
- r, err := b.getBytesReader(ctx, desc.Digest) |
|
| 116 |
+ r, err := b.getBytesReader(desc.Digest) |
|
| 117 | 117 |
if err != nil {
|
| 118 | 118 |
return nil, err |
| 119 | 119 |
} |
| 120 | 120 |
return &readerAt{Reader: r, Closer: io.NopCloser(r), size: int64(r.Len())}, nil
|
| 121 | 121 |
} |
| 122 | 122 |
|
| 123 |
-func (b *buffer) getBytesReader(ctx context.Context, dgst digest.Digest) (*bytes.Reader, error) {
|
|
| 123 |
+func (b *buffer) getBytesReader(dgst digest.Digest) (*bytes.Reader, error) {
|
|
| 124 | 124 |
b.mu.Lock() |
| 125 | 125 |
defer b.mu.Unlock() |
| 126 | 126 |
|
| ... | ... |
@@ -128,7 +128,7 @@ func (b *buffer) getBytesReader(ctx context.Context, dgst digest.Digest) (*bytes |
| 128 | 128 |
return bytes.NewReader(dt), nil |
| 129 | 129 |
} |
| 130 | 130 |
|
| 131 |
- return nil, errors.Wrapf(errdefs.ErrNotFound, "content %v", dgst) |
|
| 131 |
+ return nil, errors.Wrapf(cerrdefs.ErrNotFound, "content %v", dgst) |
|
| 132 | 132 |
} |
| 133 | 133 |
|
| 134 | 134 |
func (b *buffer) addValue(k digest.Digest, dt []byte) {
|
| ... | ... |
@@ -5,7 +5,7 @@ import ( |
| 5 | 5 |
"sync" |
| 6 | 6 |
|
| 7 | 7 |
"github.com/containerd/containerd/content" |
| 8 |
- "github.com/containerd/containerd/errdefs" |
|
| 8 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 9 | 9 |
"github.com/moby/buildkit/session" |
| 10 | 10 |
digest "github.com/opencontainers/go-digest" |
| 11 | 11 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -51,26 +51,6 @@ func (mp *MultiProvider) SnapshotLabels(descs []ocispecs.Descriptor, index int) |
| 51 | 51 |
return nil |
| 52 | 52 |
} |
| 53 | 53 |
|
| 54 |
-func (mp *MultiProvider) CheckDescriptor(ctx context.Context, desc ocispecs.Descriptor) error {
|
|
| 55 |
- type checkDescriptor interface {
|
|
| 56 |
- CheckDescriptor(context.Context, ocispecs.Descriptor) error |
|
| 57 |
- } |
|
| 58 |
- |
|
| 59 |
- mp.mu.RLock() |
|
| 60 |
- if p, ok := mp.sub[desc.Digest]; ok {
|
|
| 61 |
- mp.mu.RUnlock() |
|
| 62 |
- if cd, ok := p.(checkDescriptor); ok {
|
|
| 63 |
- return cd.CheckDescriptor(ctx, desc) |
|
| 64 |
- } |
|
| 65 |
- } else {
|
|
| 66 |
- mp.mu.RUnlock() |
|
| 67 |
- } |
|
| 68 |
- if cd, ok := mp.base.(checkDescriptor); ok {
|
|
| 69 |
- return cd.CheckDescriptor(ctx, desc) |
|
| 70 |
- } |
|
| 71 |
- return nil |
|
| 72 |
-} |
|
| 73 |
- |
|
| 74 | 54 |
// ReaderAt returns a content.ReaderAt |
| 75 | 55 |
func (mp *MultiProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
|
| 76 | 56 |
mp.mu.RLock() |
| ... | ... |
@@ -80,7 +60,7 @@ func (mp *MultiProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) |
| 80 | 80 |
} |
| 81 | 81 |
mp.mu.RUnlock() |
| 82 | 82 |
if mp.base == nil {
|
| 83 |
- return nil, errors.Wrapf(errdefs.ErrNotFound, "content %v", desc.Digest) |
|
| 83 |
+ return nil, errors.Wrapf(cerrdefs.ErrNotFound, "content %v", desc.Digest) |
|
| 84 | 84 |
} |
| 85 | 85 |
return mp.base.ReaderAt(ctx, desc) |
| 86 | 86 |
} |
| ... | ... |
@@ -94,7 +74,7 @@ func (mp *MultiProvider) Info(ctx context.Context, dgst digest.Digest) (content. |
| 94 | 94 |
} |
| 95 | 95 |
mp.mu.RUnlock() |
| 96 | 96 |
if mp.base == nil {
|
| 97 |
- return content.Info{}, errors.Wrapf(errdefs.ErrNotFound, "content %v", dgst)
|
|
| 97 |
+ return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "content %v", dgst)
|
|
| 98 | 98 |
} |
| 99 | 99 |
return mp.base.Info(ctx, dgst) |
| 100 | 100 |
} |
| ... | ... |
@@ -7,8 +7,8 @@ import ( |
| 7 | 7 |
"time" |
| 8 | 8 |
|
| 9 | 9 |
"github.com/containerd/containerd/content" |
| 10 |
- "github.com/containerd/containerd/errdefs" |
|
| 11 | 10 |
"github.com/containerd/containerd/remotes" |
| 11 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 12 | 12 |
digest "github.com/opencontainers/go-digest" |
| 13 | 13 |
"github.com/pkg/errors" |
| 14 | 14 |
) |
| ... | ... |
@@ -41,7 +41,7 @@ func (i *pushingIngester) Writer(ctx context.Context, opts ...content.WriterOpt) |
| 41 | 41 |
} |
| 42 | 42 |
} |
| 43 | 43 |
if wOpts.Ref == "" {
|
| 44 |
- return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty") |
|
| 44 |
+ return nil, errors.Wrap(cerrdefs.ErrInvalidArgument, "ref must not be empty") |
|
| 45 | 45 |
} |
| 46 | 46 |
|
| 47 | 47 |
st := time.Now() |
| ... | ... |
@@ -50,7 +50,7 @@ func (i *pushingIngester) Writer(ctx context.Context, opts ...content.WriterOpt) |
| 50 | 50 |
for {
|
| 51 | 51 |
if time.Since(st) > time.Hour {
|
| 52 | 52 |
i.mu.Unlock() |
| 53 |
- return nil, errors.Wrapf(errdefs.ErrUnavailable, "ref %v locked", wOpts.Desc.Digest) |
|
| 53 |
+ return nil, errors.Wrapf(cerrdefs.ErrUnavailable, "ref %v locked", wOpts.Desc.Digest) |
|
| 54 | 54 |
} |
| 55 | 55 |
if _, ok := i.active[wOpts.Desc.Digest]; ok {
|
| 56 | 56 |
i.c.Wait() |
| ... | ... |
@@ -6,9 +6,9 @@ import ( |
| 6 | 6 |
"sync" |
| 7 | 7 |
|
| 8 | 8 |
"github.com/containerd/containerd/content" |
| 9 |
- "github.com/containerd/containerd/errdefs" |
|
| 10 | 9 |
"github.com/containerd/containerd/remotes" |
| 11 | 10 |
"github.com/containerd/containerd/remotes/docker" |
| 11 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 12 | 12 |
"github.com/moby/buildkit/version" |
| 13 | 13 |
"github.com/moby/locker" |
| 14 | 14 |
digest "github.com/opencontainers/go-digest" |
| ... | ... |
@@ -70,7 +70,7 @@ func (w *ingester) Writer(ctx context.Context, opts ...content.WriterOpt) (conte |
| 70 | 70 |
} |
| 71 | 71 |
} |
| 72 | 72 |
if wo.Ref == "" {
|
| 73 |
- return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty") |
|
| 73 |
+ return nil, errors.Wrap(cerrdefs.ErrInvalidArgument, "ref must not be empty") |
|
| 74 | 74 |
} |
| 75 | 75 |
w.locker.Lock(wo.Ref) |
| 76 | 76 |
var once sync.Once |
| ... | ... |
@@ -10,9 +10,9 @@ import ( |
| 10 | 10 |
"time" |
| 11 | 11 |
|
| 12 | 12 |
"github.com/containerd/containerd/content" |
| 13 |
- "github.com/containerd/containerd/errdefs" |
|
| 14 | 13 |
"github.com/containerd/containerd/images/converter" |
| 15 | 14 |
"github.com/containerd/containerd/labels" |
| 15 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 16 | 16 |
"github.com/moby/buildkit/identity" |
| 17 | 17 |
"github.com/moby/buildkit/util/bklog" |
| 18 | 18 |
"github.com/moby/buildkit/util/compression" |
| ... | ... |
@@ -148,7 +148,7 @@ func (c *conversion) convert(ctx context.Context, cs content.Store, desc ocispec |
| 148 | 148 |
if c.rewriteTimestamp != nil {
|
| 149 | 149 |
labelz[labelRewrittenTimestamp] = fmt.Sprintf("%d", c.rewriteTimestamp.UTC().Unix())
|
| 150 | 150 |
} |
| 151 |
- if err = w.Commit(ctx, 0, "", content.WithLabels(labelz)); err != nil && !errdefs.IsAlreadyExists(err) {
|
|
| 151 |
+ if err = w.Commit(ctx, 0, "", content.WithLabels(labelz)); err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 152 | 152 |
return nil, err |
| 153 | 153 |
} |
| 154 | 154 |
if err := w.Close(); err != nil {
|
| ... | ... |
@@ -8,6 +8,7 @@ import ( |
| 8 | 8 |
type HeaderConverter func(*tar.Header) |
| 9 | 9 |
|
| 10 | 10 |
// NewReader returns a reader that applies headerConverter. |
| 11 |
+// srcContent is drained until hitting EOF. |
|
| 11 | 12 |
// Forked from https://github.com/moby/moby/blob/v24.0.6/pkg/archive/copy.go#L308-L373 . |
| 12 | 13 |
func NewReader(srcContent io.Reader, headerConverter HeaderConverter) io.ReadCloser {
|
| 13 | 14 |
rebased, w := io.Pipe() |
| ... | ... |
@@ -21,7 +22,14 @@ func NewReader(srcContent io.Reader, headerConverter HeaderConverter) io.ReadClo |
| 21 | 21 |
if err == io.EOF {
|
| 22 | 22 |
// Signals end of archive. |
| 23 | 23 |
rebasedTar.Close() |
| 24 |
- w.Close() |
|
| 24 |
+ // drain the reader into io.Discard, until hitting EOF |
|
| 25 |
+ // https://github.com/moby/buildkit/pull/4807#discussion_r1544621787 |
|
| 26 |
+ _, err = io.Copy(io.Discard, srcContent) |
|
| 27 |
+ if err != nil {
|
|
| 28 |
+ w.CloseWithError(err) |
|
| 29 |
+ } else {
|
|
| 30 |
+ w.Close() |
|
| 31 |
+ } |
|
| 25 | 32 |
return |
| 26 | 33 |
} |
| 27 | 34 |
if err != nil {
|
| ... | ... |
@@ -210,13 +210,23 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error |
| 210 | 210 |
} |
| 211 | 211 |
|
| 212 | 212 |
if err != nil {
|
| 213 |
+ select {
|
|
| 214 |
+ case <-ctx.Done(): |
|
| 215 |
+ cerr := context.Cause(ctx) |
|
| 216 |
+ if cerr != nil {
|
|
| 217 |
+ return buf.Bytes(), errors.Wrapf(cerr, "context completed: git stderr:\n%s", errbuf.String()) |
|
| 218 |
+ } |
|
| 219 |
+ default: |
|
| 220 |
+ } |
|
| 221 |
+ |
|
| 213 | 222 |
if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") {
|
| 214 | 223 |
if newArgs := argsNoDepth(args); len(args) > len(newArgs) {
|
| 215 | 224 |
args = newArgs |
| 216 | 225 |
continue |
| 217 | 226 |
} |
| 218 | 227 |
} |
| 219 |
- return buf.Bytes(), errors.Errorf("git error: %s\nstderr:\n%s", err, errbuf.String())
|
|
| 228 |
+ |
|
| 229 |
+ return buf.Bytes(), errors.Wrapf(err, "git stderr:\n%s", errbuf.String()) |
|
| 220 | 230 |
} |
| 221 | 231 |
return buf.Bytes(), nil |
| 222 | 232 |
} |
| ... | ... |
@@ -4,7 +4,7 @@ import ( |
| 4 | 4 |
"net/url" |
| 5 | 5 |
"strings" |
| 6 | 6 |
|
| 7 |
- "github.com/containerd/containerd/errdefs" |
|
| 7 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 8 | 8 |
"github.com/pkg/errors" |
| 9 | 9 |
) |
| 10 | 10 |
|
| ... | ... |
@@ -58,7 +58,7 @@ func ParseGitRef(ref string) (*GitRef, error) {
|
| 58 | 58 |
) |
| 59 | 59 |
|
| 60 | 60 |
if strings.HasPrefix(ref, "./") || strings.HasPrefix(ref, "../") {
|
| 61 |
- return nil, errdefs.ErrInvalidArgument |
|
| 61 |
+ return nil, cerrdefs.ErrInvalidArgument |
|
| 62 | 62 |
} else if strings.HasPrefix(ref, "github.com/") {
|
| 63 | 63 |
res.IndistinguishableFromLocal = true // Deprecated |
| 64 | 64 |
remote = fromURL(&url.URL{
|
| ... | ... |
@@ -69,7 +69,7 @@ func ParseGitRef(ref string) (*GitRef, error) {
|
| 69 | 69 |
} else {
|
| 70 | 70 |
remote, err = ParseURL(ref) |
| 71 | 71 |
if errors.Is(err, ErrUnknownProtocol) {
|
| 72 |
- remote, err = ParseURL("https://" + ref)
|
|
| 72 |
+ return nil, err |
|
| 73 | 73 |
} |
| 74 | 74 |
if err != nil {
|
| 75 | 75 |
return nil, err |
| ... | ... |
@@ -84,7 +84,7 @@ func ParseGitRef(ref string) (*GitRef, error) {
|
| 84 | 84 |
// An HTTP(S) URL is considered to be a valid git ref only when it has the ".git[...]" suffix. |
| 85 | 85 |
case HTTPProtocol, HTTPSProtocol: |
| 86 | 86 |
if !strings.HasSuffix(remote.Path, ".git") {
|
| 87 |
- return nil, errdefs.ErrInvalidArgument |
|
| 87 |
+ return nil, cerrdefs.ErrInvalidArgument |
|
| 88 | 88 |
} |
| 89 | 89 |
} |
| 90 | 90 |
} |
| ... | ... |
@@ -115,7 +115,7 @@ func Config(ctx context.Context, str string, resolver remotes.Resolver, cache Co |
| 115 | 115 |
} |
| 116 | 116 |
|
| 117 | 117 |
if desc.MediaType == images.MediaTypeDockerSchema1Manifest {
|
| 118 |
- dgst, dt, err := readSchema1Config(ctx, ref.String(), desc, fetcher, cache) |
|
| 118 |
+ dgst, dt, err := readSchema1Config(ctx, desc, fetcher) |
|
| 119 | 119 |
return dgst, dt, err |
| 120 | 120 |
} |
| 121 | 121 |
|
| ... | ... |
@@ -14,7 +14,7 @@ import ( |
| 14 | 14 |
"github.com/pkg/errors" |
| 15 | 15 |
) |
| 16 | 16 |
|
| 17 |
-func readSchema1Config(ctx context.Context, ref string, desc ocispecs.Descriptor, fetcher remotes.Fetcher, cache ContentCache) (digest.Digest, []byte, error) {
|
|
| 17 |
+func readSchema1Config(ctx context.Context, desc ocispecs.Descriptor, fetcher remotes.Fetcher) (digest.Digest, []byte, error) {
|
|
| 18 | 18 |
rc, err := fetcher.Fetch(ctx, desc) |
| 19 | 19 |
if err != nil {
|
| 20 | 20 |
return "", nil, err |
| ... | ... |
@@ -15,18 +15,26 @@ func (w *NopWriteCloser) Close() error {
|
| 15 | 15 |
return nil |
| 16 | 16 |
} |
| 17 | 17 |
|
| 18 |
-type ReadCloser struct {
|
|
| 19 |
- io.ReadCloser |
|
| 20 |
- CloseFunc func() error |
|
| 18 |
+type closeFunc func() error |
|
| 19 |
+ |
|
| 20 |
+func (c closeFunc) Close() error {
|
|
| 21 |
+ return c() |
|
| 21 | 22 |
} |
| 22 | 23 |
|
| 23 |
-func (rc *ReadCloser) Close() error {
|
|
| 24 |
- err1 := rc.ReadCloser.Close() |
|
| 25 |
- err2 := rc.CloseFunc() |
|
| 26 |
- if err1 != nil {
|
|
| 27 |
- return errors.Wrapf(err1, "failed to close: %v", err2) |
|
| 24 |
+// WithCloser returns a ReadCloser with additional closer function. |
|
| 25 |
+func WithCloser(r io.ReadCloser, closer func() error) io.ReadCloser {
|
|
| 26 |
+ var f closeFunc = func() error {
|
|
| 27 |
+ err1 := r.Close() |
|
| 28 |
+ err2 := closer() |
|
| 29 |
+ if err1 != nil {
|
|
| 30 |
+ return errors.Wrapf(err1, "failed to close: %v", err2) |
|
| 31 |
+ } |
|
| 32 |
+ return err2 |
|
| 33 |
+ } |
|
| 34 |
+ return &readCloser{
|
|
| 35 |
+ Reader: r, |
|
| 36 |
+ Closer: f, |
|
| 28 | 37 |
} |
| 29 |
- return err2 |
|
| 30 | 38 |
} |
| 31 | 39 |
|
| 32 | 40 |
type WriteCloser struct {
|
| ... | ... |
@@ -61,3 +69,22 @@ func (c *Counter) Size() (n int64) {
|
| 61 | 61 |
c.mu.Unlock() |
| 62 | 62 |
return |
| 63 | 63 |
} |
| 64 |
+ |
|
| 65 |
+type ReaderAtCloser interface {
|
|
| 66 |
+ io.ReaderAt |
|
| 67 |
+ io.Closer |
|
| 68 |
+ Size() int64 |
|
| 69 |
+} |
|
| 70 |
+ |
|
| 71 |
+// ReadCloser returns a ReadCloser from ReaderAtCloser. |
|
| 72 |
+func ReadCloser(in ReaderAtCloser) io.ReadCloser {
|
|
| 73 |
+ return &readCloser{
|
|
| 74 |
+ Reader: io.NewSectionReader(in, 0, in.Size()), |
|
| 75 |
+ Closer: in, |
|
| 76 |
+ } |
|
| 77 |
+} |
|
| 78 |
+ |
|
| 79 |
+type readCloser struct {
|
|
| 80 |
+ io.Reader |
|
| 81 |
+ io.Closer |
|
| 82 |
+} |
| ... | ... |
@@ -8,19 +8,19 @@ import ( |
| 8 | 8 |
"github.com/pkg/errors" |
| 9 | 9 |
) |
| 10 | 10 |
|
| 11 |
-func createNetNS(c *cniProvider, id string) (string, error) {
|
|
| 11 |
+func createNetNS(_ *cniProvider, _ string) (string, error) {
|
|
| 12 | 12 |
return "", errors.New("creating netns for cni not supported")
|
| 13 | 13 |
} |
| 14 | 14 |
|
| 15 |
-func setNetNS(s *specs.Spec, nativeID string) error {
|
|
| 15 |
+func setNetNS(_ *specs.Spec, _ string) error {
|
|
| 16 | 16 |
return errors.New("enabling netns for cni not supported")
|
| 17 | 17 |
} |
| 18 | 18 |
|
| 19 |
-func unmountNetNS(nativeID string) error {
|
|
| 19 |
+func unmountNetNS(_ string) error {
|
|
| 20 | 20 |
return errors.New("unmounting netns for cni not supported")
|
| 21 | 21 |
} |
| 22 | 22 |
|
| 23 |
-func deleteNetNS(nativeID string) error {
|
|
| 23 |
+func deleteNetNS(_ string) error {
|
|
| 24 | 24 |
return errors.New("deleting netns for cni not supported")
|
| 25 | 25 |
} |
| 26 | 26 |
|
| ... | ... |
@@ -9,7 +9,7 @@ import ( |
| 9 | 9 |
"github.com/pkg/errors" |
| 10 | 10 |
) |
| 11 | 11 |
|
| 12 |
-func createNetNS(_ *cniProvider, id string) (string, error) {
|
|
| 12 |
+func createNetNS(_ *cniProvider, _ string) (string, error) {
|
|
| 13 | 13 |
nsTemplate := hcn.NewNamespace(hcn.NamespaceTypeGuest) |
| 14 | 14 |
ns, err := nsTemplate.Create() |
| 15 | 15 |
if err != nil {
|
| ... | ... |
@@ -34,7 +34,7 @@ func setNetNS(s *specs.Spec, nativeID string) error {
|
| 34 | 34 |
return nil |
| 35 | 35 |
} |
| 36 | 36 |
|
| 37 |
-func unmountNetNS(nativeID string) error {
|
|
| 37 |
+func unmountNetNS(_ string) error {
|
|
| 38 | 38 |
// We don't need to unmount the NS. |
| 39 | 39 |
return nil |
| 40 | 40 |
} |
| ... | ... |
@@ -11,6 +11,6 @@ import ( |
| 11 | 11 |
"github.com/pkg/errors" |
| 12 | 12 |
) |
| 13 | 13 |
|
| 14 |
-func getBridgeProvider(opt cniprovider.Opt) (network.Provider, error) {
|
|
| 14 |
+func getBridgeProvider(_ cniprovider.Opt) (network.Provider, error) {
|
|
| 15 | 15 |
return nil, errors.Errorf("bridge network is not supported on %s yet", runtime.GOOS)
|
| 16 | 16 |
} |
| ... | ... |
@@ -189,7 +189,7 @@ func Changes(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView |
| 189 | 189 |
} |
| 190 | 190 |
|
| 191 | 191 |
// Check if this is a deleted entry |
| 192 |
- isDelete, skip, err := checkDelete(upperdir, path, base, f) |
|
| 192 |
+ isDelete, skip, err := checkDelete(path, base, f) |
|
| 193 | 193 |
if err != nil {
|
| 194 | 194 |
return err |
| 195 | 195 |
} else if skip {
|
| ... | ... |
@@ -216,7 +216,7 @@ func Changes(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView |
| 216 | 216 |
} else if os.IsNotExist(err) || errors.Is(err, unix.ENOTDIR) {
|
| 217 | 217 |
// File doesn't exist in the base layer. Thus this is added. |
| 218 | 218 |
kind = fs.ChangeKindAdd |
| 219 |
- } else if err != nil {
|
|
| 219 |
+ } else {
|
|
| 220 | 220 |
return errors.Wrap(err, "failed to stat base file during overlay diff") |
| 221 | 221 |
} |
| 222 | 222 |
|
| ... | ... |
@@ -247,7 +247,7 @@ func Changes(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView |
| 247 | 247 |
} |
| 248 | 248 |
|
| 249 | 249 |
// checkDelete checks if the specified file is a whiteout |
| 250 |
-func checkDelete(upperdir string, path string, base string, f os.FileInfo) (delete, skip bool, _ error) {
|
|
| 250 |
+func checkDelete(path string, base string, f os.FileInfo) (delete, skip bool, _ error) {
|
|
| 251 | 251 |
if f.Mode()&os.ModeCharDevice != 0 {
|
| 252 | 252 |
if _, ok := f.Sys().(*syscall.Stat_t); ok {
|
| 253 | 253 |
maj, min, err := devices.DeviceInfo(f) |
| ... | ... |
@@ -271,7 +271,7 @@ func checkDelete(upperdir string, path string, base string, f os.FileInfo) (dele |
| 271 | 271 |
return false, false, nil |
| 272 | 272 |
} |
| 273 | 273 |
|
| 274 |
-// checkDelete checks if the specified file is an opaque directory |
|
| 274 |
+// checkOpaque checks if the specified file is an opaque directory |
|
| 275 | 275 |
func checkOpaque(upperdir string, path string, base string, f os.FileInfo) (isOpaque bool, _ error) {
|
| 276 | 276 |
if f.IsDir() {
|
| 277 | 277 |
for _, oKey := range []string{"trusted.overlay.opaque", "user.overlay.opaque"} {
|
| ... | ... |
@@ -155,7 +155,7 @@ func NewDisplay(out io.Writer, mode DisplayMode, opts ...DisplayOpt) (Display, e |
| 155 | 155 |
case PlainMode: |
| 156 | 156 |
return newPlainDisplay(out, opts...), nil |
| 157 | 157 |
case RawJSONMode: |
| 158 |
- return newRawJSONDisplay(out, opts...), nil |
|
| 158 |
+ return newRawJSONDisplay(out), nil |
|
| 159 | 159 |
case QuietMode: |
| 160 | 160 |
return newDiscardDisplay(), nil |
| 161 | 161 |
default: |
| ... | ... |
@@ -283,9 +283,8 @@ type rawJSONDisplay struct {
|
| 283 | 283 |
|
| 284 | 284 |
// newRawJSONDisplay creates a new Display that outputs an unbuffered |
| 285 | 285 |
// output of status update events. |
| 286 |
-func newRawJSONDisplay(w io.Writer, opts ...DisplayOpt) Display {
|
|
| 286 |
+func newRawJSONDisplay(w io.Writer) Display {
|
|
| 287 | 287 |
enc := json.NewEncoder(w) |
| 288 |
- enc.SetIndent("", " ")
|
|
| 289 | 288 |
return Display{
|
| 290 | 289 |
disp: &rawJSONDisplay{
|
| 291 | 290 |
enc: enc, |
| ... | ... |
@@ -744,6 +743,7 @@ func (t *trace) update(s *client.SolveStatus, termWidth int) {
|
| 744 | 744 |
v.jobCached = false |
| 745 | 745 |
if v.term != nil {
|
| 746 | 746 |
if v.term.Width != termWidth {
|
| 747 |
+ termHeight = max(termHeightMin, min(termHeightInitial, v.term.Height-termHeightMin-1)) |
|
| 747 | 748 |
v.term.Resize(termHeight, termWidth-termPad) |
| 748 | 749 |
} |
| 749 | 750 |
v.termBytes += len(l.Data) |
| ... | ... |
@@ -823,7 +823,7 @@ func (t *trace) displayInfo() (d displayInfo) {
|
| 823 | 823 |
} |
| 824 | 824 |
var jobs []*job |
| 825 | 825 |
j := &job{
|
| 826 |
- name: strings.Replace(v.Name, "\t", " ", -1), |
|
| 826 |
+ name: strings.ReplaceAll(v.Name, "\t", " "), |
|
| 827 | 827 |
vertex: v, |
| 828 | 828 |
isCompleted: true, |
| 829 | 829 |
} |
| ... | ... |
@@ -913,7 +913,7 @@ func addTime(tm *time.Time, d time.Duration) *time.Time {
|
| 913 | 913 |
if tm == nil {
|
| 914 | 914 |
return nil |
| 915 | 915 |
} |
| 916 |
- t := (*tm).Add(d) |
|
| 916 |
+ t := tm.Add(d) |
|
| 917 | 917 |
return &t |
| 918 | 918 |
} |
| 919 | 919 |
|
| ... | ... |
@@ -957,6 +957,7 @@ func setupTerminals(jobs []*job, height int, all bool) []*job {
|
| 957 | 957 |
|
| 958 | 958 |
numFree := height - 2 - numInUse |
| 959 | 959 |
numToHide := 0 |
| 960 |
+ termHeight = max(termHeightMin, min(termHeightInitial, height-termHeightMin-1)) |
|
| 960 | 961 |
termLimit := termHeight + 3 |
| 961 | 962 |
|
| 962 | 963 |
for i := 0; numFree > termLimit && i < len(candidates); i++ {
|
| ... | ... |
@@ -13,7 +13,10 @@ var colorCancel aec.ANSI |
| 13 | 13 |
var colorWarning aec.ANSI |
| 14 | 14 |
var colorError aec.ANSI |
| 15 | 15 |
|
| 16 |
-var termHeight = 6 |
|
| 16 |
+const termHeightMin = 6 |
|
| 17 |
+ |
|
| 18 |
+var termHeightInitial = termHeightMin |
|
| 19 |
+var termHeight = termHeightMin |
|
| 17 | 20 |
|
| 18 | 21 |
func init() {
|
| 19 | 22 |
// As recommended on https://no-color.org/ |
| ... | ... |
@@ -43,6 +46,7 @@ func init() {
|
| 43 | 43 |
if termHeightStr != "" {
|
| 44 | 44 |
termHeightVal, err := strconv.Atoi(termHeightStr) |
| 45 | 45 |
if err == nil && termHeightVal > 0 {
|
| 46 |
+ termHeightInitial = termHeightVal |
|
| 46 | 47 |
termHeight = termHeightVal |
| 47 | 48 |
} |
| 48 | 49 |
} |
| ... | ... |
@@ -6,8 +6,8 @@ import ( |
| 6 | 6 |
"time" |
| 7 | 7 |
|
| 8 | 8 |
"github.com/containerd/containerd/content" |
| 9 |
- "github.com/containerd/containerd/errdefs" |
|
| 10 | 9 |
"github.com/containerd/containerd/remotes" |
| 10 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 11 | 11 |
"github.com/moby/buildkit/util/bklog" |
| 12 | 12 |
"github.com/moby/buildkit/util/progress" |
| 13 | 13 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -122,7 +122,7 @@ func trackProgress(ctx context.Context, desc ocispecs.Descriptor, manager PullMa |
| 122 | 122 |
Started: &started, |
| 123 | 123 |
}) |
| 124 | 124 |
continue |
| 125 |
- } else if !errors.Is(err, errdefs.ErrNotFound) {
|
|
| 125 |
+ } else if !errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 126 | 126 |
bklog.G(ctx).Errorf("unexpected error getting ingest status of %q: %v", ingestRef, err)
|
| 127 | 127 |
return |
| 128 | 128 |
} |
| ... | ... |
@@ -8,10 +8,10 @@ import ( |
| 8 | 8 |
"sync" |
| 9 | 9 |
|
| 10 | 10 |
"github.com/containerd/containerd/content" |
| 11 |
- "github.com/containerd/containerd/errdefs" |
|
| 12 | 11 |
"github.com/containerd/containerd/images" |
| 13 | 12 |
"github.com/containerd/containerd/remotes" |
| 14 | 13 |
"github.com/containerd/containerd/remotes/docker" |
| 14 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 15 | 15 |
"github.com/containerd/log" |
| 16 | 16 |
"github.com/distribution/reference" |
| 17 | 17 |
intoto "github.com/in-toto/in-toto-golang/in_toto" |
| ... | ... |
@@ -191,7 +191,7 @@ func annotateDistributionSourceHandler(manager content.Manager, annotations map[ |
| 191 | 191 |
children[i] = child |
| 192 | 192 |
|
| 193 | 193 |
info, err := manager.Info(ctx, child.Digest) |
| 194 |
- if errors.Is(err, errdefs.ErrNotFound) {
|
|
| 194 |
+ if errors.Is(err, cerrdefs.ErrNotFound) {
|
|
| 195 | 195 |
continue |
| 196 | 196 |
} else if err != nil {
|
| 197 | 197 |
return nil, err |
| ... | ... |
@@ -10,10 +10,10 @@ import ( |
| 10 | 10 |
"sync" |
| 11 | 11 |
"time" |
| 12 | 12 |
|
| 13 |
- "github.com/containerd/containerd/errdefs" |
|
| 14 | 13 |
"github.com/containerd/containerd/remotes/docker" |
| 15 | 14 |
"github.com/containerd/containerd/remotes/docker/auth" |
| 16 | 15 |
remoteserrors "github.com/containerd/containerd/remotes/errors" |
| 16 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 17 | 17 |
"github.com/moby/buildkit/session" |
| 18 | 18 |
sessionauth "github.com/moby/buildkit/session/auth" |
| 19 | 19 |
log "github.com/moby/buildkit/util/bklog" |
| ... | ... |
@@ -219,7 +219,7 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R |
| 219 | 219 |
} |
| 220 | 220 |
} |
| 221 | 221 |
} |
| 222 |
- return errors.Wrap(errdefs.ErrNotImplemented, "failed to find supported auth scheme") |
|
| 222 |
+ return errors.Wrap(cerrdefs.ErrNotImplemented, "failed to find supported auth scheme") |
|
| 223 | 223 |
} |
| 224 | 224 |
|
| 225 | 225 |
// authResult is used to control limit rate. |
| ... | ... |
@@ -267,15 +267,15 @@ func newAuthHandler(host string, client *http.Client, scheme auth.Authentication |
| 267 | 267 |
func (ah *authHandler) authorize(ctx context.Context, sm *session.Manager, g session.Group) (string, error) {
|
| 268 | 268 |
switch ah.scheme {
|
| 269 | 269 |
case auth.BasicAuth: |
| 270 |
- return ah.doBasicAuth(ctx) |
|
| 270 |
+ return ah.doBasicAuth() |
|
| 271 | 271 |
case auth.BearerAuth: |
| 272 | 272 |
return ah.doBearerAuth(ctx, sm, g) |
| 273 | 273 |
default: |
| 274 |
- return "", errors.Wrapf(errdefs.ErrNotImplemented, "failed to find supported auth scheme: %s", string(ah.scheme)) |
|
| 274 |
+ return "", errors.Wrapf(cerrdefs.ErrNotImplemented, "failed to find supported auth scheme: %s", string(ah.scheme)) |
|
| 275 | 275 |
} |
| 276 | 276 |
} |
| 277 | 277 |
|
| 278 |
-func (ah *authHandler) doBasicAuth(ctx context.Context) (string, error) {
|
|
| 278 |
+func (ah *authHandler) doBasicAuth() (string, error) {
|
|
| 279 | 279 |
username, secret := ah.common.Username, ah.common.Secret |
| 280 | 280 |
|
| 281 | 281 |
if username == "" || secret == "" {
|
| ... | ... |
@@ -10,12 +10,14 @@ import ( |
| 10 | 10 |
"path/filepath" |
| 11 | 11 |
"runtime" |
| 12 | 12 |
"strings" |
| 13 |
+ "syscall" |
|
| 13 | 14 |
"time" |
| 14 | 15 |
|
| 15 | 16 |
"github.com/containerd/containerd/remotes/docker" |
| 17 |
+ "github.com/pkg/errors" |
|
| 18 |
+ |
|
| 16 | 19 |
"github.com/moby/buildkit/util/resolver/config" |
| 17 | 20 |
"github.com/moby/buildkit/util/tracing" |
| 18 |
- "github.com/pkg/errors" |
|
| 19 | 21 |
) |
| 20 | 22 |
|
| 21 | 23 |
const ( |
| ... | ... |
@@ -46,6 +48,8 @@ func fillInsecureOpts(host string, c config.RegistryConfig, h docker.RegistryHos |
| 46 | 46 |
|
| 47 | 47 |
var transport http.RoundTripper = httpsTransport |
| 48 | 48 |
if isHTTP {
|
| 49 |
+ // TODO: Replace this with [docker.NewHTTPFallback] once |
|
| 50 |
+ // backported to vendored version of containerd |
|
| 49 | 51 |
transport = &httpFallback{super: transport}
|
| 50 | 52 |
} |
| 51 | 53 |
h2.Client = &http.Client{
|
| ... | ... |
@@ -217,10 +221,7 @@ func (f *httpFallback) RoundTrip(r *http.Request) (*http.Response, error) {
|
| 217 | 217 |
// only fall back if the same host had previously fell back |
| 218 | 218 |
if f.host != r.URL.Host {
|
| 219 | 219 |
resp, err := f.super.RoundTrip(r) |
| 220 |
- var tlsErr tls.RecordHeaderError |
|
| 221 |
- if errors.As(err, &tlsErr) && string(tlsErr.RecordHeader[:]) == "HTTP/" {
|
|
| 222 |
- f.host = r.URL.Host |
|
| 223 |
- } else {
|
|
| 220 |
+ if !isTLSError(err) && !isPortError(err, r.URL.Host) {
|
|
| 224 | 221 |
return resp, err |
| 225 | 222 |
} |
| 226 | 223 |
} |
| ... | ... |
@@ -231,5 +232,45 @@ func (f *httpFallback) RoundTrip(r *http.Request) (*http.Response, error) {
|
| 231 | 231 |
plainHTTPRequest := *r |
| 232 | 232 |
plainHTTPRequest.URL = &plainHTTPUrl |
| 233 | 233 |
|
| 234 |
+ if f.host != r.URL.Host {
|
|
| 235 |
+ f.host = r.URL.Host |
|
| 236 |
+ |
|
| 237 |
+ // update body on the second attempt |
|
| 238 |
+ if r.Body != nil && r.GetBody != nil {
|
|
| 239 |
+ body, err := r.GetBody() |
|
| 240 |
+ if err != nil {
|
|
| 241 |
+ return nil, err |
|
| 242 |
+ } |
|
| 243 |
+ plainHTTPRequest.Body = body |
|
| 244 |
+ } |
|
| 245 |
+ } |
|
| 246 |
+ |
|
| 234 | 247 |
return f.super.RoundTrip(&plainHTTPRequest) |
| 235 | 248 |
} |
| 249 |
+ |
|
| 250 |
+func isTLSError(err error) bool {
|
|
| 251 |
+ if err == nil {
|
|
| 252 |
+ return false |
|
| 253 |
+ } |
|
| 254 |
+ var tlsErr tls.RecordHeaderError |
|
| 255 |
+ if errors.As(err, &tlsErr) && string(tlsErr.RecordHeader[:]) == "HTTP/" {
|
|
| 256 |
+ return true |
|
| 257 |
+ } |
|
| 258 |
+ if strings.Contains(err.Error(), "TLS handshake timeout") {
|
|
| 259 |
+ return true |
|
| 260 |
+ } |
|
| 261 |
+ |
|
| 262 |
+ return false |
|
| 263 |
+} |
|
| 264 |
+ |
|
| 265 |
+func isPortError(err error, host string) bool {
|
|
| 266 |
+ if errors.Is(err, syscall.ECONNREFUSED) || os.IsTimeout(err) {
|
|
| 267 |
+ if _, port, _ := net.SplitHostPort(host); port != "" {
|
|
| 268 |
+ // Port is specified, will not retry on different port with scheme change |
|
| 269 |
+ return false |
|
| 270 |
+ } |
|
| 271 |
+ return true |
|
| 272 |
+ } |
|
| 273 |
+ |
|
| 274 |
+ return false |
|
| 275 |
+} |
| ... | ... |
@@ -46,22 +46,23 @@ func Helper() {
|
| 46 | 46 |
func Traces(err error) []*Stack {
|
| 47 | 47 |
var st []*Stack |
| 48 | 48 |
|
| 49 |
- wrapped, ok := err.(interface {
|
|
| 50 |
- Unwrap() error |
|
| 51 |
- }) |
|
| 52 |
- if ok {
|
|
| 53 |
- st = Traces(wrapped.Unwrap()) |
|
| 49 |
+ switch e := err.(type) {
|
|
| 50 |
+ case interface{ Unwrap() error }:
|
|
| 51 |
+ st = Traces(e.Unwrap()) |
|
| 52 |
+ case interface{ Unwrap() []error }:
|
|
| 53 |
+ for _, ue := range e.Unwrap() {
|
|
| 54 |
+ st = Traces(ue) |
|
| 55 |
+ // Only take first stack |
|
| 56 |
+ if len(st) > 0 {
|
|
| 57 |
+ break |
|
| 58 |
+ } |
|
| 59 |
+ } |
|
| 54 | 60 |
} |
| 55 | 61 |
|
| 56 |
- if ste, ok := err.(interface {
|
|
| 57 |
- StackTrace() errors.StackTrace |
|
| 58 |
- }); ok {
|
|
| 62 |
+ switch ste := err.(type) {
|
|
| 63 |
+ case interface{ StackTrace() errors.StackTrace }:
|
|
| 59 | 64 |
st = append(st, convertStack(ste.StackTrace())) |
| 60 |
- } |
|
| 61 |
- |
|
| 62 |
- if ste, ok := err.(interface {
|
|
| 63 |
- StackTrace() *Stack |
|
| 64 |
- }); ok {
|
|
| 65 |
+ case interface{ StackTrace() *Stack }:
|
|
| 65 | 66 |
st = append(st, ste.StackTrace()) |
| 66 | 67 |
} |
| 67 | 68 |
|
| ... | ... |
@@ -71,9 +71,9 @@ func (fs *FS) Open(p string) (io.ReadCloser, error) {
|
| 71 | 71 |
} |
| 72 | 72 |
|
| 73 | 73 |
func convertPathToKey(p string) string {
|
| 74 |
- return strings.Replace(p, "/", "\x00", -1) |
|
| 74 |
+ return strings.ReplaceAll(p, "/", "\x00") |
|
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 | 77 |
func convertKeyToPath(p string) string {
|
| 78 |
- return strings.Replace(p, "\x00", "/", -1) |
|
| 78 |
+ return strings.ReplaceAll(p, "\x00", "/") |
|
| 79 | 79 |
} |
| ... | ... |
@@ -6,11 +6,7 @@ import ( |
| 6 | 6 |
"github.com/agext/levenshtein" |
| 7 | 7 |
) |
| 8 | 8 |
|
| 9 |
-// WrapError wraps error with a suggestion for fixing it |
|
| 10 |
-func WrapError(err error, val string, options []string, caseSensitive bool) error {
|
|
| 11 |
- if err == nil {
|
|
| 12 |
- return nil |
|
| 13 |
- } |
|
| 9 |
+func Search(val string, options []string, caseSensitive bool) (string, bool) {
|
|
| 14 | 10 |
orig := val |
| 15 | 11 |
if !caseSensitive {
|
| 16 | 12 |
val = strings.ToLower(val) |
| ... | ... |
@@ -23,7 +19,7 @@ func WrapError(err error, val string, options []string, caseSensitive bool) erro |
| 23 | 23 |
} |
| 24 | 24 |
if val == opt {
|
| 25 | 25 |
// exact match means error was unrelated to the value |
| 26 |
- return err |
|
| 26 |
+ return "", false |
|
| 27 | 27 |
} |
| 28 | 28 |
dist := levenshtein.Distance(val, opt, nil) |
| 29 | 29 |
if dist < mindist {
|
| ... | ... |
@@ -35,12 +31,25 @@ func WrapError(err error, val string, options []string, caseSensitive bool) erro |
| 35 | 35 |
mindist = dist |
| 36 | 36 |
} |
| 37 | 37 |
} |
| 38 |
+ return match, match != "" |
|
| 39 |
+} |
|
| 38 | 40 |
|
| 39 |
- if match == "" {
|
|
| 40 |
- return err |
|
| 41 |
+// WrapError wraps error with a suggestion for fixing it |
|
| 42 |
+func WrapError(err error, val string, options []string, caseSensitive bool) error {
|
|
| 43 |
+ _, err = WrapErrorMaybe(err, val, options, caseSensitive) |
|
| 44 |
+ return err |
|
| 45 |
+} |
|
| 46 |
+ |
|
| 47 |
+func WrapErrorMaybe(err error, val string, options []string, caseSensitive bool) (bool, error) {
|
|
| 48 |
+ if err == nil {
|
|
| 49 |
+ return false, nil |
|
| 50 |
+ } |
|
| 51 |
+ match, ok := Search(val, options, caseSensitive) |
|
| 52 |
+ if match == "" || !ok {
|
|
| 53 |
+ return false, err |
|
| 41 | 54 |
} |
| 42 | 55 |
|
| 43 |
- return &suggestError{
|
|
| 56 |
+ return true, &suggestError{
|
|
| 44 | 57 |
err: err, |
| 45 | 58 |
match: match, |
| 46 | 59 |
} |
| ... | ... |
@@ -89,7 +89,7 @@ func ToSlash(inputPath, inputOS string) string {
|
| 89 | 89 |
if inputOS != "windows" {
|
| 90 | 90 |
return inputPath |
| 91 | 91 |
} |
| 92 |
- return strings.Replace(inputPath, "\\", "/", -1) |
|
| 92 |
+ return strings.ReplaceAll(inputPath, "\\", "/") |
|
| 93 | 93 |
} |
| 94 | 94 |
|
| 95 | 95 |
func FromSlash(inputPath, inputOS string) string {
|
| ... | ... |
@@ -97,7 +97,7 @@ func FromSlash(inputPath, inputOS string) string {
|
| 97 | 97 |
if inputOS == "windows" {
|
| 98 | 98 |
separator = "\\" |
| 99 | 99 |
} |
| 100 |
- return strings.Replace(inputPath, "/", separator, -1) |
|
| 100 |
+ return strings.ReplaceAll(inputPath, "/", separator) |
|
| 101 | 101 |
} |
| 102 | 102 |
|
| 103 | 103 |
// NormalizeWorkdir will return a normalized version of the new workdir, given |
| ... | ... |
@@ -3,44 +3,26 @@ package detect |
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 | 5 |
"os" |
| 6 |
- "path/filepath" |
|
| 7 | 6 |
"sort" |
| 8 | 7 |
"strconv" |
| 9 |
- "sync" |
|
| 10 | 8 |
|
| 11 |
- "github.com/moby/buildkit/util/bklog" |
|
| 12 | 9 |
"github.com/pkg/errors" |
| 13 |
- "go.opentelemetry.io/otel" |
|
| 14 |
- "go.opentelemetry.io/otel/exporters/prometheus" |
|
| 15 |
- "go.opentelemetry.io/otel/metric" |
|
| 16 | 10 |
sdkmetric "go.opentelemetry.io/otel/sdk/metric" |
| 17 |
- "go.opentelemetry.io/otel/sdk/resource" |
|
| 11 |
+ "go.opentelemetry.io/otel/sdk/metric/metricdata" |
|
| 18 | 12 |
sdktrace "go.opentelemetry.io/otel/sdk/trace" |
| 19 |
- semconv "go.opentelemetry.io/otel/semconv/v1.21.0" |
|
| 20 |
- "go.opentelemetry.io/otel/trace" |
|
| 21 |
- "go.opentelemetry.io/otel/trace/noop" |
|
| 22 | 13 |
) |
| 23 | 14 |
|
| 24 |
-type ExporterDetector func() (sdktrace.SpanExporter, sdkmetric.Exporter, error) |
|
| 15 |
+type ExporterDetector interface {
|
|
| 16 |
+ DetectTraceExporter() (sdktrace.SpanExporter, error) |
|
| 17 |
+ DetectMetricExporter() (sdkmetric.Exporter, error) |
|
| 18 |
+} |
|
| 25 | 19 |
|
| 26 | 20 |
type detector struct {
|
| 27 | 21 |
f ExporterDetector |
| 28 | 22 |
priority int |
| 29 | 23 |
} |
| 30 | 24 |
|
| 31 |
-var ServiceName string |
|
| 32 |
-var Recorder *TraceRecorder |
|
| 33 |
- |
|
| 34 | 25 |
var detectors map[string]detector |
| 35 |
-var once sync.Once |
|
| 36 |
-var tp trace.TracerProvider |
|
| 37 |
-var mp metric.MeterProvider |
|
| 38 |
-var exporter struct {
|
|
| 39 |
- SpanExporter sdktrace.SpanExporter |
|
| 40 |
- MetricExporter sdkmetric.Exporter |
|
| 41 |
-} |
|
| 42 |
-var closers []func(context.Context) error |
|
| 43 |
-var err error |
|
| 44 | 26 |
|
| 45 | 27 |
func Register(name string, exp ExporterDetector, priority int) {
|
| 46 | 28 |
if detectors == nil {
|
| ... | ... |
@@ -52,17 +34,34 @@ func Register(name string, exp ExporterDetector, priority int) {
|
| 52 | 52 |
} |
| 53 | 53 |
} |
| 54 | 54 |
|
| 55 |
-func detectExporter() (texp sdktrace.SpanExporter, mexp sdkmetric.Exporter, err error) {
|
|
| 56 |
- if n := os.Getenv("OTEL_TRACES_EXPORTER"); n != "" {
|
|
| 55 |
+type TraceExporterDetector func() (sdktrace.SpanExporter, error) |
|
| 56 |
+ |
|
| 57 |
+func (fn TraceExporterDetector) DetectTraceExporter() (sdktrace.SpanExporter, error) {
|
|
| 58 |
+ return fn() |
|
| 59 |
+} |
|
| 60 |
+ |
|
| 61 |
+func (fn TraceExporterDetector) DetectMetricExporter() (sdkmetric.Exporter, error) {
|
|
| 62 |
+ return nil, nil |
|
| 63 |
+} |
|
| 64 |
+ |
|
| 65 |
+func detectExporter[T any](envVar string, fn func(d ExporterDetector) (T, bool, error)) (exp T, err error) {
|
|
| 66 |
+ ignoreErrors, _ := strconv.ParseBool("OTEL_IGNORE_ERROR")
|
|
| 67 |
+ |
|
| 68 |
+ if n := os.Getenv(envVar); n != "" {
|
|
| 57 | 69 |
d, ok := detectors[n] |
| 58 | 70 |
if !ok {
|
| 59 |
- if n == "none" {
|
|
| 60 |
- return nil, nil, nil |
|
| 71 |
+ if !ignoreErrors {
|
|
| 72 |
+ err = errors.Errorf("unsupported opentelemetry exporter %v", n)
|
|
| 61 | 73 |
} |
| 62 |
- return nil, nil, errors.Errorf("unsupported opentelemetry tracer %v", n)
|
|
| 74 |
+ return exp, err |
|
| 63 | 75 |
} |
| 64 |
- return d.f() |
|
| 76 |
+ exp, _, err = fn(d.f) |
|
| 77 |
+ if err != nil && ignoreErrors {
|
|
| 78 |
+ err = nil |
|
| 79 |
+ } |
|
| 80 |
+ return exp, err |
|
| 65 | 81 |
} |
| 82 |
+ |
|
| 66 | 83 |
arr := make([]detector, 0, len(detectors)) |
| 67 | 84 |
for _, d := range detectors {
|
| 68 | 85 |
arr = append(arr, d) |
| ... | ... |
@@ -71,185 +70,89 @@ func detectExporter() (texp sdktrace.SpanExporter, mexp sdkmetric.Exporter, err |
| 71 | 71 |
return arr[i].priority < arr[j].priority |
| 72 | 72 |
}) |
| 73 | 73 |
|
| 74 |
+ var ok bool |
|
| 74 | 75 |
for _, d := range arr {
|
| 75 |
- t, m, err := d.f() |
|
| 76 |
- if err != nil {
|
|
| 77 |
- return nil, nil, err |
|
| 78 |
- } |
|
| 79 |
- if texp == nil {
|
|
| 80 |
- texp = t |
|
| 81 |
- } |
|
| 82 |
- if mexp == nil {
|
|
| 83 |
- mexp = m |
|
| 76 |
+ exp, ok, err = fn(d.f) |
|
| 77 |
+ if err != nil && !ignoreErrors {
|
|
| 78 |
+ return exp, err |
|
| 84 | 79 |
} |
| 85 | 80 |
|
| 86 |
- // Found a candidate for both exporters so just return now. |
|
| 87 |
- if texp != nil && mexp != nil {
|
|
| 88 |
- return texp, mexp, nil |
|
| 81 |
+ if ok {
|
|
| 82 |
+ break |
|
| 89 | 83 |
} |
| 90 | 84 |
} |
| 91 |
- return texp, mexp, nil |
|
| 85 |
+ return exp, nil |
|
| 92 | 86 |
} |
| 93 | 87 |
|
| 94 |
-func getExporters() (sdktrace.SpanExporter, sdkmetric.Exporter, error) {
|
|
| 95 |
- texp, mexp, err := detectExporter() |
|
| 96 |
- if err != nil {
|
|
| 97 |
- return nil, nil, err |
|
| 98 |
- } |
|
| 99 |
- |
|
| 100 |
- if Recorder != nil {
|
|
| 101 |
- Recorder.SpanExporter = texp |
|
| 102 |
- texp = Recorder |
|
| 103 |
- } |
|
| 104 |
- |
|
| 105 |
- return texp, mexp, nil |
|
| 88 |
+func NewSpanExporter(_ context.Context) (sdktrace.SpanExporter, error) {
|
|
| 89 |
+ return detectExporter("OTEL_TRACES_EXPORTER", func(d ExporterDetector) (sdktrace.SpanExporter, bool, error) {
|
|
| 90 |
+ exp, err := d.DetectTraceExporter() |
|
| 91 |
+ return exp, exp != nil, err |
|
| 92 |
+ }) |
|
| 106 | 93 |
} |
| 107 | 94 |
|
| 108 |
-func detect() error {
|
|
| 109 |
- tp = noop.NewTracerProvider() |
|
| 110 |
- mp = sdkmetric.NewMeterProvider() |
|
| 111 |
- |
|
| 112 |
- texp, mexp, err := getExporters() |
|
| 113 |
- if err != nil || (texp == nil && mexp == nil) {
|
|
| 114 |
- return err |
|
| 115 |
- } |
|
| 116 |
- |
|
| 117 |
- res := Resource() |
|
| 118 |
- |
|
| 119 |
- // enable log with traceID when valid exporter |
|
| 120 |
- if texp != nil {
|
|
| 121 |
- bklog.EnableLogWithTraceID(true) |
|
| 122 |
- |
|
| 123 |
- sp := sdktrace.NewBatchSpanProcessor(texp) |
|
| 124 |
- |
|
| 125 |
- if Recorder != nil {
|
|
| 126 |
- Recorder.flush = sp.ForceFlush |
|
| 127 |
- } |
|
| 128 |
- |
|
| 129 |
- sdktp := sdktrace.NewTracerProvider( |
|
| 130 |
- sdktrace.WithSpanProcessor(sp), |
|
| 131 |
- sdktrace.WithResource(res), |
|
| 132 |
- ) |
|
| 133 |
- closers = append(closers, sdktp.Shutdown) |
|
| 95 |
+func NewMetricExporter(_ context.Context) (sdkmetric.Exporter, error) {
|
|
| 96 |
+ return detectExporter("OTEL_METRICS_EXPORTER", func(d ExporterDetector) (sdkmetric.Exporter, bool, error) {
|
|
| 97 |
+ exp, err := d.DetectMetricExporter() |
|
| 98 |
+ return exp, exp != nil, err |
|
| 99 |
+ }) |
|
| 100 |
+} |
|
| 134 | 101 |
|
| 135 |
- exporter.SpanExporter = texp |
|
| 136 |
- tp = sdktp |
|
| 137 |
- } |
|
| 102 |
+type noneDetector struct{}
|
|
| 138 | 103 |
|
| 139 |
- var readers []sdkmetric.Reader |
|
| 140 |
- if mexp != nil {
|
|
| 141 |
- // Create a new periodic reader using any configured metric exporter. |
|
| 142 |
- readers = append(readers, sdkmetric.NewPeriodicReader(mexp)) |
|
| 143 |
- } |
|
| 104 |
+func (n noneDetector) DetectTraceExporter() (sdktrace.SpanExporter, error) {
|
|
| 105 |
+ return noneSpanExporter{}, nil
|
|
| 106 |
+} |
|
| 144 | 107 |
|
| 145 |
- if r, err := prometheus.New(); err != nil {
|
|
| 146 |
- // Log the error but do not fail if we could not configure the prometheus metrics. |
|
| 147 |
- bklog.G(context.Background()). |
|
| 148 |
- WithError(err). |
|
| 149 |
- Error("failed prometheus metrics configuration")
|
|
| 150 |
- } else {
|
|
| 151 |
- // Register the prometheus reader if there was no error. |
|
| 152 |
- readers = append(readers, r) |
|
| 153 |
- } |
|
| 108 |
+func (n noneDetector) DetectMetricExporter() (sdkmetric.Exporter, error) {
|
|
| 109 |
+ return noneMetricExporter{}, nil
|
|
| 110 |
+} |
|
| 154 | 111 |
|
| 155 |
- if len(readers) > 0 {
|
|
| 156 |
- opts := make([]sdkmetric.Option, 0, len(readers)+1) |
|
| 157 |
- opts = append(opts, sdkmetric.WithResource(res)) |
|
| 158 |
- for _, r := range readers {
|
|
| 159 |
- opts = append(opts, sdkmetric.WithReader(r)) |
|
| 160 |
- } |
|
| 161 |
- sdkmp := sdkmetric.NewMeterProvider(opts...) |
|
| 162 |
- closers = append(closers, sdkmp.Shutdown) |
|
| 112 |
+type noneSpanExporter struct{}
|
|
| 163 | 113 |
|
| 164 |
- exporter.MetricExporter = mexp |
|
| 165 |
- mp = sdkmp |
|
| 166 |
- } |
|
| 114 |
+func (n noneSpanExporter) ExportSpans(_ context.Context, _ []sdktrace.ReadOnlySpan) error {
|
|
| 167 | 115 |
return nil |
| 168 | 116 |
} |
| 169 | 117 |
|
| 170 |
-func TracerProvider() (trace.TracerProvider, error) {
|
|
| 171 |
- if err := detectOnce(); err != nil {
|
|
| 172 |
- return nil, err |
|
| 173 |
- } |
|
| 174 |
- return tp, nil |
|
| 118 |
+func (n noneSpanExporter) Shutdown(_ context.Context) error {
|
|
| 119 |
+ return nil |
|
| 175 | 120 |
} |
| 176 | 121 |
|
| 177 |
-func MeterProvider() (metric.MeterProvider, error) {
|
|
| 178 |
- if err := detectOnce(); err != nil {
|
|
| 179 |
- return nil, err |
|
| 180 |
- } |
|
| 181 |
- return mp, nil |
|
| 122 |
+func IsNoneSpanExporter(exp sdktrace.SpanExporter) bool {
|
|
| 123 |
+ _, ok := exp.(noneSpanExporter) |
|
| 124 |
+ return ok |
|
| 182 | 125 |
} |
| 183 | 126 |
|
| 184 |
-func detectOnce() error {
|
|
| 185 |
- once.Do(func() {
|
|
| 186 |
- if err1 := detect(); err1 != nil {
|
|
| 187 |
- b, _ := strconv.ParseBool(os.Getenv("OTEL_IGNORE_ERROR"))
|
|
| 188 |
- if !b {
|
|
| 189 |
- err = err1 |
|
| 190 |
- } |
|
| 191 |
- } |
|
| 192 |
- }) |
|
| 193 |
- return err |
|
| 127 |
+type noneMetricExporter struct{}
|
|
| 128 |
+ |
|
| 129 |
+func (n noneMetricExporter) Temporality(kind sdkmetric.InstrumentKind) metricdata.Temporality {
|
|
| 130 |
+ return sdkmetric.DefaultTemporalitySelector(kind) |
|
| 194 | 131 |
} |
| 195 | 132 |
|
| 196 |
-func Exporter() (sdktrace.SpanExporter, sdkmetric.Exporter, error) {
|
|
| 197 |
- _, err := TracerProvider() |
|
| 198 |
- if err != nil {
|
|
| 199 |
- return nil, nil, err |
|
| 200 |
- } |
|
| 201 |
- return exporter.SpanExporter, exporter.MetricExporter, nil |
|
| 133 |
+func (n noneMetricExporter) Aggregation(kind sdkmetric.InstrumentKind) sdkmetric.Aggregation {
|
|
| 134 |
+ return sdkmetric.DefaultAggregationSelector(kind) |
|
| 202 | 135 |
} |
| 203 | 136 |
|
| 204 |
-func Shutdown(ctx context.Context) error {
|
|
| 205 |
- for _, c := range closers {
|
|
| 206 |
- if err := c(ctx); err != nil {
|
|
| 207 |
- return err |
|
| 208 |
- } |
|
| 209 |
- } |
|
| 137 |
+func (n noneMetricExporter) Export(_ context.Context, _ *metricdata.ResourceMetrics) error {
|
|
| 210 | 138 |
return nil |
| 211 | 139 |
} |
| 212 | 140 |
|
| 213 |
-var ( |
|
| 214 |
- detectedResource *resource.Resource |
|
| 215 |
- detectedResourceOnce sync.Once |
|
| 216 |
-) |
|
| 217 |
- |
|
| 218 |
-func Resource() *resource.Resource {
|
|
| 219 |
- detectedResourceOnce.Do(func() {
|
|
| 220 |
- res, err := resource.New(context.Background(), |
|
| 221 |
- resource.WithDetectors(serviceNameDetector{}),
|
|
| 222 |
- resource.WithFromEnv(), |
|
| 223 |
- resource.WithTelemetrySDK(), |
|
| 224 |
- ) |
|
| 225 |
- if err != nil {
|
|
| 226 |
- otel.Handle(err) |
|
| 227 |
- } |
|
| 228 |
- detectedResource = res |
|
| 229 |
- }) |
|
| 230 |
- return detectedResource |
|
| 141 |
+func (n noneMetricExporter) ForceFlush(_ context.Context) error {
|
|
| 142 |
+ return nil |
|
| 231 | 143 |
} |
| 232 | 144 |
|
| 233 |
-// OverrideResource overrides the resource returned from Resource. |
|
| 234 |
-// |
|
| 235 |
-// This must be invoked before Resource is called otherwise it is a no-op. |
|
| 236 |
-func OverrideResource(res *resource.Resource) {
|
|
| 237 |
- detectedResourceOnce.Do(func() {
|
|
| 238 |
- detectedResource = res |
|
| 239 |
- }) |
|
| 145 |
+func (n noneMetricExporter) Shutdown(_ context.Context) error {
|
|
| 146 |
+ return nil |
|
| 240 | 147 |
} |
| 241 | 148 |
|
| 242 |
-type serviceNameDetector struct{}
|
|
| 149 |
+func IsNoneMetricExporter(exp sdkmetric.Exporter) bool {
|
|
| 150 |
+ _, ok := exp.(noneMetricExporter) |
|
| 151 |
+ return ok |
|
| 152 |
+} |
|
| 243 | 153 |
|
| 244 |
-func (serviceNameDetector) Detect(ctx context.Context) (*resource.Resource, error) {
|
|
| 245 |
- return resource.StringDetector( |
|
| 246 |
- semconv.SchemaURL, |
|
| 247 |
- semconv.ServiceNameKey, |
|
| 248 |
- func() (string, error) {
|
|
| 249 |
- if ServiceName != "" {
|
|
| 250 |
- return ServiceName, nil |
|
| 251 |
- } |
|
| 252 |
- return filepath.Base(os.Args[0]), nil |
|
| 253 |
- }, |
|
| 254 |
- ).Detect(ctx) |
|
| 154 |
+func init() {
|
|
| 155 |
+ // Register a none detector. This will never be chosen if there's another suitable |
|
| 156 |
+ // exporter that can be detected, but exists to allow telemetry to be explicitly |
|
| 157 |
+ // disabled. |
|
| 158 |
+ Register("none", noneDetector{}, 1000)
|
|
| 255 | 159 |
} |
| ... | ... |
@@ -15,24 +15,15 @@ import ( |
| 15 | 15 |
sdktrace "go.opentelemetry.io/otel/sdk/trace" |
| 16 | 16 |
) |
| 17 | 17 |
|
| 18 |
+var otlpExporter = otlpExporterDetector{}
|
|
| 19 |
+ |
|
| 18 | 20 |
func init() {
|
| 19 | 21 |
Register("otlp", otlpExporter, 10)
|
| 20 | 22 |
} |
| 21 | 23 |
|
| 22 |
-func otlpExporter() (sdktrace.SpanExporter, sdkmetric.Exporter, error) {
|
|
| 23 |
- texp, err := otlpSpanExporter() |
|
| 24 |
- if err != nil {
|
|
| 25 |
- return nil, nil, err |
|
| 26 |
- } |
|
| 27 |
- |
|
| 28 |
- mexp, err := otlpMetricExporter() |
|
| 29 |
- if err != nil {
|
|
| 30 |
- return nil, nil, err |
|
| 31 |
- } |
|
| 32 |
- return texp, mexp, nil |
|
| 33 |
-} |
|
| 24 |
+type otlpExporterDetector struct{}
|
|
| 34 | 25 |
|
| 35 |
-func otlpSpanExporter() (sdktrace.SpanExporter, error) {
|
|
| 26 |
+func (otlpExporterDetector) DetectTraceExporter() (sdktrace.SpanExporter, error) {
|
|
| 36 | 27 |
set := os.Getenv("OTEL_TRACES_EXPORTER") == "otlp" || os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" || os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") != ""
|
| 37 | 28 |
if !set {
|
| 38 | 29 |
return nil, nil |
| ... | ... |
@@ -61,7 +52,7 @@ func otlpSpanExporter() (sdktrace.SpanExporter, error) {
|
| 61 | 61 |
return otlptrace.New(context.Background(), c) |
| 62 | 62 |
} |
| 63 | 63 |
|
| 64 |
-func otlpMetricExporter() (sdkmetric.Exporter, error) {
|
|
| 64 |
+func (otlpExporterDetector) DetectMetricExporter() (sdkmetric.Exporter, error) {
|
|
| 65 | 65 |
set := os.Getenv("OTEL_METRICS_EXPORTER") == "otlp" || os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" || os.Getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") != ""
|
| 66 | 66 |
if !set {
|
| 67 | 67 |
return nil, nil |
| ... | ... |
@@ -5,18 +5,31 @@ import ( |
| 5 | 5 |
"sync" |
| 6 | 6 |
"time" |
| 7 | 7 |
|
| 8 |
+ "github.com/pkg/errors" |
|
| 8 | 9 |
sdktrace "go.opentelemetry.io/otel/sdk/trace" |
| 9 | 10 |
"go.opentelemetry.io/otel/sdk/trace/tracetest" |
| 10 | 11 |
"go.opentelemetry.io/otel/trace" |
| 12 |
+ "golang.org/x/sync/semaphore" |
|
| 11 | 13 |
) |
| 12 | 14 |
|
| 15 |
+var Recorder *TraceRecorder |
|
| 16 |
+ |
|
| 13 | 17 |
type TraceRecorder struct {
|
| 14 |
- sdktrace.SpanExporter |
|
| 18 |
+ // sem is a binary semaphore for this struct. |
|
| 19 |
+ // This is used instead of sync.Mutex because it allows |
|
| 20 |
+ // for context cancellation to work properly. |
|
| 21 |
+ sem *semaphore.Weighted |
|
| 22 |
+ |
|
| 23 |
+ // shutdown function for the gc. |
|
| 24 |
+ shutdownGC func(err error) |
|
| 25 |
+ |
|
| 26 |
+ // done channel that marks when background goroutines |
|
| 27 |
+ // are closed. |
|
| 28 |
+ done chan struct{}
|
|
| 15 | 29 |
|
| 16 |
- mu sync.Mutex |
|
| 30 |
+ // track traces and listeners for traces. |
|
| 17 | 31 |
m map[trace.TraceID]*stubs |
| 18 | 32 |
listeners map[trace.TraceID]int |
| 19 |
- flush func(context.Context) error |
|
| 20 | 33 |
} |
| 21 | 34 |
|
| 22 | 35 |
type stubs struct {
|
| ... | ... |
@@ -26,37 +39,52 @@ type stubs struct {
|
| 26 | 26 |
|
| 27 | 27 |
func NewTraceRecorder() *TraceRecorder {
|
| 28 | 28 |
tr := &TraceRecorder{
|
| 29 |
+ sem: semaphore.NewWeighted(1), |
|
| 30 |
+ done: make(chan struct{}),
|
|
| 29 | 31 |
m: map[trace.TraceID]*stubs{},
|
| 30 | 32 |
listeners: map[trace.TraceID]int{},
|
| 31 | 33 |
} |
| 32 | 34 |
|
| 33 |
- go func() {
|
|
| 34 |
- t := time.NewTimer(60 * time.Second) |
|
| 35 |
- for {
|
|
| 36 |
- <-t.C |
|
| 37 |
- tr.gc() |
|
| 38 |
- t.Reset(50 * time.Second) |
|
| 39 |
- } |
|
| 40 |
- }() |
|
| 35 |
+ ctx, cancel := context.WithCancelCause(context.Background()) |
|
| 36 |
+ go tr.gcLoop(ctx) |
|
| 37 |
+ tr.shutdownGC = cancel |
|
| 41 | 38 |
|
| 42 | 39 |
return tr |
| 43 | 40 |
} |
| 44 | 41 |
|
| 45 |
-func (r *TraceRecorder) Record(traceID trace.TraceID) func() []tracetest.SpanStub {
|
|
| 46 |
- r.mu.Lock() |
|
| 47 |
- defer r.mu.Unlock() |
|
| 42 |
+// Record signals to the TraceRecorder that it should track spans associated with the current |
|
| 43 |
+// trace and returns a function that will return these spans. |
|
| 44 |
+// |
|
| 45 |
+// If the TraceRecorder is nil or there is no valid active span, the returned function |
|
| 46 |
+// will be nil to signal that the trace cannot be recorded. |
|
| 47 |
+func (r *TraceRecorder) Record(ctx context.Context) (func() []tracetest.SpanStub, error) {
|
|
| 48 |
+ if r == nil {
|
|
| 49 |
+ return nil, nil |
|
| 50 |
+ } |
|
| 51 |
+ |
|
| 52 |
+ spanCtx := trace.SpanContextFromContext(ctx) |
|
| 53 |
+ if !spanCtx.IsValid() {
|
|
| 54 |
+ return nil, nil |
|
| 55 |
+ } |
|
| 56 |
+ |
|
| 57 |
+ if err := r.sem.Acquire(ctx, 1); err != nil {
|
|
| 58 |
+ return nil, err |
|
| 59 |
+ } |
|
| 60 |
+ defer r.sem.Release(1) |
|
| 48 | 61 |
|
| 62 |
+ traceID := spanCtx.TraceID() |
|
| 49 | 63 |
r.listeners[traceID]++ |
| 50 |
- var once sync.Once |
|
| 51 |
- var spans []tracetest.SpanStub |
|
| 64 |
+ |
|
| 65 |
+ var ( |
|
| 66 |
+ once sync.Once |
|
| 67 |
+ spans []tracetest.SpanStub |
|
| 68 |
+ ) |
|
| 52 | 69 |
return func() []tracetest.SpanStub {
|
| 53 | 70 |
once.Do(func() {
|
| 54 |
- if r.flush != nil {
|
|
| 55 |
- r.flush(context.TODO()) |
|
| 71 |
+ if err := r.sem.Acquire(context.Background(), 1); err != nil {
|
|
| 72 |
+ return |
|
| 56 | 73 |
} |
| 57 |
- |
|
| 58 |
- r.mu.Lock() |
|
| 59 |
- defer r.mu.Unlock() |
|
| 74 |
+ defer r.sem.Release(1) |
|
| 60 | 75 |
|
| 61 | 76 |
if v, ok := r.m[traceID]; ok {
|
| 62 | 77 |
spans = v.spans |
| ... | ... |
@@ -67,26 +95,46 @@ func (r *TraceRecorder) Record(traceID trace.TraceID) func() []tracetest.SpanStu |
| 67 | 67 |
} |
| 68 | 68 |
}) |
| 69 | 69 |
return spans |
| 70 |
+ }, nil |
|
| 71 |
+} |
|
| 72 |
+ |
|
| 73 |
+func (r *TraceRecorder) gcLoop(ctx context.Context) {
|
|
| 74 |
+ defer close(r.done) |
|
| 75 |
+ |
|
| 76 |
+ ticker := time.NewTicker(time.Minute) |
|
| 77 |
+ defer ticker.Stop() |
|
| 78 |
+ |
|
| 79 |
+ for {
|
|
| 80 |
+ select {
|
|
| 81 |
+ case <-ctx.Done(): |
|
| 82 |
+ return |
|
| 83 |
+ case now := <-ticker.C: |
|
| 84 |
+ r.gc(ctx, now) |
|
| 85 |
+ } |
|
| 70 | 86 |
} |
| 71 | 87 |
} |
| 72 | 88 |
|
| 73 |
-func (r *TraceRecorder) gc() {
|
|
| 74 |
- r.mu.Lock() |
|
| 75 |
- defer r.mu.Unlock() |
|
| 89 |
+func (r *TraceRecorder) gc(ctx context.Context, now time.Time) {
|
|
| 90 |
+ if err := r.sem.Acquire(ctx, 1); err != nil {
|
|
| 91 |
+ return |
|
| 92 |
+ } |
|
| 93 |
+ defer r.sem.Release(1) |
|
| 76 | 94 |
|
| 77 |
- now := time.Now() |
|
| 78 | 95 |
for k, s := range r.m {
|
| 79 | 96 |
if _, ok := r.listeners[k]; ok {
|
| 80 | 97 |
continue |
| 81 | 98 |
} |
| 82 |
- if now.Sub(s.last) > 60*time.Second {
|
|
| 99 |
+ if now.Sub(s.last) > time.Minute {
|
|
| 83 | 100 |
delete(r.m, k) |
| 84 | 101 |
} |
| 85 | 102 |
} |
| 86 | 103 |
} |
| 87 | 104 |
|
| 88 | 105 |
func (r *TraceRecorder) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error {
|
| 89 |
- r.mu.Lock() |
|
| 106 |
+ if err := r.sem.Acquire(ctx, 1); err != nil {
|
|
| 107 |
+ return err |
|
| 108 |
+ } |
|
| 109 |
+ defer r.sem.Release(1) |
|
| 90 | 110 |
|
| 91 | 111 |
now := time.Now() |
| 92 | 112 |
for _, s := range spans {
|
| ... | ... |
@@ -99,17 +147,18 @@ func (r *TraceRecorder) ExportSpans(ctx context.Context, spans []sdktrace.ReadOn |
| 99 | 99 |
v.last = now |
| 100 | 100 |
v.spans = append(v.spans, ss) |
| 101 | 101 |
} |
| 102 |
- r.mu.Unlock() |
|
| 103 |
- |
|
| 104 |
- if r.SpanExporter == nil {
|
|
| 105 |
- return nil |
|
| 106 |
- } |
|
| 107 |
- return r.SpanExporter.ExportSpans(ctx, spans) |
|
| 102 |
+ return nil |
|
| 108 | 103 |
} |
| 109 | 104 |
|
| 110 | 105 |
func (r *TraceRecorder) Shutdown(ctx context.Context) error {
|
| 111 |
- if r.SpanExporter == nil {
|
|
| 106 |
+ // Initiate the shutdown of the gc loop. |
|
| 107 |
+ r.shutdownGC(errors.WithStack(context.Canceled)) |
|
| 108 |
+ |
|
| 109 |
+ // Wait for it to be done or the context is canceled. |
|
| 110 |
+ select {
|
|
| 111 |
+ case <-r.done: |
|
| 112 | 112 |
return nil |
| 113 |
+ case <-ctx.Done(): |
|
| 114 |
+ return context.Cause(ctx) |
|
| 113 | 115 |
} |
| 114 |
- return r.SpanExporter.Shutdown(ctx) |
|
| 115 | 116 |
} |
| 116 | 117 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,58 @@ |
| 0 |
+package detect |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "os" |
|
| 5 |
+ "path/filepath" |
|
| 6 |
+ "sync" |
|
| 7 |
+ |
|
| 8 |
+ "go.opentelemetry.io/otel" |
|
| 9 |
+ "go.opentelemetry.io/otel/sdk/resource" |
|
| 10 |
+ semconv "go.opentelemetry.io/otel/semconv/v1.21.0" |
|
| 11 |
+) |
|
| 12 |
+ |
|
| 13 |
+var ( |
|
| 14 |
+ ServiceName string |
|
| 15 |
+ |
|
| 16 |
+ detectedResource *resource.Resource |
|
| 17 |
+ detectedResourceOnce sync.Once |
|
| 18 |
+) |
|
| 19 |
+ |
|
| 20 |
+func Resource() *resource.Resource {
|
|
| 21 |
+ detectedResourceOnce.Do(func() {
|
|
| 22 |
+ res, err := resource.New(context.Background(), |
|
| 23 |
+ resource.WithDetectors(serviceNameDetector{}),
|
|
| 24 |
+ resource.WithFromEnv(), |
|
| 25 |
+ resource.WithTelemetrySDK(), |
|
| 26 |
+ ) |
|
| 27 |
+ if err != nil {
|
|
| 28 |
+ otel.Handle(err) |
|
| 29 |
+ } |
|
| 30 |
+ detectedResource = res |
|
| 31 |
+ }) |
|
| 32 |
+ return detectedResource |
|
| 33 |
+} |
|
| 34 |
+ |
|
| 35 |
+// OverrideResource overrides the resource returned from Resource. |
|
| 36 |
+// |
|
| 37 |
+// This must be invoked before Resource is called otherwise it is a no-op. |
|
| 38 |
+func OverrideResource(res *resource.Resource) {
|
|
| 39 |
+ detectedResourceOnce.Do(func() {
|
|
| 40 |
+ detectedResource = res |
|
| 41 |
+ }) |
|
| 42 |
+} |
|
| 43 |
+ |
|
| 44 |
+type serviceNameDetector struct{}
|
|
| 45 |
+ |
|
| 46 |
+func (serviceNameDetector) Detect(ctx context.Context) (*resource.Resource, error) {
|
|
| 47 |
+ return resource.StringDetector( |
|
| 48 |
+ semconv.SchemaURL, |
|
| 49 |
+ semconv.ServiceNameKey, |
|
| 50 |
+ func() (string, error) {
|
|
| 51 |
+ if ServiceName != "" {
|
|
| 52 |
+ return ServiceName, nil |
|
| 53 |
+ } |
|
| 54 |
+ return filepath.Base(os.Args[0]), nil |
|
| 55 |
+ }, |
|
| 56 |
+ ).Detect(ctx) |
|
| 57 |
+} |
| 0 | 58 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,61 @@ |
| 0 |
+package tracing |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "strings" |
|
| 5 |
+ |
|
| 6 |
+ "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" |
|
| 7 |
+ "google.golang.org/grpc/stats" |
|
| 8 |
+) |
|
| 9 |
+ |
|
| 10 |
+func ServerStatsHandler(opts ...otelgrpc.Option) stats.Handler {
|
|
| 11 |
+ handler := otelgrpc.NewServerHandler(opts...) |
|
| 12 |
+ return &statsFilter{
|
|
| 13 |
+ inner: handler, |
|
| 14 |
+ filter: defaultStatsFilter, |
|
| 15 |
+ } |
|
| 16 |
+} |
|
| 17 |
+ |
|
| 18 |
+func ClientStatsHandler(opts ...otelgrpc.Option) stats.Handler {
|
|
| 19 |
+ handler := otelgrpc.NewClientHandler(opts...) |
|
| 20 |
+ return &statsFilter{
|
|
| 21 |
+ inner: handler, |
|
| 22 |
+ filter: defaultStatsFilter, |
|
| 23 |
+ } |
|
| 24 |
+} |
|
| 25 |
+ |
|
| 26 |
+type contextKey int |
|
| 27 |
+ |
|
| 28 |
+const filterContextKey contextKey = iota |
|
| 29 |
+ |
|
| 30 |
+type statsFilter struct {
|
|
| 31 |
+ inner stats.Handler |
|
| 32 |
+ filter func(info *stats.RPCTagInfo) bool |
|
| 33 |
+} |
|
| 34 |
+ |
|
| 35 |
+func (s *statsFilter) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
|
|
| 36 |
+ if s.filter(info) {
|
|
| 37 |
+ return context.WithValue(ctx, filterContextKey, struct{}{})
|
|
| 38 |
+ } |
|
| 39 |
+ return s.inner.TagRPC(ctx, info) |
|
| 40 |
+} |
|
| 41 |
+ |
|
| 42 |
+func (s *statsFilter) HandleRPC(ctx context.Context, rpcStats stats.RPCStats) {
|
|
| 43 |
+ if ctx.Value(filterContextKey) != nil {
|
|
| 44 |
+ return |
|
| 45 |
+ } |
|
| 46 |
+ s.inner.HandleRPC(ctx, rpcStats) |
|
| 47 |
+} |
|
| 48 |
+ |
|
| 49 |
+func (s *statsFilter) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
|
|
| 50 |
+ return s.inner.TagConn(ctx, info) |
|
| 51 |
+} |
|
| 52 |
+ |
|
| 53 |
+func (s *statsFilter) HandleConn(ctx context.Context, connStats stats.ConnStats) {
|
|
| 54 |
+ s.inner.HandleConn(ctx, connStats) |
|
| 55 |
+} |
|
| 56 |
+ |
|
| 57 |
+func defaultStatsFilter(info *stats.RPCTagInfo) bool {
|
|
| 58 |
+ return strings.HasSuffix(info.FullMethodName, "opentelemetry.proto.collector.trace.v1.TraceService/Export") || |
|
| 59 |
+ strings.HasSuffix(info.FullMethodName, "Health/Check") |
|
| 60 |
+} |
| 0 | 61 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,36 @@ |
| 0 |
+package tracing |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/hashicorp/go-multierror" |
|
| 6 |
+ sdktrace "go.opentelemetry.io/otel/sdk/trace" |
|
| 7 |
+) |
|
| 8 |
+ |
|
| 9 |
+type MultiSpanExporter []sdktrace.SpanExporter |
|
| 10 |
+ |
|
| 11 |
+func (m MultiSpanExporter) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) (err error) {
|
|
| 12 |
+ for _, exp := range m {
|
|
| 13 |
+ if e := exp.ExportSpans(ctx, spans); e != nil {
|
|
| 14 |
+ if err != nil {
|
|
| 15 |
+ err = multierror.Append(err, e) |
|
| 16 |
+ continue |
|
| 17 |
+ } |
|
| 18 |
+ err = e |
|
| 19 |
+ } |
|
| 20 |
+ } |
|
| 21 |
+ return err |
|
| 22 |
+} |
|
| 23 |
+ |
|
| 24 |
+func (m MultiSpanExporter) Shutdown(ctx context.Context) (err error) {
|
|
| 25 |
+ for _, exp := range m {
|
|
| 26 |
+ if e := exp.Shutdown(ctx); e != nil {
|
|
| 27 |
+ if err != nil {
|
|
| 28 |
+ err = multierror.Append(err, e) |
|
| 29 |
+ continue |
|
| 30 |
+ } |
|
| 31 |
+ err = e |
|
| 32 |
+ } |
|
| 33 |
+ } |
|
| 34 |
+ return err |
|
| 35 |
+} |
| ... | ... |
@@ -66,7 +66,7 @@ func (c *Connection) StartConnection(ctx context.Context) error {
|
| 66 | 66 |
c.disconnectedCh = make(chan bool, 1) |
| 67 | 67 |
c.backgroundConnectionDoneCh = make(chan struct{})
|
| 68 | 68 |
|
| 69 |
- if err := c.connect(ctx); err == nil {
|
|
| 69 |
+ if err := c.connect(); err == nil {
|
|
| 70 | 70 |
c.setStateConnected() |
| 71 | 71 |
} else {
|
| 72 | 72 |
c.SetStateDisconnected(err) |
| ... | ... |
@@ -148,7 +148,7 @@ func (c *Connection) indefiniteBackgroundConnection() {
|
| 148 | 148 |
// Normal scenario that we'll wait for |
| 149 | 149 |
} |
| 150 | 150 |
|
| 151 |
- if err := c.connect(context.Background()); err == nil {
|
|
| 151 |
+ if err := c.connect(); err == nil {
|
|
| 152 | 152 |
c.setStateConnected() |
| 153 | 153 |
} else {
|
| 154 | 154 |
// this code is unreachable in most cases |
| ... | ... |
@@ -168,7 +168,7 @@ func (c *Connection) indefiniteBackgroundConnection() {
|
| 168 | 168 |
} |
| 169 | 169 |
} |
| 170 | 170 |
|
| 171 |
-func (c *Connection) connect(ctx context.Context) error {
|
|
| 171 |
+func (c *Connection) connect() error {
|
|
| 172 | 172 |
c.newConnectionHandler(c.cc) |
| 173 | 173 |
return nil |
| 174 | 174 |
} |
| ... | ... |
@@ -6,7 +6,6 @@ import ( |
| 6 | 6 |
"net/http" |
| 7 | 7 |
"net/http/httptrace" |
| 8 | 8 |
|
| 9 |
- "github.com/moby/buildkit/util/bklog" |
|
| 10 | 9 |
"go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" |
| 11 | 10 |
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" |
| 12 | 11 |
"go.opentelemetry.io/otel/attribute" |
| ... | ... |
@@ -15,6 +14,11 @@ import ( |
| 15 | 15 |
semconv "go.opentelemetry.io/otel/semconv/v1.17.0" |
| 16 | 16 |
"go.opentelemetry.io/otel/trace" |
| 17 | 17 |
"go.opentelemetry.io/otel/trace/noop" |
| 18 |
+ |
|
| 19 |
+ "github.com/pkg/errors" |
|
| 20 |
+ |
|
| 21 |
+ "github.com/moby/buildkit/util/bklog" |
|
| 22 |
+ "github.com/moby/buildkit/util/stack" |
|
| 18 | 23 |
) |
| 19 | 24 |
|
| 20 | 25 |
// StartSpan starts a new span as a child of the span in context. |
| ... | ... |
@@ -30,14 +34,30 @@ func StartSpan(ctx context.Context, operationName string, opts ...trace.SpanStar |
| 30 | 30 |
return span, ctx |
| 31 | 31 |
} |
| 32 | 32 |
|
| 33 |
+func hasStacktrace(err error) bool {
|
|
| 34 |
+ switch e := err.(type) {
|
|
| 35 |
+ case interface{ StackTrace() *stack.Stack }:
|
|
| 36 |
+ return true |
|
| 37 |
+ case interface{ StackTrace() errors.StackTrace }:
|
|
| 38 |
+ return true |
|
| 39 |
+ case interface{ Unwrap() error }:
|
|
| 40 |
+ return hasStacktrace(e.Unwrap()) |
|
| 41 |
+ case interface{ Unwrap() []error }:
|
|
| 42 |
+ for _, ue := range e.Unwrap() {
|
|
| 43 |
+ if hasStacktrace(ue) {
|
|
| 44 |
+ return true |
|
| 45 |
+ } |
|
| 46 |
+ } |
|
| 47 |
+ } |
|
| 48 |
+ return false |
|
| 49 |
+} |
|
| 50 |
+ |
|
| 33 | 51 |
// FinishWithError finalizes the span and sets the error if one is passed |
| 34 | 52 |
func FinishWithError(span trace.Span, err error) {
|
| 35 | 53 |
if err != nil {
|
| 36 | 54 |
span.RecordError(err) |
| 37 |
- if _, ok := err.(interface {
|
|
| 38 |
- Cause() error |
|
| 39 |
- }); ok {
|
|
| 40 |
- span.SetAttributes(attribute.String(string(semconv.ExceptionStacktraceKey), fmt.Sprintf("%+v", err)))
|
|
| 55 |
+ if hasStacktrace(err) {
|
|
| 56 |
+ span.SetAttributes(attribute.String(string(semconv.ExceptionStacktraceKey), fmt.Sprintf("%+v", stack.Formatter(err))))
|
|
| 41 | 57 |
} |
| 42 | 58 |
span.SetStatus(codes.Error, err.Error()) |
| 43 | 59 |
} |
| ... | ... |
@@ -12,9 +12,9 @@ import ( |
| 12 | 12 |
"github.com/containerd/containerd/archive/compression" |
| 13 | 13 |
"github.com/containerd/containerd/content" |
| 14 | 14 |
"github.com/containerd/containerd/diff" |
| 15 |
- "github.com/containerd/containerd/errdefs" |
|
| 16 | 15 |
"github.com/containerd/containerd/images" |
| 17 | 16 |
"github.com/containerd/containerd/mount" |
| 17 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 18 | 18 |
digest "github.com/opencontainers/go-digest" |
| 19 | 19 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| 20 | 20 |
"github.com/pkg/errors" |
| ... | ... |
@@ -49,7 +49,7 @@ func (s *winApplier) Apply(ctx context.Context, desc ocispecs.Descriptor, mounts |
| 49 | 49 |
|
| 50 | 50 |
compressed, err := images.DiffCompression(ctx, desc.MediaType) |
| 51 | 51 |
if err != nil {
|
| 52 |
- return ocispecs.Descriptor{}, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType)
|
|
| 52 |
+ return ocispecs.Descriptor{}, errors.Wrapf(cerrdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType)
|
|
| 53 | 53 |
} |
| 54 | 54 |
|
| 55 | 55 |
var ocidesc ocispecs.Descriptor |
| ... | ... |
@@ -148,12 +148,10 @@ func filter(in io.Reader, f func(*tar.Header) bool) (io.Reader, func(error)) {
|
| 148 | 148 |
return err |
| 149 | 149 |
} |
| 150 | 150 |
} |
| 151 |
- } else {
|
|
| 152 |
- if h.Size > 0 {
|
|
| 153 |
- //nolint:gosec // never read into memory |
|
| 154 |
- if _, err := io.Copy(io.Discard, tarReader); err != nil {
|
|
| 155 |
- return err |
|
| 156 |
- } |
|
| 151 |
+ } else if h.Size > 0 {
|
|
| 152 |
+ //nolint:gosec // never read into memory |
|
| 153 |
+ if _, err := io.Copy(io.Discard, tarReader); err != nil {
|
|
| 154 |
+ return err |
|
| 157 | 155 |
} |
| 158 | 156 |
} |
| 159 | 157 |
} |
| ... | ... |
@@ -13,9 +13,9 @@ import ( |
| 13 | 13 |
"github.com/containerd/containerd/archive/compression" |
| 14 | 14 |
"github.com/containerd/containerd/content" |
| 15 | 15 |
"github.com/containerd/containerd/diff" |
| 16 |
- "github.com/containerd/containerd/errdefs" |
|
| 17 | 16 |
"github.com/containerd/containerd/labels" |
| 18 | 17 |
"github.com/containerd/containerd/mount" |
| 18 |
+ cerrdefs "github.com/containerd/errdefs" |
|
| 19 | 19 |
log "github.com/moby/buildkit/util/bklog" |
| 20 | 20 |
digest "github.com/opencontainers/go-digest" |
| 21 | 21 |
ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -66,7 +66,7 @@ func (s *winDiffer) Compare(ctx context.Context, lower, upper []mount.Mount, opt |
| 66 | 66 |
case ocispecs.MediaTypeImageLayerGzip: |
| 67 | 67 |
isCompressed = true |
| 68 | 68 |
default: |
| 69 |
- return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType) |
|
| 69 |
+ return emptyDesc, errors.Wrapf(cerrdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType) |
|
| 70 | 70 |
} |
| 71 | 71 |
|
| 72 | 72 |
var ocidesc ocispecs.Descriptor |
| ... | ... |
@@ -245,10 +245,14 @@ func (w *Worker) Labels() map[string]string {
|
| 245 | 245 |
|
| 246 | 246 |
func (w *Worker) Platforms(noCache bool) []ocispecs.Platform {
|
| 247 | 247 |
if noCache {
|
| 248 |
+ matchers := make([]platforms.MatchComparer, len(w.WorkerOpt.Platforms)) |
|
| 249 |
+ for i, p := range w.WorkerOpt.Platforms {
|
|
| 250 |
+ matchers[i] = platforms.Only(p) |
|
| 251 |
+ } |
|
| 248 | 252 |
for _, p := range archutil.SupportedPlatforms(noCache) {
|
| 249 | 253 |
exists := false |
| 250 |
- for _, pp := range w.WorkerOpt.Platforms {
|
|
| 251 |
- if platforms.Only(pp).Match(p) {
|
|
| 254 |
+ for _, m := range matchers {
|
|
| 255 |
+ if m.Match(p) {
|
|
| 252 | 256 |
exists = true |
| 253 | 257 |
break |
| 254 | 258 |
} |
| ... | ... |
@@ -472,14 +476,12 @@ func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter, |
| 472 | 472 |
} |
| 473 | 473 |
|
| 474 | 474 |
func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (ref cache.ImmutableRef, err error) {
|
| 475 |
- if cd, ok := remote.Provider.(interface {
|
|
| 476 |
- CheckDescriptor(context.Context, ocispecs.Descriptor) error |
|
| 477 |
- }); ok && len(remote.Descriptors) > 0 {
|
|
| 475 |
+ if len(remote.Descriptors) > 0 {
|
|
| 478 | 476 |
var eg errgroup.Group |
| 479 | 477 |
for _, desc := range remote.Descriptors {
|
| 480 | 478 |
desc := desc |
| 481 | 479 |
eg.Go(func() error {
|
| 482 |
- if err := cd.CheckDescriptor(ctx, desc); err != nil {
|
|
| 480 |
+ if _, err := remote.Provider.Info(ctx, desc.Digest); err != nil {
|
|
| 483 | 481 |
return err |
| 484 | 482 |
} |
| 485 | 483 |
return nil |
| ... | ... |
@@ -111,7 +111,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil |
| 111 | 111 |
if filter != nil {
|
| 112 | 112 |
filter(f2.path, &statCopy) |
| 113 | 113 |
} |
| 114 |
- f2copy = ¤tPath{path: filepath.FromSlash(f2.path), stat: &statCopy}
|
|
| 114 |
+ f2copy = ¤tPath{path: f2.path, stat: &statCopy}
|
|
| 115 | 115 |
} |
| 116 | 116 |
k, p := pathChange(f1, f2copy) |
| 117 | 117 |
switch k {
|
| ... | ... |
@@ -37,6 +37,7 @@ type DiskWriter struct {
|
| 37 | 37 |
ctx context.Context |
| 38 | 38 |
cancel func() |
| 39 | 39 |
eg *errgroup.Group |
| 40 |
+ egCtx context.Context |
|
| 40 | 41 |
filter FilterFunc |
| 41 | 42 |
dirModTimes map[string]int64 |
| 42 | 43 |
} |
| ... | ... |
@@ -50,13 +51,14 @@ func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWr |
| 50 | 50 |
} |
| 51 | 51 |
|
| 52 | 52 |
ctx, cancel := context.WithCancel(ctx) |
| 53 |
- eg, ctx := errgroup.WithContext(ctx) |
|
| 53 |
+ eg, egCtx := errgroup.WithContext(ctx) |
|
| 54 | 54 |
|
| 55 | 55 |
return &DiskWriter{
|
| 56 | 56 |
opt: opt, |
| 57 | 57 |
dest: dest, |
| 58 | 58 |
eg: eg, |
| 59 | 59 |
ctx: ctx, |
| 60 |
+ egCtx: egCtx, |
|
| 60 | 61 |
cancel: cancel, |
| 61 | 62 |
filter: opt.Filter, |
| 62 | 63 |
dirModTimes: map[string]int64{},
|
| ... | ... |
@@ -98,7 +100,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er |
| 98 | 98 |
} |
| 99 | 99 |
}() |
| 100 | 100 |
|
| 101 |
- destPath := filepath.Join(dw.dest, filepath.FromSlash(p)) |
|
| 101 |
+ destPath := filepath.Join(dw.dest, p) |
|
| 102 | 102 |
|
| 103 | 103 |
if kind == ChangeKindDelete {
|
| 104 | 104 |
if dw.filter != nil {
|
| ... | ... |
@@ -183,12 +185,12 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er |
| 183 | 183 |
} |
| 184 | 184 |
default: |
| 185 | 185 |
isRegularFile = true |
| 186 |
- file, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, fi.Mode()) //todo: windows |
|
| 186 |
+ file, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, fi.Mode()) |
|
| 187 | 187 |
if err != nil {
|
| 188 | 188 |
return errors.Wrapf(err, "failed to create %s", newPath) |
| 189 | 189 |
} |
| 190 | 190 |
if dw.opt.SyncDataCb != nil {
|
| 191 |
- if err := dw.processChange(ChangeKindAdd, p, fi, file); err != nil {
|
|
| 191 |
+ if err := dw.processChange(dw.ctx, ChangeKindAdd, p, fi, file); err != nil {
|
|
| 192 | 192 |
file.Close() |
| 193 | 193 |
return err |
| 194 | 194 |
} |
| ... | ... |
@@ -219,7 +221,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er |
| 219 | 219 |
dw.requestAsyncFileData(p, destPath, fi, &statCopy) |
| 220 | 220 |
} |
| 221 | 221 |
} else {
|
| 222 |
- return dw.processChange(kind, p, fi, nil) |
|
| 222 |
+ return dw.processChange(dw.ctx, kind, p, fi, nil) |
|
| 223 | 223 |
} |
| 224 | 224 |
|
| 225 | 225 |
return nil |
| ... | ... |
@@ -228,7 +230,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er |
| 228 | 228 |
func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo, st *types.Stat) {
|
| 229 | 229 |
// todo: limit worker threads |
| 230 | 230 |
dw.eg.Go(func() error {
|
| 231 |
- if err := dw.processChange(ChangeKindAdd, p, fi, &lazyFileWriter{
|
|
| 231 |
+ if err := dw.processChange(dw.egCtx, ChangeKindAdd, p, fi, &lazyFileWriter{
|
|
| 232 | 232 |
dest: dest, |
| 233 | 233 |
}); err != nil {
|
| 234 | 234 |
return err |
| ... | ... |
@@ -237,7 +239,7 @@ func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo, st *t |
| 237 | 237 |
}) |
| 238 | 238 |
} |
| 239 | 239 |
|
| 240 |
-func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w io.WriteCloser) error {
|
|
| 240 |
+func (dw *DiskWriter) processChange(ctx context.Context, kind ChangeKind, p string, fi os.FileInfo, w io.WriteCloser) error {
|
|
| 241 | 241 |
origw := w |
| 242 | 242 |
var hw *hashedWriter |
| 243 | 243 |
if dw.opt.NotifyCb != nil {
|
| ... | ... |
@@ -252,7 +254,7 @@ func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w |
| 252 | 252 |
if fn == nil && dw.opt.AsyncDataCb != nil {
|
| 253 | 253 |
fn = dw.opt.AsyncDataCb |
| 254 | 254 |
} |
| 255 |
- if err := fn(dw.ctx, p, w); err != nil {
|
|
| 255 |
+ if err := fn(ctx, p, w); err != nil {
|
|
| 256 | 256 |
return err |
| 257 | 257 |
} |
| 258 | 258 |
} else {
|
| ... | ... |
@@ -313,7 +315,7 @@ type lazyFileWriter struct {
|
| 313 | 313 |
|
| 314 | 314 |
func (lfw *lazyFileWriter) Write(dt []byte) (int, error) {
|
| 315 | 315 |
if lfw.f == nil {
|
| 316 |
- file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) //todo: windows |
|
| 316 |
+ file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) |
|
| 317 | 317 |
if os.IsPermission(err) {
|
| 318 | 318 |
// retry after chmod |
| 319 | 319 |
fi, er := os.Stat(lfw.dest) |
| ... | ... |
@@ -1,6 +1,9 @@ |
| 1 | 1 |
package fsutil |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "context" |
|
| 5 |
+ "io" |
|
| 6 |
+ gofs "io/fs" |
|
| 4 | 7 |
"os" |
| 5 | 8 |
"syscall" |
| 6 | 9 |
|
| ... | ... |
@@ -46,3 +49,68 @@ func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err |
| 46 | 46 |
|
| 47 | 47 |
return nil |
| 48 | 48 |
} |
| 49 |
+ |
|
| 50 |
+// WithHardlinkReset returns a FS that fixes hardlinks for FS that has been filtered |
|
| 51 |
+// so that original hardlink sources might be missing |
|
| 52 |
+func WithHardlinkReset(fs FS) FS {
|
|
| 53 |
+ return &hardlinkFilter{fs: fs}
|
|
| 54 |
+} |
|
| 55 |
+ |
|
| 56 |
+type hardlinkFilter struct {
|
|
| 57 |
+ fs FS |
|
| 58 |
+} |
|
| 59 |
+ |
|
| 60 |
+var _ FS = &hardlinkFilter{}
|
|
| 61 |
+ |
|
| 62 |
+func (r *hardlinkFilter) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
|
|
| 63 |
+ seenFiles := make(map[string]string) |
|
| 64 |
+ return r.fs.Walk(ctx, target, func(path string, entry gofs.DirEntry, err error) error {
|
|
| 65 |
+ if err != nil {
|
|
| 66 |
+ return err |
|
| 67 |
+ } |
|
| 68 |
+ |
|
| 69 |
+ fi, err := entry.Info() |
|
| 70 |
+ if err != nil {
|
|
| 71 |
+ return err |
|
| 72 |
+ } |
|
| 73 |
+ |
|
| 74 |
+ if fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 {
|
|
| 75 |
+ return fn(path, entry, nil) |
|
| 76 |
+ } |
|
| 77 |
+ |
|
| 78 |
+ stat, ok := fi.Sys().(*types.Stat) |
|
| 79 |
+ if !ok {
|
|
| 80 |
+ return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
|
|
| 81 |
+ } |
|
| 82 |
+ |
|
| 83 |
+ if stat.Linkname != "" {
|
|
| 84 |
+ if v, ok := seenFiles[stat.Linkname]; !ok {
|
|
| 85 |
+ seenFiles[stat.Linkname] = stat.Path |
|
| 86 |
+ stat.Linkname = "" |
|
| 87 |
+ entry = &dirEntryWithStat{DirEntry: entry, stat: stat}
|
|
| 88 |
+ } else {
|
|
| 89 |
+ if v != stat.Path {
|
|
| 90 |
+ stat.Linkname = v |
|
| 91 |
+ entry = &dirEntryWithStat{DirEntry: entry, stat: stat}
|
|
| 92 |
+ } |
|
| 93 |
+ } |
|
| 94 |
+ } |
|
| 95 |
+ |
|
| 96 |
+ seenFiles[path] = stat.Path |
|
| 97 |
+ |
|
| 98 |
+ return fn(path, entry, nil) |
|
| 99 |
+ }) |
|
| 100 |
+} |
|
| 101 |
+ |
|
| 102 |
+func (r *hardlinkFilter) Open(p string) (io.ReadCloser, error) {
|
|
| 103 |
+ return r.fs.Open(p) |
|
| 104 |
+} |
|
| 105 |
+ |
|
| 106 |
+type dirEntryWithStat struct {
|
|
| 107 |
+ gofs.DirEntry |
|
| 108 |
+ stat *types.Stat |
|
| 109 |
+} |
|
| 110 |
+ |
|
| 111 |
+func (d *dirEntryWithStat) Info() (gofs.FileInfo, error) {
|
|
| 112 |
+ return &StatInfo{d.stat}, nil
|
|
| 113 |
+} |
| ... | ... |
@@ -1,10 +1,42 @@ |
| 1 |
+// send.go and receive.go describe the fsutil file-transfer protocol, which |
|
| 2 |
+// allows transferring file trees across a network connection. |
|
| 3 |
+// |
|
| 4 |
+// The protocol operates as follows: |
|
| 5 |
+// - The client (the receiver) connects to the server (the sender). |
|
| 6 |
+// - The sender walks the target tree lexicographically and sends a series of |
|
| 7 |
+// STAT packets that describe each file (an empty stat indicates EOF). |
|
| 8 |
+// - The receiver sends a REQ packet for each file it requires the contents for, |
|
| 9 |
+// using the ID for the file (determined as its index in the STAT sequence). |
|
| 10 |
+// - The sender sends a DATA packet with byte arrays for the contents of the |
|
| 11 |
+// file, associated with an ID (an empty array indicates EOF). |
|
| 12 |
+// - Once the receiver has received all files it wants, it sends a FIN packet, |
|
| 13 |
+// and the file transfer is complete. |
|
| 14 |
+// If an error is encountered on either side, an ERR packet is sent containing |
|
| 15 |
+// a human-readable error. |
|
| 16 |
+// |
|
| 17 |
+// All paths transferred over the protocol are normalized to unix-style paths, |
|
| 18 |
+// regardless of which platforms are present on either side. These path |
|
| 19 |
+// conversions are performed right before sending a STAT packet (for the |
|
| 20 |
+// sender) or right after receiving the corresponding STAT packet (for the |
|
| 21 |
+// receiver); this abstraction doesn't leak into the rest of fsutil, which |
|
| 22 |
+// operates on native platform-specific paths. |
|
| 23 |
+// |
|
| 24 |
+// Note that in the case of cross-platform file transfers, the transfer is |
|
| 25 |
+// best-effort. Some filenames that are valid on a unix sender would not be |
|
| 26 |
+// valid on a windows receiver, so these paths are rejected as they are |
|
| 27 |
+// received. Additionally, file metadata, like user/group owners and xattrs do |
|
| 28 |
+// not have an exact correspondence on windows, and so would be discarded by |
|
| 29 |
+// a windows receiver. |
|
| 30 |
+ |
|
| 1 | 31 |
package fsutil |
| 2 | 32 |
|
| 3 | 33 |
import ( |
| 4 | 34 |
"context" |
| 5 | 35 |
"io" |
| 6 | 36 |
"os" |
| 37 |
+ "path/filepath" |
|
| 7 | 38 |
"sync" |
| 39 |
+ "syscall" |
|
| 8 | 40 |
|
| 9 | 41 |
"github.com/pkg/errors" |
| 10 | 42 |
"github.com/tonistiigi/fsutil/types" |
| ... | ... |
@@ -184,13 +216,24 @@ func (r *receiver) run(ctx context.Context) error {
|
| 184 | 184 |
} |
| 185 | 185 |
break |
| 186 | 186 |
} |
| 187 |
+ |
|
| 188 |
+ // normalize unix wire-specific paths to platform-specific paths |
|
| 189 |
+ path := filepath.FromSlash(p.Stat.Path) |
|
| 190 |
+ if filepath.ToSlash(path) != p.Stat.Path {
|
|
| 191 |
+ // e.g. a linux path foo/bar\baz cannot be represented on windows |
|
| 192 |
+ return errors.WithStack(&os.PathError{Path: p.Stat.Path, Err: syscall.EINVAL, Op: "unrepresentable path"})
|
|
| 193 |
+ } |
|
| 194 |
+ p.Stat.Path = path |
|
| 195 |
+ p.Stat.Linkname = filepath.FromSlash(p.Stat.Linkname) |
|
| 196 |
+ |
|
| 187 | 197 |
if fileCanRequestData(os.FileMode(p.Stat.Mode)) {
|
| 188 | 198 |
r.mu.Lock() |
| 189 | 199 |
r.files[p.Stat.Path] = i |
| 190 | 200 |
r.mu.Unlock() |
| 191 | 201 |
} |
| 192 | 202 |
i++ |
| 193 |
- cp := ¤tPath{path: p.Stat.Path, stat: p.Stat}
|
|
| 203 |
+ |
|
| 204 |
+ cp := ¤tPath{path: path, stat: p.Stat}
|
|
| 194 | 205 |
if err := r.orderValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil {
|
| 195 | 206 |
return err |
| 196 | 207 |
} |
| ... | ... |
@@ -29,7 +29,7 @@ type Stream interface {
|
| 29 | 29 |
func Send(ctx context.Context, conn Stream, fs FS, progressCb func(int, bool)) error {
|
| 30 | 30 |
s := &sender{
|
| 31 | 31 |
conn: &syncStream{Stream: conn},
|
| 32 |
- fs: fs, |
|
| 32 |
+ fs: WithHardlinkReset(fs), |
|
| 33 | 33 |
files: make(map[uint32]string), |
| 34 | 34 |
progressCb: progressCb, |
| 35 | 35 |
sendpipeline: make(chan *sendHandle, 128), |
| ... | ... |
@@ -161,6 +161,7 @@ func (s *sender) walk(ctx context.Context) error {
|
| 161 | 161 |
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
|
| 162 | 162 |
} |
| 163 | 163 |
stat.Path = filepath.ToSlash(stat.Path) |
| 164 |
+ stat.Linkname = filepath.ToSlash(stat.Linkname) |
|
| 164 | 165 |
p := &types.Packet{
|
| 165 | 166 |
Type: types.PACKET_STAT, |
| 166 | 167 |
Stat: stat, |
| ... | ... |
@@ -138,16 +138,7 @@ func New(token, url string, opt Opt) (*Cache, error) {
|
| 138 | 138 |
} |
| 139 | 139 |
Log("parsed token: scopes: %+v, issued: %v, expires: %v", scopes, nbft, expt)
|
| 140 | 140 |
|
| 141 |
- if opt.Client == nil {
|
|
| 142 |
- opt.Client = http.DefaultClient |
|
| 143 |
- } |
|
| 144 |
- if opt.Timeout == 0 {
|
|
| 145 |
- opt.Timeout = 5 * time.Minute |
|
| 146 |
- } |
|
| 147 |
- |
|
| 148 |
- if opt.BackoffPool == nil {
|
|
| 149 |
- opt.BackoffPool = defaultBackoffPool |
|
| 150 |
- } |
|
| 141 |
+ opt = optsWithDefaults(opt) |
|
| 151 | 142 |
|
| 152 | 143 |
return &Cache{
|
| 153 | 144 |
opt: opt, |
| ... | ... |
@@ -159,6 +150,19 @@ func New(token, url string, opt Opt) (*Cache, error) {
|
| 159 | 159 |
}, nil |
| 160 | 160 |
} |
| 161 | 161 |
|
| 162 |
+func optsWithDefaults(opt Opt) Opt {
|
|
| 163 |
+ if opt.Client == nil {
|
|
| 164 |
+ opt.Client = http.DefaultClient |
|
| 165 |
+ } |
|
| 166 |
+ if opt.Timeout == 0 {
|
|
| 167 |
+ opt.Timeout = 5 * time.Minute |
|
| 168 |
+ } |
|
| 169 |
+ if opt.BackoffPool == nil {
|
|
| 170 |
+ opt.BackoffPool = defaultBackoffPool |
|
| 171 |
+ } |
|
| 172 |
+ return opt |
|
| 173 |
+} |
|
| 174 |
+ |
|
| 162 | 175 |
type Scope struct {
|
| 163 | 176 |
Scope string |
| 164 | 177 |
Permission Permission |
| ... | ... |
@@ -470,6 +474,31 @@ func (c *Cache) url(p string) string {
|
| 470 | 470 |
return c.URL + "_apis/artifactcache/" + p |
| 471 | 471 |
} |
| 472 | 472 |
|
| 473 |
+func (c *Cache) AllKeys(ctx context.Context, api *RestAPI, prefix string) (map[string]struct{}, error) {
|
|
| 474 |
+ m := map[string]struct{}{}
|
|
| 475 |
+ var mu sync.Mutex |
|
| 476 |
+ eg, ctx := errgroup.WithContext(ctx) |
|
| 477 |
+ for _, s := range c.scopes {
|
|
| 478 |
+ s := s |
|
| 479 |
+ eg.Go(func() error {
|
|
| 480 |
+ keys, err := api.ListKeys(ctx, prefix, s.Scope) |
|
| 481 |
+ if err != nil {
|
|
| 482 |
+ return err |
|
| 483 |
+ } |
|
| 484 |
+ mu.Lock() |
|
| 485 |
+ for _, k := range keys {
|
|
| 486 |
+ m[k.Key] = struct{}{}
|
|
| 487 |
+ } |
|
| 488 |
+ mu.Unlock() |
|
| 489 |
+ return nil |
|
| 490 |
+ }) |
|
| 491 |
+ } |
|
| 492 |
+ if err := eg.Wait(); err != nil {
|
|
| 493 |
+ return nil, err |
|
| 494 |
+ } |
|
| 495 |
+ return m, nil |
|
| 496 |
+} |
|
| 497 |
+ |
|
| 473 | 498 |
type ReserveCacheReq struct {
|
| 474 | 499 |
Key string `json:"key"` |
| 475 | 500 |
Version string `json:"version"` |
| 476 | 501 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,111 @@ |
| 0 |
+package actionscache |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "encoding/json" |
|
| 5 |
+ "net/http" |
|
| 6 |
+ "net/url" |
|
| 7 |
+ "strconv" |
|
| 8 |
+) |
|
| 9 |
+ |
|
| 10 |
+const ( |
|
| 11 |
+ apiURL = "https://api.github.com" |
|
| 12 |
+ perPage = 100 |
|
| 13 |
+) |
|
| 14 |
+ |
|
| 15 |
+type RestAPI struct {
|
|
| 16 |
+ repo string |
|
| 17 |
+ token string |
|
| 18 |
+ opt Opt |
|
| 19 |
+} |
|
| 20 |
+ |
|
| 21 |
+type CacheKey struct {
|
|
| 22 |
+ ID int `json:"id"` |
|
| 23 |
+ Ref string `json:"ref"` |
|
| 24 |
+ Key string `json:"key"` |
|
| 25 |
+ Version string `json:"version"` |
|
| 26 |
+ LastAccessed string `json:"last_accessed_at"` |
|
| 27 |
+ CreatedAt string `json:"created_at"` |
|
| 28 |
+ SizeInBytes int `json:"size_in_bytes"` |
|
| 29 |
+} |
|
| 30 |
+ |
|
| 31 |
+func NewRestAPI(repo, token string, opt Opt) (*RestAPI, error) {
|
|
| 32 |
+ opt = optsWithDefaults(opt) |
|
| 33 |
+ return &RestAPI{
|
|
| 34 |
+ repo: repo, |
|
| 35 |
+ token: token, |
|
| 36 |
+ opt: opt, |
|
| 37 |
+ }, nil |
|
| 38 |
+} |
|
| 39 |
+ |
|
| 40 |
+func (r *RestAPI) httpReq(ctx context.Context, method string, url *url.URL) (*http.Request, error) {
|
|
| 41 |
+ req, err := http.NewRequest(method, url.String(), nil) |
|
| 42 |
+ if err != nil {
|
|
| 43 |
+ return nil, err |
|
| 44 |
+ } |
|
| 45 |
+ req = req.WithContext(ctx) |
|
| 46 |
+ req.Header.Set("Accept", "application/vnd.github+json")
|
|
| 47 |
+ req.Header.Set("Authorization", "Bearer "+r.token)
|
|
| 48 |
+ req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
|
|
| 49 |
+ return req, nil |
|
| 50 |
+} |
|
| 51 |
+ |
|
| 52 |
+func (r *RestAPI) ListKeys(ctx context.Context, prefix, ref string) ([]CacheKey, error) {
|
|
| 53 |
+ var out []CacheKey |
|
| 54 |
+ page := 1 |
|
| 55 |
+ for {
|
|
| 56 |
+ keys, total, err := r.listKeysPage(ctx, prefix, ref, page) |
|
| 57 |
+ if err != nil {
|
|
| 58 |
+ return nil, err |
|
| 59 |
+ } |
|
| 60 |
+ out = append(out, keys...) |
|
| 61 |
+ if total > page*perPage {
|
|
| 62 |
+ page++ |
|
| 63 |
+ } else {
|
|
| 64 |
+ break |
|
| 65 |
+ } |
|
| 66 |
+ } |
|
| 67 |
+ return out, nil |
|
| 68 |
+} |
|
| 69 |
+ |
|
| 70 |
+func (r *RestAPI) listKeysPage(ctx context.Context, prefix, ref string, page int) ([]CacheKey, int, error) {
|
|
| 71 |
+ u, err := url.Parse(apiURL + "/repos/" + r.repo + "/actions/caches") |
|
| 72 |
+ if err != nil {
|
|
| 73 |
+ return nil, 0, err |
|
| 74 |
+ } |
|
| 75 |
+ q := u.Query() |
|
| 76 |
+ q.Set("per_page", strconv.Itoa(perPage))
|
|
| 77 |
+ if page > 0 {
|
|
| 78 |
+ q.Set("page", strconv.Itoa(page))
|
|
| 79 |
+ } |
|
| 80 |
+ if prefix != "" {
|
|
| 81 |
+ q.Set("key", prefix)
|
|
| 82 |
+ } |
|
| 83 |
+ if ref != "" {
|
|
| 84 |
+ q.Set("ref", ref)
|
|
| 85 |
+ } |
|
| 86 |
+ u.RawQuery = q.Encode() |
|
| 87 |
+ |
|
| 88 |
+ req, err := r.httpReq(ctx, "GET", u) |
|
| 89 |
+ if err != nil {
|
|
| 90 |
+ return nil, 0, err |
|
| 91 |
+ } |
|
| 92 |
+ |
|
| 93 |
+ resp, err := r.opt.Client.Do(req) |
|
| 94 |
+ if err != nil {
|
|
| 95 |
+ return nil, 0, err |
|
| 96 |
+ } |
|
| 97 |
+ |
|
| 98 |
+ dec := json.NewDecoder(resp.Body) |
|
| 99 |
+ var keys struct {
|
|
| 100 |
+ Total int `json:"total_count"` |
|
| 101 |
+ Caches []CacheKey `json:"actions_caches"` |
|
| 102 |
+ } |
|
| 103 |
+ |
|
| 104 |
+ if err := dec.Decode(&keys); err != nil {
|
|
| 105 |
+ return nil, 0, err |
|
| 106 |
+ } |
|
| 107 |
+ |
|
| 108 |
+ resp.Body.Close() |
|
| 109 |
+ return keys.Caches, keys.Total, nil |
|
| 110 |
+} |
| 206 | 207 |
deleted file mode 100644 |
| ... | ... |
@@ -1,201 +0,0 @@ |
| 1 |
- Apache License |
|
| 2 |
- Version 2.0, January 2004 |
|
| 3 |
- http://www.apache.org/licenses/ |
|
| 4 |
- |
|
| 5 |
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
|
| 6 |
- |
|
| 7 |
- 1. Definitions. |
|
| 8 |
- |
|
| 9 |
- "License" shall mean the terms and conditions for use, reproduction, |
|
| 10 |
- and distribution as defined by Sections 1 through 9 of this document. |
|
| 11 |
- |
|
| 12 |
- "Licensor" shall mean the copyright owner or entity authorized by |
|
| 13 |
- the copyright owner that is granting the License. |
|
| 14 |
- |
|
| 15 |
- "Legal Entity" shall mean the union of the acting entity and all |
|
| 16 |
- other entities that control, are controlled by, or are under common |
|
| 17 |
- control with that entity. For the purposes of this definition, |
|
| 18 |
- "control" means (i) the power, direct or indirect, to cause the |
|
| 19 |
- direction or management of such entity, whether by contract or |
|
| 20 |
- otherwise, or (ii) ownership of fifty percent (50%) or more of the |
|
| 21 |
- outstanding shares, or (iii) beneficial ownership of such entity. |
|
| 22 |
- |
|
| 23 |
- "You" (or "Your") shall mean an individual or Legal Entity |
|
| 24 |
- exercising permissions granted by this License. |
|
| 25 |
- |
|
| 26 |
- "Source" form shall mean the preferred form for making modifications, |
|
| 27 |
- including but not limited to software source code, documentation |
|
| 28 |
- source, and configuration files. |
|
| 29 |
- |
|
| 30 |
- "Object" form shall mean any form resulting from mechanical |
|
| 31 |
- transformation or translation of a Source form, including but |
|
| 32 |
- not limited to compiled object code, generated documentation, |
|
| 33 |
- and conversions to other media types. |
|
| 34 |
- |
|
| 35 |
- "Work" shall mean the work of authorship, whether in Source or |
|
| 36 |
- Object form, made available under the License, as indicated by a |
|
| 37 |
- copyright notice that is included in or attached to the work |
|
| 38 |
- (an example is provided in the Appendix below). |
|
| 39 |
- |
|
| 40 |
- "Derivative Works" shall mean any work, whether in Source or Object |
|
| 41 |
- form, that is based on (or derived from) the Work and for which the |
|
| 42 |
- editorial revisions, annotations, elaborations, or other modifications |
|
| 43 |
- represent, as a whole, an original work of authorship. For the purposes |
|
| 44 |
- of this License, Derivative Works shall not include works that remain |
|
| 45 |
- separable from, or merely link (or bind by name) to the interfaces of, |
|
| 46 |
- the Work and Derivative Works thereof. |
|
| 47 |
- |
|
| 48 |
- "Contribution" shall mean any work of authorship, including |
|
| 49 |
- the original version of the Work and any modifications or additions |
|
| 50 |
- to that Work or Derivative Works thereof, that is intentionally |
|
| 51 |
- submitted to Licensor for inclusion in the Work by the copyright owner |
|
| 52 |
- or by an individual or Legal Entity authorized to submit on behalf of |
|
| 53 |
- the copyright owner. For the purposes of this definition, "submitted" |
|
| 54 |
- means any form of electronic, verbal, or written communication sent |
|
| 55 |
- to the Licensor or its representatives, including but not limited to |
|
| 56 |
- communication on electronic mailing lists, source code control systems, |
|
| 57 |
- and issue tracking systems that are managed by, or on behalf of, the |
|
| 58 |
- Licensor for the purpose of discussing and improving the Work, but |
|
| 59 |
- excluding communication that is conspicuously marked or otherwise |
|
| 60 |
- designated in writing by the copyright owner as "Not a Contribution." |
|
| 61 |
- |
|
| 62 |
- "Contributor" shall mean Licensor and any individual or Legal Entity |
|
| 63 |
- on behalf of whom a Contribution has been received by Licensor and |
|
| 64 |
- subsequently incorporated within the Work. |
|
| 65 |
- |
|
| 66 |
- 2. Grant of Copyright License. Subject to the terms and conditions of |
|
| 67 |
- this License, each Contributor hereby grants to You a perpetual, |
|
| 68 |
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
|
| 69 |
- copyright license to reproduce, prepare Derivative Works of, |
|
| 70 |
- publicly display, publicly perform, sublicense, and distribute the |
|
| 71 |
- Work and such Derivative Works in Source or Object form. |
|
| 72 |
- |
|
| 73 |
- 3. Grant of Patent License. Subject to the terms and conditions of |
|
| 74 |
- this License, each Contributor hereby grants to You a perpetual, |
|
| 75 |
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
|
| 76 |
- (except as stated in this section) patent license to make, have made, |
|
| 77 |
- use, offer to sell, sell, import, and otherwise transfer the Work, |
|
| 78 |
- where such license applies only to those patent claims licensable |
|
| 79 |
- by such Contributor that are necessarily infringed by their |
|
| 80 |
- Contribution(s) alone or by combination of their Contribution(s) |
|
| 81 |
- with the Work to which such Contribution(s) was submitted. If You |
|
| 82 |
- institute patent litigation against any entity (including a |
|
| 83 |
- cross-claim or counterclaim in a lawsuit) alleging that the Work |
|
| 84 |
- or a Contribution incorporated within the Work constitutes direct |
|
| 85 |
- or contributory patent infringement, then any patent licenses |
|
| 86 |
- granted to You under this License for that Work shall terminate |
|
| 87 |
- as of the date such litigation is filed. |
|
| 88 |
- |
|
| 89 |
- 4. Redistribution. You may reproduce and distribute copies of the |
|
| 90 |
- Work or Derivative Works thereof in any medium, with or without |
|
| 91 |
- modifications, and in Source or Object form, provided that You |
|
| 92 |
- meet the following conditions: |
|
| 93 |
- |
|
| 94 |
- (a) You must give any other recipients of the Work or |
|
| 95 |
- Derivative Works a copy of this License; and |
|
| 96 |
- |
|
| 97 |
- (b) You must cause any modified files to carry prominent notices |
|
| 98 |
- stating that You changed the files; and |
|
| 99 |
- |
|
| 100 |
- (c) You must retain, in the Source form of any Derivative Works |
|
| 101 |
- that You distribute, all copyright, patent, trademark, and |
|
| 102 |
- attribution notices from the Source form of the Work, |
|
| 103 |
- excluding those notices that do not pertain to any part of |
|
| 104 |
- the Derivative Works; and |
|
| 105 |
- |
|
| 106 |
- (d) If the Work includes a "NOTICE" text file as part of its |
|
| 107 |
- distribution, then any Derivative Works that You distribute must |
|
| 108 |
- include a readable copy of the attribution notices contained |
|
| 109 |
- within such NOTICE file, excluding those notices that do not |
|
| 110 |
- pertain to any part of the Derivative Works, in at least one |
|
| 111 |
- of the following places: within a NOTICE text file distributed |
|
| 112 |
- as part of the Derivative Works; within the Source form or |
|
| 113 |
- documentation, if provided along with the Derivative Works; or, |
|
| 114 |
- within a display generated by the Derivative Works, if and |
|
| 115 |
- wherever such third-party notices normally appear. The contents |
|
| 116 |
- of the NOTICE file are for informational purposes only and |
|
| 117 |
- do not modify the License. You may add Your own attribution |
|
| 118 |
- notices within Derivative Works that You distribute, alongside |
|
| 119 |
- or as an addendum to the NOTICE text from the Work, provided |
|
| 120 |
- that such additional attribution notices cannot be construed |
|
| 121 |
- as modifying the License. |
|
| 122 |
- |
|
| 123 |
- You may add Your own copyright statement to Your modifications and |
|
| 124 |
- may provide additional or different license terms and conditions |
|
| 125 |
- for use, reproduction, or distribution of Your modifications, or |
|
| 126 |
- for any such Derivative Works as a whole, provided Your use, |
|
| 127 |
- reproduction, and distribution of the Work otherwise complies with |
|
| 128 |
- the conditions stated in this License. |
|
| 129 |
- |
|
| 130 |
- 5. Submission of Contributions. Unless You explicitly state otherwise, |
|
| 131 |
- any Contribution intentionally submitted for inclusion in the Work |
|
| 132 |
- by You to the Licensor shall be under the terms and conditions of |
|
| 133 |
- this License, without any additional terms or conditions. |
|
| 134 |
- Notwithstanding the above, nothing herein shall supersede or modify |
|
| 135 |
- the terms of any separate license agreement you may have executed |
|
| 136 |
- with Licensor regarding such Contributions. |
|
| 137 |
- |
|
| 138 |
- 6. Trademarks. This License does not grant permission to use the trade |
|
| 139 |
- names, trademarks, service marks, or product names of the Licensor, |
|
| 140 |
- except as required for reasonable and customary use in describing the |
|
| 141 |
- origin of the Work and reproducing the content of the NOTICE file. |
|
| 142 |
- |
|
| 143 |
- 7. Disclaimer of Warranty. Unless required by applicable law or |
|
| 144 |
- agreed to in writing, Licensor provides the Work (and each |
|
| 145 |
- Contributor provides its Contributions) on an "AS IS" BASIS, |
|
| 146 |
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
|
| 147 |
- implied, including, without limitation, any warranties or conditions |
|
| 148 |
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
|
| 149 |
- PARTICULAR PURPOSE. You are solely responsible for determining the |
|
| 150 |
- appropriateness of using or redistributing the Work and assume any |
|
| 151 |
- risks associated with Your exercise of permissions under this License. |
|
| 152 |
- |
|
| 153 |
- 8. Limitation of Liability. In no event and under no legal theory, |
|
| 154 |
- whether in tort (including negligence), contract, or otherwise, |
|
| 155 |
- unless required by applicable law (such as deliberate and grossly |
|
| 156 |
- negligent acts) or agreed to in writing, shall any Contributor be |
|
| 157 |
- liable to You for damages, including any direct, indirect, special, |
|
| 158 |
- incidental, or consequential damages of any character arising as a |
|
| 159 |
- result of this License or out of the use or inability to use the |
|
| 160 |
- Work (including but not limited to damages for loss of goodwill, |
|
| 161 |
- work stoppage, computer failure or malfunction, or any and all |
|
| 162 |
- other commercial damages or losses), even if such Contributor |
|
| 163 |
- has been advised of the possibility of such damages. |
|
| 164 |
- |
|
| 165 |
- 9. Accepting Warranty or Additional Liability. While redistributing |
|
| 166 |
- the Work or Derivative Works thereof, You may choose to offer, |
|
| 167 |
- and charge a fee for, acceptance of support, warranty, indemnity, |
|
| 168 |
- or other liability obligations and/or rights consistent with this |
|
| 169 |
- License. However, in accepting such obligations, You may act only |
|
| 170 |
- on Your own behalf and on Your sole responsibility, not on behalf |
|
| 171 |
- of any other Contributor, and only if You agree to indemnify, |
|
| 172 |
- defend, and hold each Contributor harmless for any liability |
|
| 173 |
- incurred by, or claims asserted against, such Contributor by reason |
|
| 174 |
- of your accepting any such warranty or additional liability. |
|
| 175 |
- |
|
| 176 |
- END OF TERMS AND CONDITIONS |
|
| 177 |
- |
|
| 178 |
- APPENDIX: How to apply the Apache License to your work. |
|
| 179 |
- |
|
| 180 |
- To apply the Apache License to your work, attach the following |
|
| 181 |
- boilerplate notice, with the fields enclosed by brackets "[]" |
|
| 182 |
- replaced with your own identifying information. (Don't include |
|
| 183 |
- the brackets!) The text should be enclosed in the appropriate |
|
| 184 |
- comment syntax for the file format. We also recommend that a |
|
| 185 |
- file or class name and description of purpose be included on the |
|
| 186 |
- same "printed page" as the copyright notice for easier |
|
| 187 |
- identification within third-party archives. |
|
| 188 |
- |
|
| 189 |
- Copyright [yyyy] [name of copyright owner] |
|
| 190 |
- |
|
| 191 |
- Licensed under the Apache License, Version 2.0 (the "License"); |
|
| 192 |
- you may not use this file except in compliance with the License. |
|
| 193 |
- You may obtain a copy of the License at |
|
| 194 |
- |
|
| 195 |
- http://www.apache.org/licenses/LICENSE-2.0 |
|
| 196 |
- |
|
| 197 |
- Unless required by applicable law or agreed to in writing, software |
|
| 198 |
- distributed under the License is distributed on an "AS IS" BASIS, |
|
| 199 |
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
| 200 |
- See the License for the specific language governing permissions and |
|
| 201 |
- limitations under the License. |
| 202 | 1 |
deleted file mode 100644 |
| ... | ... |
@@ -1,153 +0,0 @@ |
| 1 |
-// Copyright The OpenTelemetry Authors |
|
| 2 |
-// |
|
| 3 |
-// Licensed under the Apache License, Version 2.0 (the "License"); |
|
| 4 |
-// you may not use this file except in compliance with the License. |
|
| 5 |
-// You may obtain a copy of the License at |
|
| 6 |
-// |
|
| 7 |
-// http://www.apache.org/licenses/LICENSE-2.0 |
|
| 8 |
-// |
|
| 9 |
-// Unless required by applicable law or agreed to in writing, software |
|
| 10 |
-// distributed under the License is distributed on an "AS IS" BASIS, |
|
| 11 |
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
| 12 |
-// See the License for the specific language governing permissions and |
|
| 13 |
-// limitations under the License. |
|
| 14 |
- |
|
| 15 |
-package prometheus // import "go.opentelemetry.io/otel/exporters/prometheus" |
|
| 16 |
- |
|
| 17 |
-import ( |
|
| 18 |
- "strings" |
|
| 19 |
- |
|
| 20 |
- "github.com/prometheus/client_golang/prometheus" |
|
| 21 |
- |
|
| 22 |
- "go.opentelemetry.io/otel/sdk/metric" |
|
| 23 |
-) |
|
| 24 |
- |
|
| 25 |
-// config contains options for the exporter. |
|
| 26 |
-type config struct {
|
|
| 27 |
- registerer prometheus.Registerer |
|
| 28 |
- disableTargetInfo bool |
|
| 29 |
- withoutUnits bool |
|
| 30 |
- withoutCounterSuffixes bool |
|
| 31 |
- readerOpts []metric.ManualReaderOption |
|
| 32 |
- disableScopeInfo bool |
|
| 33 |
- namespace string |
|
| 34 |
-} |
|
| 35 |
- |
|
| 36 |
-// newConfig creates a validated config configured with options. |
|
| 37 |
-func newConfig(opts ...Option) config {
|
|
| 38 |
- cfg := config{}
|
|
| 39 |
- for _, opt := range opts {
|
|
| 40 |
- cfg = opt.apply(cfg) |
|
| 41 |
- } |
|
| 42 |
- |
|
| 43 |
- if cfg.registerer == nil {
|
|
| 44 |
- cfg.registerer = prometheus.DefaultRegisterer |
|
| 45 |
- } |
|
| 46 |
- |
|
| 47 |
- return cfg |
|
| 48 |
-} |
|
| 49 |
- |
|
| 50 |
-// Option sets exporter option values. |
|
| 51 |
-type Option interface {
|
|
| 52 |
- apply(config) config |
|
| 53 |
-} |
|
| 54 |
- |
|
| 55 |
-type optionFunc func(config) config |
|
| 56 |
- |
|
| 57 |
-func (fn optionFunc) apply(cfg config) config {
|
|
| 58 |
- return fn(cfg) |
|
| 59 |
-} |
|
| 60 |
- |
|
| 61 |
-// WithRegisterer configures which prometheus Registerer the Exporter will |
|
| 62 |
-// register with. If no registerer is used the prometheus DefaultRegisterer is |
|
| 63 |
-// used. |
|
| 64 |
-func WithRegisterer(reg prometheus.Registerer) Option {
|
|
| 65 |
- return optionFunc(func(cfg config) config {
|
|
| 66 |
- cfg.registerer = reg |
|
| 67 |
- return cfg |
|
| 68 |
- }) |
|
| 69 |
-} |
|
| 70 |
- |
|
| 71 |
-// WithAggregationSelector configure the Aggregation Selector the exporter will |
|
| 72 |
-// use. If no AggregationSelector is provided the DefaultAggregationSelector is |
|
| 73 |
-// used. |
|
| 74 |
-func WithAggregationSelector(agg metric.AggregationSelector) Option {
|
|
| 75 |
- return optionFunc(func(cfg config) config {
|
|
| 76 |
- cfg.readerOpts = append(cfg.readerOpts, metric.WithAggregationSelector(agg)) |
|
| 77 |
- return cfg |
|
| 78 |
- }) |
|
| 79 |
-} |
|
| 80 |
- |
|
| 81 |
-// WithProducer configure the metric Producer the exporter will use as a source |
|
| 82 |
-// of external metric data. |
|
| 83 |
-func WithProducer(producer metric.Producer) Option {
|
|
| 84 |
- return optionFunc(func(cfg config) config {
|
|
| 85 |
- cfg.readerOpts = append(cfg.readerOpts, metric.WithProducer(producer)) |
|
| 86 |
- return cfg |
|
| 87 |
- }) |
|
| 88 |
-} |
|
| 89 |
- |
|
| 90 |
-// WithoutTargetInfo configures the Exporter to not export the resource target_info metric. |
|
| 91 |
-// If not specified, the Exporter will create a target_info metric containing |
|
| 92 |
-// the metrics' resource.Resource attributes. |
|
| 93 |
-func WithoutTargetInfo() Option {
|
|
| 94 |
- return optionFunc(func(cfg config) config {
|
|
| 95 |
- cfg.disableTargetInfo = true |
|
| 96 |
- return cfg |
|
| 97 |
- }) |
|
| 98 |
-} |
|
| 99 |
- |
|
| 100 |
-// WithoutUnits disables exporter's addition of unit suffixes to metric names, |
|
| 101 |
-// and will also prevent unit comments from being added in OpenMetrics once |
|
| 102 |
-// unit comments are supported. |
|
| 103 |
-// |
|
| 104 |
-// By default, metric names include a unit suffix to follow Prometheus naming |
|
| 105 |
-// conventions. For example, the counter metric request.duration, with unit |
|
| 106 |
-// milliseconds would become request_duration_milliseconds_total. |
|
| 107 |
-// With this option set, the name would instead be request_duration_total. |
|
| 108 |
-func WithoutUnits() Option {
|
|
| 109 |
- return optionFunc(func(cfg config) config {
|
|
| 110 |
- cfg.withoutUnits = true |
|
| 111 |
- return cfg |
|
| 112 |
- }) |
|
| 113 |
-} |
|
| 114 |
- |
|
| 115 |
-// WithoutUnits disables exporter's addition _total suffixes on counters. |
|
| 116 |
-// |
|
| 117 |
-// By default, metric names include a _total suffix to follow Prometheus naming |
|
| 118 |
-// conventions. For example, the counter metric happy.people would become |
|
| 119 |
-// happy_people_total. With this option set, the name would instead be |
|
| 120 |
-// happy_people. |
|
| 121 |
-func WithoutCounterSuffixes() Option {
|
|
| 122 |
- return optionFunc(func(cfg config) config {
|
|
| 123 |
- cfg.withoutCounterSuffixes = true |
|
| 124 |
- return cfg |
|
| 125 |
- }) |
|
| 126 |
-} |
|
| 127 |
- |
|
| 128 |
-// WithoutScopeInfo configures the Exporter to not export the otel_scope_info metric. |
|
| 129 |
-// If not specified, the Exporter will create a otel_scope_info metric containing |
|
| 130 |
-// the metrics' Instrumentation Scope, and also add labels about Instrumentation Scope to all metric points. |
|
| 131 |
-func WithoutScopeInfo() Option {
|
|
| 132 |
- return optionFunc(func(cfg config) config {
|
|
| 133 |
- cfg.disableScopeInfo = true |
|
| 134 |
- return cfg |
|
| 135 |
- }) |
|
| 136 |
-} |
|
| 137 |
- |
|
| 138 |
-// WithNamespace configures the Exporter to prefix metric with the given namespace. |
|
| 139 |
-// Metadata metrics such as target_info and otel_scope_info are not prefixed since these |
|
| 140 |
-// have special behavior based on their name. |
|
| 141 |
-func WithNamespace(ns string) Option {
|
|
| 142 |
- return optionFunc(func(cfg config) config {
|
|
| 143 |
- ns = sanitizeName(ns) |
|
| 144 |
- if !strings.HasSuffix(ns, "_") {
|
|
| 145 |
- // namespace and metric names should be separated with an underscore, |
|
| 146 |
- // adds a trailing underscore if there is not one already. |
|
| 147 |
- ns = ns + "_" |
|
| 148 |
- } |
|
| 149 |
- |
|
| 150 |
- cfg.namespace = ns |
|
| 151 |
- return cfg |
|
| 152 |
- }) |
|
| 153 |
-} |
| 154 | 1 |
deleted file mode 100644 |
| ... | ... |
@@ -1,18 +0,0 @@ |
| 1 |
-// Copyright The OpenTelemetry Authors |
|
| 2 |
-// |
|
| 3 |
-// Licensed under the Apache License, Version 2.0 (the "License"); |
|
| 4 |
-// you may not use this file except in compliance with the License. |
|
| 5 |
-// You may obtain a copy of the License at |
|
| 6 |
-// |
|
| 7 |
-// http://www.apache.org/licenses/LICENSE-2.0 |
|
| 8 |
-// |
|
| 9 |
-// Unless required by applicable law or agreed to in writing, software |
|
| 10 |
-// distributed under the License is distributed on an "AS IS" BASIS, |
|
| 11 |
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
| 12 |
-// See the License for the specific language governing permissions and |
|
| 13 |
-// limitations under the License. |
|
| 14 |
- |
|
| 15 |
-// Package prometheus provides a Prometheus Exporter that converts |
|
| 16 |
-// OTLP metrics into the Prometheus exposition format and implements |
|
| 17 |
-// prometheus.Collector to provide a handler for these metrics. |
|
| 18 |
-package prometheus // import "go.opentelemetry.io/otel/exporters/prometheus" |
| 19 | 1 |
deleted file mode 100644 |
| ... | ... |
@@ -1,533 +0,0 @@ |
| 1 |
-// Copyright The OpenTelemetry Authors |
|
| 2 |
-// |
|
| 3 |
-// Licensed under the Apache License, Version 2.0 (the "License"); |
|
| 4 |
-// you may not use this file except in compliance with the License. |
|
| 5 |
-// You may obtain a copy of the License at |
|
| 6 |
-// |
|
| 7 |
-// http://www.apache.org/licenses/LICENSE-2.0 |
|
| 8 |
-// |
|
| 9 |
-// Unless required by applicable law or agreed to in writing, software |
|
| 10 |
-// distributed under the License is distributed on an "AS IS" BASIS, |
|
| 11 |
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
| 12 |
-// See the License for the specific language governing permissions and |
|
| 13 |
-// limitations under the License. |
|
| 14 |
- |
|
| 15 |
-package prometheus // import "go.opentelemetry.io/otel/exporters/prometheus" |
|
| 16 |
- |
|
| 17 |
-import ( |
|
| 18 |
- "context" |
|
| 19 |
- "errors" |
|
| 20 |
- "fmt" |
|
| 21 |
- "sort" |
|
| 22 |
- "strings" |
|
| 23 |
- "sync" |
|
| 24 |
- "unicode" |
|
| 25 |
- "unicode/utf8" |
|
| 26 |
- |
|
| 27 |
- "github.com/prometheus/client_golang/prometheus" |
|
| 28 |
- dto "github.com/prometheus/client_model/go" |
|
| 29 |
- "google.golang.org/protobuf/proto" |
|
| 30 |
- |
|
| 31 |
- "go.opentelemetry.io/otel" |
|
| 32 |
- "go.opentelemetry.io/otel/attribute" |
|
| 33 |
- "go.opentelemetry.io/otel/internal/global" |
|
| 34 |
- "go.opentelemetry.io/otel/sdk/instrumentation" |
|
| 35 |
- "go.opentelemetry.io/otel/sdk/metric" |
|
| 36 |
- "go.opentelemetry.io/otel/sdk/metric/metricdata" |
|
| 37 |
- "go.opentelemetry.io/otel/sdk/resource" |
|
| 38 |
-) |
|
| 39 |
- |
|
| 40 |
-const ( |
|
| 41 |
- targetInfoMetricName = "target_info" |
|
| 42 |
- targetInfoDescription = "Target metadata" |
|
| 43 |
- |
|
| 44 |
- scopeInfoMetricName = "otel_scope_info" |
|
| 45 |
- scopeInfoDescription = "Instrumentation Scope metadata" |
|
| 46 |
-) |
|
| 47 |
- |
|
| 48 |
-var ( |
|
| 49 |
- scopeInfoKeys = [2]string{"otel_scope_name", "otel_scope_version"}
|
|
| 50 |
- |
|
| 51 |
- errScopeInvalid = errors.New("invalid scope")
|
|
| 52 |
-) |
|
| 53 |
- |
|
| 54 |
-// Exporter is a Prometheus Exporter that embeds the OTel metric.Reader |
|
| 55 |
-// interface for easy instantiation with a MeterProvider. |
|
| 56 |
-type Exporter struct {
|
|
| 57 |
- metric.Reader |
|
| 58 |
-} |
|
| 59 |
- |
|
| 60 |
-// MarshalLog returns logging data about the Exporter. |
|
| 61 |
-func (e *Exporter) MarshalLog() interface{} {
|
|
| 62 |
- const t = "Prometheus exporter" |
|
| 63 |
- |
|
| 64 |
- if r, ok := e.Reader.(*metric.ManualReader); ok {
|
|
| 65 |
- under := r.MarshalLog() |
|
| 66 |
- if data, ok := under.(struct {
|
|
| 67 |
- Type string |
|
| 68 |
- Registered bool |
|
| 69 |
- Shutdown bool |
|
| 70 |
- }); ok {
|
|
| 71 |
- data.Type = t |
|
| 72 |
- return data |
|
| 73 |
- } |
|
| 74 |
- } |
|
| 75 |
- |
|
| 76 |
- return struct{ Type string }{Type: t}
|
|
| 77 |
-} |
|
| 78 |
- |
|
| 79 |
-var _ metric.Reader = &Exporter{}
|
|
| 80 |
- |
|
| 81 |
-// collector is used to implement prometheus.Collector. |
|
| 82 |
-type collector struct {
|
|
| 83 |
- reader metric.Reader |
|
| 84 |
- |
|
| 85 |
- withoutUnits bool |
|
| 86 |
- withoutCounterSuffixes bool |
|
| 87 |
- disableScopeInfo bool |
|
| 88 |
- namespace string |
|
| 89 |
- |
|
| 90 |
- mu sync.Mutex // mu protects all members below from the concurrent access. |
|
| 91 |
- disableTargetInfo bool |
|
| 92 |
- targetInfo prometheus.Metric |
|
| 93 |
- scopeInfos map[instrumentation.Scope]prometheus.Metric |
|
| 94 |
- scopeInfosInvalid map[instrumentation.Scope]struct{}
|
|
| 95 |
- metricFamilies map[string]*dto.MetricFamily |
|
| 96 |
-} |
|
| 97 |
- |
|
| 98 |
-// prometheus counters MUST have a _total suffix by default: |
|
| 99 |
-// https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/specification/compatibility/prometheus_and_openmetrics.md |
|
| 100 |
-const counterSuffix = "_total" |
|
| 101 |
- |
|
| 102 |
-// New returns a Prometheus Exporter. |
|
| 103 |
-func New(opts ...Option) (*Exporter, error) {
|
|
| 104 |
- cfg := newConfig(opts...) |
|
| 105 |
- |
|
| 106 |
- // this assumes that the default temporality selector will always return cumulative. |
|
| 107 |
- // we only support cumulative temporality, so building our own reader enforces this. |
|
| 108 |
- // TODO (#3244): Enable some way to configure the reader, but not change temporality. |
|
| 109 |
- reader := metric.NewManualReader(cfg.readerOpts...) |
|
| 110 |
- |
|
| 111 |
- collector := &collector{
|
|
| 112 |
- reader: reader, |
|
| 113 |
- disableTargetInfo: cfg.disableTargetInfo, |
|
| 114 |
- withoutUnits: cfg.withoutUnits, |
|
| 115 |
- withoutCounterSuffixes: cfg.withoutCounterSuffixes, |
|
| 116 |
- disableScopeInfo: cfg.disableScopeInfo, |
|
| 117 |
- scopeInfos: make(map[instrumentation.Scope]prometheus.Metric), |
|
| 118 |
- scopeInfosInvalid: make(map[instrumentation.Scope]struct{}),
|
|
| 119 |
- metricFamilies: make(map[string]*dto.MetricFamily), |
|
| 120 |
- namespace: cfg.namespace, |
|
| 121 |
- } |
|
| 122 |
- |
|
| 123 |
- if err := cfg.registerer.Register(collector); err != nil {
|
|
| 124 |
- return nil, fmt.Errorf("cannot register the collector: %w", err)
|
|
| 125 |
- } |
|
| 126 |
- |
|
| 127 |
- e := &Exporter{
|
|
| 128 |
- Reader: reader, |
|
| 129 |
- } |
|
| 130 |
- |
|
| 131 |
- return e, nil |
|
| 132 |
-} |
|
| 133 |
- |
|
| 134 |
-// Describe implements prometheus.Collector. |
|
| 135 |
-func (c *collector) Describe(ch chan<- *prometheus.Desc) {
|
|
| 136 |
- // The Opentelemetry SDK doesn't have information on which will exist when the collector |
|
| 137 |
- // is registered. By returning nothing we are an "unchecked" collector in Prometheus, |
|
| 138 |
- // and assume responsibility for consistency of the metrics produced. |
|
| 139 |
- // |
|
| 140 |
- // See https://pkg.go.dev/github.com/prometheus/client_golang@v1.13.0/prometheus#hdr-Custom_Collectors_and_constant_Metrics |
|
| 141 |
-} |
|
| 142 |
- |
|
| 143 |
-// Collect implements prometheus.Collector. |
|
| 144 |
-// |
|
| 145 |
-// This method is safe to call concurrently. |
|
| 146 |
-func (c *collector) Collect(ch chan<- prometheus.Metric) {
|
|
| 147 |
- // TODO (#3047): Use a sync.Pool instead of allocating metrics every Collect. |
|
| 148 |
- metrics := metricdata.ResourceMetrics{}
|
|
| 149 |
- err := c.reader.Collect(context.TODO(), &metrics) |
|
| 150 |
- if err != nil {
|
|
| 151 |
- otel.Handle(err) |
|
| 152 |
- if err == metric.ErrReaderNotRegistered {
|
|
| 153 |
- return |
|
| 154 |
- } |
|
| 155 |
- } |
|
| 156 |
- |
|
| 157 |
- global.Debug("Prometheus exporter export", "Data", metrics)
|
|
| 158 |
- |
|
| 159 |
- // Initialize (once) targetInfo and disableTargetInfo. |
|
| 160 |
- func() {
|
|
| 161 |
- c.mu.Lock() |
|
| 162 |
- defer c.mu.Unlock() |
|
| 163 |
- |
|
| 164 |
- if c.targetInfo == nil && !c.disableTargetInfo {
|
|
| 165 |
- targetInfo, err := createInfoMetric(targetInfoMetricName, targetInfoDescription, metrics.Resource) |
|
| 166 |
- if err != nil {
|
|
| 167 |
- // If the target info metric is invalid, disable sending it. |
|
| 168 |
- c.disableTargetInfo = true |
|
| 169 |
- otel.Handle(err) |
|
| 170 |
- return |
|
| 171 |
- } |
|
| 172 |
- |
|
| 173 |
- c.targetInfo = targetInfo |
|
| 174 |
- } |
|
| 175 |
- }() |
|
| 176 |
- |
|
| 177 |
- if !c.disableTargetInfo {
|
|
| 178 |
- ch <- c.targetInfo |
|
| 179 |
- } |
|
| 180 |
- |
|
| 181 |
- for _, scopeMetrics := range metrics.ScopeMetrics {
|
|
| 182 |
- var keys, values [2]string |
|
| 183 |
- |
|
| 184 |
- if !c.disableScopeInfo {
|
|
| 185 |
- scopeInfo, err := c.scopeInfo(scopeMetrics.Scope) |
|
| 186 |
- if err == errScopeInvalid {
|
|
| 187 |
- // Do not report the same error multiple times. |
|
| 188 |
- continue |
|
| 189 |
- } |
|
| 190 |
- if err != nil {
|
|
| 191 |
- otel.Handle(err) |
|
| 192 |
- continue |
|
| 193 |
- } |
|
| 194 |
- |
|
| 195 |
- ch <- scopeInfo |
|
| 196 |
- |
|
| 197 |
- keys = scopeInfoKeys |
|
| 198 |
- values = [2]string{scopeMetrics.Scope.Name, scopeMetrics.Scope.Version}
|
|
| 199 |
- } |
|
| 200 |
- |
|
| 201 |
- for _, m := range scopeMetrics.Metrics {
|
|
| 202 |
- typ := c.metricType(m) |
|
| 203 |
- if typ == nil {
|
|
| 204 |
- continue |
|
| 205 |
- } |
|
| 206 |
- name := c.getName(m, typ) |
|
| 207 |
- |
|
| 208 |
- drop, help := c.validateMetrics(name, m.Description, typ) |
|
| 209 |
- if drop {
|
|
| 210 |
- continue |
|
| 211 |
- } |
|
| 212 |
- |
|
| 213 |
- if help != "" {
|
|
| 214 |
- m.Description = help |
|
| 215 |
- } |
|
| 216 |
- |
|
| 217 |
- switch v := m.Data.(type) {
|
|
| 218 |
- case metricdata.Histogram[int64]: |
|
| 219 |
- addHistogramMetric(ch, v, m, keys, values, name) |
|
| 220 |
- case metricdata.Histogram[float64]: |
|
| 221 |
- addHistogramMetric(ch, v, m, keys, values, name) |
|
| 222 |
- case metricdata.Sum[int64]: |
|
| 223 |
- addSumMetric(ch, v, m, keys, values, name) |
|
| 224 |
- case metricdata.Sum[float64]: |
|
| 225 |
- addSumMetric(ch, v, m, keys, values, name) |
|
| 226 |
- case metricdata.Gauge[int64]: |
|
| 227 |
- addGaugeMetric(ch, v, m, keys, values, name) |
|
| 228 |
- case metricdata.Gauge[float64]: |
|
| 229 |
- addGaugeMetric(ch, v, m, keys, values, name) |
|
| 230 |
- } |
|
| 231 |
- } |
|
| 232 |
- } |
|
| 233 |
-} |
|
| 234 |
- |
|
| 235 |
-func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogram metricdata.Histogram[N], m metricdata.Metrics, ks, vs [2]string, name string) {
|
|
| 236 |
- // TODO(https://github.com/open-telemetry/opentelemetry-go/issues/3163): support exemplars |
|
| 237 |
- for _, dp := range histogram.DataPoints {
|
|
| 238 |
- keys, values := getAttrs(dp.Attributes, ks, vs) |
|
| 239 |
- |
|
| 240 |
- desc := prometheus.NewDesc(name, m.Description, keys, nil) |
|
| 241 |
- buckets := make(map[float64]uint64, len(dp.Bounds)) |
|
| 242 |
- |
|
| 243 |
- cumulativeCount := uint64(0) |
|
| 244 |
- for i, bound := range dp.Bounds {
|
|
| 245 |
- cumulativeCount += dp.BucketCounts[i] |
|
| 246 |
- buckets[bound] = cumulativeCount |
|
| 247 |
- } |
|
| 248 |
- m, err := prometheus.NewConstHistogram(desc, dp.Count, float64(dp.Sum), buckets, values...) |
|
| 249 |
- if err != nil {
|
|
| 250 |
- otel.Handle(err) |
|
| 251 |
- continue |
|
| 252 |
- } |
|
| 253 |
- ch <- m |
|
| 254 |
- } |
|
| 255 |
-} |
|
| 256 |
- |
|
| 257 |
-func addSumMetric[N int64 | float64](ch chan<- prometheus.Metric, sum metricdata.Sum[N], m metricdata.Metrics, ks, vs [2]string, name string) {
|
|
| 258 |
- valueType := prometheus.CounterValue |
|
| 259 |
- if !sum.IsMonotonic {
|
|
| 260 |
- valueType = prometheus.GaugeValue |
|
| 261 |
- } |
|
| 262 |
- |
|
| 263 |
- for _, dp := range sum.DataPoints {
|
|
| 264 |
- keys, values := getAttrs(dp.Attributes, ks, vs) |
|
| 265 |
- |
|
| 266 |
- desc := prometheus.NewDesc(name, m.Description, keys, nil) |
|
| 267 |
- m, err := prometheus.NewConstMetric(desc, valueType, float64(dp.Value), values...) |
|
| 268 |
- if err != nil {
|
|
| 269 |
- otel.Handle(err) |
|
| 270 |
- continue |
|
| 271 |
- } |
|
| 272 |
- ch <- m |
|
| 273 |
- } |
|
| 274 |
-} |
|
| 275 |
- |
|
| 276 |
-func addGaugeMetric[N int64 | float64](ch chan<- prometheus.Metric, gauge metricdata.Gauge[N], m metricdata.Metrics, ks, vs [2]string, name string) {
|
|
| 277 |
- for _, dp := range gauge.DataPoints {
|
|
| 278 |
- keys, values := getAttrs(dp.Attributes, ks, vs) |
|
| 279 |
- |
|
| 280 |
- desc := prometheus.NewDesc(name, m.Description, keys, nil) |
|
| 281 |
- m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, float64(dp.Value), values...) |
|
| 282 |
- if err != nil {
|
|
| 283 |
- otel.Handle(err) |
|
| 284 |
- continue |
|
| 285 |
- } |
|
| 286 |
- ch <- m |
|
| 287 |
- } |
|
| 288 |
-} |
|
| 289 |
- |
|
| 290 |
-// getAttrs parses the attribute.Set to two lists of matching Prometheus-style |
|
| 291 |
-// keys and values. It sanitizes invalid characters and handles duplicate keys |
|
| 292 |
-// (due to sanitization) by sorting and concatenating the values following the spec. |
|
| 293 |
-func getAttrs(attrs attribute.Set, ks, vs [2]string) ([]string, []string) {
|
|
| 294 |
- keysMap := make(map[string][]string) |
|
| 295 |
- itr := attrs.Iter() |
|
| 296 |
- for itr.Next() {
|
|
| 297 |
- kv := itr.Attribute() |
|
| 298 |
- key := strings.Map(sanitizeRune, string(kv.Key)) |
|
| 299 |
- if _, ok := keysMap[key]; !ok {
|
|
| 300 |
- keysMap[key] = []string{kv.Value.Emit()}
|
|
| 301 |
- } else {
|
|
| 302 |
- // if the sanitized key is a duplicate, append to the list of keys |
|
| 303 |
- keysMap[key] = append(keysMap[key], kv.Value.Emit()) |
|
| 304 |
- } |
|
| 305 |
- } |
|
| 306 |
- |
|
| 307 |
- keys := make([]string, 0, attrs.Len()) |
|
| 308 |
- values := make([]string, 0, attrs.Len()) |
|
| 309 |
- for key, vals := range keysMap {
|
|
| 310 |
- keys = append(keys, key) |
|
| 311 |
- sort.Slice(vals, func(i, j int) bool {
|
|
| 312 |
- return i < j |
|
| 313 |
- }) |
|
| 314 |
- values = append(values, strings.Join(vals, ";")) |
|
| 315 |
- } |
|
| 316 |
- |
|
| 317 |
- if ks[0] != "" {
|
|
| 318 |
- keys = append(keys, ks[:]...) |
|
| 319 |
- values = append(values, vs[:]...) |
|
| 320 |
- } |
|
| 321 |
- return keys, values |
|
| 322 |
-} |
|
| 323 |
- |
|
| 324 |
-func createInfoMetric(name, description string, res *resource.Resource) (prometheus.Metric, error) {
|
|
| 325 |
- keys, values := getAttrs(*res.Set(), [2]string{}, [2]string{})
|
|
| 326 |
- desc := prometheus.NewDesc(name, description, keys, nil) |
|
| 327 |
- return prometheus.NewConstMetric(desc, prometheus.GaugeValue, float64(1), values...) |
|
| 328 |
-} |
|
| 329 |
- |
|
| 330 |
-func createScopeInfoMetric(scope instrumentation.Scope) (prometheus.Metric, error) {
|
|
| 331 |
- keys := scopeInfoKeys[:] |
|
| 332 |
- desc := prometheus.NewDesc(scopeInfoMetricName, scopeInfoDescription, keys, nil) |
|
| 333 |
- return prometheus.NewConstMetric(desc, prometheus.GaugeValue, float64(1), scope.Name, scope.Version) |
|
| 334 |
-} |
|
| 335 |
- |
|
| 336 |
-func sanitizeRune(r rune) rune {
|
|
| 337 |
- if unicode.IsLetter(r) || unicode.IsDigit(r) || r == ':' || r == '_' {
|
|
| 338 |
- return r |
|
| 339 |
- } |
|
| 340 |
- return '_' |
|
| 341 |
-} |
|
| 342 |
- |
|
| 343 |
-var unitSuffixes = map[string]string{
|
|
| 344 |
- // Time |
|
| 345 |
- "d": "_days", |
|
| 346 |
- "h": "_hours", |
|
| 347 |
- "min": "_minutes", |
|
| 348 |
- "s": "_seconds", |
|
| 349 |
- "ms": "_milliseconds", |
|
| 350 |
- "us": "_microseconds", |
|
| 351 |
- "ns": "_nanoseconds", |
|
| 352 |
- |
|
| 353 |
- // Bytes |
|
| 354 |
- "By": "_bytes", |
|
| 355 |
- "KiBy": "_kibibytes", |
|
| 356 |
- "MiBy": "_mebibytes", |
|
| 357 |
- "GiBy": "_gibibytes", |
|
| 358 |
- "TiBy": "_tibibytes", |
|
| 359 |
- "KBy": "_kilobytes", |
|
| 360 |
- "MBy": "_megabytes", |
|
| 361 |
- "GBy": "_gigabytes", |
|
| 362 |
- "TBy": "_terabytes", |
|
| 363 |
- |
|
| 364 |
- // SI |
|
| 365 |
- "m": "_meters", |
|
| 366 |
- "V": "_volts", |
|
| 367 |
- "A": "_amperes", |
|
| 368 |
- "J": "_joules", |
|
| 369 |
- "W": "_watts", |
|
| 370 |
- "g": "_grams", |
|
| 371 |
- |
|
| 372 |
- // Misc |
|
| 373 |
- "Cel": "_celsius", |
|
| 374 |
- "Hz": "_hertz", |
|
| 375 |
- "1": "_ratio", |
|
| 376 |
- "%": "_percent", |
|
| 377 |
-} |
|
| 378 |
- |
|
| 379 |
-// getName returns the sanitized name, prefixed with the namespace and suffixed with unit. |
|
| 380 |
-func (c *collector) getName(m metricdata.Metrics, typ *dto.MetricType) string {
|
|
| 381 |
- name := sanitizeName(m.Name) |
|
| 382 |
- addCounterSuffix := !c.withoutCounterSuffixes && *typ == dto.MetricType_COUNTER |
|
| 383 |
- if addCounterSuffix {
|
|
| 384 |
- // Remove the _total suffix here, as we will re-add the total suffix |
|
| 385 |
- // later, and it needs to come after the unit suffix. |
|
| 386 |
- name = strings.TrimSuffix(name, counterSuffix) |
|
| 387 |
- } |
|
| 388 |
- if c.namespace != "" {
|
|
| 389 |
- name = c.namespace + name |
|
| 390 |
- } |
|
| 391 |
- if suffix, ok := unitSuffixes[m.Unit]; ok && !c.withoutUnits && !strings.HasSuffix(name, suffix) {
|
|
| 392 |
- name += suffix |
|
| 393 |
- } |
|
| 394 |
- if addCounterSuffix {
|
|
| 395 |
- name += counterSuffix |
|
| 396 |
- } |
|
| 397 |
- return name |
|
| 398 |
-} |
|
| 399 |
- |
|
| 400 |
-func sanitizeName(n string) string {
|
|
| 401 |
- // This algorithm is based on strings.Map from Go 1.19. |
|
| 402 |
- const replacement = '_' |
|
| 403 |
- |
|
| 404 |
- valid := func(i int, r rune) bool {
|
|
| 405 |
- // Taken from |
|
| 406 |
- // https://github.com/prometheus/common/blob/dfbc25bd00225c70aca0d94c3c4bb7744f28ace0/model/metric.go#L92-L102 |
|
| 407 |
- if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_' || r == ':' || (r >= '0' && r <= '9' && i > 0) {
|
|
| 408 |
- return true |
|
| 409 |
- } |
|
| 410 |
- return false |
|
| 411 |
- } |
|
| 412 |
- |
|
| 413 |
- // This output buffer b is initialized on demand, the first time a |
|
| 414 |
- // character needs to be replaced. |
|
| 415 |
- var b strings.Builder |
|
| 416 |
- for i, c := range n {
|
|
| 417 |
- if valid(i, c) {
|
|
| 418 |
- continue |
|
| 419 |
- } |
|
| 420 |
- |
|
| 421 |
- if i == 0 && c >= '0' && c <= '9' {
|
|
| 422 |
- // Prefix leading number with replacement character. |
|
| 423 |
- b.Grow(len(n) + 1) |
|
| 424 |
- _ = b.WriteByte(byte(replacement)) |
|
| 425 |
- break |
|
| 426 |
- } |
|
| 427 |
- b.Grow(len(n)) |
|
| 428 |
- _, _ = b.WriteString(n[:i]) |
|
| 429 |
- _ = b.WriteByte(byte(replacement)) |
|
| 430 |
- width := utf8.RuneLen(c) |
|
| 431 |
- n = n[i+width:] |
|
| 432 |
- break |
|
| 433 |
- } |
|
| 434 |
- |
|
| 435 |
- // Fast path for unchanged input. |
|
| 436 |
- if b.Cap() == 0 { // b.Grow was not called above.
|
|
| 437 |
- return n |
|
| 438 |
- } |
|
| 439 |
- |
|
| 440 |
- for _, c := range n {
|
|
| 441 |
- // Due to inlining, it is more performant to invoke WriteByte rather then |
|
| 442 |
- // WriteRune. |
|
| 443 |
- if valid(1, c) { // We are guaranteed to not be at the start.
|
|
| 444 |
- _ = b.WriteByte(byte(c)) |
|
| 445 |
- } else {
|
|
| 446 |
- _ = b.WriteByte(byte(replacement)) |
|
| 447 |
- } |
|
| 448 |
- } |
|
| 449 |
- |
|
| 450 |
- return b.String() |
|
| 451 |
-} |
|
| 452 |
- |
|
| 453 |
-func (c *collector) metricType(m metricdata.Metrics) *dto.MetricType {
|
|
| 454 |
- switch v := m.Data.(type) {
|
|
| 455 |
- case metricdata.Histogram[int64], metricdata.Histogram[float64]: |
|
| 456 |
- return dto.MetricType_HISTOGRAM.Enum() |
|
| 457 |
- case metricdata.Sum[float64]: |
|
| 458 |
- if v.IsMonotonic {
|
|
| 459 |
- return dto.MetricType_COUNTER.Enum() |
|
| 460 |
- } |
|
| 461 |
- return dto.MetricType_GAUGE.Enum() |
|
| 462 |
- case metricdata.Sum[int64]: |
|
| 463 |
- if v.IsMonotonic {
|
|
| 464 |
- return dto.MetricType_COUNTER.Enum() |
|
| 465 |
- } |
|
| 466 |
- return dto.MetricType_GAUGE.Enum() |
|
| 467 |
- case metricdata.Gauge[int64], metricdata.Gauge[float64]: |
|
| 468 |
- return dto.MetricType_GAUGE.Enum() |
|
| 469 |
- } |
|
| 470 |
- return nil |
|
| 471 |
-} |
|
| 472 |
- |
|
| 473 |
-func (c *collector) scopeInfo(scope instrumentation.Scope) (prometheus.Metric, error) {
|
|
| 474 |
- c.mu.Lock() |
|
| 475 |
- defer c.mu.Unlock() |
|
| 476 |
- |
|
| 477 |
- scopeInfo, ok := c.scopeInfos[scope] |
|
| 478 |
- if ok {
|
|
| 479 |
- return scopeInfo, nil |
|
| 480 |
- } |
|
| 481 |
- |
|
| 482 |
- if _, ok := c.scopeInfosInvalid[scope]; ok {
|
|
| 483 |
- return nil, errScopeInvalid |
|
| 484 |
- } |
|
| 485 |
- |
|
| 486 |
- scopeInfo, err := createScopeInfoMetric(scope) |
|
| 487 |
- if err != nil {
|
|
| 488 |
- c.scopeInfosInvalid[scope] = struct{}{}
|
|
| 489 |
- return nil, fmt.Errorf("cannot create scope info metric: %w", err)
|
|
| 490 |
- } |
|
| 491 |
- |
|
| 492 |
- c.scopeInfos[scope] = scopeInfo |
|
| 493 |
- |
|
| 494 |
- return scopeInfo, nil |
|
| 495 |
-} |
|
| 496 |
- |
|
| 497 |
-func (c *collector) validateMetrics(name, description string, metricType *dto.MetricType) (drop bool, help string) {
|
|
| 498 |
- c.mu.Lock() |
|
| 499 |
- defer c.mu.Unlock() |
|
| 500 |
- |
|
| 501 |
- emf, exist := c.metricFamilies[name] |
|
| 502 |
- |
|
| 503 |
- if !exist {
|
|
| 504 |
- c.metricFamilies[name] = &dto.MetricFamily{
|
|
| 505 |
- Name: proto.String(name), |
|
| 506 |
- Help: proto.String(description), |
|
| 507 |
- Type: metricType, |
|
| 508 |
- } |
|
| 509 |
- return false, "" |
|
| 510 |
- } |
|
| 511 |
- |
|
| 512 |
- if emf.GetType() != *metricType {
|
|
| 513 |
- global.Error( |
|
| 514 |
- errors.New("instrument type conflict"),
|
|
| 515 |
- "Using existing type definition.", |
|
| 516 |
- "instrument", name, |
|
| 517 |
- "existing", emf.GetType(), |
|
| 518 |
- "dropped", *metricType, |
|
| 519 |
- ) |
|
| 520 |
- return true, "" |
|
| 521 |
- } |
|
| 522 |
- if emf.GetHelp() != description {
|
|
| 523 |
- global.Info( |
|
| 524 |
- "Instrument description conflict, using existing", |
|
| 525 |
- "instrument", name, |
|
| 526 |
- "existing", emf.GetHelp(), |
|
| 527 |
- "dropped", description, |
|
| 528 |
- ) |
|
| 529 |
- return false, emf.GetHelp() |
|
| 530 |
- } |
|
| 531 |
- |
|
| 532 |
- return false, "" |
|
| 533 |
-} |
| ... | ... |
@@ -712,7 +712,7 @@ github.com/mitchellh/hashstructure/v2 |
| 712 | 712 |
# github.com/mitchellh/reflectwalk v1.0.2 |
| 713 | 713 |
## explicit |
| 714 | 714 |
github.com/mitchellh/reflectwalk |
| 715 |
-# github.com/moby/buildkit v0.13.1 |
|
| 715 |
+# github.com/moby/buildkit v0.14.0-rc1.0.20240605195929-c1f5352a1b7e |
|
| 716 | 716 |
## explicit; go 1.21 |
| 717 | 717 |
github.com/moby/buildkit/api/services/control |
| 718 | 718 |
github.com/moby/buildkit/api/types |
| ... | ... |
@@ -752,6 +752,7 @@ github.com/moby/buildkit/exporter/local |
| 752 | 752 |
github.com/moby/buildkit/exporter/oci |
| 753 | 753 |
github.com/moby/buildkit/exporter/tar |
| 754 | 754 |
github.com/moby/buildkit/exporter/util/epoch |
| 755 |
+github.com/moby/buildkit/exporter/verifier |
|
| 755 | 756 |
github.com/moby/buildkit/frontend |
| 756 | 757 |
github.com/moby/buildkit/frontend/attestations |
| 757 | 758 |
github.com/moby/buildkit/frontend/attestations/sbom |
| ... | ... |
@@ -759,6 +760,7 @@ github.com/moby/buildkit/frontend/dockerfile/builder |
| 759 | 759 |
github.com/moby/buildkit/frontend/dockerfile/command |
| 760 | 760 |
github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb |
| 761 | 761 |
github.com/moby/buildkit/frontend/dockerfile/instructions |
| 762 |
+github.com/moby/buildkit/frontend/dockerfile/linter |
|
| 762 | 763 |
github.com/moby/buildkit/frontend/dockerfile/parser |
| 763 | 764 |
github.com/moby/buildkit/frontend/dockerfile/shell |
| 764 | 765 |
github.com/moby/buildkit/frontend/dockerui |
| ... | ... |
@@ -769,6 +771,7 @@ github.com/moby/buildkit/frontend/gateway/forwarder |
| 769 | 769 |
github.com/moby/buildkit/frontend/gateway/grpcclient |
| 770 | 770 |
github.com/moby/buildkit/frontend/gateway/pb |
| 771 | 771 |
github.com/moby/buildkit/frontend/subrequests |
| 772 |
+github.com/moby/buildkit/frontend/subrequests/lint |
|
| 772 | 773 |
github.com/moby/buildkit/frontend/subrequests/outline |
| 773 | 774 |
github.com/moby/buildkit/frontend/subrequests/targets |
| 774 | 775 |
github.com/moby/buildkit/identity |
| ... | ... |
@@ -1089,12 +1092,12 @@ github.com/syndtr/gocapability/capability |
| 1089 | 1089 |
# github.com/tinylib/msgp v1.1.8 |
| 1090 | 1090 |
## explicit; go 1.15 |
| 1091 | 1091 |
github.com/tinylib/msgp/msgp |
| 1092 |
-# github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 |
|
| 1092 |
+# github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c |
|
| 1093 | 1093 |
## explicit; go 1.20 |
| 1094 | 1094 |
github.com/tonistiigi/fsutil |
| 1095 | 1095 |
github.com/tonistiigi/fsutil/copy |
| 1096 | 1096 |
github.com/tonistiigi/fsutil/types |
| 1097 |
-# github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 |
|
| 1097 |
+# github.com/tonistiigi/go-actions-cache v0.0.0-20240320205438-9794bdbb2fb4 |
|
| 1098 | 1098 |
## explicit; go 1.20 |
| 1099 | 1099 |
github.com/tonistiigi/go-actions-cache |
| 1100 | 1100 |
# github.com/tonistiigi/go-archvariant v1.0.0 |
| ... | ... |
@@ -1103,7 +1106,7 @@ github.com/tonistiigi/go-archvariant |
| 1103 | 1103 |
# github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea |
| 1104 | 1104 |
## explicit |
| 1105 | 1105 |
github.com/tonistiigi/units |
| 1106 |
-# github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 |
|
| 1106 |
+# github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab |
|
| 1107 | 1107 |
## explicit; go 1.12 |
| 1108 | 1108 |
github.com/tonistiigi/vt100 |
| 1109 | 1109 |
# github.com/vbatts/tar-split v0.11.5 |
| ... | ... |
@@ -1248,9 +1251,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal |
| 1248 | 1248 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/envconfig |
| 1249 | 1249 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/otlpconfig |
| 1250 | 1250 |
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/retry |
| 1251 |
-# go.opentelemetry.io/otel/exporters/prometheus v0.42.0 |
|
| 1252 |
-## explicit; go 1.20 |
|
| 1253 |
-go.opentelemetry.io/otel/exporters/prometheus |
|
| 1254 | 1251 |
# go.opentelemetry.io/otel/metric v1.21.0 |
| 1255 | 1252 |
## explicit; go 1.20 |
| 1256 | 1253 |
go.opentelemetry.io/otel/metric |