Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
| ... | ... |
@@ -440,6 +440,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt |
| 440 | 440 |
ImageSource: src, |
| 441 | 441 |
DownloadManager: dist.DownloadManager, |
| 442 | 442 |
V2MetadataService: dist.V2MetadataService, |
| 443 |
+ RegistryHosts: opt.RegistryHosts, |
|
| 443 | 444 |
Exporter: exp, |
| 444 | 445 |
Transport: rt, |
| 445 | 446 |
Layers: layers, |
| ... | ... |
@@ -10,6 +10,7 @@ import ( |
| 10 | 10 |
|
| 11 | 11 |
"github.com/containerd/containerd/v2/core/content" |
| 12 | 12 |
c8dimages "github.com/containerd/containerd/v2/core/images" |
| 13 |
+ "github.com/containerd/containerd/v2/core/remotes/docker" |
|
| 13 | 14 |
"github.com/containerd/containerd/v2/pkg/gc" |
| 14 | 15 |
"github.com/containerd/containerd/v2/pkg/rootfs" |
| 15 | 16 |
cerrdefs "github.com/containerd/errdefs" |
| ... | ... |
@@ -84,6 +85,7 @@ type Opt struct {
|
| 84 | 84 |
|
| 85 | 85 |
DownloadManager *xfer.LayerDownloadManager |
| 86 | 86 |
V2MetadataService distmetadata.V2MetadataService |
| 87 |
+ RegistryHosts docker.RegistryHosts |
|
| 87 | 88 |
Transport nethttp.RoundTripper |
| 88 | 89 |
Exporter exporter.Exporter |
| 89 | 90 |
Layers LayerAccess |
| ... | ... |
@@ -116,6 +118,7 @@ func NewWorker(opt Opt) (*Worker, error) {
|
| 116 | 116 |
|
| 117 | 117 |
gs, err := git.NewSource(git.Opt{
|
| 118 | 118 |
CacheAccessor: cm, |
| 119 |
+ RegistryHosts: opt.RegistryHosts, |
|
| 119 | 120 |
}) |
| 120 | 121 |
if err == nil {
|
| 121 | 122 |
sm.Register(gs) |
| ... | ... |
@@ -55,7 +55,7 @@ require ( |
| 55 | 55 |
github.com/miekg/dns v1.1.72 |
| 56 | 56 |
github.com/mistifyio/go-zfs/v4 v4.0.0 |
| 57 | 57 |
github.com/mitchellh/copystructure v1.2.0 |
| 58 |
- github.com/moby/buildkit v0.29.0 |
|
| 58 |
+ github.com/moby/buildkit v0.30.0-rc1 |
|
| 59 | 59 |
github.com/moby/docker-image-spec v1.3.1 |
| 60 | 60 |
github.com/moby/go-archive v0.2.0 |
| 61 | 61 |
github.com/moby/ipvs v1.1.0 |
| ... | ... |
@@ -108,7 +108,7 @@ require ( |
| 108 | 108 |
golang.org/x/sys v0.43.0 |
| 109 | 109 |
golang.org/x/text v0.36.0 |
| 110 | 110 |
golang.org/x/time v0.15.0 |
| 111 |
- google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 |
|
| 111 |
+ google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d |
|
| 112 | 112 |
google.golang.org/grpc v1.80.0 |
| 113 | 113 |
google.golang.org/protobuf v1.36.11 |
| 114 | 114 |
gotest.tools/v3 v3.5.2 |
| ... | ... |
@@ -222,7 +222,7 @@ require ( |
| 222 | 222 |
github.com/in-toto/attestation v1.1.2 // indirect |
| 223 | 223 |
github.com/inconshreveable/mousetrap v1.1.0 // indirect |
| 224 | 224 |
github.com/jmoiron/sqlx v1.4.0 // indirect |
| 225 |
- github.com/klauspost/compress v1.18.5 // indirect |
|
| 225 |
+ github.com/klauspost/compress v1.18.6 // indirect |
|
| 226 | 226 |
github.com/knqyf263/go-plugin v0.9.0 // indirect |
| 227 | 227 |
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect |
| 228 | 228 |
github.com/mitchellh/reflectwalk v1.0.2 // indirect |
| ... | ... |
@@ -226,8 +226,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi |
| 226 | 226 |
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= |
| 227 | 227 |
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= |
| 228 | 228 |
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= |
| 229 |
-github.com/docker/cli v29.3.1+incompatible h1:M04FDj2TRehDacrosh7Vlkgc7AuQoWloQkf1PA5hmoI= |
|
| 230 |
-github.com/docker/cli v29.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= |
|
| 229 |
+github.com/docker/cli v29.4.2+incompatible h1:nhxMY4v7wB0QMMc5ppeqV6FBMwzqv0n4t2gogu/R2DQ= |
|
| 230 |
+github.com/docker/cli v29.4.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= |
|
| 231 | 231 |
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= |
| 232 | 232 |
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= |
| 233 | 233 |
github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= |
| ... | ... |
@@ -505,8 +505,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V |
| 505 | 505 |
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= |
| 506 | 506 |
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= |
| 507 | 507 |
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= |
| 508 |
-github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= |
|
| 509 |
-github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= |
|
| 508 |
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= |
|
| 509 |
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= |
|
| 510 | 510 |
github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= |
| 511 | 511 |
github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= |
| 512 | 512 |
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= |
| ... | ... |
@@ -549,8 +549,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua |
| 549 | 549 |
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= |
| 550 | 550 |
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= |
| 551 | 551 |
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= |
| 552 |
-github.com/moby/buildkit v0.29.0 h1:wxLEFbCOJntEDjSNNN2YWd8zxltZxT5muDQ0LzpbtpU= |
|
| 553 |
-github.com/moby/buildkit v0.29.0/go.mod h1:Dmv2FeDe34t75QuzeU87rBoZpAAkcpT5zeu4hXzmASc= |
|
| 552 |
+github.com/moby/buildkit v0.30.0-rc1 h1:meXm1eqqweZ7EPnTbRiU8c4BjgjDKs7rUczHUsU0nJo= |
|
| 553 |
+github.com/moby/buildkit v0.30.0-rc1/go.mod h1:4ha5jioeEAREXOpmpEUSQ4bKZON1yEc/co9I57+zCZs= |
|
| 554 | 554 |
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= |
| 555 | 555 |
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= |
| 556 | 556 |
github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= |
| ... | ... |
@@ -1043,8 +1043,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 |
| 1043 | 1043 |
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= |
| 1044 | 1044 |
google.golang.org/genproto v0.0.0-20260401024825-9d38bb4040a9 h1:w8JYjr7zHemS95YA5FFwk+fUv5tdQU4I8twN9bFdxVU= |
| 1045 | 1045 |
google.golang.org/genproto v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:YCEC8W7HTtK7iBv+pI7g7hGAi7qdGB6bQXw3BIYAusM= |
| 1046 |
-google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= |
|
| 1047 |
-google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= |
|
| 1046 |
+google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d h1:/aDRtSZJjyLQzm75d+a1wOJaqyKBMvIAfeQmoa3ORiI= |
|
| 1047 |
+google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:etfGUgejTiadZAUaEP14NP97xi1RGeawqkjDARA/UOs= |
|
| 1048 | 1048 |
google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= |
| 1049 | 1049 |
google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= |
| 1050 | 1050 |
google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= |
| ... | ... |
@@ -1090,10 +1090,10 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh |
| 1090 | 1090 |
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= |
| 1091 | 1091 |
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= |
| 1092 | 1092 |
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= |
| 1093 |
-kernel.org/pub/linux/libs/security/libcap/cap v1.2.77 h1:iQtQTjFUOcTT19fI8sTCzYXsjeVs56et3D8AbKS2Uks= |
|
| 1094 |
-kernel.org/pub/linux/libs/security/libcap/cap v1.2.77/go.mod h1:oV+IO8kGh0B7TxErbydDe2+BRmi9g/W0CkpVV+QBTJU= |
|
| 1095 |
-kernel.org/pub/linux/libs/security/libcap/psx v1.2.77 h1:Z06sMOzc0GNCwp6efaVrIrz4ywGJ1v+DP0pjVkOfDuA= |
|
| 1096 |
-kernel.org/pub/linux/libs/security/libcap/psx v1.2.77/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= |
|
| 1093 |
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.78 h1:jgqg4gyu2BaYW9L6uzEtGLf8GNREwk/z4UFdwt5F3pE= |
|
| 1094 |
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.78/go.mod h1:VjuVda6m2qGkpCVfrFkpTGyvkdlZ2N5/yfo89tujlg8= |
|
| 1095 |
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.78 h1:PC3yNs51cX5LZ7U57a7xielBcoXB3xnV+rXD8V0H0DQ= |
|
| 1096 |
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.78/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= |
|
| 1097 | 1097 |
pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc= |
| 1098 | 1098 |
pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk= |
| 1099 | 1099 |
resenje.org/singleflight v0.4.3 h1:l7foFYg8X/VEHPxWs1K/Pw77807RMVzvXgWGb0J1sdM= |
| ... | ... |
@@ -1,700 +1,700 @@ |
| 1 |
-# compress |
|
| 2 |
- |
|
| 3 |
-This package provides various compression algorithms. |
|
| 4 |
- |
|
| 5 |
-* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. |
|
| 6 |
-* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. |
|
| 7 |
-* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). |
|
| 8 |
-* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. |
|
| 9 |
-* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. |
|
| 10 |
-* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. |
|
| 11 |
-* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. |
|
| 12 |
- |
|
| 13 |
-[](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) |
|
| 14 |
-[](https://github.com/klauspost/compress/actions/workflows/go.yml) |
|
| 15 |
-[](https://sourcegraph.com/github.com/klauspost/compress?badge) |
|
| 16 |
- |
|
| 17 |
-# package usage |
|
| 18 |
- |
|
| 19 |
-Use `go get github.com/klauspost/compress@latest` to add it to your project. |
|
| 20 |
- |
|
| 21 |
-This package will support the current Go version and 2 versions back. |
|
| 22 |
- |
|
| 23 |
-* Use the `nounsafe` tag to disable all use of the "unsafe" package. |
|
| 24 |
-* Use the `noasm` tag to disable all assembly across packages. |
|
| 25 |
- |
|
| 26 |
-Use the links above for more information on each. |
|
| 27 |
- |
|
| 28 |
-# changelog |
|
| 29 |
- |
|
| 30 |
-* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) |
|
| 31 |
- * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 |
|
| 32 |
- * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 |
|
| 33 |
- * gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116 |
|
| 34 |
- |
|
| 35 |
-* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) |
|
| 36 |
- * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). |
|
| 37 |
- |
|
| 38 |
-* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) |
|
| 39 |
- * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 |
|
| 40 |
- * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 |
|
| 41 |
- |
|
| 42 |
-* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED |
|
| 43 |
- * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 |
|
| 44 |
- * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 |
|
| 45 |
- * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 |
|
| 46 |
- * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 |
|
| 47 |
- * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 |
|
| 48 |
- * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 |
|
| 49 |
- * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 |
|
| 50 |
- * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 |
|
| 51 |
- |
|
| 52 |
-* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) |
|
| 53 |
- * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 |
|
| 54 |
- * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 |
|
| 55 |
- * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 |
|
| 56 |
- * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 |
|
| 57 |
- * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 |
|
| 58 |
- * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 |
|
| 59 |
- * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 |
|
| 60 |
- |
|
| 61 |
-<details> |
|
| 62 |
- <summary>See changes to v1.17.x</summary> |
|
| 63 |
- |
|
| 64 |
-* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) |
|
| 65 |
- * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 |
|
| 66 |
- * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 |
|
| 67 |
- * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 |
|
| 68 |
- * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 |
|
| 69 |
- |
|
| 70 |
-* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) |
|
| 71 |
- * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 |
|
| 72 |
- * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 |
|
| 73 |
- * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 |
|
| 74 |
- * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 |
|
| 75 |
- * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 |
|
| 76 |
- |
|
| 77 |
-* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) |
|
| 78 |
- * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 |
|
| 79 |
- * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 |
|
| 80 |
- * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 |
|
| 81 |
- * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 |
|
| 82 |
- |
|
| 83 |
-* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) |
|
| 84 |
- * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 |
|
| 85 |
- * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 |
|
| 86 |
- |
|
| 87 |
-* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) |
|
| 88 |
- * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 |
|
| 89 |
- * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 |
|
| 90 |
- |
|
| 91 |
-* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) |
|
| 92 |
- * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 |
|
| 93 |
- * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 |
|
| 94 |
- |
|
| 95 |
-* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) |
|
| 96 |
- * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 |
|
| 97 |
- * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 |
|
| 98 |
- * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 |
|
| 99 |
- * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 |
|
| 100 |
- * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 |
|
| 101 |
-https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 |
|
| 102 |
- |
|
| 103 |
-* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) |
|
| 104 |
- * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 |
|
| 105 |
- * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 |
|
| 106 |
- * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 |
|
| 107 |
- * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 |
|
| 108 |
- * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 |
|
| 109 |
- |
|
| 110 |
-* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) |
|
| 111 |
- * fse: Fix max header size https://github.com/klauspost/compress/pull/881 |
|
| 112 |
- * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 |
|
| 113 |
- * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 |
|
| 114 |
- |
|
| 115 |
-* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) |
|
| 116 |
- * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 |
|
| 117 |
- |
|
| 118 |
-* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) |
|
| 119 |
- * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 |
|
| 120 |
- * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 |
|
| 121 |
- * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 |
|
| 122 |
- |
|
| 123 |
-* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) |
|
| 124 |
- * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 |
|
| 125 |
- * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 |
|
| 126 |
- * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 |
|
| 127 |
- * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 |
|
| 128 |
- * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 |
|
| 129 |
- * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 |
|
| 130 |
- |
|
| 131 |
-</details> |
|
| 132 |
-<details> |
|
| 133 |
- <summary>See changes to v1.16.x</summary> |
|
| 134 |
- |
|
| 135 |
- |
|
| 136 |
-* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) |
|
| 137 |
- * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 |
|
| 138 |
- * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 |
|
| 139 |
- |
|
| 140 |
-* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) |
|
| 141 |
- * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 |
|
| 142 |
- * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 |
|
| 143 |
- * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 |
|
| 144 |
- * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 |
|
| 145 |
- |
|
| 146 |
-* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) |
|
| 147 |
- * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 |
|
| 148 |
- * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 |
|
| 149 |
- |
|
| 150 |
-* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) |
|
| 151 |
- * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 |
|
| 152 |
- * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 |
|
| 153 |
- * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 |
|
| 154 |
- * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 |
|
| 155 |
- * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 |
|
| 156 |
- * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 |
|
| 157 |
- * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 |
|
| 158 |
- |
|
| 159 |
-* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) |
|
| 160 |
- * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 |
|
| 161 |
- * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 |
|
| 162 |
- * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 |
|
| 163 |
- * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 |
|
| 164 |
- * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 |
|
| 165 |
- |
|
| 166 |
-* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) |
|
| 167 |
- * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 |
|
| 168 |
- * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 |
|
| 169 |
- * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 |
|
| 170 |
- * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 |
|
| 171 |
- * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 |
|
| 172 |
- * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 |
|
| 173 |
-</details> |
|
| 174 |
- |
|
| 175 |
-<details> |
|
| 176 |
- <summary>See changes to v1.15.x</summary> |
|
| 177 |
- |
|
| 178 |
-* Jan 21st, 2023 (v1.15.15) |
|
| 179 |
- * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 |
|
| 180 |
- * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 |
|
| 181 |
- * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 |
|
| 182 |
- * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 |
|
| 183 |
- |
|
| 184 |
-* Jan 3rd, 2023 (v1.15.14) |
|
| 185 |
- |
|
| 186 |
- * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 |
|
| 187 |
- * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 |
|
| 188 |
- * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 |
|
| 189 |
- * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 |
|
| 190 |
- |
|
| 191 |
-* Dec 11, 2022 (v1.15.13) |
|
| 192 |
- * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 |
|
| 193 |
- * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 |
|
| 194 |
- |
|
| 195 |
-* Oct 26, 2022 (v1.15.12) |
|
| 196 |
- |
|
| 197 |
- * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 |
|
| 198 |
- * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 |
|
| 199 |
- |
|
| 200 |
-* Sept 26, 2022 (v1.15.11) |
|
| 201 |
- |
|
| 202 |
- * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 |
|
| 203 |
- * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 |
|
| 204 |
- * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 |
|
| 205 |
- * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 |
|
| 206 |
- |
|
| 207 |
-* Sept 16, 2022 (v1.15.10) |
|
| 208 |
- |
|
| 209 |
- * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 |
|
| 210 |
- * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 |
|
| 211 |
- * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 |
|
| 212 |
- * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 |
|
| 213 |
- * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 |
|
| 214 |
- * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 |
|
| 215 |
- * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 |
|
| 216 |
- * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 |
|
| 217 |
- |
|
| 218 |
-* July 21, 2022 (v1.15.9) |
|
| 219 |
- |
|
| 220 |
- * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 |
|
| 221 |
- * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 |
|
| 222 |
- * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 |
|
| 223 |
- |
|
| 224 |
-* July 13, 2022 (v1.15.8) |
|
| 225 |
- |
|
| 226 |
- * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 |
|
| 227 |
- * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 |
|
| 228 |
- * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 |
|
| 229 |
- * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 |
|
| 230 |
- * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 |
|
| 231 |
- * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 |
|
| 232 |
- * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 |
|
| 233 |
- |
|
| 234 |
-* June 29, 2022 (v1.15.7) |
|
| 235 |
- |
|
| 236 |
- * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 |
|
| 237 |
- * zip: Merge upstream https://github.com/klauspost/compress/pull/631 |
|
| 238 |
- * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 |
|
| 239 |
- * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 |
|
| 240 |
- * flate: Faster histograms https://github.com/klauspost/compress/pull/620 |
|
| 241 |
- * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 |
|
| 242 |
- |
|
| 243 |
-* June 3, 2022 (v1.15.6) |
|
| 244 |
- * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 |
|
| 245 |
- * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 |
|
| 246 |
- * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 |
|
| 247 |
- * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 |
|
| 248 |
- * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 |
|
| 249 |
- * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 |
|
| 250 |
- * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 |
|
| 251 |
- * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 |
|
| 252 |
- * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 |
|
| 253 |
- * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 |
|
| 254 |
- |
|
| 255 |
-* May 25, 2022 (v1.15.5) |
|
| 256 |
- * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 |
|
| 257 |
- * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 |
|
| 258 |
- * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 |
|
| 259 |
- * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 |
|
| 260 |
- * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 |
|
| 261 |
- * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 |
|
| 262 |
- * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 |
|
| 263 |
- * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 |
|
| 264 |
- * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 |
|
| 265 |
- |
|
| 266 |
- |
|
| 267 |
-* May 11, 2022 (v1.15.4) |
|
| 268 |
- * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) |
|
| 269 |
- * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) |
|
| 270 |
- * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) |
|
| 271 |
- * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) |
|
| 272 |
- |
|
| 273 |
-* May 5, 2022 (v1.15.3) |
|
| 274 |
- * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) |
|
| 275 |
- * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) |
|
| 276 |
- |
|
| 277 |
-* Apr 26, 2022 (v1.15.2) |
|
| 278 |
- * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) |
|
| 279 |
- * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) |
|
| 280 |
- * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) |
|
| 281 |
- * Minimum version is Go 1.16, added CI test on 1.18. |
|
| 282 |
- |
|
| 283 |
-* Mar 11, 2022 (v1.15.1) |
|
| 284 |
- * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) |
|
| 285 |
- * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) |
|
| 286 |
- * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) |
|
| 287 |
- * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) |
|
| 288 |
- * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) |
|
| 289 |
- |
|
| 290 |
-* Mar 3, 2022 (v1.15.0) |
|
| 291 |
- * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) |
|
| 292 |
- * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) |
|
| 293 |
- * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) |
|
| 294 |
- * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) |
|
| 295 |
- * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) |
|
| 296 |
- * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) |
|
| 297 |
- |
|
| 298 |
-Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. |
|
| 299 |
- |
|
| 300 |
-Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. |
|
| 301 |
- |
|
| 302 |
-While the release has been extensively tested, it is recommended to testing when upgrading. |
|
| 303 |
- |
|
| 304 |
-</details> |
|
| 305 |
- |
|
| 306 |
-<details> |
|
| 307 |
- <summary>See changes to v1.14.x</summary> |
|
| 308 |
- |
|
| 309 |
-* Feb 22, 2022 (v1.14.4) |
|
| 310 |
- * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) |
|
| 311 |
- * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) |
|
| 312 |
- * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 |
|
| 313 |
- * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) |
|
| 314 |
- |
|
| 315 |
-* Feb 17, 2022 (v1.14.3) |
|
| 316 |
- * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) |
|
| 317 |
- * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) |
|
| 318 |
- * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) |
|
| 319 |
- |
|
| 320 |
-* Jan 25, 2022 (v1.14.2) |
|
| 321 |
- * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) |
|
| 322 |
- * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) |
|
| 323 |
- * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) |
|
| 324 |
- * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) |
|
| 325 |
- * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) |
|
| 326 |
- * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) |
|
| 327 |
- |
|
| 328 |
-* Jan 11, 2022 (v1.14.1) |
|
| 329 |
- * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) |
|
| 330 |
- * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) |
|
| 331 |
- * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) |
|
| 332 |
- * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) |
|
| 333 |
- * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) |
|
| 334 |
-</details> |
|
| 335 |
- |
|
| 336 |
-<details> |
|
| 337 |
- <summary>See changes to v1.13.x</summary> |
|
| 338 |
- |
|
| 339 |
-* Aug 30, 2021 (v1.13.5) |
|
| 340 |
- * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) |
|
| 341 |
- * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) |
|
| 342 |
- * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) |
|
| 343 |
- * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) |
|
| 344 |
- |
|
| 345 |
-* Aug 12, 2021 (v1.13.4) |
|
| 346 |
- * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). |
|
| 347 |
- * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) |
|
| 348 |
- |
|
| 349 |
-* Aug 3, 2021 (v1.13.3) |
|
| 350 |
- * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) |
|
| 351 |
- * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) |
|
| 352 |
- * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) |
|
| 353 |
- * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) |
|
| 354 |
- * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) |
|
| 355 |
- * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) |
|
| 356 |
- |
|
| 357 |
-* Jun 14, 2021 (v1.13.1) |
|
| 358 |
- * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) |
|
| 359 |
- * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) |
|
| 360 |
- * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) |
|
| 361 |
- * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) |
|
| 362 |
- |
|
| 363 |
-* Jun 3, 2021 (v1.13.0) |
|
| 364 |
- * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. |
|
| 365 |
- * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) |
|
| 366 |
- * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) |
|
| 367 |
-</details> |
|
| 368 |
- |
|
| 369 |
- |
|
| 370 |
-<details> |
|
| 371 |
- <summary>See changes to v1.12.x</summary> |
|
| 372 |
- |
|
| 373 |
-* May 25, 2021 (v1.12.3) |
|
| 374 |
- * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) |
|
| 375 |
- * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) |
|
| 376 |
- * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) |
|
| 377 |
- |
|
| 378 |
-* Apr 27, 2021 (v1.12.2) |
|
| 379 |
- * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) |
|
| 380 |
- * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) |
|
| 381 |
- * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) |
|
| 382 |
- * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) |
|
| 383 |
- * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) |
|
| 384 |
- * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) |
|
| 385 |
- |
|
| 386 |
-* Apr 14, 2021 (v1.12.1) |
|
| 387 |
- * snappy package removed. Upstream added as dependency. |
|
| 388 |
- * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) |
|
| 389 |
- * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) |
|
| 390 |
- * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) |
|
| 391 |
- * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) |
|
| 392 |
- * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) |
|
| 393 |
- * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) |
|
| 394 |
-</details> |
|
| 395 |
- |
|
| 396 |
-<details> |
|
| 397 |
- <summary>See changes to v1.11.x</summary> |
|
| 398 |
- |
|
| 399 |
-* Mar 26, 2021 (v1.11.13) |
|
| 400 |
- * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) |
|
| 401 |
- * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) |
|
| 402 |
- * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) |
|
| 403 |
- * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) |
|
| 404 |
- |
|
| 405 |
-* Mar 5, 2021 (v1.11.12) |
|
| 406 |
- * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). |
|
| 407 |
- * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) |
|
| 408 |
- |
|
| 409 |
-* Mar 1, 2021 (v1.11.9) |
|
| 410 |
- * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) |
|
| 411 |
- * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) |
|
| 412 |
- * s2: Fix binaries. |
|
| 413 |
- |
|
| 414 |
-* Feb 25, 2021 (v1.11.8) |
|
| 415 |
- * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. |
|
| 416 |
- * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) |
|
| 417 |
- * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) |
|
| 418 |
- * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) |
|
| 419 |
- * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) |
|
| 420 |
- |
|
| 421 |
-* Jan 14, 2021 (v1.11.7) |
|
| 422 |
- * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) |
|
| 423 |
- * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) |
|
| 424 |
- * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) |
|
| 425 |
- * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) |
|
| 426 |
- * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) |
|
| 427 |
- |
|
| 428 |
-* Jan 7, 2021 (v1.11.6) |
|
| 429 |
- * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) |
|
| 430 |
- * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) |
|
| 431 |
- |
|
| 432 |
-* Dec 20, 2020 (v1.11.4) |
|
| 433 |
- * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) |
|
| 434 |
- * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) |
|
| 435 |
- * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) |
|
| 436 |
- * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) |
|
| 437 |
- * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) |
|
| 438 |
- |
|
| 439 |
-* Nov 15, 2020 (v1.11.3) |
|
| 440 |
- * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) |
|
| 441 |
- * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) |
|
| 442 |
- |
|
| 443 |
-* Oct 11, 2020 (v1.11.2) |
|
| 444 |
- * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) |
|
| 445 |
- |
|
| 446 |
-* Oct 1, 2020 (v1.11.1) |
|
| 447 |
- * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) |
|
| 448 |
- |
|
| 449 |
-* Sept 8, 2020 (v1.11.0) |
|
| 450 |
- * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) |
|
| 451 |
- * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) |
|
| 452 |
- * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) |
|
| 453 |
-</details> |
|
| 454 |
- |
|
| 455 |
-<details> |
|
| 456 |
- <summary>See changes to v1.10.x</summary> |
|
| 457 |
- |
|
| 458 |
-* July 8, 2020 (v1.10.11) |
|
| 459 |
- * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) |
|
| 460 |
- * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) |
|
| 461 |
- |
|
| 462 |
-* June 23, 2020 (v1.10.10) |
|
| 463 |
- * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) |
|
| 464 |
- |
|
| 465 |
-* June 16, 2020 (v1.10.9): |
|
| 466 |
- * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) |
|
| 467 |
- * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) |
|
| 468 |
- * Fuzzit tests removed. The service has been purchased and is no longer available. |
|
| 469 |
- |
|
| 470 |
-* June 5, 2020 (v1.10.8): |
|
| 471 |
- * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) |
|
| 472 |
- |
|
| 473 |
-* June 1, 2020 (v1.10.7): |
|
| 474 |
- * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) |
|
| 475 |
- * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) |
|
| 476 |
- * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) |
|
| 477 |
- |
|
| 478 |
-* May 21, 2020: (v1.10.6) |
|
| 479 |
- * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) |
|
| 480 |
- * zstd: Stricter decompression checks. |
|
| 481 |
- |
|
| 482 |
-* April 12, 2020: (v1.10.5) |
|
| 483 |
- * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) |
|
| 484 |
- |
|
| 485 |
-* Apr 8, 2020: (v1.10.4) |
|
| 486 |
- * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) |
|
| 487 |
-* Mar 11, 2020: (v1.10.3) |
|
| 488 |
- * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) |
|
| 489 |
- * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) |
|
| 490 |
- * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) |
|
| 491 |
- * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) |
|
| 492 |
- * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) |
|
| 493 |
- |
|
| 494 |
-* Feb 27, 2020: (v1.10.2) |
|
| 495 |
- * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) |
|
| 496 |
- * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) |
|
| 497 |
- |
|
| 498 |
-* Feb 18, 2020: (v1.10.1) |
|
| 499 |
- * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) |
|
| 500 |
- * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) |
|
| 501 |
- * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) |
|
| 502 |
- |
|
| 503 |
-* Feb 4, 2020: (v1.10.0) |
|
| 504 |
- * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) |
|
| 505 |
- * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) |
|
| 506 |
- * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) |
|
| 507 |
- * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) |
|
| 508 |
- |
|
| 509 |
-</details> |
|
| 510 |
- |
|
| 511 |
-<details> |
|
| 512 |
- <summary>See changes prior to v1.10.0</summary> |
|
| 513 |
- |
|
| 514 |
-* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). |
|
| 515 |
-* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) |
|
| 516 |
-* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. |
|
| 517 |
-* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. |
|
| 518 |
-* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) |
|
| 519 |
-* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. |
|
| 520 |
-* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) |
|
| 521 |
-* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features |
|
| 522 |
-* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) |
|
| 523 |
-* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) |
|
| 524 |
-* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. |
|
| 525 |
-* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) |
|
| 526 |
-* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) |
|
| 527 |
-* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) |
|
| 528 |
-* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. |
|
| 529 |
-* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. |
|
| 530 |
-* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) |
|
| 531 |
-* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. |
|
| 532 |
-* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) |
|
| 533 |
-* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. |
|
| 534 |
-* Nov 11, 2019: Reduce inflate memory use by 1KB. |
|
| 535 |
-* Nov 10, 2019: Less allocations in deflate bit writer. |
|
| 536 |
-* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. |
|
| 537 |
-* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) |
|
| 538 |
-* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) |
|
| 539 |
-* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) |
|
| 540 |
-* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) |
|
| 541 |
- |
|
| 542 |
-</details> |
|
| 543 |
- |
|
| 544 |
-<details> |
|
| 545 |
- <summary>See changes prior to v1.9.0</summary> |
|
| 546 |
- |
|
| 547 |
-* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) |
|
| 548 |
-* Oct 3, 2019: Fix inconsistent results on broken zstd streams. |
|
| 549 |
-* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) |
|
| 550 |
-* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). |
|
| 551 |
-* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). |
|
| 552 |
-* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). |
|
| 553 |
-* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. |
|
| 554 |
-* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. |
|
| 555 |
-* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. |
|
| 556 |
-* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. |
|
| 557 |
-* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. |
|
| 558 |
-* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. |
|
| 559 |
-* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. |
|
| 560 |
-* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) |
|
| 561 |
-* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) |
|
| 562 |
-* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) |
|
| 563 |
-* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) |
|
| 564 |
-* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. |
|
| 565 |
-* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. |
|
| 566 |
-* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. |
|
| 567 |
-* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. |
|
| 568 |
-* June 17, 2019: zstd decompression bugfix. |
|
| 569 |
-* June 17, 2019: fix 32 bit builds. |
|
| 570 |
-* June 17, 2019: Easier use in modules (less dependencies). |
|
| 571 |
-* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. |
|
| 572 |
-* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. |
|
| 573 |
-* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. |
|
| 574 |
-* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! |
|
| 575 |
-* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. |
|
| 576 |
-* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. |
|
| 577 |
-* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). |
|
| 578 |
-* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. |
|
| 579 |
-* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). |
|
| 580 |
-* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. |
|
| 581 |
-* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. |
|
| 582 |
-* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. |
|
| 583 |
-* May 28, 2017: Reduce allocations when resetting decoder. |
|
| 584 |
-* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. |
|
| 585 |
-* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). |
|
| 586 |
-* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. |
|
| 587 |
-* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. |
|
| 588 |
-* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. |
|
| 589 |
-* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. |
|
| 590 |
-* Mar 24, 2016: Small speedup for level 1-3. |
|
| 591 |
-* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. |
|
| 592 |
-* Feb 19, 2016: Handle small payloads faster in level 1-3. |
|
| 593 |
-* Feb 19, 2016: Added faster level 2 + 3 compression modes. |
|
| 594 |
-* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. |
|
| 595 |
-* Feb 14, 2016: Snappy: Merge upstream changes. |
|
| 596 |
-* Feb 14, 2016: Snappy: Fix aggressive skipping. |
|
| 597 |
-* Feb 14, 2016: Snappy: Update benchmark. |
|
| 598 |
-* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. |
|
| 599 |
-* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. |
|
| 600 |
-* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. |
|
| 601 |
-* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. |
|
| 602 |
-* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. |
|
| 603 |
-* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. |
|
| 604 |
-* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. |
|
| 605 |
-* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. |
|
| 606 |
-* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! |
|
| 607 |
-* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). |
|
| 608 |
-* Nov 20 2015: Small optimization to bit writer on 64 bit systems. |
|
| 609 |
-* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). |
|
| 610 |
-* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. |
|
| 611 |
-* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file |
|
| 612 |
-* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. |
|
| 613 |
- |
|
| 614 |
-</details> |
|
| 615 |
- |
|
| 616 |
-# deflate usage |
|
| 617 |
- |
|
| 618 |
-The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: |
|
| 619 |
- |
|
| 620 |
-Typical speed is about 2x of the standard library packages. |
|
| 621 |
- |
|
| 622 |
-| old import | new import | Documentation | |
|
| 623 |
-|------------------|---------------------------------------|-------------------------------------------------------------------------| |
|
| 624 |
-| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | |
|
| 625 |
-| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | |
|
| 626 |
-| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | |
|
| 627 |
-| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | |
|
| 628 |
- |
|
| 629 |
-You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. |
|
| 630 |
- |
|
| 631 |
-The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). |
|
| 632 |
- |
|
| 633 |
-Currently there is only minor speedup on decompression (mostly CRC32 calculation). |
|
| 634 |
- |
|
| 635 |
-Memory usage is typically 1MB for a Writer. stdlib is in the same range. |
|
| 636 |
-If you expect to have a lot of concurrently allocated Writers consider using |
|
| 637 |
-the stateless compression described below. |
|
| 638 |
- |
|
| 639 |
-For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). |
|
| 640 |
- |
|
| 641 |
-To disable all assembly add `-tags=noasm`. This works across all packages. |
|
| 642 |
- |
|
| 643 |
-# Stateless compression |
|
| 644 |
- |
|
| 645 |
-This package offers stateless compression as a special option for gzip/deflate. |
|
| 646 |
-It will do compression but without maintaining any state between Write calls. |
|
| 647 |
- |
|
| 648 |
-This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. |
|
| 649 |
- |
|
| 650 |
-This is only relevant in cases where you expect to run many thousands of compressors concurrently, |
|
| 651 |
-but with very little activity. This is *not* intended for regular web servers serving individual requests. |
|
| 652 |
- |
|
| 653 |
-Because of this, the size of actual Write calls will affect output size. |
|
| 654 |
- |
|
| 655 |
-In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. |
|
| 656 |
- |
|
| 657 |
-For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) |
|
| 658 |
- |
|
| 659 |
-A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: |
|
| 660 |
- |
|
| 661 |
-```go |
|
| 662 |
- // replace 'ioutil.Discard' with your output. |
|
| 663 |
- gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) |
|
| 664 |
- if err != nil {
|
|
| 665 |
- return err |
|
| 666 |
- } |
|
| 667 |
- defer gzw.Close() |
|
| 668 |
- |
|
| 669 |
- w := bufio.NewWriterSize(gzw, 4096) |
|
| 670 |
- defer w.Flush() |
|
| 671 |
- |
|
| 672 |
- // Write to 'w' |
|
| 673 |
-``` |
|
| 674 |
- |
|
| 675 |
-This will only use up to 4KB in memory when the writer is idle. |
|
| 676 |
- |
|
| 677 |
-Compression is almost always worse than the fastest compression level |
|
| 678 |
-and each write will allocate (a little) memory. |
|
| 679 |
- |
|
| 680 |
- |
|
| 681 |
-# Other packages |
|
| 682 |
- |
|
| 683 |
-Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): |
|
| 684 |
- |
|
| 685 |
-* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. |
|
| 686 |
-* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. |
|
| 687 |
-* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. |
|
| 688 |
-* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. |
|
| 689 |
-* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. |
|
| 690 |
-* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. |
|
| 691 |
-* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. |
|
| 692 |
- |
|
| 693 |
-# license |
|
| 694 |
- |
|
| 695 |
-This code is licensed under the same conditions as the original Go code. See LICENSE file. |
|
| 696 |
- |
|
| 697 |
- |
|
| 698 |
- |
|
| 699 |
- |
|
| 700 |
- |
|
| 1 |
+# compress |
|
| 2 |
+ |
|
| 3 |
+This package provides various compression algorithms. |
|
| 4 |
+ |
|
| 5 |
+* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. |
|
| 6 |
+* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. |
|
| 7 |
+* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). |
|
| 8 |
+* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. |
|
| 9 |
+* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. |
|
| 10 |
+* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. |
|
| 11 |
+* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. |
|
| 12 |
+ |
|
| 13 |
+[](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) |
|
| 14 |
+[](https://github.com/klauspost/compress/actions/workflows/go.yml) |
|
| 15 |
+[](https://sourcegraph.com/github.com/klauspost/compress?badge) |
|
| 16 |
+ |
|
| 17 |
+# package usage |
|
| 18 |
+ |
|
| 19 |
+Use `go get github.com/klauspost/compress@latest` to add it to your project. |
|
| 20 |
+ |
|
| 21 |
+This package will support the current Go version and 2 versions back. |
|
| 22 |
+ |
|
| 23 |
+* Use the `nounsafe` tag to disable all use of the "unsafe" package. |
|
| 24 |
+* Use the `noasm` tag to disable all assembly across packages. |
|
| 25 |
+ |
|
| 26 |
+Use the links above for more information on each. |
|
| 27 |
+ |
|
| 28 |
+# changelog |
|
| 29 |
+ |
|
| 30 |
+* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) |
|
| 31 |
+ * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 |
|
| 32 |
+ * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 |
|
| 33 |
+ * gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116 |
|
| 34 |
+ |
|
| 35 |
+* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) |
|
| 36 |
+ * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). |
|
| 37 |
+ |
|
| 38 |
+* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) |
|
| 39 |
+ * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 |
|
| 40 |
+ * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 |
|
| 41 |
+ |
|
| 42 |
+* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED |
|
| 43 |
+ * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 |
|
| 44 |
+ * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 |
|
| 45 |
+ * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 |
|
| 46 |
+ * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 |
|
| 47 |
+ * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 |
|
| 48 |
+ * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 |
|
| 49 |
+ * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 |
|
| 50 |
+ * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 |
|
| 51 |
+ |
|
| 52 |
+* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) |
|
| 53 |
+ * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 |
|
| 54 |
+ * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 |
|
| 55 |
+ * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 |
|
| 56 |
+ * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 |
|
| 57 |
+ * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 |
|
| 58 |
+ * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 |
|
| 59 |
+ * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 |
|
| 60 |
+ |
|
| 61 |
+<details> |
|
| 62 |
+ <summary>See changes to v1.17.x</summary> |
|
| 63 |
+ |
|
| 64 |
+* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) |
|
| 65 |
+ * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 |
|
| 66 |
+ * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 |
|
| 67 |
+ * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 |
|
| 68 |
+ * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 |
|
| 69 |
+ |
|
| 70 |
+* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) |
|
| 71 |
+ * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 |
|
| 72 |
+ * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 |
|
| 73 |
+ * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 |
|
| 74 |
+ * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 |
|
| 75 |
+ * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 |
|
| 76 |
+ |
|
| 77 |
+* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) |
|
| 78 |
+ * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 |
|
| 79 |
+ * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 |
|
| 80 |
+ * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 |
|
| 81 |
+ * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 |
|
| 82 |
+ |
|
| 83 |
+* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) |
|
| 84 |
+ * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 |
|
| 85 |
+ * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 |
|
| 86 |
+ |
|
| 87 |
+* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) |
|
| 88 |
+ * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 |
|
| 89 |
+ * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 |
|
| 90 |
+ |
|
| 91 |
+* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) |
|
| 92 |
+ * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 |
|
| 93 |
+ * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 |
|
| 94 |
+ |
|
| 95 |
+* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) |
|
| 96 |
+ * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 |
|
| 97 |
+ * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 |
|
| 98 |
+ * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 |
|
| 99 |
+ * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 |
|
| 100 |
+ * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 |
|
| 101 |
+https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 |
|
| 102 |
+ |
|
| 103 |
+* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) |
|
| 104 |
+ * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 |
|
| 105 |
+ * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 |
|
| 106 |
+ * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 |
|
| 107 |
+ * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 |
|
| 108 |
+ * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 |
|
| 109 |
+ |
|
| 110 |
+* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) |
|
| 111 |
+ * fse: Fix max header size https://github.com/klauspost/compress/pull/881 |
|
| 112 |
+ * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 |
|
| 113 |
+ * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 |
|
| 114 |
+ |
|
| 115 |
+* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) |
|
| 116 |
+ * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 |
|
| 117 |
+ |
|
| 118 |
+* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) |
|
| 119 |
+ * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 |
|
| 120 |
+ * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 |
|
| 121 |
+ * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 |
|
| 122 |
+ |
|
| 123 |
+* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) |
|
| 124 |
+ * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 |
|
| 125 |
+ * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 |
|
| 126 |
+ * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 |
|
| 127 |
+ * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 |
|
| 128 |
+ * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 |
|
| 129 |
+ * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 |
|
| 130 |
+ |
|
| 131 |
+</details> |
|
| 132 |
+<details> |
|
| 133 |
+ <summary>See changes to v1.16.x</summary> |
|
| 134 |
+ |
|
| 135 |
+ |
|
| 136 |
+* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) |
|
| 137 |
+ * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 |
|
| 138 |
+ * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 |
|
| 139 |
+ |
|
| 140 |
+* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) |
|
| 141 |
+ * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 |
|
| 142 |
+ * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 |
|
| 143 |
+ * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 |
|
| 144 |
+ * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 |
|
| 145 |
+ |
|
| 146 |
+* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) |
|
| 147 |
+ * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 |
|
| 148 |
+ * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 |
|
| 149 |
+ |
|
| 150 |
+* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) |
|
| 151 |
+ * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 |
|
| 152 |
+ * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 |
|
| 153 |
+ * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 |
|
| 154 |
+ * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 |
|
| 155 |
+ * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 |
|
| 156 |
+ * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 |
|
| 157 |
+ * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 |
|
| 158 |
+ |
|
| 159 |
+* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) |
|
| 160 |
+ * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 |
|
| 161 |
+ * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 |
|
| 162 |
+ * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 |
|
| 163 |
+ * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 |
|
| 164 |
+ * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 |
|
| 165 |
+ |
|
| 166 |
+* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) |
|
| 167 |
+ * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 |
|
| 168 |
+ * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 |
|
| 169 |
+ * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 |
|
| 170 |
+ * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 |
|
| 171 |
+ * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 |
|
| 172 |
+ * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 |
|
| 173 |
+</details> |
|
| 174 |
+ |
|
| 175 |
+<details> |
|
| 176 |
+ <summary>See changes to v1.15.x</summary> |
|
| 177 |
+ |
|
| 178 |
+* Jan 21st, 2023 (v1.15.15) |
|
| 179 |
+ * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 |
|
| 180 |
+ * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 |
|
| 181 |
+ * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 |
|
| 182 |
+ * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 |
|
| 183 |
+ |
|
| 184 |
+* Jan 3rd, 2023 (v1.15.14) |
|
| 185 |
+ |
|
| 186 |
+ * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 |
|
| 187 |
+ * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 |
|
| 188 |
+ * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 |
|
| 189 |
+ * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 |
|
| 190 |
+ |
|
| 191 |
+* Dec 11, 2022 (v1.15.13) |
|
| 192 |
+ * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 |
|
| 193 |
+ * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 |
|
| 194 |
+ |
|
| 195 |
+* Oct 26, 2022 (v1.15.12) |
|
| 196 |
+ |
|
| 197 |
+ * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 |
|
| 198 |
+ * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 |
|
| 199 |
+ |
|
| 200 |
+* Sept 26, 2022 (v1.15.11) |
|
| 201 |
+ |
|
| 202 |
+ * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 |
|
| 203 |
+ * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 |
|
| 204 |
+ * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 |
|
| 205 |
+ * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 |
|
| 206 |
+ |
|
| 207 |
+* Sept 16, 2022 (v1.15.10) |
|
| 208 |
+ |
|
| 209 |
+ * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 |
|
| 210 |
+ * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 |
|
| 211 |
+ * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 |
|
| 212 |
+ * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 |
|
| 213 |
+ * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 |
|
| 214 |
+ * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 |
|
| 215 |
+ * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 |
|
| 216 |
+ * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 |
|
| 217 |
+ |
|
| 218 |
+* July 21, 2022 (v1.15.9) |
|
| 219 |
+ |
|
| 220 |
+ * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 |
|
| 221 |
+ * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 |
|
| 222 |
+ * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 |
|
| 223 |
+ |
|
| 224 |
+* July 13, 2022 (v1.15.8) |
|
| 225 |
+ |
|
| 226 |
+ * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 |
|
| 227 |
+ * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 |
|
| 228 |
+ * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 |
|
| 229 |
+ * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 |
|
| 230 |
+ * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 |
|
| 231 |
+ * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 |
|
| 232 |
+ * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 |
|
| 233 |
+ |
|
| 234 |
+* June 29, 2022 (v1.15.7) |
|
| 235 |
+ |
|
| 236 |
+ * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 |
|
| 237 |
+ * zip: Merge upstream https://github.com/klauspost/compress/pull/631 |
|
| 238 |
+ * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 |
|
| 239 |
+ * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 |
|
| 240 |
+ * flate: Faster histograms https://github.com/klauspost/compress/pull/620 |
|
| 241 |
+ * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 |
|
| 242 |
+ |
|
| 243 |
+* June 3, 2022 (v1.15.6) |
|
| 244 |
+ * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 |
|
| 245 |
+ * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 |
|
| 246 |
+ * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 |
|
| 247 |
+ * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 |
|
| 248 |
+ * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 |
|
| 249 |
+ * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 |
|
| 250 |
+ * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 |
|
| 251 |
+ * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 |
|
| 252 |
+ * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 |
|
| 253 |
+ * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 |
|
| 254 |
+ |
|
| 255 |
+* May 25, 2022 (v1.15.5) |
|
| 256 |
+ * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 |
|
| 257 |
+ * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 |
|
| 258 |
+ * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 |
|
| 259 |
+ * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 |
|
| 260 |
+ * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 |
|
| 261 |
+ * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 |
|
| 262 |
+ * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 |
|
| 263 |
+ * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 |
|
| 264 |
+ * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 |
|
| 265 |
+ |
|
| 266 |
+ |
|
| 267 |
+* May 11, 2022 (v1.15.4) |
|
| 268 |
+ * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) |
|
| 269 |
+ * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) |
|
| 270 |
+ * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) |
|
| 271 |
+ * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) |
|
| 272 |
+ |
|
| 273 |
+* May 5, 2022 (v1.15.3) |
|
| 274 |
+ * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) |
|
| 275 |
+ * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) |
|
| 276 |
+ |
|
| 277 |
+* Apr 26, 2022 (v1.15.2) |
|
| 278 |
+ * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) |
|
| 279 |
+ * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) |
|
| 280 |
+ * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) |
|
| 281 |
+ * Minimum version is Go 1.16, added CI test on 1.18. |
|
| 282 |
+ |
|
| 283 |
+* Mar 11, 2022 (v1.15.1) |
|
| 284 |
+ * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) |
|
| 285 |
+ * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) |
|
| 286 |
+ * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) |
|
| 287 |
+ * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) |
|
| 288 |
+ * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) |
|
| 289 |
+ |
|
| 290 |
+* Mar 3, 2022 (v1.15.0) |
|
| 291 |
+ * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) |
|
| 292 |
+ * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) |
|
| 293 |
+ * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) |
|
| 294 |
+ * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) |
|
| 295 |
+ * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) |
|
| 296 |
+ * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) |
|
| 297 |
+ |
|
| 298 |
+Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. |
|
| 299 |
+ |
|
| 300 |
+Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. |
|
| 301 |
+ |
|
| 302 |
+While the release has been extensively tested, it is recommended to testing when upgrading. |
|
| 303 |
+ |
|
| 304 |
+</details> |
|
| 305 |
+ |
|
| 306 |
+<details> |
|
| 307 |
+ <summary>See changes to v1.14.x</summary> |
|
| 308 |
+ |
|
| 309 |
+* Feb 22, 2022 (v1.14.4) |
|
| 310 |
+ * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) |
|
| 311 |
+ * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) |
|
| 312 |
+ * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 |
|
| 313 |
+ * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) |
|
| 314 |
+ |
|
| 315 |
+* Feb 17, 2022 (v1.14.3) |
|
| 316 |
+ * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) |
|
| 317 |
+ * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) |
|
| 318 |
+ * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) |
|
| 319 |
+ |
|
| 320 |
+* Jan 25, 2022 (v1.14.2) |
|
| 321 |
+ * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) |
|
| 322 |
+ * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) |
|
| 323 |
+ * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) |
|
| 324 |
+ * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) |
|
| 325 |
+ * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) |
|
| 326 |
+ * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) |
|
| 327 |
+ |
|
| 328 |
+* Jan 11, 2022 (v1.14.1) |
|
| 329 |
+ * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) |
|
| 330 |
+ * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) |
|
| 331 |
+ * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) |
|
| 332 |
+ * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) |
|
| 333 |
+ * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) |
|
| 334 |
+</details> |
|
| 335 |
+ |
|
| 336 |
+<details> |
|
| 337 |
+ <summary>See changes to v1.13.x</summary> |
|
| 338 |
+ |
|
| 339 |
+* Aug 30, 2021 (v1.13.5) |
|
| 340 |
+ * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) |
|
| 341 |
+ * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) |
|
| 342 |
+ * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) |
|
| 343 |
+ * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) |
|
| 344 |
+ |
|
| 345 |
+* Aug 12, 2021 (v1.13.4) |
|
| 346 |
+ * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). |
|
| 347 |
+ * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) |
|
| 348 |
+ |
|
| 349 |
+* Aug 3, 2021 (v1.13.3) |
|
| 350 |
+ * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) |
|
| 351 |
+ * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) |
|
| 352 |
+ * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) |
|
| 353 |
+ * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) |
|
| 354 |
+ * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) |
|
| 355 |
+ * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) |
|
| 356 |
+ |
|
| 357 |
+* Jun 14, 2021 (v1.13.1) |
|
| 358 |
+ * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) |
|
| 359 |
+ * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) |
|
| 360 |
+ * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) |
|
| 361 |
+ * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) |
|
| 362 |
+ |
|
| 363 |
+* Jun 3, 2021 (v1.13.0) |
|
| 364 |
+ * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. |
|
| 365 |
+ * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) |
|
| 366 |
+ * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) |
|
| 367 |
+</details> |
|
| 368 |
+ |
|
| 369 |
+ |
|
| 370 |
+<details> |
|
| 371 |
+ <summary>See changes to v1.12.x</summary> |
|
| 372 |
+ |
|
| 373 |
+* May 25, 2021 (v1.12.3) |
|
| 374 |
+ * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) |
|
| 375 |
+ * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) |
|
| 376 |
+ * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) |
|
| 377 |
+ |
|
| 378 |
+* Apr 27, 2021 (v1.12.2) |
|
| 379 |
+ * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) |
|
| 380 |
+ * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) |
|
| 381 |
+ * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) |
|
| 382 |
+ * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) |
|
| 383 |
+ * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) |
|
| 384 |
+ * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) |
|
| 385 |
+ |
|
| 386 |
+* Apr 14, 2021 (v1.12.1) |
|
| 387 |
+ * snappy package removed. Upstream added as dependency. |
|
| 388 |
+ * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) |
|
| 389 |
+ * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) |
|
| 390 |
+ * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) |
|
| 391 |
+ * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) |
|
| 392 |
+ * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) |
|
| 393 |
+ * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) |
|
| 394 |
+</details> |
|
| 395 |
+ |
|
| 396 |
+<details> |
|
| 397 |
+ <summary>See changes to v1.11.x</summary> |
|
| 398 |
+ |
|
| 399 |
+* Mar 26, 2021 (v1.11.13) |
|
| 400 |
+ * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) |
|
| 401 |
+ * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) |
|
| 402 |
+ * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) |
|
| 403 |
+ * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) |
|
| 404 |
+ |
|
| 405 |
+* Mar 5, 2021 (v1.11.12) |
|
| 406 |
+ * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). |
|
| 407 |
+ * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) |
|
| 408 |
+ |
|
| 409 |
+* Mar 1, 2021 (v1.11.9) |
|
| 410 |
+ * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) |
|
| 411 |
+ * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) |
|
| 412 |
+ * s2: Fix binaries. |
|
| 413 |
+ |
|
| 414 |
+* Feb 25, 2021 (v1.11.8) |
|
| 415 |
+ * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. |
|
| 416 |
+ * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) |
|
| 417 |
+ * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) |
|
| 418 |
+ * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) |
|
| 419 |
+ * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) |
|
| 420 |
+ |
|
| 421 |
+* Jan 14, 2021 (v1.11.7) |
|
| 422 |
+ * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) |
|
| 423 |
+ * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) |
|
| 424 |
+ * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) |
|
| 425 |
+ * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) |
|
| 426 |
+ * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) |
|
| 427 |
+ |
|
| 428 |
+* Jan 7, 2021 (v1.11.6) |
|
| 429 |
+ * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) |
|
| 430 |
+ * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) |
|
| 431 |
+ |
|
| 432 |
+* Dec 20, 2020 (v1.11.4) |
|
| 433 |
+ * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) |
|
| 434 |
+ * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) |
|
| 435 |
+ * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) |
|
| 436 |
+ * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) |
|
| 437 |
+ * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) |
|
| 438 |
+ |
|
| 439 |
+* Nov 15, 2020 (v1.11.3) |
|
| 440 |
+ * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) |
|
| 441 |
+ * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) |
|
| 442 |
+ |
|
| 443 |
+* Oct 11, 2020 (v1.11.2) |
|
| 444 |
+ * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) |
|
| 445 |
+ |
|
| 446 |
+* Oct 1, 2020 (v1.11.1) |
|
| 447 |
+ * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) |
|
| 448 |
+ |
|
| 449 |
+* Sept 8, 2020 (v1.11.0) |
|
| 450 |
+ * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) |
|
| 451 |
+ * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) |
|
| 452 |
+ * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) |
|
| 453 |
+</details> |
|
| 454 |
+ |
|
| 455 |
+<details> |
|
| 456 |
+ <summary>See changes to v1.10.x</summary> |
|
| 457 |
+ |
|
| 458 |
+* July 8, 2020 (v1.10.11) |
|
| 459 |
+ * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) |
|
| 460 |
+ * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) |
|
| 461 |
+ |
|
| 462 |
+* June 23, 2020 (v1.10.10) |
|
| 463 |
+ * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) |
|
| 464 |
+ |
|
| 465 |
+* June 16, 2020 (v1.10.9): |
|
| 466 |
+ * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) |
|
| 467 |
+ * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) |
|
| 468 |
+ * Fuzzit tests removed. The service has been purchased and is no longer available. |
|
| 469 |
+ |
|
| 470 |
+* June 5, 2020 (v1.10.8): |
|
| 471 |
+ * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) |
|
| 472 |
+ |
|
| 473 |
+* June 1, 2020 (v1.10.7): |
|
| 474 |
+ * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) |
|
| 475 |
+ * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) |
|
| 476 |
+ * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) |
|
| 477 |
+ |
|
| 478 |
+* May 21, 2020: (v1.10.6) |
|
| 479 |
+ * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) |
|
| 480 |
+ * zstd: Stricter decompression checks. |
|
| 481 |
+ |
|
| 482 |
+* April 12, 2020: (v1.10.5) |
|
| 483 |
+ * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) |
|
| 484 |
+ |
|
| 485 |
+* Apr 8, 2020: (v1.10.4) |
|
| 486 |
+ * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) |
|
| 487 |
+* Mar 11, 2020: (v1.10.3) |
|
| 488 |
+ * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) |
|
| 489 |
+ * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) |
|
| 490 |
+ * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) |
|
| 491 |
+ * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) |
|
| 492 |
+ * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) |
|
| 493 |
+ |
|
| 494 |
+* Feb 27, 2020: (v1.10.2) |
|
| 495 |
+ * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) |
|
| 496 |
+ * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) |
|
| 497 |
+ |
|
| 498 |
+* Feb 18, 2020: (v1.10.1) |
|
| 499 |
+ * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) |
|
| 500 |
+ * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) |
|
| 501 |
+ * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) |
|
| 502 |
+ |
|
| 503 |
+* Feb 4, 2020: (v1.10.0) |
|
| 504 |
+ * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) |
|
| 505 |
+ * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) |
|
| 506 |
+ * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) |
|
| 507 |
+ * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) |
|
| 508 |
+ |
|
| 509 |
+</details> |
|
| 510 |
+ |
|
| 511 |
+<details> |
|
| 512 |
+ <summary>See changes prior to v1.10.0</summary> |
|
| 513 |
+ |
|
| 514 |
+* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). |
|
| 515 |
+* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) |
|
| 516 |
+* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. |
|
| 517 |
+* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. |
|
| 518 |
+* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) |
|
| 519 |
+* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. |
|
| 520 |
+* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) |
|
| 521 |
+* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features |
|
| 522 |
+* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) |
|
| 523 |
+* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) |
|
| 524 |
+* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. |
|
| 525 |
+* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) |
|
| 526 |
+* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) |
|
| 527 |
+* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) |
|
| 528 |
+* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. |
|
| 529 |
+* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. |
|
| 530 |
+* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) |
|
| 531 |
+* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. |
|
| 532 |
+* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) |
|
| 533 |
+* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. |
|
| 534 |
+* Nov 11, 2019: Reduce inflate memory use by 1KB. |
|
| 535 |
+* Nov 10, 2019: Less allocations in deflate bit writer. |
|
| 536 |
+* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. |
|
| 537 |
+* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) |
|
| 538 |
+* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) |
|
| 539 |
+* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) |
|
| 540 |
+* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) |
|
| 541 |
+ |
|
| 542 |
+</details> |
|
| 543 |
+ |
|
| 544 |
+<details> |
|
| 545 |
+ <summary>See changes prior to v1.9.0</summary> |
|
| 546 |
+ |
|
| 547 |
+* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) |
|
| 548 |
+* Oct 3, 2019: Fix inconsistent results on broken zstd streams. |
|
| 549 |
+* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) |
|
| 550 |
+* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). |
|
| 551 |
+* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). |
|
| 552 |
+* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). |
|
| 553 |
+* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. |
|
| 554 |
+* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. |
|
| 555 |
+* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. |
|
| 556 |
+* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. |
|
| 557 |
+* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. |
|
| 558 |
+* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. |
|
| 559 |
+* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. |
|
| 560 |
+* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) |
|
| 561 |
+* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) |
|
| 562 |
+* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) |
|
| 563 |
+* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) |
|
| 564 |
+* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. |
|
| 565 |
+* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. |
|
| 566 |
+* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. |
|
| 567 |
+* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. |
|
| 568 |
+* June 17, 2019: zstd decompression bugfix. |
|
| 569 |
+* June 17, 2019: fix 32 bit builds. |
|
| 570 |
+* June 17, 2019: Easier use in modules (less dependencies). |
|
| 571 |
+* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. |
|
| 572 |
+* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. |
|
| 573 |
+* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. |
|
| 574 |
+* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! |
|
| 575 |
+* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. |
|
| 576 |
+* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. |
|
| 577 |
+* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). |
|
| 578 |
+* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. |
|
| 579 |
+* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). |
|
| 580 |
+* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. |
|
| 581 |
+* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. |
|
| 582 |
+* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. |
|
| 583 |
+* May 28, 2017: Reduce allocations when resetting decoder. |
|
| 584 |
+* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. |
|
| 585 |
+* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). |
|
| 586 |
+* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. |
|
| 587 |
+* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. |
|
| 588 |
+* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. |
|
| 589 |
+* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. |
|
| 590 |
+* Mar 24, 2016: Small speedup for level 1-3. |
|
| 591 |
+* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. |
|
| 592 |
+* Feb 19, 2016: Handle small payloads faster in level 1-3. |
|
| 593 |
+* Feb 19, 2016: Added faster level 2 + 3 compression modes. |
|
| 594 |
+* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. |
|
| 595 |
+* Feb 14, 2016: Snappy: Merge upstream changes. |
|
| 596 |
+* Feb 14, 2016: Snappy: Fix aggressive skipping. |
|
| 597 |
+* Feb 14, 2016: Snappy: Update benchmark. |
|
| 598 |
+* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. |
|
| 599 |
+* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. |
|
| 600 |
+* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. |
|
| 601 |
+* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. |
|
| 602 |
+* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. |
|
| 603 |
+* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. |
|
| 604 |
+* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. |
|
| 605 |
+* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. |
|
| 606 |
+* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! |
|
| 607 |
+* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). |
|
| 608 |
+* Nov 20 2015: Small optimization to bit writer on 64 bit systems. |
|
| 609 |
+* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). |
|
| 610 |
+* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. |
|
| 611 |
+* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file |
|
| 612 |
+* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. |
|
| 613 |
+ |
|
| 614 |
+</details> |
|
| 615 |
+ |
|
| 616 |
+# deflate usage |
|
| 617 |
+ |
|
| 618 |
+The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: |
|
| 619 |
+ |
|
| 620 |
+Typical speed is about 2x of the standard library packages. |
|
| 621 |
+ |
|
| 622 |
+| old import | new import | Documentation | |
|
| 623 |
+|------------------|---------------------------------------|-------------------------------------------------------------------------| |
|
| 624 |
+| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | |
|
| 625 |
+| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | |
|
| 626 |
+| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | |
|
| 627 |
+| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | |
|
| 628 |
+ |
|
| 629 |
+You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. |
|
| 630 |
+ |
|
| 631 |
+The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). |
|
| 632 |
+ |
|
| 633 |
+Currently there is only minor speedup on decompression (mostly CRC32 calculation). |
|
| 634 |
+ |
|
| 635 |
+Memory usage is typically 1MB for a Writer. stdlib is in the same range. |
|
| 636 |
+If you expect to have a lot of concurrently allocated Writers consider using |
|
| 637 |
+the stateless compression described below. |
|
| 638 |
+ |
|
| 639 |
+For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). |
|
| 640 |
+ |
|
| 641 |
+To disable all assembly add `-tags=noasm`. This works across all packages. |
|
| 642 |
+ |
|
| 643 |
+# Stateless compression |
|
| 644 |
+ |
|
| 645 |
+This package offers stateless compression as a special option for gzip/deflate. |
|
| 646 |
+It will do compression but without maintaining any state between Write calls. |
|
| 647 |
+ |
|
| 648 |
+This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. |
|
| 649 |
+ |
|
| 650 |
+This is only relevant in cases where you expect to run many thousands of compressors concurrently, |
|
| 651 |
+but with very little activity. This is *not* intended for regular web servers serving individual requests. |
|
| 652 |
+ |
|
| 653 |
+Because of this, the size of actual Write calls will affect output size. |
|
| 654 |
+ |
|
| 655 |
+In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. |
|
| 656 |
+ |
|
| 657 |
+For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) |
|
| 658 |
+ |
|
| 659 |
+A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: |
|
| 660 |
+ |
|
| 661 |
+```go |
|
| 662 |
+ // replace 'ioutil.Discard' with your output. |
|
| 663 |
+ gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) |
|
| 664 |
+ if err != nil {
|
|
| 665 |
+ return err |
|
| 666 |
+ } |
|
| 667 |
+ defer gzw.Close() |
|
| 668 |
+ |
|
| 669 |
+ w := bufio.NewWriterSize(gzw, 4096) |
|
| 670 |
+ defer w.Flush() |
|
| 671 |
+ |
|
| 672 |
+ // Write to 'w' |
|
| 673 |
+``` |
|
| 674 |
+ |
|
| 675 |
+This will only use up to 4KB in memory when the writer is idle. |
|
| 676 |
+ |
|
| 677 |
+Compression is almost always worse than the fastest compression level |
|
| 678 |
+and each write will allocate (a little) memory. |
|
| 679 |
+ |
|
| 680 |
+ |
|
| 681 |
+# Other packages |
|
| 682 |
+ |
|
| 683 |
+Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): |
|
| 684 |
+ |
|
| 685 |
+* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. |
|
| 686 |
+* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. |
|
| 687 |
+* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. |
|
| 688 |
+* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. |
|
| 689 |
+* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. |
|
| 690 |
+* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. |
|
| 691 |
+* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. |
|
| 692 |
+ |
|
| 693 |
+# license |
|
| 694 |
+ |
|
| 695 |
+This code is licensed under the same conditions as the original Go code. See LICENSE file. |
|
| 696 |
+ |
|
| 697 |
+ |
|
| 698 |
+ |
|
| 699 |
+ |
|
| 700 |
+ |
| ... | ... |
@@ -1,79 +1,79 @@ |
| 1 |
-# Finite State Entropy |
|
| 2 |
- |
|
| 3 |
-This package provides Finite State Entropy encoding and decoding. |
|
| 4 |
- |
|
| 5 |
-Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) |
|
| 6 |
-encoding provides a fast near-optimal symbol encoding/decoding |
|
| 7 |
-for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). |
|
| 8 |
- |
|
| 9 |
-This can be used for compressing input with a lot of similar input values to the smallest number of bytes. |
|
| 10 |
-This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, |
|
| 11 |
-but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. |
|
| 12 |
- |
|
| 13 |
-* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) |
|
| 14 |
- |
|
| 15 |
-## News |
|
| 16 |
- |
|
| 17 |
- * Feb 2018: First implementation released. Consider this beta software for now. |
|
| 18 |
- |
|
| 19 |
-# Usage |
|
| 20 |
- |
|
| 21 |
-This package provides a low level interface that allows to compress single independent blocks. |
|
| 22 |
- |
|
| 23 |
-Each block is separate, and there is no built in integrity checks. |
|
| 24 |
-This means that the caller should keep track of block sizes and also do checksums if needed. |
|
| 25 |
- |
|
| 26 |
-Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. |
|
| 27 |
-You must provide input and will receive the output and maybe an error. |
|
| 28 |
- |
|
| 29 |
-These error values can be returned: |
|
| 30 |
- |
|
| 31 |
-| Error | Description | |
|
| 32 |
-|---------------------|-----------------------------------------------------------------------------| |
|
| 33 |
-| `<nil>` | Everything ok, output is returned | |
|
| 34 |
-| `ErrIncompressible` | Returned when input is judged to be too hard to compress | |
|
| 35 |
-| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | |
|
| 36 |
-| `(error)` | An internal error occurred. | |
|
| 37 |
- |
|
| 38 |
-As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. |
|
| 39 |
- |
|
| 40 |
-To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object |
|
| 41 |
-that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same |
|
| 42 |
-object can be used for both. |
|
| 43 |
- |
|
| 44 |
-Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this |
|
| 45 |
-you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. |
|
| 46 |
- |
|
| 47 |
-Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. |
|
| 48 |
-You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back |
|
| 49 |
-your input was likely corrupted. |
|
| 50 |
- |
|
| 51 |
-It is important to note that a successful decoding does *not* mean your output matches your original input. |
|
| 52 |
-There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. |
|
| 53 |
- |
|
| 54 |
-For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). |
|
| 55 |
- |
|
| 56 |
-# Performance |
|
| 57 |
- |
|
| 58 |
-A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. |
|
| 59 |
-All compression functions are currently only running on the calling goroutine so only one core will be used per block. |
|
| 60 |
- |
|
| 61 |
-The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input |
|
| 62 |
-is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be |
|
| 63 |
-beneficial to transpose all your input values down by 64. |
|
| 64 |
- |
|
| 65 |
-With moderate block sizes around 64k speed are typically 200MB/s per core for compression and |
|
| 66 |
-around 300MB/s decompression speed. |
|
| 67 |
- |
|
| 68 |
-The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. |
|
| 69 |
- |
|
| 70 |
-# Plans |
|
| 71 |
- |
|
| 72 |
-At one point, more internals will be exposed to facilitate more "expert" usage of the components. |
|
| 73 |
- |
|
| 74 |
-A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). |
|
| 75 |
- |
|
| 76 |
-# Contributing |
|
| 77 |
- |
|
| 78 |
-Contributions are always welcome. Be aware that adding public functions will require good justification and breaking |
|
| 1 |
+# Finite State Entropy |
|
| 2 |
+ |
|
| 3 |
+This package provides Finite State Entropy encoding and decoding. |
|
| 4 |
+ |
|
| 5 |
+Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) |
|
| 6 |
+encoding provides a fast near-optimal symbol encoding/decoding |
|
| 7 |
+for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). |
|
| 8 |
+ |
|
| 9 |
+This can be used for compressing input with a lot of similar input values to the smallest number of bytes. |
|
| 10 |
+This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, |
|
| 11 |
+but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. |
|
| 12 |
+ |
|
| 13 |
+* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) |
|
| 14 |
+ |
|
| 15 |
+## News |
|
| 16 |
+ |
|
| 17 |
+ * Feb 2018: First implementation released. Consider this beta software for now. |
|
| 18 |
+ |
|
| 19 |
+# Usage |
|
| 20 |
+ |
|
| 21 |
+This package provides a low level interface that allows to compress single independent blocks. |
|
| 22 |
+ |
|
| 23 |
+Each block is separate, and there is no built in integrity checks. |
|
| 24 |
+This means that the caller should keep track of block sizes and also do checksums if needed. |
|
| 25 |
+ |
|
| 26 |
+Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. |
|
| 27 |
+You must provide input and will receive the output and maybe an error. |
|
| 28 |
+ |
|
| 29 |
+These error values can be returned: |
|
| 30 |
+ |
|
| 31 |
+| Error | Description | |
|
| 32 |
+|---------------------|-----------------------------------------------------------------------------| |
|
| 33 |
+| `<nil>` | Everything ok, output is returned | |
|
| 34 |
+| `ErrIncompressible` | Returned when input is judged to be too hard to compress | |
|
| 35 |
+| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | |
|
| 36 |
+| `(error)` | An internal error occurred. | |
|
| 37 |
+ |
|
| 38 |
+As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. |
|
| 39 |
+ |
|
| 40 |
+To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object |
|
| 41 |
+that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same |
|
| 42 |
+object can be used for both. |
|
| 43 |
+ |
|
| 44 |
+Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this |
|
| 45 |
+you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. |
|
| 46 |
+ |
|
| 47 |
+Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. |
|
| 48 |
+You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back |
|
| 49 |
+your input was likely corrupted. |
|
| 50 |
+ |
|
| 51 |
+It is important to note that a successful decoding does *not* mean your output matches your original input. |
|
| 52 |
+There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. |
|
| 53 |
+ |
|
| 54 |
+For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). |
|
| 55 |
+ |
|
| 56 |
+# Performance |
|
| 57 |
+ |
|
| 58 |
+A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. |
|
| 59 |
+All compression functions are currently only running on the calling goroutine so only one core will be used per block. |
|
| 60 |
+ |
|
| 61 |
+The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input |
|
| 62 |
+is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be |
|
| 63 |
+beneficial to transpose all your input values down by 64. |
|
| 64 |
+ |
|
| 65 |
+With moderate block sizes around 64k speed are typically 200MB/s per core for compression and |
|
| 66 |
+around 300MB/s decompression speed. |
|
| 67 |
+ |
|
| 68 |
+The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. |
|
| 69 |
+ |
|
| 70 |
+# Plans |
|
| 71 |
+ |
|
| 72 |
+At one point, more internals will be exposed to facilitate more "expert" usage of the components. |
|
| 73 |
+ |
|
| 74 |
+A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). |
|
| 75 |
+ |
|
| 76 |
+# Contributing |
|
| 77 |
+ |
|
| 78 |
+Contributions are always welcome. Be aware that adding public functions will require good justification and breaking |
|
| 79 | 79 |
changes will likely not be accepted. If in doubt open an issue before writing the PR. |
| 80 | 80 |
\ No newline at end of file |
| ... | ... |
@@ -1,89 +1,89 @@ |
| 1 |
-# Huff0 entropy compression |
|
| 2 |
- |
|
| 3 |
-This package provides Huff0 encoding and decoding as used in zstd. |
|
| 4 |
- |
|
| 5 |
-[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), |
|
| 6 |
-a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU |
|
| 7 |
-(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. |
|
| 8 |
- |
|
| 9 |
-This can be used for compressing input with a lot of similar input values to the smallest number of bytes. |
|
| 10 |
-This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, |
|
| 11 |
-but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. |
|
| 12 |
- |
|
| 13 |
-* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) |
|
| 14 |
- |
|
| 15 |
-## News |
|
| 16 |
- |
|
| 17 |
-This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. |
|
| 18 |
- |
|
| 19 |
-This ensures that most functionality is well tested. |
|
| 20 |
- |
|
| 21 |
-# Usage |
|
| 22 |
- |
|
| 23 |
-This package provides a low level interface that allows to compress single independent blocks. |
|
| 24 |
- |
|
| 25 |
-Each block is separate, and there is no built in integrity checks. |
|
| 26 |
-This means that the caller should keep track of block sizes and also do checksums if needed. |
|
| 27 |
- |
|
| 28 |
-Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and |
|
| 29 |
-[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. |
|
| 30 |
-You must provide input and will receive the output and maybe an error. |
|
| 31 |
- |
|
| 32 |
-These error values can be returned: |
|
| 33 |
- |
|
| 34 |
-| Error | Description | |
|
| 35 |
-|---------------------|-----------------------------------------------------------------------------| |
|
| 36 |
-| `<nil>` | Everything ok, output is returned | |
|
| 37 |
-| `ErrIncompressible` | Returned when input is judged to be too hard to compress | |
|
| 38 |
-| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | |
|
| 39 |
-| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | |
|
| 40 |
-| `(error)` | An internal error occurred. | |
|
| 41 |
- |
|
| 42 |
- |
|
| 43 |
-As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. |
|
| 44 |
- |
|
| 45 |
-To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object |
|
| 46 |
-that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same |
|
| 47 |
-object can be used for both. |
|
| 48 |
- |
|
| 49 |
-Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this |
|
| 50 |
-you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. |
|
| 51 |
- |
|
| 52 |
-The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. |
|
| 53 |
- |
|
| 54 |
-## Tables and re-use |
|
| 55 |
- |
|
| 56 |
-Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. |
|
| 57 |
- |
|
| 58 |
-The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) |
|
| 59 |
-that controls this behaviour. See the documentation for details. This can be altered between each block. |
|
| 60 |
- |
|
| 61 |
-Do however note that this information is *not* stored in the output block and it is up to the users of the package to |
|
| 62 |
-record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, |
|
| 63 |
-based on the boolean reported back from the CompressXX call. |
|
| 64 |
- |
|
| 65 |
-If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the |
|
| 66 |
-[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. |
|
| 67 |
- |
|
| 68 |
-## Decompressing |
|
| 69 |
- |
|
| 70 |
-The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). |
|
| 71 |
-This will initialize the decoding tables. |
|
| 72 |
-You can supply the complete block to `ReadTable` and it will return the data part of the block |
|
| 73 |
-which can be given to the decompressor. |
|
| 74 |
- |
|
| 75 |
-Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) |
|
| 76 |
-or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. |
|
| 77 |
- |
|
| 78 |
-For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. |
|
| 79 |
- |
|
| 80 |
-You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back |
|
| 81 |
-your input was likely corrupted. |
|
| 82 |
- |
|
| 83 |
-It is important to note that a successful decoding does *not* mean your output matches your original input. |
|
| 84 |
-There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. |
|
| 85 |
- |
|
| 86 |
-# Contributing |
|
| 87 |
- |
|
| 88 |
-Contributions are always welcome. Be aware that adding public functions will require good justification and breaking |
|
| 89 |
-changes will likely not be accepted. If in doubt open an issue before writing the PR. |
|
| 1 |
+# Huff0 entropy compression |
|
| 2 |
+ |
|
| 3 |
+This package provides Huff0 encoding and decoding as used in zstd. |
|
| 4 |
+ |
|
| 5 |
+[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), |
|
| 6 |
+a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU |
|
| 7 |
+(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. |
|
| 8 |
+ |
|
| 9 |
+This can be used for compressing input with a lot of similar input values to the smallest number of bytes. |
|
| 10 |
+This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, |
|
| 11 |
+but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. |
|
| 12 |
+ |
|
| 13 |
+* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) |
|
| 14 |
+ |
|
| 15 |
+## News |
|
| 16 |
+ |
|
| 17 |
+This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. |
|
| 18 |
+ |
|
| 19 |
+This ensures that most functionality is well tested. |
|
| 20 |
+ |
|
| 21 |
+# Usage |
|
| 22 |
+ |
|
| 23 |
+This package provides a low level interface that allows to compress single independent blocks. |
|
| 24 |
+ |
|
| 25 |
+Each block is separate, and there is no built in integrity checks. |
|
| 26 |
+This means that the caller should keep track of block sizes and also do checksums if needed. |
|
| 27 |
+ |
|
| 28 |
+Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and |
|
| 29 |
+[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. |
|
| 30 |
+You must provide input and will receive the output and maybe an error. |
|
| 31 |
+ |
|
| 32 |
+These error values can be returned: |
|
| 33 |
+ |
|
| 34 |
+| Error | Description | |
|
| 35 |
+|---------------------|-----------------------------------------------------------------------------| |
|
| 36 |
+| `<nil>` | Everything ok, output is returned | |
|
| 37 |
+| `ErrIncompressible` | Returned when input is judged to be too hard to compress | |
|
| 38 |
+| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | |
|
| 39 |
+| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | |
|
| 40 |
+| `(error)` | An internal error occurred. | |
|
| 41 |
+ |
|
| 42 |
+ |
|
| 43 |
+As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. |
|
| 44 |
+ |
|
| 45 |
+To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object |
|
| 46 |
+that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same |
|
| 47 |
+object can be used for both. |
|
| 48 |
+ |
|
| 49 |
+Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this |
|
| 50 |
+you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. |
|
| 51 |
+ |
|
| 52 |
+The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. |
|
| 53 |
+ |
|
| 54 |
+## Tables and re-use |
|
| 55 |
+ |
|
| 56 |
+Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. |
|
| 57 |
+ |
|
| 58 |
+The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) |
|
| 59 |
+that controls this behaviour. See the documentation for details. This can be altered between each block. |
|
| 60 |
+ |
|
| 61 |
+Do however note that this information is *not* stored in the output block and it is up to the users of the package to |
|
| 62 |
+record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, |
|
| 63 |
+based on the boolean reported back from the CompressXX call. |
|
| 64 |
+ |
|
| 65 |
+If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the |
|
| 66 |
+[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. |
|
| 67 |
+ |
|
| 68 |
+## Decompressing |
|
| 69 |
+ |
|
| 70 |
+The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). |
|
| 71 |
+This will initialize the decoding tables. |
|
| 72 |
+You can supply the complete block to `ReadTable` and it will return the data part of the block |
|
| 73 |
+which can be given to the decompressor. |
|
| 74 |
+ |
|
| 75 |
+Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) |
|
| 76 |
+or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. |
|
| 77 |
+ |
|
| 78 |
+For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. |
|
| 79 |
+ |
|
| 80 |
+You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back |
|
| 81 |
+your input was likely corrupted. |
|
| 82 |
+ |
|
| 83 |
+It is important to note that a successful decoding does *not* mean your output matches your original input. |
|
| 84 |
+There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. |
|
| 85 |
+ |
|
| 86 |
+# Contributing |
|
| 87 |
+ |
|
| 88 |
+Contributions are always welcome. Be aware that adding public functions will require good justification and breaking |
|
| 89 |
+changes will likely not be accepted. If in doubt open an issue before writing the PR. |
| ... | ... |
@@ -409,6 +409,7 @@ type SolveRequest struct {
|
| 409 | 409 |
Exporters []*Exporter `protobuf:"bytes,13,rep,name=Exporters,proto3" json:"Exporters,omitempty"` |
| 410 | 410 |
EnableSessionExporter bool `protobuf:"varint,14,opt,name=EnableSessionExporter,proto3" json:"EnableSessionExporter,omitempty"` |
| 411 | 411 |
SourcePolicySession string `protobuf:"bytes,15,opt,name=SourcePolicySession,proto3" json:"SourcePolicySession,omitempty"` |
| 412 |
+ CompatibilityVersion int64 `protobuf:"varint,16,opt,name=CompatibilityVersion,proto3" json:"CompatibilityVersion,omitempty"` |
|
| 412 | 413 |
unknownFields protoimpl.UnknownFields |
| 413 | 414 |
sizeCache protoimpl.SizeCache |
| 414 | 415 |
} |
| ... | ... |
@@ -548,6 +549,13 @@ func (x *SolveRequest) GetSourcePolicySession() string {
|
| 548 | 548 |
return "" |
| 549 | 549 |
} |
| 550 | 550 |
|
| 551 |
+func (x *SolveRequest) GetCompatibilityVersion() int64 {
|
|
| 552 |
+ if x != nil {
|
|
| 553 |
+ return x.CompatibilityVersion |
|
| 554 |
+ } |
|
| 555 |
+ return 0 |
|
| 556 |
+} |
|
| 557 |
+ |
|
| 551 | 558 |
type CacheOptions struct {
|
| 552 | 559 |
state protoimpl.MessageState `protogen:"open.v1"` |
| 553 | 560 |
// ExportRefDeprecated is deprecated in favor or the new Exports since BuildKit v0.4.0. |
| ... | ... |
@@ -2058,7 +2066,7 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = |
| 2058 | 2058 |
" \x01(\tR\n" + |
| 2059 | 2059 |
"RecordType\x12\x16\n" + |
| 2060 | 2060 |
"\x06Shared\x18\v \x01(\bR\x06Shared\x12\x18\n" + |
| 2061 |
- "\aParents\x18\f \x03(\tR\aParents\"\xa6\b\n" + |
|
| 2061 |
+ "\aParents\x18\f \x03(\tR\aParents\"\xda\b\n" + |
|
| 2062 | 2062 |
"\fSolveRequest\x12\x10\n" + |
| 2063 | 2063 |
"\x03Ref\x18\x01 \x01(\tR\x03Ref\x12.\n" + |
| 2064 | 2064 |
"\n" + |
| ... | ... |
@@ -2077,7 +2085,8 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = |
| 2077 | 2077 |
"\fSourcePolicy\x18\f \x01(\v2%.moby.buildkit.v1.sourcepolicy.PolicyR\fSourcePolicy\x128\n" + |
| 2078 | 2078 |
"\tExporters\x18\r \x03(\v2\x1a.moby.buildkit.v1.ExporterR\tExporters\x124\n" + |
| 2079 | 2079 |
"\x15EnableSessionExporter\x18\x0e \x01(\bR\x15EnableSessionExporter\x120\n" + |
| 2080 |
- "\x13SourcePolicySession\x18\x0f \x01(\tR\x13SourcePolicySession\x1aJ\n" + |
|
| 2080 |
+ "\x13SourcePolicySession\x18\x0f \x01(\tR\x13SourcePolicySession\x122\n" + |
|
| 2081 |
+ "\x14CompatibilityVersion\x18\x10 \x01(\x03R\x14CompatibilityVersion\x1aJ\n" + |
|
| 2081 | 2082 |
"\x1cExporterAttrsDeprecatedEntry\x12\x10\n" + |
| 2082 | 2083 |
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + |
| 2083 | 2084 |
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a@\n" + |
| ... | ... |
@@ -143,6 +143,7 @@ func (m *SolveRequest) CloneVT() *SolveRequest {
|
| 143 | 143 |
r.SourcePolicy = m.SourcePolicy.CloneVT() |
| 144 | 144 |
r.EnableSessionExporter = m.EnableSessionExporter |
| 145 | 145 |
r.SourcePolicySession = m.SourcePolicySession |
| 146 |
+ r.CompatibilityVersion = m.CompatibilityVersion |
|
| 146 | 147 |
if rhs := m.ExporterAttrsDeprecated; rhs != nil {
|
| 147 | 148 |
tmpContainer := make(map[string]string, len(rhs)) |
| 148 | 149 |
for k, v := range rhs {
|
| ... | ... |
@@ -1043,6 +1044,9 @@ func (this *SolveRequest) EqualVT(that *SolveRequest) bool {
|
| 1043 | 1043 |
if this.SourcePolicySession != that.SourcePolicySession {
|
| 1044 | 1044 |
return false |
| 1045 | 1045 |
} |
| 1046 |
+ if this.CompatibilityVersion != that.CompatibilityVersion {
|
|
| 1047 |
+ return false |
|
| 1048 |
+ } |
|
| 1046 | 1049 |
return string(this.unknownFields) == string(that.unknownFields) |
| 1047 | 1050 |
} |
| 1048 | 1051 |
|
| ... | ... |
@@ -2249,6 +2253,13 @@ func (m *SolveRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
|
| 2249 | 2249 |
i -= len(m.unknownFields) |
| 2250 | 2250 |
copy(dAtA[i:], m.unknownFields) |
| 2251 | 2251 |
} |
| 2252 |
+ if m.CompatibilityVersion != 0 {
|
|
| 2253 |
+ i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CompatibilityVersion)) |
|
| 2254 |
+ i-- |
|
| 2255 |
+ dAtA[i] = 0x1 |
|
| 2256 |
+ i-- |
|
| 2257 |
+ dAtA[i] = 0x80 |
|
| 2258 |
+ } |
|
| 2252 | 2259 |
if len(m.SourcePolicySession) > 0 {
|
| 2253 | 2260 |
i -= len(m.SourcePolicySession) |
| 2254 | 2261 |
copy(dAtA[i:], m.SourcePolicySession) |
| ... | ... |
@@ -4174,6 +4185,9 @@ func (m *SolveRequest) SizeVT() (n int) {
|
| 4174 | 4174 |
if l > 0 {
|
| 4175 | 4175 |
n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) |
| 4176 | 4176 |
} |
| 4177 |
+ if m.CompatibilityVersion != 0 {
|
|
| 4178 |
+ n += 2 + protohelpers.SizeOfVarint(uint64(m.CompatibilityVersion)) |
|
| 4179 |
+ } |
|
| 4177 | 4180 |
n += len(m.unknownFields) |
| 4178 | 4181 |
return n |
| 4179 | 4182 |
} |
| ... | ... |
@@ -6326,6 +6340,25 @@ func (m *SolveRequest) UnmarshalVT(dAtA []byte) error {
|
| 6326 | 6326 |
} |
| 6327 | 6327 |
m.SourcePolicySession = string(dAtA[iNdEx:postIndex]) |
| 6328 | 6328 |
iNdEx = postIndex |
| 6329 |
+ case 16: |
|
| 6330 |
+ if wireType != 0 {
|
|
| 6331 |
+ return fmt.Errorf("proto: wrong wireType = %d for field CompatibilityVersion", wireType)
|
|
| 6332 |
+ } |
|
| 6333 |
+ m.CompatibilityVersion = 0 |
|
| 6334 |
+ for shift := uint(0); ; shift += 7 {
|
|
| 6335 |
+ if shift >= 64 {
|
|
| 6336 |
+ return protohelpers.ErrIntOverflow |
|
| 6337 |
+ } |
|
| 6338 |
+ if iNdEx >= l {
|
|
| 6339 |
+ return io.ErrUnexpectedEOF |
|
| 6340 |
+ } |
|
| 6341 |
+ b := dAtA[iNdEx] |
|
| 6342 |
+ iNdEx++ |
|
| 6343 |
+ m.CompatibilityVersion |= int64(b&0x7F) << shift |
|
| 6344 |
+ if b < 0x80 {
|
|
| 6345 |
+ break |
|
| 6346 |
+ } |
|
| 6347 |
+ } |
|
| 6329 | 6348 |
default: |
| 6330 | 6349 |
iNdEx = preIndex |
| 6331 | 6350 |
skippy, err := protohelpers.Skip(dAtA[iNdEx:]) |
| ... | ... |
@@ -483,6 +483,26 @@ func Git(url, fragment string, opts ...GitOption) State {
|
| 483 | 483 |
addCap(&gi.Constraints, pb.CapSourceGitMTime) |
| 484 | 484 |
} |
| 485 | 485 |
|
| 486 |
+ if gi.FetchByCommit {
|
|
| 487 |
+ attrs[pb.AttrGitFetchByCommit] = "true" |
|
| 488 |
+ addCap(&gi.Constraints, pb.CapSourceGitFetchByCommit) |
|
| 489 |
+ } |
|
| 490 |
+ if gi.Bundle != "" {
|
|
| 491 |
+ attrs[pb.AttrGitBundle] = gi.Bundle |
|
| 492 |
+ addCap(&gi.Constraints, pb.CapSourceGitBundle) |
|
| 493 |
+ } |
|
| 494 |
+ if gi.BundleOCISessionID != "" {
|
|
| 495 |
+ attrs[pb.AttrOCILayoutSessionID] = gi.BundleOCISessionID |
|
| 496 |
+ } |
|
| 497 |
+ if gi.BundleOCIStoreID != "" {
|
|
| 498 |
+ attrs[pb.AttrOCILayoutStoreID] = gi.BundleOCIStoreID |
|
| 499 |
+ } |
|
| 500 |
+ |
|
| 501 |
+ if gi.CheckoutBundle {
|
|
| 502 |
+ attrs[pb.AttrGitCheckoutBundle] = "true" |
|
| 503 |
+ addCap(&gi.Constraints, pb.CapSourceGitCheckoutBundle) |
|
| 504 |
+ } |
|
| 505 |
+ |
|
| 486 | 506 |
addCap(&gi.Constraints, pb.CapSourceGit) |
| 487 | 507 |
|
| 488 | 508 |
source := NewSource("git://"+id, attrs, gi.Constraints)
|
| ... | ... |
@@ -500,17 +520,22 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) {
|
| 500 | 500 |
|
| 501 | 501 |
type GitInfo struct {
|
| 502 | 502 |
constraintsWrapper |
| 503 |
- KeepGitDir bool |
|
| 504 |
- AuthTokenSecret string |
|
| 505 |
- AuthHeaderSecret string |
|
| 506 |
- addAuthCap bool |
|
| 507 |
- KnownSSHHosts string |
|
| 508 |
- MountSSHSock string |
|
| 509 |
- Checksum string |
|
| 510 |
- Ref string |
|
| 511 |
- SubDir string |
|
| 512 |
- SkipSubmodules bool |
|
| 513 |
- MTime string |
|
| 503 |
+ KeepGitDir bool |
|
| 504 |
+ AuthTokenSecret string |
|
| 505 |
+ AuthHeaderSecret string |
|
| 506 |
+ addAuthCap bool |
|
| 507 |
+ KnownSSHHosts string |
|
| 508 |
+ MountSSHSock string |
|
| 509 |
+ Checksum string |
|
| 510 |
+ Ref string |
|
| 511 |
+ SubDir string |
|
| 512 |
+ SkipSubmodules bool |
|
| 513 |
+ MTime string |
|
| 514 |
+ Bundle string |
|
| 515 |
+ BundleOCISessionID string |
|
| 516 |
+ BundleOCIStoreID string |
|
| 517 |
+ CheckoutBundle bool |
|
| 518 |
+ FetchByCommit bool |
|
| 514 | 519 |
} |
| 515 | 520 |
|
| 516 | 521 |
func GitRef(v string) GitOption {
|
| ... | ... |
@@ -577,6 +602,99 @@ func GitChecksum(v string) GitOption {
|
| 577 | 577 |
}) |
| 578 | 578 |
} |
| 579 | 579 |
|
| 580 |
+// GitFetchByCommit makes the git source trust the provided checksum as the |
|
| 581 |
+// commit to fetch, without resolving the ref against the remote. The ref, if |
|
| 582 |
+// set, is applied locally after the commit is fetched so that cache keys |
|
| 583 |
+// still depend on it. This is useful when the remote ref may have moved |
|
| 584 |
+// since the original resolution. |
|
| 585 |
+// |
|
| 586 |
+// Unqualified ref names are canonicalized to "refs/heads/<name>" to match |
|
| 587 |
+// the normal path's cache keys for branches. Tags must be passed fully |
|
| 588 |
+// qualified ("refs/tags/<name>").
|
|
| 589 |
+func GitFetchByCommit() GitOption {
|
|
| 590 |
+ return gitOptionFunc(func(gi *GitInfo) {
|
|
| 591 |
+ gi.FetchByCommit = true |
|
| 592 |
+ }) |
|
| 593 |
+} |
|
| 594 |
+ |
|
| 595 |
+// GitBundleInfo carries scheme-specific configuration for [GitBundleURL]. |
|
| 596 |
+// It is populated by [GitBundleOption] values and consumed by GitBundleURL. |
|
| 597 |
+type GitBundleInfo struct {
|
|
| 598 |
+ // OCISessionID pins the OCI-layout bundle fetch to a specific client |
|
| 599 |
+ // session. Only meaningful when the locator uses the |
|
| 600 |
+ // "oci-layout+blob://" scheme. |
|
| 601 |
+ OCISessionID string |
|
| 602 |
+ // OCIStoreID overrides the OCI-layout store name derived from the |
|
| 603 |
+ // bundle locator body. Only meaningful when the locator uses the |
|
| 604 |
+ // "oci-layout+blob://" scheme. |
|
| 605 |
+ OCIStoreID string |
|
| 606 |
+} |
|
| 607 |
+ |
|
| 608 |
+// GitBundleOption configures bundle-specific behavior for [GitBundleURL]. |
|
| 609 |
+type GitBundleOption interface {
|
|
| 610 |
+ SetGitBundleOption(*GitBundleInfo) |
|
| 611 |
+} |
|
| 612 |
+ |
|
| 613 |
+type gitBundleOptionFunc func(*GitBundleInfo) |
|
| 614 |
+ |
|
| 615 |
+func (fn gitBundleOptionFunc) SetGitBundleOption(bi *GitBundleInfo) {
|
|
| 616 |
+ fn(bi) |
|
| 617 |
+} |
|
| 618 |
+ |
|
| 619 |
+// GitBundleURL configures the git source to import the commit history from a |
|
| 620 |
+// pre-built git bundle instead of fetching from the upstream Git remote. The |
|
| 621 |
+// locator must use one of the following schemes: |
|
| 622 |
+// |
|
| 623 |
+// docker-image+blob://<registry-ref>@sha256:<digest> |
|
| 624 |
+// oci-layout+blob://<ref>@sha256:<digest> |
|
| 625 |
+// |
|
| 626 |
+// [GitChecksum] is required when GitBundleURL is used and the commit it |
|
| 627 |
+// identifies must be reachable from a ref inside the bundle (i.e. the bundle |
|
| 628 |
+// must contain that commit; BuildKit validates this after import). Auth and |
|
| 629 |
+// SSH options are ignored in bundle mode. Submodules, if present, still fetch |
|
| 630 |
+// from their own remotes. See [GitCheckoutBundle] for the checkout side; it |
|
| 631 |
+// is mutually exclusive with [KeepGitDir] and [GitSubDir]. |
|
| 632 |
+// |
|
| 633 |
+// Scheme-specific behavior can be tuned via [GitBundleOption] values, e.g. |
|
| 634 |
+// [GitBundleOCIStore] for the "oci-layout+blob://" scheme. |
|
| 635 |
+func GitBundleURL(locator string, opts ...GitBundleOption) GitOption {
|
|
| 636 |
+ bi := &GitBundleInfo{}
|
|
| 637 |
+ for _, o := range opts {
|
|
| 638 |
+ o.SetGitBundleOption(bi) |
|
| 639 |
+ } |
|
| 640 |
+ return gitOptionFunc(func(gi *GitInfo) {
|
|
| 641 |
+ gi.Bundle = locator |
|
| 642 |
+ gi.BundleOCISessionID = bi.OCISessionID |
|
| 643 |
+ gi.BundleOCIStoreID = bi.OCIStoreID |
|
| 644 |
+ }) |
|
| 645 |
+} |
|
| 646 |
+ |
|
| 647 |
+// GitBundleOCIStore configures the OCI-layout session and store id used to |
|
| 648 |
+// resolve a bundle locator with the "oci-layout+blob://" scheme. When unset, |
|
| 649 |
+// the locator body (the part before "@digest") is used as the store id and no |
|
| 650 |
+// session is pinned. This mirrors [ImageBlobOCIStore] for regular OCI-layout |
|
| 651 |
+// blobs. |
|
| 652 |
+// |
|
| 653 |
+// Only meaningful when passed to [GitBundleURL] together with a locator that |
|
| 654 |
+// uses the "oci-layout+blob://" scheme. |
|
| 655 |
+func GitBundleOCIStore(sessionID, storeID string) GitBundleOption {
|
|
| 656 |
+ return gitBundleOptionFunc(func(bi *GitBundleInfo) {
|
|
| 657 |
+ bi.OCISessionID = sessionID |
|
| 658 |
+ bi.OCIStoreID = storeID |
|
| 659 |
+ }) |
|
| 660 |
+} |
|
| 661 |
+ |
|
| 662 |
+// GitCheckoutBundle produces a single-file git bundle at the checkout mount |
|
| 663 |
+// root (filename "bundle") containing the resolved commit, instead of a |
|
| 664 |
+// worktree. Mutually exclusive with [KeepGitDir] and [GitSubDir]. Submodules |
|
| 665 |
+// are not included in the bundle. Pair with [GitBundleURL] to re-import the |
|
| 666 |
+// bundle back into a later build. |
|
| 667 |
+func GitCheckoutBundle() GitOption {
|
|
| 668 |
+ return gitOptionFunc(func(gi *GitInfo) {
|
|
| 669 |
+ gi.CheckoutBundle = true |
|
| 670 |
+ }) |
|
| 671 |
+} |
|
| 672 |
+ |
|
| 580 | 673 |
// AuthOption can be used with either HTTP or Git sources. |
| 581 | 674 |
type AuthOption interface {
|
| 582 | 675 |
GitOption |
| ... | ... |
@@ -36,8 +36,8 @@ import ( |
| 36 | 36 |
|
| 37 | 37 |
type SolveOpt struct {
|
| 38 | 38 |
Exports []ExportEntry |
| 39 |
+ CompatibilityVersion int |
|
| 39 | 40 |
EnableSessionExporter bool |
| 40 |
- LocalDirs map[string]string // Deprecated: use LocalMounts |
|
| 41 | 41 |
LocalMounts map[string]fsutil.FS |
| 42 | 42 |
OCIStores map[string]content.Store |
| 43 | 43 |
SharedKey string |
| ... | ... |
@@ -95,11 +95,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG |
| 95 | 95 |
return nil, errors.New("invalid with def and cb")
|
| 96 | 96 |
} |
| 97 | 97 |
|
| 98 |
- mounts, err := prepareMounts(&opt) |
|
| 99 |
- if err != nil {
|
|
| 100 |
- return nil, err |
|
| 101 |
- } |
|
| 102 |
- syncedDirs, err := prepareSyncedFiles(def, mounts) |
|
| 98 |
+ syncedDirs, err := prepareSyncedFiles(def, opt.LocalMounts) |
|
| 103 | 99 |
if err != nil {
|
| 104 | 100 |
return nil, err |
| 105 | 101 |
} |
| ... | ... |
@@ -311,6 +307,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG |
| 311 | 311 |
Cache: &cacheOpt.options, |
| 312 | 312 |
Entitlements: slices.Clone(opt.AllowedEntitlements), |
| 313 | 313 |
Internal: opt.Internal, |
| 314 |
+ CompatibilityVersion: int64(opt.CompatibilityVersion), |
|
| 314 | 315 |
SourcePolicy: opt.SourcePolicy, |
| 315 | 316 |
} |
| 316 | 317 |
if opt.SourcePolicyProvider != nil {
|
| ... | ... |
@@ -586,20 +583,3 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach |
| 586 | 586 |
} |
| 587 | 587 |
return &res, nil |
| 588 | 588 |
} |
| 589 |
- |
|
| 590 |
-func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
|
|
| 591 |
- // merge local mounts and fallback local directories together |
|
| 592 |
- mounts := make(map[string]fsutil.FS) |
|
| 593 |
- maps.Copy(mounts, opt.LocalMounts) |
|
| 594 |
- for k, dir := range opt.LocalDirs {
|
|
| 595 |
- mount, err := fsutil.NewFS(dir) |
|
| 596 |
- if err != nil {
|
|
| 597 |
- return nil, err |
|
| 598 |
- } |
|
| 599 |
- if _, ok := mounts[k]; ok {
|
|
| 600 |
- return nil, errors.Errorf("local mount %s already exists", k)
|
|
| 601 |
- } |
|
| 602 |
- mounts[k] = mount |
|
| 603 |
- } |
|
| 604 |
- return mounts, nil |
|
| 605 |
-} |
| ... | ... |
@@ -7,7 +7,10 @@ import ( |
| 7 | 7 |
|
| 8 | 8 |
// Config provides containerd configuration data for the server |
| 9 | 9 |
type Config struct {
|
| 10 |
+ // Deprecated: Use Log.Level with "debug" set instead. |
|
| 10 | 11 |
Debug bool `toml:"debug"` |
| 12 |
+ |
|
| 13 |
+ // Deprecated: Use Log.Level with "trace" set instead. |
|
| 11 | 14 |
Trace bool `toml:"trace"` |
| 12 | 15 |
|
| 13 | 16 |
// Root is the path to a directory where buildkit will store persistent data |
| ... | ... |
@@ -63,6 +66,7 @@ type SystemConfig struct {
|
| 63 | 63 |
|
| 64 | 64 |
type LogConfig struct {
|
| 65 | 65 |
Format string `toml:"format"` |
| 66 |
+ Level string `toml:"level"` |
|
| 66 | 67 |
} |
| 67 | 68 |
|
| 68 | 69 |
type GRPCConfig struct {
|
| ... | ... |
@@ -34,6 +34,7 @@ import ( |
| 34 | 34 |
"github.com/moby/buildkit/solver/bboltcachestorage" |
| 35 | 35 |
"github.com/moby/buildkit/solver/llbsolver" |
| 36 | 36 |
"github.com/moby/buildkit/solver/llbsolver/cdidevices" |
| 37 |
+ "github.com/moby/buildkit/solver/llbsolver/compat" |
|
| 37 | 38 |
"github.com/moby/buildkit/solver/llbsolver/history" |
| 38 | 39 |
"github.com/moby/buildkit/solver/llbsolver/proc" |
| 39 | 40 |
provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types" |
| ... | ... |
@@ -389,6 +390,14 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* |
| 389 | 389 |
} |
| 390 | 390 |
translateLegacySolveRequest(req) |
| 391 | 391 |
|
| 392 |
+ compatibilityVersion := int(req.CompatibilityVersion) |
|
| 393 |
+ if compatibilityVersion == 0 {
|
|
| 394 |
+ compatibilityVersion = compat.CompatibilityVersionCurrent |
|
| 395 |
+ } |
|
| 396 |
+ if err := compat.ValidateCompatibilityVersion(compatibilityVersion); err != nil {
|
|
| 397 |
+ return nil, err |
|
| 398 |
+ } |
|
| 399 |
+ |
|
| 392 | 400 |
defer func() {
|
| 393 | 401 |
time.AfterFunc(time.Second, c.throttledGC) |
| 394 | 402 |
}() |
| ... | ... |
@@ -538,7 +547,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* |
| 538 | 538 |
FrontendOpt: req.FrontendAttrs, |
| 539 | 539 |
FrontendInputs: req.FrontendInputs, |
| 540 | 540 |
CacheImports: cacheImports, |
| 541 |
- }, llbsolver.ExporterRequest{
|
|
| 541 |
+ }, compatibilityVersion, llbsolver.ExporterRequest{
|
|
| 542 | 542 |
Exporters: expis, |
| 543 | 543 |
CacheExporters: cacheExporters, |
| 544 | 544 |
EnableSessionExporter: req.EnableSessionExporter, |
| ... | ... |
@@ -2,55 +2,36 @@ package gateway |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 |
- "sync" |
|
| 6 |
- "time" |
|
| 7 | 5 |
|
| 8 | 6 |
"github.com/moby/buildkit/client/buildid" |
| 9 | 7 |
"github.com/moby/buildkit/frontend/gateway" |
| 10 | 8 |
gwapi "github.com/moby/buildkit/frontend/gateway/pb" |
| 11 | 9 |
"github.com/moby/buildkit/solver/errdefs" |
| 10 |
+ "github.com/moby/buildkit/util/registrar" |
|
| 12 | 11 |
"github.com/pkg/errors" |
| 13 | 12 |
"google.golang.org/grpc" |
| 14 | 13 |
) |
| 15 | 14 |
|
| 16 | 15 |
type GatewayForwarder struct {
|
| 17 |
- mu sync.RWMutex |
|
| 18 |
- updateCond *sync.Cond |
|
| 19 |
- builds map[string]gateway.LLBBridgeForwarder |
|
| 16 |
+ registrar *registrar.Registrar[string, gateway.LLBBridgeForwarder] |
|
| 20 | 17 |
} |
| 21 | 18 |
|
| 22 | 19 |
func NewGatewayForwarder() *GatewayForwarder {
|
| 23 |
- gwf := &GatewayForwarder{
|
|
| 24 |
- builds: map[string]gateway.LLBBridgeForwarder{},
|
|
| 20 |
+ return &GatewayForwarder{
|
|
| 21 |
+ registrar: registrar.New[string, gateway.LLBBridgeForwarder](), |
|
| 25 | 22 |
} |
| 26 |
- gwf.updateCond = sync.NewCond(gwf.mu.RLocker()) |
|
| 27 |
- return gwf |
|
| 28 | 23 |
} |
| 29 | 24 |
|
| 30 | 25 |
func (gwf *GatewayForwarder) Register(server *grpc.Server) {
|
| 31 | 26 |
gwapi.RegisterLLBBridgeServer(server, gwf) |
| 32 | 27 |
} |
| 33 | 28 |
|
| 34 |
-func (gwf *GatewayForwarder) RegisterBuild(ctx context.Context, id string, bridge gateway.LLBBridgeForwarder) error {
|
|
| 35 |
- gwf.mu.Lock() |
|
| 36 |
- defer gwf.mu.Unlock() |
|
| 37 |
- |
|
| 38 |
- if _, ok := gwf.builds[id]; ok {
|
|
| 39 |
- return errors.Errorf("build ID %s exists", id)
|
|
| 40 |
- } |
|
| 41 |
- |
|
| 42 |
- gwf.builds[id] = bridge |
|
| 43 |
- gwf.updateCond.Broadcast() |
|
| 44 |
- |
|
| 45 |
- return nil |
|
| 29 |
+func (gwf *GatewayForwarder) RegisterBuild(ctx context.Context, id string, bridge gateway.LLBBridgeForwarder) {
|
|
| 30 |
+ gwf.registrar.Register(id, bridge) |
|
| 46 | 31 |
} |
| 47 | 32 |
|
| 48 | 33 |
func (gwf *GatewayForwarder) UnregisterBuild(ctx context.Context, id string) {
|
| 49 |
- gwf.mu.Lock() |
|
| 50 |
- defer gwf.mu.Unlock() |
|
| 51 |
- |
|
| 52 |
- delete(gwf.builds, id) |
|
| 53 |
- gwf.updateCond.Broadcast() |
|
| 34 |
+ gwf.registrar.Discard(id) |
|
| 54 | 35 |
} |
| 55 | 36 |
|
| 56 | 37 |
func (gwf *GatewayForwarder) lookupForwarder(ctx context.Context) (gateway.LLBBridgeForwarder, error) {
|
| ... | ... |
@@ -59,32 +40,14 @@ func (gwf *GatewayForwarder) lookupForwarder(ctx context.Context) (gateway.LLBBr |
| 59 | 59 |
return nil, errors.New("no buildid found in context")
|
| 60 | 60 |
} |
| 61 | 61 |
|
| 62 |
- ctx, cancel := context.WithCancelCause(ctx) |
|
| 63 |
- ctx, _ = context.WithTimeoutCause(ctx, 3*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet |
|
| 64 |
- defer func() { cancel(errors.WithStack(context.Canceled)) }()
|
|
| 65 |
- |
|
| 66 |
- go func() {
|
|
| 67 |
- <-ctx.Done() |
|
| 68 |
- gwf.mu.Lock() |
|
| 69 |
- gwf.updateCond.Broadcast() |
|
| 70 |
- gwf.mu.Unlock() |
|
| 71 |
- }() |
|
| 72 |
- |
|
| 73 |
- gwf.mu.RLock() |
|
| 74 |
- defer gwf.mu.RUnlock() |
|
| 75 |
- for {
|
|
| 76 |
- select {
|
|
| 77 |
- case <-ctx.Done(): |
|
| 78 |
- return nil, errdefs.NewUnknownJobError(bid) |
|
| 79 |
- default: |
|
| 80 |
- } |
|
| 81 |
- fwd, ok := gwf.builds[bid] |
|
| 82 |
- if !ok {
|
|
| 83 |
- gwf.updateCond.Wait() |
|
| 84 |
- continue |
|
| 62 |
+ fwd, err := gwf.registrar.Get(ctx, bid) |
|
| 63 |
+ if err != nil {
|
|
| 64 |
+ if errors.Is(err, context.Canceled) {
|
|
| 65 |
+ return nil, errors.WithStack(errdefs.NewUnknownJobError(bid)) |
|
| 85 | 66 |
} |
| 86 |
- return fwd, nil |
|
| 67 |
+ return nil, err |
|
| 87 | 68 |
} |
| 69 |
+ return fwd, nil |
|
| 88 | 70 |
} |
| 89 | 71 |
|
| 90 | 72 |
func (gwf *GatewayForwarder) ResolveImageConfig(ctx context.Context, req *gwapi.ResolveImageConfigRequest) (*gwapi.ResolveImageConfigResponse, error) {
|
| ... | ... |
@@ -20,7 +20,7 @@ import ( |
| 20 | 20 |
"github.com/moby/buildkit/util/network" |
| 21 | 21 |
rootlessmountopts "github.com/moby/buildkit/util/rootless/mountopts" |
| 22 | 22 |
"github.com/moby/buildkit/util/system" |
| 23 |
- traceexec "github.com/moby/buildkit/util/tracing/exec" |
|
| 23 |
+ "github.com/moby/buildkit/util/tracing/childprocess" |
|
| 24 | 24 |
"github.com/moby/sys/user" |
| 25 | 25 |
"github.com/moby/sys/userns" |
| 26 | 26 |
specs "github.com/opencontainers/runtime-spec/specs-go" |
| ... | ... |
@@ -120,7 +120,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou |
| 120 | 120 |
if tracingSocket != "" {
|
| 121 | 121 |
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md |
| 122 | 122 |
meta.Env = append(meta.Env, tracingEnvVars...) |
| 123 |
- meta.Env = append(meta.Env, traceexec.Environ(ctx)...) |
|
| 123 |
+ meta.Env = append(meta.Env, childprocess.Environ(ctx)...) |
|
| 124 | 124 |
} |
| 125 | 125 |
|
| 126 | 126 |
opts = append(opts, |
| ... | ... |
@@ -246,7 +246,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source |
| 246 | 246 |
} |
| 247 | 247 |
}() |
| 248 | 248 |
|
| 249 |
- desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts) |
|
| 249 |
+ desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts, buildInfo.CompatibilityVersion, e.Type()) |
|
| 250 | 250 |
if err != nil {
|
| 251 | 251 |
return nil, nil, nil, err |
| 252 | 252 |
} |
| ... | ... |
@@ -25,6 +25,8 @@ import ( |
| 25 | 25 |
"github.com/moby/buildkit/session" |
| 26 | 26 |
"github.com/moby/buildkit/snapshot" |
| 27 | 27 |
"github.com/moby/buildkit/solver" |
| 28 |
+ solvererrdefs "github.com/moby/buildkit/solver/errdefs" |
|
| 29 |
+ "github.com/moby/buildkit/solver/llbsolver/compat" |
|
| 28 | 30 |
"github.com/moby/buildkit/solver/result" |
| 29 | 31 |
attestationTypes "github.com/moby/buildkit/util/attestation" |
| 30 | 32 |
"github.com/moby/buildkit/util/bklog" |
| ... | ... |
@@ -63,10 +65,14 @@ type ImageWriter struct {
|
| 63 | 63 |
opt WriterOpt |
| 64 | 64 |
} |
| 65 | 65 |
|
| 66 |
-func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, sessionID string, inlineCache exptypes.InlineCache, opts *ImageCommitOpts) (*ocispecs.Descriptor, error) {
|
|
| 66 |
+func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, sessionID string, inlineCache exptypes.InlineCache, opts *ImageCommitOpts, compatibilityVersion int, exporterType string) (*ocispecs.Descriptor, error) {
|
|
| 67 | 67 |
if _, ok := inp.Metadata[exptypes.ExporterPlatformsKey]; len(inp.Refs) > 0 && !ok {
|
| 68 | 68 |
return nil, errors.Errorf("unable to export multiple refs, missing platforms mapping")
|
| 69 | 69 |
} |
| 70 |
+ if compatibilityVersion == compat.CompatibilityVersion013 && opts.RefCfg.Compression.Type == compression.Zstd {
|
|
| 71 |
+ feature := fmt.Sprintf("%s exporter compression=%s", exporterType, opts.RefCfg.Compression.Type.String())
|
|
| 72 |
+ return nil, solvererrdefs.NewUnsupportedCompatibilityFeatureError(compatibilityVersion, feature) |
|
| 73 |
+ } |
|
| 70 | 74 |
|
| 71 | 75 |
isMap := len(inp.Refs) > 0 |
| 72 | 76 |
|
| ... | ... |
@@ -50,9 +50,10 @@ type ExporterInstance interface {
|
| 50 | 50 |
} |
| 51 | 51 |
|
| 52 | 52 |
type ExportBuildInfo struct {
|
| 53 |
- Ref string |
|
| 54 |
- InlineCache exptypes.InlineCache |
|
| 55 |
- SessionID string |
|
| 53 |
+ Ref string |
|
| 54 |
+ InlineCache exptypes.InlineCache |
|
| 55 |
+ SessionID string |
|
| 56 |
+ CompatibilityVersion int |
|
| 56 | 57 |
} |
| 57 | 58 |
|
| 58 | 59 |
type DescriptorReference interface {
|
| ... | ... |
@@ -159,7 +159,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source |
| 159 | 159 |
} |
| 160 | 160 |
}() |
| 161 | 161 |
|
| 162 |
- desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts) |
|
| 162 |
+ desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts, buildInfo.CompatibilityVersion, e.Type()) |
|
| 163 | 163 |
if err != nil {
|
| 164 | 164 |
return nil, nil, nil, err |
| 165 | 165 |
} |
| ... | ... |
@@ -21,7 +21,13 @@ type Epoch struct {
|
| 21 | 21 |
|
| 22 | 22 |
func ParseBuildArgs(opt map[string]string) (string, bool) {
|
| 23 | 23 |
v, ok := opt[frontendSourceDateEpochArg] |
| 24 |
- return v, ok |
|
| 24 |
+ if !ok {
|
|
| 25 |
+ return "", false |
|
| 26 |
+ } |
|
| 27 |
+ if _, err := parseTime(frontendSourceDateEpochArg, v); err != nil {
|
|
| 28 |
+ return "", false |
|
| 29 |
+ } |
|
| 30 |
+ return v, true |
|
| 25 | 31 |
} |
| 26 | 32 |
|
| 27 | 33 |
func ParseExporterAttrs(opt map[string]string) (*Epoch, map[string]string, error) {
|
| ... | ... |
@@ -2,6 +2,7 @@ package builder |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 |
+ "maps" |
|
| 5 | 6 |
"strings" |
| 6 | 7 |
|
| 7 | 8 |
"github.com/containerd/platforms" |
| ... | ... |
@@ -131,6 +132,7 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
|
| 131 | 131 |
|
| 132 | 132 |
rb, err := bc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (*dockerui.BuildResult, error) {
|
| 133 | 133 |
opt := convertOpt |
| 134 |
+ opt.BuildArgs = maps.Clone(opt.BuildArgs) |
|
| 134 | 135 |
opt.TargetPlatform = platform |
| 135 | 136 |
if idx != 0 {
|
| 136 | 137 |
opt.Warn = nil |
| ... | ... |
@@ -3,6 +3,7 @@ package dfgitutil |
| 3 | 3 |
|
| 4 | 4 |
import ( |
| 5 | 5 |
"net/url" |
| 6 |
+ "path" |
|
| 6 | 7 |
"strconv" |
| 7 | 8 |
"strings" |
| 8 | 9 |
|
| ... | ... |
@@ -59,6 +60,11 @@ type GitRef struct {
|
| 59 | 59 |
|
| 60 | 60 |
// MTime controls file modification time policy: "checkout" (default) or "commit". |
| 61 | 61 |
MTime string |
| 62 |
+ |
|
| 63 |
+ // FetchByCommit, when true, trusts Checksum as the commit and skips comparing |
|
| 64 |
+ // it against the remote ref. The commit is fetched directly and the ref name |
|
| 65 |
+ // (if any) is applied locally. |
|
| 66 |
+ FetchByCommit bool |
|
| 62 | 67 |
} |
| 63 | 68 |
|
| 64 | 69 |
// ParseGitRef parses a git ref. |
| ... | ... |
@@ -134,7 +140,7 @@ func (gf *GitRef) loadQuery(query url.Values) error {
|
| 134 | 134 |
case 0, 1: |
| 135 | 135 |
if len(v) == 0 || v[0] == "" {
|
| 136 | 136 |
switch k {
|
| 137 |
- case "submodules", "keep-git-dir": |
|
| 137 |
+ case "submodules", "keep-git-dir", "fetch-by-commit": |
|
| 138 | 138 |
v = nil |
| 139 | 139 |
default: |
| 140 | 140 |
return errors.Errorf("query %q has no value", k)
|
| ... | ... |
@@ -192,6 +198,18 @@ func (gf *GitRef) loadQuery(query url.Values) error {
|
| 192 | 192 |
default: |
| 193 | 193 |
return errors.Errorf("invalid mtime value: %q (must be \"checkout\" or \"commit\")", v[0])
|
| 194 | 194 |
} |
| 195 |
+ case "fetch-by-commit": |
|
| 196 |
+ var vv bool |
|
| 197 |
+ if len(v) == 0 {
|
|
| 198 |
+ vv = true |
|
| 199 |
+ } else {
|
|
| 200 |
+ var err error |
|
| 201 |
+ vv, err = strconv.ParseBool(v[0]) |
|
| 202 |
+ if err != nil {
|
|
| 203 |
+ return errors.Errorf("invalid fetch-by-commit value: %q", v[0])
|
|
| 204 |
+ } |
|
| 205 |
+ } |
|
| 206 |
+ gf.FetchByCommit = vv |
|
| 195 | 207 |
default: |
| 196 | 208 |
return errors.Errorf("unexpected query %q", k)
|
| 197 | 209 |
} |
| ... | ... |
@@ -223,16 +241,23 @@ func (gf *GitRef) loadQuery(query url.Values) error {
|
| 223 | 223 |
return nil |
| 224 | 224 |
} |
| 225 | 225 |
|
| 226 |
-// FragmentFormat returns a simplified git URL in fragment format with only ref. |
|
| 226 |
+// FragmentFormat returns a simplified git URL in fragment format. |
|
| 227 | 227 |
// If the URL cannot be parsed, the original string is returned with false. |
| 228 |
-func FragmentFormat(remote string) (string, bool) {
|
|
| 228 |
+func FragmentFormat(remote string, withSubdir bool) (string, bool) {
|
|
| 229 | 229 |
gitRef, _, err := ParseGitRef(remote) |
| 230 | 230 |
if err != nil || gitRef == nil {
|
| 231 | 231 |
return remote, false |
| 232 | 232 |
} |
| 233 | 233 |
u := gitRef.Remote |
| 234 |
- if gitRef.Ref != "" {
|
|
| 234 |
+ subdir := "" |
|
| 235 |
+ if withSubdir {
|
|
| 236 |
+ subdir = strings.TrimPrefix(path.Join("/", gitRef.SubDir), "/")
|
|
| 237 |
+ } |
|
| 238 |
+ if gitRef.Ref != "" || subdir != "" {
|
|
| 235 | 239 |
u += "#" + gitRef.Ref |
| 240 |
+ if subdir != "" {
|
|
| 241 |
+ u += ":" + subdir |
|
| 242 |
+ } |
|
| 236 | 243 |
} |
| 237 | 244 |
return u, true |
| 238 | 245 |
} |
| ... | ... |
@@ -208,6 +208,7 @@ type dispatchContext struct {
|
| 208 | 208 |
opt ConvertOpt |
| 209 | 209 |
platformOpt *platformOpt |
| 210 | 210 |
globalArgs *llb.EnvList |
| 211 |
+ epoch *time.Time |
|
| 211 | 212 |
shlex *shell.Lex |
| 212 | 213 |
outline outlineCapture |
| 213 | 214 |
lint *linter.Linter |
| ... | ... |
@@ -300,9 +301,13 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 300 | 300 |
return nil, err |
| 301 | 301 |
} |
| 302 | 302 |
|
| 303 |
- opt.Epoch, err = resolveSourceDateEpoch(opt.Epoch, globalArgs) |
|
| 304 |
- if err != nil {
|
|
| 305 |
- return nil, err |
|
| 303 |
+ var resolvedEpoch *time.Time |
|
| 304 |
+ if sourceDateEpoch, ok := getBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH"); ok {
|
|
| 305 |
+ resolvedEpoch, err = resolveSourceDateEpochValue(ctx, sourceDateEpoch, opt, stages, globalArgs, shlex) |
|
| 306 |
+ if err != nil {
|
|
| 307 |
+ return nil, err |
|
| 308 |
+ } |
|
| 309 |
+ globalArgs = setBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH", formatSourceDateEpochValue(resolvedEpoch)) |
|
| 306 | 310 |
} |
| 307 | 311 |
|
| 308 | 312 |
metaResolver := opt.MetaResolver |
| ... | ... |
@@ -314,6 +319,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 314 | 314 |
opt: opt, |
| 315 | 315 |
platformOpt: platformOpt, |
| 316 | 316 |
globalArgs: globalArgs, |
| 317 |
+ epoch: resolvedEpoch, |
|
| 317 | 318 |
shlex: shlex, |
| 318 | 319 |
outline: outline, |
| 319 | 320 |
lint: lint, |
| ... | ... |
@@ -353,25 +359,39 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS |
| 353 | 353 |
return target, nil |
| 354 | 354 |
} |
| 355 | 355 |
|
| 356 |
-func resolveSourceDateEpoch(explicit *time.Time, globalArgs *llb.EnvList) (*time.Time, error) {
|
|
| 357 |
- if explicit != nil {
|
|
| 358 |
- return explicit, nil |
|
| 356 |
+func getBuildArgValue(buildArgs map[string]string, globalArgs *llb.EnvList, key string) (string, bool) {
|
|
| 357 |
+ if v, ok := buildArgs[key]; ok {
|
|
| 358 |
+ return v, true |
|
| 359 | 359 |
} |
| 360 | 360 |
if globalArgs == nil {
|
| 361 |
- return nil, nil |
|
| 361 |
+ return "", false |
|
| 362 | 362 |
} |
| 363 |
- |
|
| 364 |
- v, ok := globalArgs.Get("SOURCE_DATE_EPOCH")
|
|
| 363 |
+ v, ok := globalArgs.Get(key) |
|
| 365 | 364 |
if !ok || v == "" {
|
| 366 |
- return nil, nil |
|
| 365 |
+ return "", false |
|
| 367 | 366 |
} |
| 367 |
+ return v, true |
|
| 368 |
+} |
|
| 368 | 369 |
|
| 369 |
- sde, err := strconv.ParseInt(v, 10, 64) |
|
| 370 |
- if err != nil {
|
|
| 371 |
- return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v) |
|
| 370 |
+func setBuildArgValue(buildArgs map[string]string, globalArgs *llb.EnvList, key, value string) *llb.EnvList {
|
|
| 371 |
+ if _, ok := buildArgs[key]; ok {
|
|
| 372 |
+ if value == "" {
|
|
| 373 |
+ delete(buildArgs, key) |
|
| 374 |
+ } else {
|
|
| 375 |
+ buildArgs[key] = value |
|
| 376 |
+ } |
|
| 377 |
+ } |
|
| 378 |
+ if globalArgs != nil {
|
|
| 379 |
+ if _, ok := globalArgs.Get(key); ok {
|
|
| 380 |
+ if value == "" {
|
|
| 381 |
+ updated := globalArgs.Delete(key) |
|
| 382 |
+ globalArgs = &updated |
|
| 383 |
+ } else {
|
|
| 384 |
+ globalArgs = globalArgs.AddOrReplace(key, value) |
|
| 385 |
+ } |
|
| 386 |
+ } |
|
| 372 | 387 |
} |
| 373 |
- tm := time.Unix(sde, 0).UTC() |
|
| 374 |
- return &tm, nil |
|
| 388 |
+ return globalArgs |
|
| 375 | 389 |
} |
| 376 | 390 |
|
| 377 | 391 |
func (dctx *dispatchContext) buildDispatchStates(stages []instructions.Stage) error {
|
| ... | ... |
@@ -402,7 +422,7 @@ func (dctx *dispatchContext) buildDispatchStates(stages []instructions.Stage) er |
| 402 | 402 |
stageName: st.Name, |
| 403 | 403 |
prefixPlatform: dctx.opt.MultiPlatformRequested, |
| 404 | 404 |
outline: dctx.outline.clone(), |
| 405 |
- epoch: dctx.opt.Epoch, |
|
| 405 |
+ epoch: dctx.epoch, |
|
| 406 | 406 |
} |
| 407 | 407 |
|
| 408 | 408 |
if v := st.Platform; v != "" {
|
| ... | ... |
@@ -1598,7 +1618,7 @@ func dispatchHealthcheck(d *dispatchState, c *instructions.HealthCheckCommand, l |
| 1598 | 1598 |
StartInterval: c.Health.StartInterval, |
| 1599 | 1599 |
Retries: c.Health.Retries, |
| 1600 | 1600 |
} |
| 1601 |
- return commitToHistory(&d.image, fmt.Sprintf("HEALTHCHECK %q", d.image.Config.Healthcheck), false, nil, d.epoch)
|
|
| 1601 |
+ return commitToHistory(&d.image, fmt.Sprintf("HEALTHCHECK %+v", *d.image.Config.Healthcheck), false, nil, d.epoch)
|
|
| 1602 | 1602 |
} |
| 1603 | 1603 |
|
| 1604 | 1604 |
func dispatchUser(d *dispatchState, c *instructions.UserCommand, commit bool) error {
|
| ... | ... |
@@ -167,6 +167,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
|
| 167 | 167 |
if gitRef.Submodules != nil && !*gitRef.Submodules {
|
| 168 | 168 |
gitOptions = append(gitOptions, llb.GitSkipSubmodules()) |
| 169 | 169 |
} |
| 170 |
+ if gitRef.FetchByCommit {
|
|
| 171 |
+ gitOptions = append(gitOptions, llb.GitFetchByCommit()) |
|
| 172 |
+ } |
|
| 170 | 173 |
|
| 171 | 174 |
st := llb.Git(gitRef.Remote, "", gitOptions...) |
| 172 | 175 |
opts := append([]llb.CopyOption{&llb.CopyInfo{
|
| 173 | 176 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,381 @@ |
| 0 |
+package dockerfile2llb |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "archive/tar" |
|
| 4 |
+ "bytes" |
|
| 5 |
+ "context" |
|
| 6 |
+ "io" |
|
| 7 |
+ "maps" |
|
| 8 |
+ "net/url" |
|
| 9 |
+ "path" |
|
| 10 |
+ "strconv" |
|
| 11 |
+ "strings" |
|
| 12 |
+ "time" |
|
| 13 |
+ |
|
| 14 |
+ "github.com/moby/buildkit/client/llb" |
|
| 15 |
+ "github.com/moby/buildkit/client/llb/sourceresolver" |
|
| 16 |
+ "github.com/moby/buildkit/frontend/dockerfile/dfgitutil" |
|
| 17 |
+ "github.com/moby/buildkit/frontend/dockerfile/instructions" |
|
| 18 |
+ "github.com/moby/buildkit/frontend/dockerfile/parser" |
|
| 19 |
+ "github.com/moby/buildkit/frontend/dockerfile/shell" |
|
| 20 |
+ "github.com/moby/buildkit/frontend/dockerui" |
|
| 21 |
+ gwclient "github.com/moby/buildkit/frontend/gateway/client" |
|
| 22 |
+ "github.com/moby/buildkit/solver/pb" |
|
| 23 |
+ "github.com/moby/buildkit/util/gitutil/gitobject" |
|
| 24 |
+ archivecompression "github.com/moby/go-archive/compression" |
|
| 25 |
+ digest "github.com/opencontainers/go-digest" |
|
| 26 |
+ "github.com/pkg/errors" |
|
| 27 |
+) |
|
| 28 |
+ |
|
| 29 |
+type sourceDateEpochStateOpt struct {
|
|
| 30 |
+ LogName string |
|
| 31 |
+} |
|
| 32 |
+ |
|
| 33 |
+func resolveSourceDateEpochValue(ctx context.Context, v string, opt ConvertOpt, stages []instructions.Stage, globalArgs *llb.EnvList, shlex *shell.Lex) (*time.Time, error) {
|
|
| 34 |
+ if v == "" {
|
|
| 35 |
+ return nil, nil |
|
| 36 |
+ } |
|
| 37 |
+ if sde, err := strconv.ParseInt(v, 10, 64); err == nil {
|
|
| 38 |
+ tm := time.Unix(sde, 0).UTC() |
|
| 39 |
+ return &tm, nil |
|
| 40 |
+ } |
|
| 41 |
+ |
|
| 42 |
+ state, stateOpt, err := resolveSourceDateEpochState(ctx, v, opt, stages, globalArgs, shlex) |
|
| 43 |
+ if err != nil {
|
|
| 44 |
+ return nil, err |
|
| 45 |
+ } |
|
| 46 |
+ if state == nil || opt.Client == nil {
|
|
| 47 |
+ return nil, nil |
|
| 48 |
+ } |
|
| 49 |
+ return resolveSourceDateEpochFromState(ctx, *state, opt.Client, stateOpt) |
|
| 50 |
+} |
|
| 51 |
+ |
|
| 52 |
+func formatSourceDateEpochValue(tm *time.Time) string {
|
|
| 53 |
+ if tm == nil {
|
|
| 54 |
+ return "" |
|
| 55 |
+ } |
|
| 56 |
+ return strconv.FormatInt(tm.Unix(), 10) |
|
| 57 |
+} |
|
| 58 |
+ |
|
| 59 |
+func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertOpt, stages []instructions.Stage, globalArgs *llb.EnvList, shlex *shell.Lex) (*llb.State, sourceDateEpochStateOpt, error) {
|
|
| 60 |
+ if value == "context" {
|
|
| 61 |
+ if opt.Client == nil {
|
|
| 62 |
+ return nil, sourceDateEpochStateOpt{}, nil
|
|
| 63 |
+ } |
|
| 64 |
+ mainContextState, err := opt.Client.MainContext(ctx) |
|
| 65 |
+ if err != nil {
|
|
| 66 |
+ return nil, sourceDateEpochStateOpt{}, err
|
|
| 67 |
+ } |
|
| 68 |
+ return mainContextState, sourceDateEpochStateOpt{
|
|
| 69 |
+ LogName: "[internal] resolve main build context metadata", |
|
| 70 |
+ }, nil |
|
| 71 |
+ } |
|
| 72 |
+ |
|
| 73 |
+ if opt.Client != nil {
|
|
| 74 |
+ nc, err := opt.Client.NamedContext(value, dockerui.ContextOpt{})
|
|
| 75 |
+ if err != nil {
|
|
| 76 |
+ return nil, sourceDateEpochStateOpt{}, err
|
|
| 77 |
+ } |
|
| 78 |
+ if nc != nil {
|
|
| 79 |
+ st, _, err := nc.Load(ctx) |
|
| 80 |
+ if err != nil {
|
|
| 81 |
+ return nil, sourceDateEpochStateOpt{}, err
|
|
| 82 |
+ } |
|
| 83 |
+ return st, sourceDateEpochStateOpt{
|
|
| 84 |
+ LogName: "[internal] resolve SOURCE_DATE_EPOCH named context " + value, |
|
| 85 |
+ }, nil |
|
| 86 |
+ } |
|
| 87 |
+ } |
|
| 88 |
+ |
|
| 89 |
+ for i := range stages {
|
|
| 90 |
+ if !strings.EqualFold(stages[i].Name, value) {
|
|
| 91 |
+ continue |
|
| 92 |
+ } |
|
| 93 |
+ |
|
| 94 |
+ args := globalArgs |
|
| 95 |
+ if globalArgs != nil {
|
|
| 96 |
+ updated := globalArgs.Delete("SOURCE_DATE_EPOCH")
|
|
| 97 |
+ args = &updated |
|
| 98 |
+ } |
|
| 99 |
+ |
|
| 100 |
+ sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex) |
|
| 101 |
+ if err != nil {
|
|
| 102 |
+ return nil, sourceDateEpochStateOpt{}, parser.WithLocation(err, stages[i].Location)
|
|
| 103 |
+ } |
|
| 104 |
+ |
|
| 105 |
+ return sourceState, sourceDateEpochStateOpt{
|
|
| 106 |
+ LogName: "[internal] resolve SOURCE_DATE_EPOCH source stage " + stages[i].Name, |
|
| 107 |
+ }, nil |
|
| 108 |
+ } |
|
| 109 |
+ return nil, sourceDateEpochStateOpt{}, errors.Errorf("invalid SOURCE_DATE_EPOCH: %s", value)
|
|
| 110 |
+} |
|
| 111 |
+ |
|
| 112 |
+func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex) (*llb.State, error) {
|
|
| 113 |
+ stageBaseName, _, err := shlex.ProcessWord(stage.BaseName, globalArgs) |
|
| 114 |
+ if err != nil {
|
|
| 115 |
+ return nil, errors.Wrapf(err, "failed to process source stage base name %q", stage.BaseName) |
|
| 116 |
+ } |
|
| 117 |
+ if stageBaseName != emptyImageName {
|
|
| 118 |
+ return nil, errors.New("SOURCE_DATE_EPOCH stage must use FROM scratch")
|
|
| 119 |
+ } |
|
| 120 |
+ |
|
| 121 |
+ env := globalArgs |
|
| 122 |
+ var sourceState *llb.State |
|
| 123 |
+ |
|
| 124 |
+ for _, cmd := range stage.Commands {
|
|
| 125 |
+ switch c := cmd.(type) {
|
|
| 126 |
+ case *instructions.ArgCommand: |
|
| 127 |
+ env, err = applySourceDateEpochStageArgs(c.Args, env, buildArgs, shlex) |
|
| 128 |
+ if err != nil {
|
|
| 129 |
+ return nil, err |
|
| 130 |
+ } |
|
| 131 |
+ case *instructions.AddCommand: |
|
| 132 |
+ if sourceState != nil {
|
|
| 133 |
+ return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD")
|
|
| 134 |
+ } |
|
| 135 |
+ sourceState, err = sourceDateEpochAddSource(c, env, shlex) |
|
| 136 |
+ if err != nil {
|
|
| 137 |
+ return nil, err |
|
| 138 |
+ } |
|
| 139 |
+ default: |
|
| 140 |
+ return nil, errors.Errorf("SOURCE_DATE_EPOCH stage does not meet source-only requirements: unsupported %s instruction", cmd.Name())
|
|
| 141 |
+ } |
|
| 142 |
+ } |
|
| 143 |
+ |
|
| 144 |
+ if sourceState == nil {
|
|
| 145 |
+ return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD")
|
|
| 146 |
+ } |
|
| 147 |
+ |
|
| 148 |
+ return sourceState, nil |
|
| 149 |
+} |
|
| 150 |
+ |
|
| 151 |
+func applySourceDateEpochStageArgs(args []instructions.KeyValuePairOptional, env *llb.EnvList, buildArgs map[string]string, shlex *shell.Lex) (*llb.EnvList, error) {
|
|
| 152 |
+ for _, arg := range args {
|
|
| 153 |
+ if v, ok := buildArgs[arg.Key]; ok {
|
|
| 154 |
+ env = env.AddOrReplace(arg.Key, v) |
|
| 155 |
+ continue |
|
| 156 |
+ } |
|
| 157 |
+ if arg.Value == nil {
|
|
| 158 |
+ continue |
|
| 159 |
+ } |
|
| 160 |
+ v, _, err := shlex.ProcessWord(*arg.Value, env) |
|
| 161 |
+ if err != nil {
|
|
| 162 |
+ return nil, err |
|
| 163 |
+ } |
|
| 164 |
+ env = env.AddOrReplace(arg.Key, v) |
|
| 165 |
+ } |
|
| 166 |
+ return env, nil |
|
| 167 |
+} |
|
| 168 |
+ |
|
| 169 |
+func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex) (*llb.State, error) {
|
|
| 170 |
+ if len(cmd.SourceContents) != 0 || len(cmd.SourcePaths) != 1 {
|
|
| 171 |
+ return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD source")
|
|
| 172 |
+ } |
|
| 173 |
+ |
|
| 174 |
+ src, _, err := shlex.ProcessWord(cmd.SourcePaths[0], env) |
|
| 175 |
+ if err != nil {
|
|
| 176 |
+ return nil, err |
|
| 177 |
+ } |
|
| 178 |
+ |
|
| 179 |
+ if isHTTPSource(src) {
|
|
| 180 |
+ var checksum digest.Digest |
|
| 181 |
+ if cmd.Checksum != "" {
|
|
| 182 |
+ expandedChecksum, _, err := shlex.ProcessWord(cmd.Checksum, env) |
|
| 183 |
+ if err != nil {
|
|
| 184 |
+ return nil, err |
|
| 185 |
+ } |
|
| 186 |
+ checksum, err = digest.Parse(expandedChecksum) |
|
| 187 |
+ if err != nil {
|
|
| 188 |
+ return nil, err |
|
| 189 |
+ } |
|
| 190 |
+ } |
|
| 191 |
+ st := llb.HTTP(src, llb.Filename(sourceDateEpochHTTPFilename(src)), llb.Checksum(checksum)) |
|
| 192 |
+ return &st, nil |
|
| 193 |
+ } |
|
| 194 |
+ |
|
| 195 |
+ gitRef, isGit, gitRefErr := dfgitutil.ParseGitRef(src) |
|
| 196 |
+ if gitRefErr != nil && isGit {
|
|
| 197 |
+ return nil, gitRefErr |
|
| 198 |
+ } |
|
| 199 |
+ if gitRefErr == nil && !gitRef.IndistinguishableFromLocal {
|
|
| 200 |
+ gitOptions := []llb.GitOption{
|
|
| 201 |
+ llb.GitRef(gitRef.Ref), |
|
| 202 |
+ } |
|
| 203 |
+ if cmd.KeepGitDir != nil && *cmd.KeepGitDir {
|
|
| 204 |
+ gitOptions = append(gitOptions, llb.KeepGitDir()) |
|
| 205 |
+ } |
|
| 206 |
+ if gitRef.KeepGitDir != nil && *gitRef.KeepGitDir {
|
|
| 207 |
+ gitOptions = append(gitOptions, llb.KeepGitDir()) |
|
| 208 |
+ } |
|
| 209 |
+ if cmd.Checksum != "" {
|
|
| 210 |
+ expandedChecksum, _, err := shlex.ProcessWord(cmd.Checksum, env) |
|
| 211 |
+ if err != nil {
|
|
| 212 |
+ return nil, err |
|
| 213 |
+ } |
|
| 214 |
+ gitOptions = append(gitOptions, llb.GitChecksum(expandedChecksum)) |
|
| 215 |
+ } else if gitRef.Checksum != "" {
|
|
| 216 |
+ gitOptions = append(gitOptions, llb.GitChecksum(gitRef.Checksum)) |
|
| 217 |
+ } |
|
| 218 |
+ if gitRef.SubDir != "" {
|
|
| 219 |
+ gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir)) |
|
| 220 |
+ } |
|
| 221 |
+ if gitRef.Submodules != nil && !*gitRef.Submodules {
|
|
| 222 |
+ gitOptions = append(gitOptions, llb.GitSkipSubmodules()) |
|
| 223 |
+ } |
|
| 224 |
+ st := llb.Git(gitRef.Remote, "", gitOptions...) |
|
| 225 |
+ return &st, nil |
|
| 226 |
+ } |
|
| 227 |
+ |
|
| 228 |
+ return nil, errors.New("SOURCE_DATE_EPOCH stage source must be a single HTTP(S) or Git ADD")
|
|
| 229 |
+} |
|
| 230 |
+ |
|
| 231 |
+func sourceDateEpochHTTPFilename(src string) string {
|
|
| 232 |
+ u, err := url.Parse(src) |
|
| 233 |
+ if err == nil {
|
|
| 234 |
+ if base := path.Base(u.Path); base != "." && base != "/" {
|
|
| 235 |
+ return base |
|
| 236 |
+ } |
|
| 237 |
+ } |
|
| 238 |
+ return "__unnamed__" |
|
| 239 |
+} |
|
| 240 |
+ |
|
| 241 |
+func resolveSourceDateEpochFromState(ctx context.Context, st llb.State, client *dockerui.Client, opt sourceDateEpochStateOpt) (*time.Time, error) {
|
|
| 242 |
+ sourceOp, err := sourceOpFromState(ctx, &st, llb.WithCaps(client.BuildOpts().Caps)) |
|
| 243 |
+ if err != nil {
|
|
| 244 |
+ return nil, err |
|
| 245 |
+ } |
|
| 246 |
+ if sourceOp == nil {
|
|
| 247 |
+ return nil, nil |
|
| 248 |
+ } |
|
| 249 |
+ |
|
| 250 |
+ metaOpt := sourceresolver.Opt{
|
|
| 251 |
+ LogName: opt.LogName, |
|
| 252 |
+ } |
|
| 253 |
+ if strings.HasPrefix(sourceOp.Identifier, "git://") {
|
|
| 254 |
+ metaOpt.GitOpt = &sourceresolver.ResolveGitOpt{ReturnObject: true}
|
|
| 255 |
+ } |
|
| 256 |
+ isHTTP := strings.HasPrefix(sourceOp.Identifier, "http://") || strings.HasPrefix(sourceOp.Identifier, "https://") |
|
| 257 |
+ md, err := client.GatewayClient().ResolveSourceMetadata(ctx, sourceOp, metaOpt) |
|
| 258 |
+ if err != nil {
|
|
| 259 |
+ return nil, err |
|
| 260 |
+ } |
|
| 261 |
+ if tm, ok, err := sourceDateEpochFromMetadata(md); ok || err != nil {
|
|
| 262 |
+ return tm, err |
|
| 263 |
+ } |
|
| 264 |
+ filename := sourceOp.Attrs[pb.AttrHTTPFilename] |
|
| 265 |
+ if !isHTTP || filename == "" {
|
|
| 266 |
+ return nil, nil |
|
| 267 |
+ } |
|
| 268 |
+ |
|
| 269 |
+ sourceState := llb.NewState(llb.NewSource(sourceOp.Identifier, maps.Clone(sourceOp.Attrs), llb.Constraints{}).Output())
|
|
| 270 |
+ def, err := sourceState.Marshal(ctx, llb.WithCaps(client.BuildOpts().Caps)) |
|
| 271 |
+ if err != nil {
|
|
| 272 |
+ return nil, err |
|
| 273 |
+ } |
|
| 274 |
+ res, err := client.GatewayClient().Solve(ctx, gwclient.SolveRequest{
|
|
| 275 |
+ Definition: def.ToPB(), |
|
| 276 |
+ }) |
|
| 277 |
+ if err != nil {
|
|
| 278 |
+ return nil, err |
|
| 279 |
+ } |
|
| 280 |
+ ref, err := res.SingleRef() |
|
| 281 |
+ if err != nil {
|
|
| 282 |
+ return nil, err |
|
| 283 |
+ } |
|
| 284 |
+ |
|
| 285 |
+ return archiveMaxTimeFromRef(ctx, ref, filename, true) |
|
| 286 |
+} |
|
| 287 |
+ |
|
| 288 |
+func sourceOpFromState(ctx context.Context, st *llb.State, opts ...llb.ConstraintsOpt) (*pb.SourceOp, error) {
|
|
| 289 |
+ if st == nil {
|
|
| 290 |
+ return nil, nil |
|
| 291 |
+ } |
|
| 292 |
+ def, err := st.Marshal(ctx, opts...) |
|
| 293 |
+ if err != nil {
|
|
| 294 |
+ return nil, err |
|
| 295 |
+ } |
|
| 296 |
+ dt := def.ToPB().Def |
|
| 297 |
+ var src *pb.SourceOp |
|
| 298 |
+ for _, d := range dt {
|
|
| 299 |
+ var op pb.Op |
|
| 300 |
+ if err := op.Unmarshal(d); err != nil {
|
|
| 301 |
+ return nil, err |
|
| 302 |
+ } |
|
| 303 |
+ opSrc := op.GetSource() |
|
| 304 |
+ if opSrc == nil {
|
|
| 305 |
+ continue |
|
| 306 |
+ } |
|
| 307 |
+ if src != nil {
|
|
| 308 |
+ return nil, nil |
|
| 309 |
+ } |
|
| 310 |
+ src = opSrc |
|
| 311 |
+ } |
|
| 312 |
+ return cloneSourceOp(src), nil |
|
| 313 |
+} |
|
| 314 |
+ |
|
| 315 |
+func cloneSourceOp(op *pb.SourceOp) *pb.SourceOp {
|
|
| 316 |
+ if op == nil {
|
|
| 317 |
+ return nil |
|
| 318 |
+ } |
|
| 319 |
+ return &pb.SourceOp{
|
|
| 320 |
+ Identifier: op.Identifier, |
|
| 321 |
+ Attrs: maps.Clone(op.Attrs), |
|
| 322 |
+ } |
|
| 323 |
+} |
|
| 324 |
+ |
|
| 325 |
+func sourceDateEpochFromMetadata(md *sourceresolver.MetaResponse) (*time.Time, bool, error) {
|
|
| 326 |
+ if md.Git != nil && len(md.Git.CommitObject) > 0 {
|
|
| 327 |
+ obj, err := gitobject.Parse(md.Git.CommitObject) |
|
| 328 |
+ if err != nil {
|
|
| 329 |
+ return nil, false, err |
|
| 330 |
+ } |
|
| 331 |
+ commit, err := obj.ToCommit() |
|
| 332 |
+ if err != nil {
|
|
| 333 |
+ return nil, false, err |
|
| 334 |
+ } |
|
| 335 |
+ return commit.Committer.When, true, nil |
|
| 336 |
+ } |
|
| 337 |
+ if md.HTTP != nil && md.HTTP.LastModified != nil {
|
|
| 338 |
+ return md.HTTP.LastModified, true, nil |
|
| 339 |
+ } |
|
| 340 |
+ return nil, false, nil |
|
| 341 |
+} |
|
| 342 |
+ |
|
| 343 |
+func archiveMaxTimeFromRef(ctx context.Context, ref gwclient.Reference, filename string, allowNonArchive bool) (*time.Time, error) {
|
|
| 344 |
+ dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
|
|
| 345 |
+ Filename: filename, |
|
| 346 |
+ }) |
|
| 347 |
+ if err != nil {
|
|
| 348 |
+ return nil, err |
|
| 349 |
+ } |
|
| 350 |
+ rc, err := archivecompression.DecompressStream(bytes.NewReader(dt)) |
|
| 351 |
+ if err != nil {
|
|
| 352 |
+ if allowNonArchive {
|
|
| 353 |
+ return nil, nil |
|
| 354 |
+ } |
|
| 355 |
+ return nil, err |
|
| 356 |
+ } |
|
| 357 |
+ defer rc.Close() |
|
| 358 |
+ |
|
| 359 |
+ tr := tar.NewReader(rc) |
|
| 360 |
+ var maxTime *time.Time |
|
| 361 |
+ for {
|
|
| 362 |
+ hdr, err := tr.Next() |
|
| 363 |
+ if err != nil {
|
|
| 364 |
+ if errors.Is(err, io.EOF) {
|
|
| 365 |
+ return maxTime, nil |
|
| 366 |
+ } |
|
| 367 |
+ if allowNonArchive {
|
|
| 368 |
+ return nil, nil |
|
| 369 |
+ } |
|
| 370 |
+ return nil, err |
|
| 371 |
+ } |
|
| 372 |
+ if !hdr.FileInfo().Mode().IsRegular() {
|
|
| 373 |
+ continue |
|
| 374 |
+ } |
|
| 375 |
+ tm := hdr.ModTime.UTC() |
|
| 376 |
+ if maxTime == nil || tm.After(*maxTime) {
|
|
| 377 |
+ maxTime = &tm |
|
| 378 |
+ } |
|
| 379 |
+ } |
|
| 380 |
+} |
| ... | ... |
@@ -1,6 +1,7 @@ |
| 1 | 1 |
package dockerfile2llb |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "maps" |
|
| 4 | 5 |
"slices" |
| 5 | 6 |
|
| 6 | 7 |
"github.com/moby/buildkit/util/system" |
| ... | ... |
@@ -15,6 +16,16 @@ func clone(src dockerspec.DockerOCIImage) dockerspec.DockerOCIImage {
|
| 15 | 15 |
img.Config.Cmd = slices.Clone(src.Config.Cmd) |
| 16 | 16 |
img.Config.Entrypoint = slices.Clone(src.Config.Entrypoint) |
| 17 | 17 |
img.Config.OnBuild = slices.Clone(src.Config.OnBuild) |
| 18 |
+ img.Config.ExposedPorts = maps.Clone(src.Config.ExposedPorts) |
|
| 19 |
+ img.Config.Volumes = maps.Clone(src.Config.Volumes) |
|
| 20 |
+ img.Config.Labels = maps.Clone(src.Config.Labels) |
|
| 21 |
+ img.Config.Shell = slices.Clone(src.Config.Shell) |
|
| 22 |
+ if src.Config.Healthcheck != nil {
|
|
| 23 |
+ hc := *src.Config.Healthcheck |
|
| 24 |
+ hc.Test = slices.Clone(src.Config.Healthcheck.Test) |
|
| 25 |
+ img.Config.Healthcheck = &hc |
|
| 26 |
+ } |
|
| 27 |
+ img.History = slices.Clone(src.History) |
|
| 18 | 28 |
return img |
| 19 | 29 |
} |
| 20 | 30 |
|
| ... | ... |
@@ -4,7 +4,6 @@ import ( |
| 4 | 4 |
"net" |
| 5 | 5 |
"strconv" |
| 6 | 6 |
"strings" |
| 7 |
- "time" |
|
| 8 | 7 |
|
| 9 | 8 |
"github.com/containerd/platforms" |
| 10 | 9 |
"github.com/docker/go-units" |
| ... | ... |
@@ -113,18 +112,6 @@ func parseNetMode(v string) (pb.NetMode, error) {
|
| 113 | 113 |
} |
| 114 | 114 |
} |
| 115 | 115 |
|
| 116 |
-func parseSourceDateEpoch(v string) (*time.Time, error) {
|
|
| 117 |
- if v == "" {
|
|
| 118 |
- return nil, nil |
|
| 119 |
- } |
|
| 120 |
- sde, err := strconv.ParseInt(v, 10, 64) |
|
| 121 |
- if err != nil {
|
|
| 122 |
- return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v) |
|
| 123 |
- } |
|
| 124 |
- tm := time.Unix(sde, 0).UTC() |
|
| 125 |
- return &tm, nil |
|
| 126 |
-} |
|
| 127 |
- |
|
| 128 | 116 |
func parseLocalSessionIDs(opt map[string]string) map[string]string {
|
| 129 | 117 |
m := map[string]string{}
|
| 130 | 118 |
for k, v := range opt {
|
| ... | ... |
@@ -8,7 +8,6 @@ import ( |
| 8 | 8 |
"strconv" |
| 9 | 9 |
"strings" |
| 10 | 10 |
"sync" |
| 11 |
- "time" |
|
| 12 | 11 |
|
| 13 | 12 |
"github.com/containerd/platforms" |
| 14 | 13 |
"github.com/distribution/reference" |
| ... | ... |
@@ -51,14 +50,12 @@ const ( |
| 51 | 51 |
keyHostnameArg = "build-arg:BUILDKIT_SANDBOX_HOSTNAME" |
| 52 | 52 |
keyDockerfileLintArg = "build-arg:BUILDKIT_DOCKERFILE_CHECK" |
| 53 | 53 |
keyContextKeepGitDirArg = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR" |
| 54 |
- keySourceDateEpoch = "build-arg:SOURCE_DATE_EPOCH" |
|
| 55 | 54 |
) |
| 56 | 55 |
|
| 57 | 56 |
type Config struct {
|
| 58 | 57 |
BuildArgs map[string]string |
| 59 | 58 |
CacheIDNamespace string |
| 60 | 59 |
CgroupParent string |
| 61 |
- Epoch *time.Time |
|
| 62 | 60 |
ExtraHosts []llb.HostIP |
| 63 | 61 |
Hostname string |
| 64 | 62 |
ImageResolveMode llb.ResolveMode |
| ... | ... |
@@ -147,6 +144,10 @@ func (bc *Client) BuildOpts() client.BuildOpts {
|
| 147 | 147 |
return bc.bopts |
| 148 | 148 |
} |
| 149 | 149 |
|
| 150 |
+func (bc *Client) GatewayClient() client.Client {
|
|
| 151 |
+ return bc.client |
|
| 152 |
+} |
|
| 153 |
+ |
|
| 150 | 154 |
func (bc *Client) init() error {
|
| 151 | 155 |
opts := bc.bopts.Opts |
| 152 | 156 |
|
| ... | ... |
@@ -251,12 +252,6 @@ func (bc *Client) init() error {
|
| 251 | 251 |
} |
| 252 | 252 |
bc.CacheImports = cacheImports |
| 253 | 253 |
|
| 254 |
- epoch, err := parseSourceDateEpoch(opts[keySourceDateEpoch]) |
|
| 255 |
- if err != nil {
|
|
| 256 |
- return err |
|
| 257 |
- } |
|
| 258 |
- bc.Epoch = epoch |
|
| 259 |
- |
|
| 260 | 254 |
attests, err := attestations.Parse(opts) |
| 261 | 255 |
if err != nil {
|
| 262 | 256 |
return err |
| ... | ... |
@@ -4,15 +4,23 @@ import ( |
| 4 | 4 |
"archive/tar" |
| 5 | 5 |
"bytes" |
| 6 | 6 |
"context" |
| 7 |
+ "io" |
|
| 8 |
+ "maps" |
|
| 7 | 9 |
"path/filepath" |
| 8 | 10 |
"regexp" |
| 9 | 11 |
"slices" |
| 10 | 12 |
"strconv" |
| 13 |
+ "strings" |
|
| 14 |
+ "time" |
|
| 11 | 15 |
|
| 12 | 16 |
"github.com/moby/buildkit/client/llb" |
| 17 |
+ "github.com/moby/buildkit/client/llb/sourceresolver" |
|
| 13 | 18 |
"github.com/moby/buildkit/frontend/dockerfile/dfgitutil" |
| 14 | 19 |
"github.com/moby/buildkit/frontend/gateway/client" |
| 15 | 20 |
gwpb "github.com/moby/buildkit/frontend/gateway/pb" |
| 21 |
+ "github.com/moby/buildkit/solver/pb" |
|
| 22 |
+ "github.com/moby/buildkit/util/gitutil/gitobject" |
|
| 23 |
+ archivecompression "github.com/moby/go-archive/compression" |
|
| 16 | 24 |
"github.com/pkg/errors" |
| 17 | 25 |
) |
| 18 | 26 |
|
| ... | ... |
@@ -36,10 +44,14 @@ var httpPrefix = regexp.MustCompile(`^https?://`) |
| 36 | 36 |
type buildContext struct {
|
| 37 | 37 |
context *llb.State // set if not local |
| 38 | 38 |
dockerfile *llb.State // override remoteContext if set |
| 39 |
+ contextRef client.Reference |
|
| 39 | 40 |
contextLocalName string |
| 40 | 41 |
dockerfileLocalName string |
| 41 | 42 |
filename string |
| 42 | 43 |
forceLocalDockerfile bool |
| 44 |
+ sourceOp *pb.SourceOp |
|
| 45 |
+ httpContextIsArchive bool |
|
| 46 |
+ httpContextFilename string |
|
| 43 | 47 |
} |
| 44 | 48 |
|
| 45 | 49 |
func (bc *Client) marshalOpts() []llb.ConstraintsOpt {
|
| ... | ... |
@@ -75,16 +87,25 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
|
| 75 | 75 |
keepGit = &v |
| 76 | 76 |
} |
| 77 | 77 |
var extraGitOpts []llb.GitOption |
| 78 |
- if opts[keySourceDateEpoch] != "" {
|
|
| 78 |
+ if opts[buildArgPrefix+"SOURCE_DATE_EPOCH"] != "" {
|
|
| 79 | 79 |
extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit()) |
| 80 | 80 |
} |
| 81 | 81 |
if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok {
|
| 82 | 82 |
if err != nil {
|
| 83 | 83 |
return nil, err |
| 84 | 84 |
} |
| 85 |
+ sourceOp, err := sourceOpFromState(ctx, st, bc.marshalOpts()...) |
|
| 86 |
+ if err != nil {
|
|
| 87 |
+ return nil, errors.Wrapf(err, "failed to derive git source op") |
|
| 88 |
+ } |
|
| 85 | 89 |
bctx.context = st |
| 86 | 90 |
bctx.dockerfile = st |
| 91 |
+ bctx.sourceOp = sourceOp |
|
| 87 | 92 |
} else if st, filename, ok := DetectHTTPContext(opts[localNameContext]); ok {
|
| 93 |
+ sourceOp, err := sourceOpFromState(ctx, st, bc.marshalOpts()...) |
|
| 94 |
+ if err != nil {
|
|
| 95 |
+ return nil, errors.Wrapf(err, "failed to derive http source op") |
|
| 96 |
+ } |
|
| 88 | 97 |
def, err := st.Marshal(ctx, bc.marshalOpts()...) |
| 89 | 98 |
if err != nil {
|
| 90 | 99 |
return nil, errors.Wrapf(err, "failed to marshal httpcontext") |
| ... | ... |
@@ -115,11 +136,15 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
|
| 115 | 115 |
AttemptUnpack: true, |
| 116 | 116 |
})) |
| 117 | 117 |
bctx.context = &bc |
| 118 |
+ bctx.httpContextIsArchive = true |
|
| 118 | 119 |
} else {
|
| 119 | 120 |
bctx.filename = filename |
| 120 | 121 |
bctx.context = st |
| 121 | 122 |
} |
| 123 |
+ bctx.contextRef = ref |
|
| 122 | 124 |
bctx.dockerfile = bctx.context |
| 125 |
+ bctx.sourceOp = sourceOp |
|
| 126 |
+ bctx.httpContextFilename = filename |
|
| 123 | 127 |
} else if (&gwcaps).Supports(gwpb.CapFrontendInputs) == nil {
|
| 124 | 128 |
inputs, err := bc.client.Inputs(ctx) |
| 125 | 129 |
if err != nil {
|
| ... | ... |
@@ -148,6 +173,123 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
|
| 148 | 148 |
return bctx, nil |
| 149 | 149 |
} |
| 150 | 150 |
|
| 151 |
+func (bc *Client) ResolveMainContextSourceDateEpoch(ctx context.Context) (*time.Time, error) {
|
|
| 152 |
+ bctx, err := bc.buildContext(ctx) |
|
| 153 |
+ if err != nil {
|
|
| 154 |
+ return nil, err |
|
| 155 |
+ } |
|
| 156 |
+ if bctx.sourceOp == nil {
|
|
| 157 |
+ return nil, nil |
|
| 158 |
+ } |
|
| 159 |
+ |
|
| 160 |
+ opt := sourceresolver.Opt{
|
|
| 161 |
+ LogName: "[internal] resolve main build context metadata", |
|
| 162 |
+ } |
|
| 163 |
+ if strings.HasPrefix(bctx.sourceOp.Identifier, "git://") {
|
|
| 164 |
+ opt.GitOpt = &sourceresolver.ResolveGitOpt{ReturnObject: true}
|
|
| 165 |
+ } |
|
| 166 |
+ md, err := bc.client.ResolveSourceMetadata(ctx, cloneSourceOp(bctx.sourceOp), opt) |
|
| 167 |
+ if err != nil {
|
|
| 168 |
+ return nil, err |
|
| 169 |
+ } |
|
| 170 |
+ if md.Git != nil && len(md.Git.CommitObject) > 0 {
|
|
| 171 |
+ obj, err := gitobject.Parse(md.Git.CommitObject) |
|
| 172 |
+ if err != nil {
|
|
| 173 |
+ return nil, err |
|
| 174 |
+ } |
|
| 175 |
+ commit, err := obj.ToCommit() |
|
| 176 |
+ if err != nil {
|
|
| 177 |
+ return nil, err |
|
| 178 |
+ } |
|
| 179 |
+ return commit.Committer.When, nil |
|
| 180 |
+ } |
|
| 181 |
+ if md.HTTP != nil {
|
|
| 182 |
+ if md.HTTP.LastModified != nil {
|
|
| 183 |
+ return md.HTTP.LastModified, nil |
|
| 184 |
+ } |
|
| 185 |
+ if bctx.httpContextIsArchive {
|
|
| 186 |
+ return archiveMaxTimeFromHTTPArchive(ctx, bctx) |
|
| 187 |
+ } |
|
| 188 |
+ } |
|
| 189 |
+ return nil, nil |
|
| 190 |
+} |
|
| 191 |
+ |
|
| 192 |
+func archiveMaxTimeFromHTTPArchive(ctx context.Context, bctx *buildContext) (*time.Time, error) {
|
|
| 193 |
+ if bctx.contextRef == nil || bctx.httpContextFilename == "" {
|
|
| 194 |
+ return nil, nil |
|
| 195 |
+ } |
|
| 196 |
+ dt, err := bctx.contextRef.ReadFile(ctx, client.ReadRequest{
|
|
| 197 |
+ Filename: bctx.httpContextFilename, |
|
| 198 |
+ }) |
|
| 199 |
+ if err != nil {
|
|
| 200 |
+ return nil, err |
|
| 201 |
+ } |
|
| 202 |
+ rc, err := archivecompression.DecompressStream(bytes.NewReader(dt)) |
|
| 203 |
+ if err != nil {
|
|
| 204 |
+ return nil, err |
|
| 205 |
+ } |
|
| 206 |
+ defer rc.Close() |
|
| 207 |
+ |
|
| 208 |
+ tr := tar.NewReader(rc) |
|
| 209 |
+ var maxTime *time.Time |
|
| 210 |
+ for {
|
|
| 211 |
+ hdr, err := tr.Next() |
|
| 212 |
+ if err != nil {
|
|
| 213 |
+ if errors.Is(err, io.EOF) {
|
|
| 214 |
+ return maxTime, nil |
|
| 215 |
+ } |
|
| 216 |
+ return nil, err |
|
| 217 |
+ } |
|
| 218 |
+ if !hdr.FileInfo().Mode().IsRegular() {
|
|
| 219 |
+ continue |
|
| 220 |
+ } |
|
| 221 |
+ tm := hdr.ModTime.UTC() |
|
| 222 |
+ if maxTime == nil || tm.After(*maxTime) {
|
|
| 223 |
+ maxTime = &tm |
|
| 224 |
+ } |
|
| 225 |
+ } |
|
| 226 |
+} |
|
| 227 |
+ |
|
| 228 |
+func cloneSourceOp(op *pb.SourceOp) *pb.SourceOp {
|
|
| 229 |
+ if op == nil {
|
|
| 230 |
+ return nil |
|
| 231 |
+ } |
|
| 232 |
+ return &pb.SourceOp{
|
|
| 233 |
+ Identifier: op.Identifier, |
|
| 234 |
+ Attrs: maps.Clone(op.Attrs), |
|
| 235 |
+ } |
|
| 236 |
+} |
|
| 237 |
+ |
|
| 238 |
+func sourceOpFromState(ctx context.Context, st *llb.State, opts ...llb.ConstraintsOpt) (*pb.SourceOp, error) {
|
|
| 239 |
+ if st == nil {
|
|
| 240 |
+ return nil, nil |
|
| 241 |
+ } |
|
| 242 |
+ def, err := st.Marshal(ctx, opts...) |
|
| 243 |
+ if err != nil {
|
|
| 244 |
+ return nil, err |
|
| 245 |
+ } |
|
| 246 |
+ dt := def.ToPB().Def |
|
| 247 |
+ var src *pb.SourceOp |
|
| 248 |
+ for _, d := range dt {
|
|
| 249 |
+ var op pb.Op |
|
| 250 |
+ if err := op.Unmarshal(d); err != nil {
|
|
| 251 |
+ return nil, err |
|
| 252 |
+ } |
|
| 253 |
+ opSrc := op.GetSource() |
|
| 254 |
+ if opSrc == nil {
|
|
| 255 |
+ continue |
|
| 256 |
+ } |
|
| 257 |
+ if src != nil {
|
|
| 258 |
+ return nil, errors.New("state marshaled to multiple source ops")
|
|
| 259 |
+ } |
|
| 260 |
+ src = opSrc |
|
| 261 |
+ } |
|
| 262 |
+ if src == nil {
|
|
| 263 |
+ return nil, errors.New("state did not marshal to a source op")
|
|
| 264 |
+ } |
|
| 265 |
+ return cloneSourceOp(src), nil |
|
| 266 |
+} |
|
| 267 |
+ |
|
| 151 | 268 |
func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.State, bool, error) {
|
| 152 | 269 |
g, isGit, err := dfgitutil.ParseGitRef(ref) |
| 153 | 270 |
if err != nil {
|
| ... | ... |
@@ -175,6 +317,9 @@ func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.St |
| 175 | 175 |
if g.MTime != "" {
|
| 176 | 176 |
gitOpts = append(gitOpts, llb.GitMTime(g.MTime)) |
| 177 | 177 |
} |
| 178 |
+ if g.FetchByCommit {
|
|
| 179 |
+ gitOpts = append(gitOpts, llb.GitFetchByCommit()) |
|
| 180 |
+ } |
|
| 178 | 181 |
|
| 179 | 182 |
st := llb.Git(g.Remote, "", gitOpts...) |
| 180 | 183 |
return &st, true, nil |
| ... | ... |
@@ -151,14 +151,14 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke |
| 151 | 151 |
return st, nil, nil |
| 152 | 152 |
case "http", "https": |
| 153 | 153 |
st, ok, err := DetectGitContext(nc.input, nil) |
| 154 |
- if !ok {
|
|
| 155 |
- httpst := llb.HTTP(nc.input, llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input))
|
|
| 156 |
- st = &httpst |
|
| 157 |
- } |
|
| 158 |
- if err != nil {
|
|
| 159 |
- return nil, nil, err |
|
| 154 |
+ if ok {
|
|
| 155 |
+ if err != nil {
|
|
| 156 |
+ return nil, nil, err |
|
| 157 |
+ } |
|
| 158 |
+ return st, nil, nil |
|
| 160 | 159 |
} |
| 161 |
- return st, nil, nil |
|
| 160 |
+ httpst := llb.HTTP(nc.input, llb.Filename("context"), llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input))
|
|
| 161 |
+ return &httpst, nil, nil |
|
| 162 | 162 |
case "oci-layout": |
| 163 | 163 |
refSpec := strings.TrimPrefix(vv[1], "//") |
| 164 | 164 |
ref, err := reference.Parse(refSpec) |
| ... | ... |
@@ -3,6 +3,8 @@ package client |
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 | 5 |
"io" |
| 6 |
+ "maps" |
|
| 7 |
+ "slices" |
|
| 6 | 8 |
"syscall" |
| 7 | 9 |
|
| 8 | 10 |
"github.com/moby/buildkit/client/llb" |
| ... | ... |
@@ -154,6 +156,40 @@ type SolveRequest struct {
|
| 154 | 154 |
SourcePolicies []*spb.Policy |
| 155 | 155 |
} |
| 156 | 156 |
|
| 157 |
+// Clone returns a deep copy of the solve request. |
|
| 158 |
+func (r SolveRequest) Clone() SolveRequest {
|
|
| 159 |
+ if r.Definition != nil {
|
|
| 160 |
+ r.Definition = r.Definition.CloneVT() |
|
| 161 |
+ } |
|
| 162 |
+ r.FrontendOpt = maps.Clone(r.FrontendOpt) |
|
| 163 |
+ if len(r.FrontendInputs) > 0 {
|
|
| 164 |
+ inputs := r.FrontendInputs |
|
| 165 |
+ r.FrontendInputs = make(map[string]*pb.Definition, len(inputs)) |
|
| 166 |
+ for k, v := range inputs {
|
|
| 167 |
+ if v != nil {
|
|
| 168 |
+ v = v.CloneVT() |
|
| 169 |
+ } |
|
| 170 |
+ r.FrontendInputs[k] = v |
|
| 171 |
+ } |
|
| 172 |
+ } |
|
| 173 |
+ if len(r.CacheImports) > 0 {
|
|
| 174 |
+ r.CacheImports = slices.Clone(r.CacheImports) |
|
| 175 |
+ for i, ci := range r.CacheImports {
|
|
| 176 |
+ ci.Attrs = maps.Clone(ci.Attrs) |
|
| 177 |
+ r.CacheImports[i] = ci |
|
| 178 |
+ } |
|
| 179 |
+ } |
|
| 180 |
+ if len(r.SourcePolicies) > 0 {
|
|
| 181 |
+ r.SourcePolicies = slices.Clone(r.SourcePolicies) |
|
| 182 |
+ for i, p := range r.SourcePolicies {
|
|
| 183 |
+ if p != nil {
|
|
| 184 |
+ r.SourcePolicies[i] = p.CloneVT() |
|
| 185 |
+ } |
|
| 186 |
+ } |
|
| 187 |
+ } |
|
| 188 |
+ return r |
|
| 189 |
+} |
|
| 190 |
+ |
|
| 157 | 191 |
type CacheOptionsEntry struct {
|
| 158 | 192 |
Type string |
| 159 | 193 |
Attrs map[string]string |
| ... | ... |
@@ -5,6 +5,7 @@ import ( |
| 5 | 5 |
"crypto/rand" |
| 6 | 6 |
"crypto/subtle" |
| 7 | 7 |
"sync" |
| 8 |
+ "time" |
|
| 8 | 9 |
|
| 9 | 10 |
"github.com/moby/buildkit/session" |
| 10 | 11 |
"github.com/moby/buildkit/util/grpcerrors" |
| ... | ... |
@@ -13,6 +14,8 @@ import ( |
| 13 | 13 |
"google.golang.org/grpc/codes" |
| 14 | 14 |
) |
| 15 | 15 |
|
| 16 |
+const sessionAuthTimeout = 60 * time.Second |
|
| 17 |
+ |
|
| 16 | 18 |
var salt []byte |
| 17 | 19 |
var saltOnce sync.Once |
| 18 | 20 |
|
| ... | ... |
@@ -25,10 +28,12 @@ func getSalt() []byte {
|
| 25 | 25 |
return salt |
| 26 | 26 |
} |
| 27 | 27 |
|
| 28 |
-func CredentialsFunc(sm *session.Manager, g session.Group) func(string) (session, username, secret string, err error) {
|
|
| 28 |
+func CredentialsFunc(ctx context.Context, sm *session.Manager, g session.Group) func(string) (session, username, secret string, err error) {
|
|
| 29 | 29 |
return func(host string) (string, string, string, error) {
|
| 30 |
+ ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "resolving credentials from session")) |
|
| 31 |
+ defer cancel() |
|
| 30 | 32 |
var sessionID, user, secret string |
| 31 |
- err := sm.Any(context.TODO(), g, func(ctx context.Context, id string, c session.Caller) error {
|
|
| 33 |
+ err := sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
|
|
| 32 | 34 |
client := NewAuthClient(c.Conn()) |
| 33 | 35 |
|
| 34 | 36 |
resp, err := client.Credentials(ctx, &CredentialsRequest{
|
| ... | ... |
@@ -53,6 +58,8 @@ func CredentialsFunc(sm *session.Manager, g session.Group) func(string) (session |
| 53 | 53 |
} |
| 54 | 54 |
|
| 55 | 55 |
func FetchToken(ctx context.Context, req *FetchTokenRequest, sm *session.Manager, g session.Group) (resp *FetchTokenResponse, err error) {
|
| 56 |
+ ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "fetching auth token from session")) |
|
| 57 |
+ defer cancel() |
|
| 56 | 58 |
err = sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
|
| 57 | 59 |
client := NewAuthClient(c.Conn()) |
| 58 | 60 |
|
| ... | ... |
@@ -69,6 +76,8 @@ func FetchToken(ctx context.Context, req *FetchTokenRequest, sm *session.Manager |
| 69 | 69 |
} |
| 70 | 70 |
|
| 71 | 71 |
func VerifyTokenAuthority(ctx context.Context, host string, pubKey *[32]byte, sm *session.Manager, g session.Group) (sessionID string, ok bool, err error) {
|
| 72 |
+ ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "verifying token authority from session")) |
|
| 73 |
+ defer cancel() |
|
| 72 | 74 |
var verified bool |
| 73 | 75 |
err = sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
|
| 74 | 76 |
client := NewAuthClient(c.Conn()) |
| ... | ... |
@@ -101,6 +110,8 @@ func VerifyTokenAuthority(ctx context.Context, host string, pubKey *[32]byte, sm |
| 101 | 101 |
} |
| 102 | 102 |
|
| 103 | 103 |
func GetTokenAuthority(ctx context.Context, host string, sm *session.Manager, g session.Group) (sessionID string, pubKey *[32]byte, err error) {
|
| 104 |
+ ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "getting token authority from session")) |
|
| 105 |
+ defer cancel() |
|
| 104 | 106 |
err = sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
|
| 105 | 107 |
client := NewAuthClient(c.Conn()) |
| 106 | 108 |
|
| ... | ... |
@@ -16,9 +16,11 @@ import ( |
| 16 | 16 |
type callerContentStore struct {
|
| 17 | 17 |
store content.Store |
| 18 | 18 |
storeID string |
| 19 |
+ caller session.Caller |
|
| 19 | 20 |
} |
| 20 | 21 |
|
| 21 | 22 |
func (cs *callerContentStore) choose(ctx context.Context) context.Context {
|
| 23 |
+ ctx = cs.caller.Context(ctx) |
|
| 22 | 24 |
nsheader := metadata.Pairs(GRPCHeaderID, cs.storeID) |
| 23 | 25 |
md, ok := metadata.FromOutgoingContext(ctx) // merge with outgoing context. |
| 24 | 26 |
if !ok {
|
| ... | ... |
@@ -87,5 +89,6 @@ func NewCallerStore(c session.Caller, storeID string) content.Store {
|
| 87 | 87 |
return &callerContentStore{
|
| 88 | 88 |
store: proxy.NewContentStore(client), |
| 89 | 89 |
storeID: storeID, |
| 90 |
+ caller: c, |
|
| 90 | 91 |
} |
| 91 | 92 |
} |
| 92 | 93 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,17 @@ |
| 0 |
+package session |
|
| 1 |
+ |
|
| 2 |
+import "context" |
|
| 3 |
+ |
|
| 4 |
+// contextWithCaller returns a context that is canceled when either the request |
|
| 5 |
+// context is done or the session context is closed. |
|
| 6 |
+func contextWithCaller(ctx context.Context, callerCtx context.Context) context.Context {
|
|
| 7 |
+ ctx, cancel := context.WithCancelCause(ctx) |
|
| 8 |
+ context.AfterFunc(callerCtx, func() {
|
|
| 9 |
+ cause := context.Cause(callerCtx) |
|
| 10 |
+ if cause == nil {
|
|
| 11 |
+ cause = context.Canceled |
|
| 12 |
+ } |
|
| 13 |
+ cancel(cause) |
|
| 14 |
+ }) |
|
| 15 |
+ return ctx |
|
| 16 |
+} |
| ... | ... |
@@ -207,6 +207,7 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
|
| 207 | 207 |
|
| 208 | 208 |
opts[keyDirName] = []string{opt.Name}
|
| 209 | 209 |
|
| 210 |
+ ctx = c.Context(ctx) |
|
| 210 | 211 |
ctx, cancel := context.WithCancelCause(ctx) |
| 211 | 212 |
defer func() { cancel(errors.WithStack(context.Canceled)) }()
|
| 212 | 213 |
|
| ... | ... |
@@ -362,6 +363,8 @@ func CopyToCaller(ctx context.Context, fs fsutil.FS, id int, c session.Caller, p |
| 362 | 362 |
return errors.Errorf("method %s not supported by the client", method)
|
| 363 | 363 |
} |
| 364 | 364 |
|
| 365 |
+ ctx = c.Context(ctx) |
|
| 366 |
+ |
|
| 365 | 367 |
client := NewFileSendClient(c.Conn()) |
| 366 | 368 |
|
| 367 | 369 |
opts, ok := metadata.FromOutgoingContext(ctx) |
| ... | ... |
@@ -388,6 +391,7 @@ func CopyFileWriter(ctx context.Context, md map[string]string, id int, c session |
| 388 | 388 |
return nil, errors.Errorf("method %s not supported by the client", method)
|
| 389 | 389 |
} |
| 390 | 390 |
|
| 391 |
+ ctx = c.Context(ctx) |
|
| 391 | 392 |
client := NewFileSendClient(c.Conn()) |
| 392 | 393 |
|
| 393 | 394 |
opts, ok := metadata.FromOutgoingContext(ctx) |
| ... | ... |
@@ -7,6 +7,10 @@ import ( |
| 7 | 7 |
"github.com/pkg/errors" |
| 8 | 8 |
) |
| 9 | 9 |
|
| 10 |
+// ErrNoActiveSessions is returned when a session group does not contain any |
|
| 11 |
+// active session IDs. |
|
| 12 |
+var ErrNoActiveSessions = errors.New("no active sessions")
|
|
| 13 |
+ |
|
| 10 | 14 |
type Group interface {
|
| 11 | 15 |
SessionIterator() Iterator |
| 12 | 16 |
} |
| ... | ... |
@@ -69,7 +73,7 @@ func (sm *Manager) Any(ctx context.Context, g Group, f func(context.Context, str |
| 69 | 69 |
if lastErr != nil {
|
| 70 | 70 |
return lastErr |
| 71 | 71 |
} |
| 72 |
- return errors.Errorf("no active sessions")
|
|
| 72 |
+ return ErrNoActiveSessions |
|
| 73 | 73 |
} |
| 74 | 74 |
|
| 75 | 75 |
timeoutCtx, cancel := context.WithCancelCause(ctx) |
| ... | ... |
@@ -80,7 +84,7 @@ func (sm *Manager) Any(ctx context.Context, g Group, f func(context.Context, str |
| 80 | 80 |
lastErr = err |
| 81 | 81 |
continue |
| 82 | 82 |
} |
| 83 |
- if err := f(ctx, id, c); err != nil {
|
|
| 83 |
+ if err := f(c.Context(ctx), id, c); err != nil {
|
|
| 84 | 84 |
lastErr = err |
| 85 | 85 |
continue |
| 86 | 86 |
} |
| ... | ... |
@@ -4,6 +4,8 @@ import ( |
| 4 | 4 |
"context" |
| 5 | 5 |
"math" |
| 6 | 6 |
"net" |
| 7 |
+ "net/http" |
|
| 8 |
+ "strconv" |
|
| 7 | 9 |
"sync/atomic" |
| 8 | 10 |
"time" |
| 9 | 11 |
|
| ... | ... |
@@ -21,6 +23,40 @@ import ( |
| 21 | 21 |
"google.golang.org/grpc/health/grpc_health_v1" |
| 22 | 22 |
) |
| 23 | 23 |
|
| 24 |
+type healthCheckConfig struct {
|
|
| 25 |
+ interval time.Duration |
|
| 26 |
+ defaultTimeout time.Duration |
|
| 27 |
+ failureThreshold int |
|
| 28 |
+ successResetThreshold int |
|
| 29 |
+} |
|
| 30 |
+ |
|
| 31 |
+var defaultHealthCheckConfig = healthCheckConfig{
|
|
| 32 |
+ interval: 5 * time.Second, |
|
| 33 |
+ defaultTimeout: 15 * time.Second, |
|
| 34 |
+ failureThreshold: 2, |
|
| 35 |
+ successResetThreshold: 1, |
|
| 36 |
+} |
|
| 37 |
+ |
|
| 38 |
+const headerSessionHealthCustomTimeout = "X-Buildkit-Session-Health-Custom-Timeout" |
|
| 39 |
+ |
|
| 40 |
+func healthCheckConfigFromHeaders(h http.Header) healthCheckConfig {
|
|
| 41 |
+ cfg := defaultHealthCheckConfig |
|
| 42 |
+ |
|
| 43 |
+ if v := h.Get(headerSessionHealthCustomTimeout); v != "" {
|
|
| 44 |
+ if ms, err := strconv.Atoi(v); err == nil && ms > 0 {
|
|
| 45 |
+ d := time.Duration(ms) * time.Millisecond |
|
| 46 |
+ // Avoid test overrides that are unrealistically aggressive for slower machines. |
|
| 47 |
+ d = max(d, time.Second) |
|
| 48 |
+ cfg.interval = d |
|
| 49 |
+ cfg.defaultTimeout = d |
|
| 50 |
+ cfg.failureThreshold = 1 |
|
| 51 |
+ cfg.successResetThreshold = 1 |
|
| 52 |
+ } |
|
| 53 |
+ } |
|
| 54 |
+ |
|
| 55 |
+ return cfg |
|
| 56 |
+} |
|
| 57 |
+ |
|
| 24 | 58 |
func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) {
|
| 25 | 59 |
go func() {
|
| 26 | 60 |
<-ctx.Done() |
| ... | ... |
@@ -30,7 +66,7 @@ func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) {
|
| 30 | 30 |
(&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: grpcServer})
|
| 31 | 31 |
} |
| 32 | 32 |
|
| 33 |
-func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc.ClientConn, error) {
|
|
| 33 |
+func grpcClientConn(ctx context.Context, conn net.Conn, opts map[string][]string) (context.Context, *grpc.ClientConn, error) {
|
|
| 34 | 34 |
var dialCount int64 |
| 35 | 35 |
dialer := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
| 36 | 36 |
if c := atomic.AddInt64(&dialCount, 1); c > 1 {
|
| ... | ... |
@@ -63,22 +99,29 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc. |
| 63 | 63 |
} |
| 64 | 64 |
|
| 65 | 65 |
ctx, cancel := context.WithCancelCause(ctx) |
| 66 |
- go monitorHealth(ctx, cc, cancel) |
|
| 66 |
+ go monitorHealth(ctx, conn, cc, cancel, healthCheckConfigFromHeaders(http.Header(opts))) |
|
| 67 | 67 |
|
| 68 | 68 |
return ctx, cc, nil |
| 69 | 69 |
} |
| 70 | 70 |
|
| 71 |
-func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(error)) {
|
|
| 72 |
- defer cancelConn(errors.WithStack(context.Canceled)) |
|
| 73 |
- defer cc.Close() |
|
| 74 |
- |
|
| 75 |
- ticker := time.NewTicker(5 * time.Second) |
|
| 71 |
+func monitorHealth(ctx context.Context, conn net.Conn, cc *grpc.ClientConn, cancelConn func(error), cfg healthCheckConfig) {
|
|
| 72 |
+ closed := false |
|
| 73 |
+ closeConn := func(err error) {
|
|
| 74 |
+ if closed {
|
|
| 75 |
+ return |
|
| 76 |
+ } |
|
| 77 |
+ closed = true |
|
| 78 |
+ cancelConn(err) |
|
| 79 |
+ cc.Close() |
|
| 80 |
+ go conn.Close() |
|
| 81 |
+ } |
|
| 82 |
+ defer closeConn(errors.WithStack(context.Canceled)) |
|
| 83 |
+ ticker := time.NewTicker(cfg.interval) |
|
| 76 | 84 |
defer ticker.Stop() |
| 77 | 85 |
healthClient := grpc_health_v1.NewHealthClient(cc) |
| 78 | 86 |
|
| 79 |
- failedBefore := false |
|
| 87 |
+ consecutiveFailures := 0 |
|
| 80 | 88 |
consecutiveSuccessful := 0 |
| 81 |
- defaultHealthcheckDuration := 30 * time.Second |
|
| 82 | 89 |
lastHealthcheckDuration := time.Duration(0) |
| 83 | 90 |
|
| 84 | 91 |
for {
|
| ... | ... |
@@ -90,13 +133,24 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(err |
| 90 | 90 |
// So, this healthcheck is purposely long, and can tolerate some failures on purpose. |
| 91 | 91 |
|
| 92 | 92 |
healthcheckStart := time.Now() |
| 93 |
- |
|
| 94 |
- timeout := time.Duration(math.Max(float64(defaultHealthcheckDuration), float64(lastHealthcheckDuration)*1.5)) |
|
| 95 |
- |
|
| 96 |
- ctx, cancel := context.WithCancelCause(ctx) |
|
| 97 |
- ctx, _ = context.WithTimeoutCause(ctx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet |
|
| 98 |
- _, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
|
|
| 99 |
- cancel(errors.WithStack(context.Canceled)) |
|
| 93 |
+ timeout := time.Duration(math.Max(float64(cfg.defaultTimeout), float64(lastHealthcheckDuration)*1.5)) |
|
| 94 |
+ resultCh := make(chan error, 1) |
|
| 95 |
+ go func() {
|
|
| 96 |
+ checkCtx, cancel := context.WithCancelCause(ctx) |
|
| 97 |
+ checkCtx, _ = context.WithTimeoutCause(checkCtx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet |
|
| 98 |
+ _, err := healthClient.Check(checkCtx, &grpc_health_v1.HealthCheckRequest{})
|
|
| 99 |
+ cancel(errors.WithStack(context.Canceled)) |
|
| 100 |
+ resultCh <- err |
|
| 101 |
+ }() |
|
| 102 |
+ |
|
| 103 |
+ var err error |
|
| 104 |
+ select {
|
|
| 105 |
+ case <-ctx.Done(): |
|
| 106 |
+ return |
|
| 107 |
+ case err = <-resultCh: |
|
| 108 |
+ case <-time.After(timeout): |
|
| 109 |
+ err = errors.WithStack(context.DeadlineExceeded) |
|
| 110 |
+ } |
|
| 100 | 111 |
|
| 101 | 112 |
lastHealthcheckDuration = time.Since(healthcheckStart) |
| 102 | 113 |
logFields := logrus.Fields{
|
| ... | ... |
@@ -110,19 +164,21 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(err |
| 110 | 110 |
return |
| 111 | 111 |
default: |
| 112 | 112 |
} |
| 113 |
- if failedBefore {
|
|
| 114 |
- bklog.G(ctx).Error("healthcheck failed fatally")
|
|
| 113 |
+ consecutiveFailures++ |
|
| 114 |
+ consecutiveSuccessful = 0 |
|
| 115 |
+ if consecutiveFailures >= cfg.failureThreshold {
|
|
| 116 |
+ err = errors.Wrap(err, "session healthcheck failed fatally") |
|
| 117 |
+ bklog.G(ctx).WithError(err).Error("healthcheck failed fatally")
|
|
| 118 |
+ closeConn(err) |
|
| 115 | 119 |
return |
| 116 | 120 |
} |
| 117 | 121 |
|
| 118 |
- failedBefore = true |
|
| 119 |
- consecutiveSuccessful = 0 |
|
| 120 |
- bklog.G(ctx).WithFields(logFields).Warn("healthcheck failed")
|
|
| 122 |
+ bklog.G(ctx).WithError(err).WithFields(logFields).Warn("healthcheck failed")
|
|
| 121 | 123 |
} else {
|
| 122 | 124 |
consecutiveSuccessful++ |
| 125 |
+ consecutiveFailures = 0 |
|
| 123 | 126 |
|
| 124 |
- if consecutiveSuccessful >= 5 && failedBefore {
|
|
| 125 |
- failedBefore = false |
|
| 127 |
+ if consecutiveSuccessful >= cfg.successResetThreshold {
|
|
| 126 | 128 |
bklog.G(ctx).WithFields(logFields).Debug("reset healthcheck failure")
|
| 127 | 129 |
} |
| 128 | 130 |
} |
| ... | ... |
@@ -13,7 +13,7 @@ import ( |
| 13 | 13 |
|
| 14 | 14 |
// Caller can invoke requests on the session |
| 15 | 15 |
type Caller interface {
|
| 16 |
- Context() context.Context |
|
| 16 |
+ Context(context.Context) context.Context |
|
| 17 | 17 |
Supports(method string) bool |
| 18 | 18 |
Conn() *grpc.ClientConn |
| 19 | 19 |
SharedKey() string |
| ... | ... |
@@ -107,7 +107,7 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin |
| 107 | 107 |
id := h.Get(headerSessionID) |
| 108 | 108 |
sharedKey := h.Get(headerSessionSharedKey) |
| 109 | 109 |
|
| 110 |
- ctx, cc, err := grpcClientConn(ctx, conn) |
|
| 110 |
+ ctx, cc, err := grpcClientConn(ctx, conn, opts) |
|
| 111 | 111 |
if err != nil {
|
| 112 | 112 |
sm.mu.Unlock() |
| 113 | 113 |
return err |
| ... | ... |
@@ -190,8 +190,8 @@ func (sm *Manager) Get(ctx context.Context, id string, noWait bool) (Caller, err |
| 190 | 190 |
return c, nil |
| 191 | 191 |
} |
| 192 | 192 |
|
| 193 |
-func (c *client) Context() context.Context {
|
|
| 194 |
- return c.context() |
|
| 193 |
+func (c *client) Context(ctx context.Context) context.Context {
|
|
| 194 |
+ return contextWithCaller(ctx, c.context()) |
|
| 195 | 195 |
} |
| 196 | 196 |
|
| 197 | 197 |
func (c *client) SharedKey() string {
|
| ... | ... |
@@ -16,6 +16,7 @@ type SecretStore interface {
|
| 16 | 16 |
var ErrNotFound = errors.Errorf("not found")
|
| 17 | 17 |
|
| 18 | 18 |
func GetSecret(ctx context.Context, c session.Caller, id string) ([]byte, error) {
|
| 19 |
+ ctx = c.Context(ctx) |
|
| 19 | 20 |
client := NewSecretsClient(c.Conn()) |
| 20 | 21 |
resp, err := client.GetSecret(ctx, &GetSecretRequest{
|
| 21 | 22 |
ID: id, |
| ... | ... |
@@ -37,18 +37,19 @@ func (s *server) run(ctx context.Context, l net.Listener, id string) error {
|
| 37 | 37 |
} |
| 38 | 38 |
|
| 39 | 39 |
client := NewSSHClient(s.caller.Conn()) |
| 40 |
+ rpcCtx := s.caller.Context(ctx) |
|
| 40 | 41 |
|
| 41 | 42 |
opts := make(map[string][]string) |
| 42 | 43 |
opts[KeySSHID] = []string{id}
|
| 43 |
- ctx := metadata.NewOutgoingContext(ctx, opts) |
|
| 44 |
+ rpcCtx = metadata.NewOutgoingContext(rpcCtx, opts) |
|
| 44 | 45 |
|
| 45 |
- stream, err := client.ForwardAgent(ctx) |
|
| 46 |
+ stream, err := client.ForwardAgent(rpcCtx) |
|
| 46 | 47 |
if err != nil {
|
| 47 | 48 |
conn.Close() |
| 48 | 49 |
return err |
| 49 | 50 |
} |
| 50 | 51 |
|
| 51 |
- go Copy(ctx, conn, stream, stream.CloseSend) |
|
| 52 |
+ go Copy(rpcCtx, conn, stream, stream.CloseSend) |
|
| 52 | 53 |
} |
| 53 | 54 |
}) |
| 54 | 55 |
|
| ... | ... |
@@ -112,6 +113,7 @@ func MountSSHSocket(ctx context.Context, c session.Caller, opt SocketOpt) (sockP |
| 112 | 112 |
} |
| 113 | 113 |
|
| 114 | 114 |
func CheckSSHID(ctx context.Context, c session.Caller, id string) error {
|
| 115 |
+ ctx = c.Context(ctx) |
|
| 115 | 116 |
client := NewSSHClient(c.Conn()) |
| 116 | 117 |
_, err := client.CheckAgent(ctx, &CheckAgentRequest{ID: id})
|
| 117 | 118 |
return errors.WithStack(err) |
| 22 | 23 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,43 @@ |
| 0 |
+package errdefs |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ fmt "fmt" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/containerd/typeurl/v2" |
|
| 6 |
+ "github.com/moby/buildkit/util/grpcerrors" |
|
| 7 |
+) |
|
| 8 |
+ |
|
| 9 |
+func init() {
|
|
| 10 |
+ typeurl.Register((*CompatibilityFeature)(nil), "github.com/moby/buildkit", "errdefs.CompatibilityFeature+json") |
|
| 11 |
+} |
|
| 12 |
+ |
|
| 13 |
+type UnsupportedCompatibilityFeatureError struct {
|
|
| 14 |
+ *CompatibilityFeature |
|
| 15 |
+ error |
|
| 16 |
+} |
|
| 17 |
+ |
|
| 18 |
+func (e *UnsupportedCompatibilityFeatureError) Error() string {
|
|
| 19 |
+ msg := fmt.Sprintf("unsupported compatibility-version %d feature %s", e.Version, e.Feature)
|
|
| 20 |
+ if e.error != nil {
|
|
| 21 |
+ msg += ": " + e.error.Error() |
|
| 22 |
+ } |
|
| 23 |
+ return msg |
|
| 24 |
+} |
|
| 25 |
+ |
|
| 26 |
+func (e *UnsupportedCompatibilityFeatureError) Unwrap() error {
|
|
| 27 |
+ return e.error |
|
| 28 |
+} |
|
| 29 |
+ |
|
| 30 |
+func (e *UnsupportedCompatibilityFeatureError) ToProto() grpcerrors.TypedErrorProto {
|
|
| 31 |
+ return e.CompatibilityFeature |
|
| 32 |
+} |
|
| 33 |
+ |
|
| 34 |
+func NewUnsupportedCompatibilityFeatureError(version int, feature string) error {
|
|
| 35 |
+ return &UnsupportedCompatibilityFeatureError{
|
|
| 36 |
+ CompatibilityFeature: &CompatibilityFeature{Version: int64(version), Feature: feature},
|
|
| 37 |
+ } |
|
| 38 |
+} |
|
| 39 |
+ |
|
| 40 |
+func (v *CompatibilityFeature) WrapError(err error) error {
|
|
| 41 |
+ return &UnsupportedCompatibilityFeatureError{error: err, CompatibilityFeature: v}
|
|
| 42 |
+} |
| ... | ... |
@@ -214,6 +214,58 @@ func (x *FrontendCap) GetName() string {
|
| 214 | 214 |
return "" |
| 215 | 215 |
} |
| 216 | 216 |
|
| 217 |
+type CompatibilityFeature struct {
|
|
| 218 |
+ state protoimpl.MessageState `protogen:"open.v1"` |
|
| 219 |
+ Version int64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` |
|
| 220 |
+ Feature string `protobuf:"bytes,2,opt,name=feature,proto3" json:"feature,omitempty"` |
|
| 221 |
+ unknownFields protoimpl.UnknownFields |
|
| 222 |
+ sizeCache protoimpl.SizeCache |
|
| 223 |
+} |
|
| 224 |
+ |
|
| 225 |
+func (x *CompatibilityFeature) Reset() {
|
|
| 226 |
+ *x = CompatibilityFeature{}
|
|
| 227 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4] |
|
| 228 |
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|
| 229 |
+ ms.StoreMessageInfo(mi) |
|
| 230 |
+} |
|
| 231 |
+ |
|
| 232 |
+func (x *CompatibilityFeature) String() string {
|
|
| 233 |
+ return protoimpl.X.MessageStringOf(x) |
|
| 234 |
+} |
|
| 235 |
+ |
|
| 236 |
+func (*CompatibilityFeature) ProtoMessage() {}
|
|
| 237 |
+ |
|
| 238 |
+func (x *CompatibilityFeature) ProtoReflect() protoreflect.Message {
|
|
| 239 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4] |
|
| 240 |
+ if x != nil {
|
|
| 241 |
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
|
| 242 |
+ if ms.LoadMessageInfo() == nil {
|
|
| 243 |
+ ms.StoreMessageInfo(mi) |
|
| 244 |
+ } |
|
| 245 |
+ return ms |
|
| 246 |
+ } |
|
| 247 |
+ return mi.MessageOf(x) |
|
| 248 |
+} |
|
| 249 |
+ |
|
| 250 |
+// Deprecated: Use CompatibilityFeature.ProtoReflect.Descriptor instead. |
|
| 251 |
+func (*CompatibilityFeature) Descriptor() ([]byte, []int) {
|
|
| 252 |
+ return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{4}
|
|
| 253 |
+} |
|
| 254 |
+ |
|
| 255 |
+func (x *CompatibilityFeature) GetVersion() int64 {
|
|
| 256 |
+ if x != nil {
|
|
| 257 |
+ return x.Version |
|
| 258 |
+ } |
|
| 259 |
+ return 0 |
|
| 260 |
+} |
|
| 261 |
+ |
|
| 262 |
+func (x *CompatibilityFeature) GetFeature() string {
|
|
| 263 |
+ if x != nil {
|
|
| 264 |
+ return x.Feature |
|
| 265 |
+ } |
|
| 266 |
+ return "" |
|
| 267 |
+} |
|
| 268 |
+ |
|
| 217 | 269 |
type Subrequest struct {
|
| 218 | 270 |
state protoimpl.MessageState `protogen:"open.v1"` |
| 219 | 271 |
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` |
| ... | ... |
@@ -223,7 +275,7 @@ type Subrequest struct {
|
| 223 | 223 |
|
| 224 | 224 |
func (x *Subrequest) Reset() {
|
| 225 | 225 |
*x = Subrequest{}
|
| 226 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4] |
|
| 226 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5] |
|
| 227 | 227 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 228 | 228 |
ms.StoreMessageInfo(mi) |
| 229 | 229 |
} |
| ... | ... |
@@ -235,7 +287,7 @@ func (x *Subrequest) String() string {
|
| 235 | 235 |
func (*Subrequest) ProtoMessage() {}
|
| 236 | 236 |
|
| 237 | 237 |
func (x *Subrequest) ProtoReflect() protoreflect.Message {
|
| 238 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4] |
|
| 238 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5] |
|
| 239 | 239 |
if x != nil {
|
| 240 | 240 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 241 | 241 |
if ms.LoadMessageInfo() == nil {
|
| ... | ... |
@@ -248,7 +300,7 @@ func (x *Subrequest) ProtoReflect() protoreflect.Message {
|
| 248 | 248 |
|
| 249 | 249 |
// Deprecated: Use Subrequest.ProtoReflect.Descriptor instead. |
| 250 | 250 |
func (*Subrequest) Descriptor() ([]byte, []int) {
|
| 251 |
- return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{4}
|
|
| 251 |
+ return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{5}
|
|
| 252 | 252 |
} |
| 253 | 253 |
|
| 254 | 254 |
func (x *Subrequest) GetName() string {
|
| ... | ... |
@@ -275,7 +327,7 @@ type Solve struct {
|
| 275 | 275 |
|
| 276 | 276 |
func (x *Solve) Reset() {
|
| 277 | 277 |
*x = Solve{}
|
| 278 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5] |
|
| 278 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6] |
|
| 279 | 279 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 280 | 280 |
ms.StoreMessageInfo(mi) |
| 281 | 281 |
} |
| ... | ... |
@@ -287,7 +339,7 @@ func (x *Solve) String() string {
|
| 287 | 287 |
func (*Solve) ProtoMessage() {}
|
| 288 | 288 |
|
| 289 | 289 |
func (x *Solve) ProtoReflect() protoreflect.Message {
|
| 290 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5] |
|
| 290 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6] |
|
| 291 | 291 |
if x != nil {
|
| 292 | 292 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 293 | 293 |
if ms.LoadMessageInfo() == nil {
|
| ... | ... |
@@ -300,7 +352,7 @@ func (x *Solve) ProtoReflect() protoreflect.Message {
|
| 300 | 300 |
|
| 301 | 301 |
// Deprecated: Use Solve.ProtoReflect.Descriptor instead. |
| 302 | 302 |
func (*Solve) Descriptor() ([]byte, []int) {
|
| 303 |
- return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{5}
|
|
| 303 |
+ return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{6}
|
|
| 304 | 304 |
} |
| 305 | 305 |
|
| 306 | 306 |
func (x *Solve) GetInputIDs() []string {
|
| ... | ... |
@@ -382,7 +434,7 @@ type FileAction struct {
|
| 382 | 382 |
|
| 383 | 383 |
func (x *FileAction) Reset() {
|
| 384 | 384 |
*x = FileAction{}
|
| 385 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6] |
|
| 385 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7] |
|
| 386 | 386 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 387 | 387 |
ms.StoreMessageInfo(mi) |
| 388 | 388 |
} |
| ... | ... |
@@ -394,7 +446,7 @@ func (x *FileAction) String() string {
|
| 394 | 394 |
func (*FileAction) ProtoMessage() {}
|
| 395 | 395 |
|
| 396 | 396 |
func (x *FileAction) ProtoReflect() protoreflect.Message {
|
| 397 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6] |
|
| 397 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7] |
|
| 398 | 398 |
if x != nil {
|
| 399 | 399 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 400 | 400 |
if ms.LoadMessageInfo() == nil {
|
| ... | ... |
@@ -407,7 +459,7 @@ func (x *FileAction) ProtoReflect() protoreflect.Message {
|
| 407 | 407 |
|
| 408 | 408 |
// Deprecated: Use FileAction.ProtoReflect.Descriptor instead. |
| 409 | 409 |
func (*FileAction) Descriptor() ([]byte, []int) {
|
| 410 |
- return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{6}
|
|
| 410 |
+ return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{7}
|
|
| 411 | 411 |
} |
| 412 | 412 |
|
| 413 | 413 |
func (x *FileAction) GetIndex() int64 {
|
| ... | ... |
@@ -427,7 +479,7 @@ type ContentCache struct {
|
| 427 | 427 |
|
| 428 | 428 |
func (x *ContentCache) Reset() {
|
| 429 | 429 |
*x = ContentCache{}
|
| 430 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7] |
|
| 430 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[8] |
|
| 431 | 431 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 432 | 432 |
ms.StoreMessageInfo(mi) |
| 433 | 433 |
} |
| ... | ... |
@@ -439,7 +491,7 @@ func (x *ContentCache) String() string {
|
| 439 | 439 |
func (*ContentCache) ProtoMessage() {}
|
| 440 | 440 |
|
| 441 | 441 |
func (x *ContentCache) ProtoReflect() protoreflect.Message {
|
| 442 |
- mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7] |
|
| 442 |
+ mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[8] |
|
| 443 | 443 |
if x != nil {
|
| 444 | 444 |
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) |
| 445 | 445 |
if ms.LoadMessageInfo() == nil {
|
| ... | ... |
@@ -452,7 +504,7 @@ func (x *ContentCache) ProtoReflect() protoreflect.Message {
|
| 452 | 452 |
|
| 453 | 453 |
// Deprecated: Use ContentCache.ProtoReflect.Descriptor instead. |
| 454 | 454 |
func (*ContentCache) Descriptor() ([]byte, []int) {
|
| 455 |
- return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{7}
|
|
| 455 |
+ return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{8}
|
|
| 456 | 456 |
} |
| 457 | 457 |
|
| 458 | 458 |
func (x *ContentCache) GetIndex() int64 {
|
| ... | ... |
@@ -476,7 +528,10 @@ const file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc = "" + |
| 476 | 476 |
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + |
| 477 | 477 |
"\x06source\x18\x02 \x01(\tR\x06source\"!\n" + |
| 478 | 478 |
"\vFrontendCap\x12\x12\n" + |
| 479 |
- "\x04name\x18\x01 \x01(\tR\x04name\" \n" + |
|
| 479 |
+ "\x04name\x18\x01 \x01(\tR\x04name\"J\n" + |
|
| 480 |
+ "\x14CompatibilityFeature\x12\x18\n" + |
|
| 481 |
+ "\aversion\x18\x01 \x01(\x03R\aversion\x12\x18\n" + |
|
| 482 |
+ "\afeature\x18\x02 \x01(\tR\afeature\" \n" + |
|
| 480 | 483 |
"\n" + |
| 481 | 484 |
"Subrequest\x12\x12\n" + |
| 482 | 485 |
"\x04name\x18\x01 \x01(\tR\x04name\"\xbf\x02\n" + |
| ... | ... |
@@ -509,28 +564,29 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP() [] |
| 509 | 509 |
return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescData |
| 510 | 510 |
} |
| 511 | 511 |
|
| 512 |
-var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes = make([]protoimpl.MessageInfo, 9) |
|
| 512 |
+var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes = make([]protoimpl.MessageInfo, 10) |
|
| 513 | 513 |
var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_goTypes = []any{
|
| 514 |
- (*Vertex)(nil), // 0: errdefs.Vertex |
|
| 515 |
- (*Source)(nil), // 1: errdefs.Source |
|
| 516 |
- (*Frontend)(nil), // 2: errdefs.Frontend |
|
| 517 |
- (*FrontendCap)(nil), // 3: errdefs.FrontendCap |
|
| 518 |
- (*Subrequest)(nil), // 4: errdefs.Subrequest |
|
| 519 |
- (*Solve)(nil), // 5: errdefs.Solve |
|
| 520 |
- (*FileAction)(nil), // 6: errdefs.FileAction |
|
| 521 |
- (*ContentCache)(nil), // 7: errdefs.ContentCache |
|
| 522 |
- nil, // 8: errdefs.Solve.DescriptionEntry |
|
| 523 |
- (*pb.SourceInfo)(nil), // 9: pb.SourceInfo |
|
| 524 |
- (*pb.Range)(nil), // 10: pb.Range |
|
| 525 |
- (*pb.Op)(nil), // 11: pb.Op |
|
| 514 |
+ (*Vertex)(nil), // 0: errdefs.Vertex |
|
| 515 |
+ (*Source)(nil), // 1: errdefs.Source |
|
| 516 |
+ (*Frontend)(nil), // 2: errdefs.Frontend |
|
| 517 |
+ (*FrontendCap)(nil), // 3: errdefs.FrontendCap |
|
| 518 |
+ (*CompatibilityFeature)(nil), // 4: errdefs.CompatibilityFeature |
|
| 519 |
+ (*Subrequest)(nil), // 5: errdefs.Subrequest |
|
| 520 |
+ (*Solve)(nil), // 6: errdefs.Solve |
|
| 521 |
+ (*FileAction)(nil), // 7: errdefs.FileAction |
|
| 522 |
+ (*ContentCache)(nil), // 8: errdefs.ContentCache |
|
| 523 |
+ nil, // 9: errdefs.Solve.DescriptionEntry |
|
| 524 |
+ (*pb.SourceInfo)(nil), // 10: pb.SourceInfo |
|
| 525 |
+ (*pb.Range)(nil), // 11: pb.Range |
|
| 526 |
+ (*pb.Op)(nil), // 12: pb.Op |
|
| 526 | 527 |
} |
| 527 | 528 |
var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_depIdxs = []int32{
|
| 528 |
- 9, // 0: errdefs.Source.info:type_name -> pb.SourceInfo |
|
| 529 |
- 10, // 1: errdefs.Source.ranges:type_name -> pb.Range |
|
| 530 |
- 11, // 2: errdefs.Solve.op:type_name -> pb.Op |
|
| 531 |
- 6, // 3: errdefs.Solve.file:type_name -> errdefs.FileAction |
|
| 532 |
- 7, // 4: errdefs.Solve.cache:type_name -> errdefs.ContentCache |
|
| 533 |
- 8, // 5: errdefs.Solve.description:type_name -> errdefs.Solve.DescriptionEntry |
|
| 529 |
+ 10, // 0: errdefs.Source.info:type_name -> pb.SourceInfo |
|
| 530 |
+ 11, // 1: errdefs.Source.ranges:type_name -> pb.Range |
|
| 531 |
+ 12, // 2: errdefs.Solve.op:type_name -> pb.Op |
|
| 532 |
+ 7, // 3: errdefs.Solve.file:type_name -> errdefs.FileAction |
|
| 533 |
+ 8, // 4: errdefs.Solve.cache:type_name -> errdefs.ContentCache |
|
| 534 |
+ 9, // 5: errdefs.Solve.description:type_name -> errdefs.Solve.DescriptionEntry |
|
| 534 | 535 |
6, // [6:6] is the sub-list for method output_type |
| 535 | 536 |
6, // [6:6] is the sub-list for method input_type |
| 536 | 537 |
6, // [6:6] is the sub-list for extension type_name |
| ... | ... |
@@ -543,7 +599,7 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_init() {
|
| 543 | 543 |
if File_github_com_moby_buildkit_solver_errdefs_errdefs_proto != nil {
|
| 544 | 544 |
return |
| 545 | 545 |
} |
| 546 |
- file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5].OneofWrappers = []any{
|
|
| 546 |
+ file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6].OneofWrappers = []any{
|
|
| 547 | 547 |
(*Solve_File)(nil), |
| 548 | 548 |
(*Solve_Cache)(nil), |
| 549 | 549 |
} |
| ... | ... |
@@ -553,7 +609,7 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_init() {
|
| 553 | 553 |
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
| 554 | 554 |
RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc), len(file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc)), |
| 555 | 555 |
NumEnums: 0, |
| 556 |
- NumMessages: 9, |
|
| 556 |
+ NumMessages: 10, |
|
| 557 | 557 |
NumExtensions: 0, |
| 558 | 558 |
NumServices: 0, |
| 559 | 559 |
}, |
| ... | ... |
@@ -96,6 +96,24 @@ func (m *FrontendCap) CloneMessageVT() proto.Message {
|
| 96 | 96 |
return m.CloneVT() |
| 97 | 97 |
} |
| 98 | 98 |
|
| 99 |
+func (m *CompatibilityFeature) CloneVT() *CompatibilityFeature {
|
|
| 100 |
+ if m == nil {
|
|
| 101 |
+ return (*CompatibilityFeature)(nil) |
|
| 102 |
+ } |
|
| 103 |
+ r := new(CompatibilityFeature) |
|
| 104 |
+ r.Version = m.Version |
|
| 105 |
+ r.Feature = m.Feature |
|
| 106 |
+ if len(m.unknownFields) > 0 {
|
|
| 107 |
+ r.unknownFields = make([]byte, len(m.unknownFields)) |
|
| 108 |
+ copy(r.unknownFields, m.unknownFields) |
|
| 109 |
+ } |
|
| 110 |
+ return r |
|
| 111 |
+} |
|
| 112 |
+ |
|
| 113 |
+func (m *CompatibilityFeature) CloneMessageVT() proto.Message {
|
|
| 114 |
+ return m.CloneVT() |
|
| 115 |
+} |
|
| 116 |
+ |
|
| 99 | 117 |
func (m *Subrequest) CloneVT() *Subrequest {
|
| 100 | 118 |
if m == nil {
|
| 101 | 119 |
return (*Subrequest)(nil) |
| ... | ... |
@@ -298,6 +316,28 @@ func (this *FrontendCap) EqualMessageVT(thatMsg proto.Message) bool {
|
| 298 | 298 |
} |
| 299 | 299 |
return this.EqualVT(that) |
| 300 | 300 |
} |
| 301 |
+func (this *CompatibilityFeature) EqualVT(that *CompatibilityFeature) bool {
|
|
| 302 |
+ if this == that {
|
|
| 303 |
+ return true |
|
| 304 |
+ } else if this == nil || that == nil {
|
|
| 305 |
+ return false |
|
| 306 |
+ } |
|
| 307 |
+ if this.Version != that.Version {
|
|
| 308 |
+ return false |
|
| 309 |
+ } |
|
| 310 |
+ if this.Feature != that.Feature {
|
|
| 311 |
+ return false |
|
| 312 |
+ } |
|
| 313 |
+ return string(this.unknownFields) == string(that.unknownFields) |
|
| 314 |
+} |
|
| 315 |
+ |
|
| 316 |
+func (this *CompatibilityFeature) EqualMessageVT(thatMsg proto.Message) bool {
|
|
| 317 |
+ that, ok := thatMsg.(*CompatibilityFeature) |
|
| 318 |
+ if !ok {
|
|
| 319 |
+ return false |
|
| 320 |
+ } |
|
| 321 |
+ return this.EqualVT(that) |
|
| 322 |
+} |
|
| 301 | 323 |
func (this *Subrequest) EqualVT(that *Subrequest) bool {
|
| 302 | 324 |
if this == that {
|
| 303 | 325 |
return true |
| ... | ... |
@@ -646,6 +686,51 @@ func (m *FrontendCap) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
|
| 646 | 646 |
return len(dAtA) - i, nil |
| 647 | 647 |
} |
| 648 | 648 |
|
| 649 |
+func (m *CompatibilityFeature) MarshalVT() (dAtA []byte, err error) {
|
|
| 650 |
+ if m == nil {
|
|
| 651 |
+ return nil, nil |
|
| 652 |
+ } |
|
| 653 |
+ size := m.SizeVT() |
|
| 654 |
+ dAtA = make([]byte, size) |
|
| 655 |
+ n, err := m.MarshalToSizedBufferVT(dAtA[:size]) |
|
| 656 |
+ if err != nil {
|
|
| 657 |
+ return nil, err |
|
| 658 |
+ } |
|
| 659 |
+ return dAtA[:n], nil |
|
| 660 |
+} |
|
| 661 |
+ |
|
| 662 |
+func (m *CompatibilityFeature) MarshalToVT(dAtA []byte) (int, error) {
|
|
| 663 |
+ size := m.SizeVT() |
|
| 664 |
+ return m.MarshalToSizedBufferVT(dAtA[:size]) |
|
| 665 |
+} |
|
| 666 |
+ |
|
| 667 |
+func (m *CompatibilityFeature) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
|
|
| 668 |
+ if m == nil {
|
|
| 669 |
+ return 0, nil |
|
| 670 |
+ } |
|
| 671 |
+ i := len(dAtA) |
|
| 672 |
+ _ = i |
|
| 673 |
+ var l int |
|
| 674 |
+ _ = l |
|
| 675 |
+ if m.unknownFields != nil {
|
|
| 676 |
+ i -= len(m.unknownFields) |
|
| 677 |
+ copy(dAtA[i:], m.unknownFields) |
|
| 678 |
+ } |
|
| 679 |
+ if len(m.Feature) > 0 {
|
|
| 680 |
+ i -= len(m.Feature) |
|
| 681 |
+ copy(dAtA[i:], m.Feature) |
|
| 682 |
+ i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Feature))) |
|
| 683 |
+ i-- |
|
| 684 |
+ dAtA[i] = 0x12 |
|
| 685 |
+ } |
|
| 686 |
+ if m.Version != 0 {
|
|
| 687 |
+ i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Version)) |
|
| 688 |
+ i-- |
|
| 689 |
+ dAtA[i] = 0x8 |
|
| 690 |
+ } |
|
| 691 |
+ return len(dAtA) - i, nil |
|
| 692 |
+} |
|
| 693 |
+ |
|
| 649 | 694 |
func (m *Subrequest) MarshalVT() (dAtA []byte, err error) {
|
| 650 | 695 |
if m == nil {
|
| 651 | 696 |
return nil, nil |
| ... | ... |
@@ -963,6 +1048,23 @@ func (m *FrontendCap) SizeVT() (n int) {
|
| 963 | 963 |
return n |
| 964 | 964 |
} |
| 965 | 965 |
|
| 966 |
+func (m *CompatibilityFeature) SizeVT() (n int) {
|
|
| 967 |
+ if m == nil {
|
|
| 968 |
+ return 0 |
|
| 969 |
+ } |
|
| 970 |
+ var l int |
|
| 971 |
+ _ = l |
|
| 972 |
+ if m.Version != 0 {
|
|
| 973 |
+ n += 1 + protohelpers.SizeOfVarint(uint64(m.Version)) |
|
| 974 |
+ } |
|
| 975 |
+ l = len(m.Feature) |
|
| 976 |
+ if l > 0 {
|
|
| 977 |
+ n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) |
|
| 978 |
+ } |
|
| 979 |
+ n += len(m.unknownFields) |
|
| 980 |
+ return n |
|
| 981 |
+} |
|
| 982 |
+ |
|
| 966 | 983 |
func (m *Subrequest) SizeVT() (n int) {
|
| 967 | 984 |
if m == nil {
|
| 968 | 985 |
return 0 |
| ... | ... |
@@ -1470,6 +1572,108 @@ func (m *FrontendCap) UnmarshalVT(dAtA []byte) error {
|
| 1470 | 1470 |
} |
| 1471 | 1471 |
return nil |
| 1472 | 1472 |
} |
| 1473 |
+func (m *CompatibilityFeature) UnmarshalVT(dAtA []byte) error {
|
|
| 1474 |
+ l := len(dAtA) |
|
| 1475 |
+ iNdEx := 0 |
|
| 1476 |
+ for iNdEx < l {
|
|
| 1477 |
+ preIndex := iNdEx |
|
| 1478 |
+ var wire uint64 |
|
| 1479 |
+ for shift := uint(0); ; shift += 7 {
|
|
| 1480 |
+ if shift >= 64 {
|
|
| 1481 |
+ return protohelpers.ErrIntOverflow |
|
| 1482 |
+ } |
|
| 1483 |
+ if iNdEx >= l {
|
|
| 1484 |
+ return io.ErrUnexpectedEOF |
|
| 1485 |
+ } |
|
| 1486 |
+ b := dAtA[iNdEx] |
|
| 1487 |
+ iNdEx++ |
|
| 1488 |
+ wire |= uint64(b&0x7F) << shift |
|
| 1489 |
+ if b < 0x80 {
|
|
| 1490 |
+ break |
|
| 1491 |
+ } |
|
| 1492 |
+ } |
|
| 1493 |
+ fieldNum := int32(wire >> 3) |
|
| 1494 |
+ wireType := int(wire & 0x7) |
|
| 1495 |
+ if wireType == 4 {
|
|
| 1496 |
+ return fmt.Errorf("proto: CompatibilityFeature: wiretype end group for non-group")
|
|
| 1497 |
+ } |
|
| 1498 |
+ if fieldNum <= 0 {
|
|
| 1499 |
+ return fmt.Errorf("proto: CompatibilityFeature: illegal tag %d (wire type %d)", fieldNum, wire)
|
|
| 1500 |
+ } |
|
| 1501 |
+ switch fieldNum {
|
|
| 1502 |
+ case 1: |
|
| 1503 |
+ if wireType != 0 {
|
|
| 1504 |
+ return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
|
|
| 1505 |
+ } |
|
| 1506 |
+ m.Version = 0 |
|
| 1507 |
+ for shift := uint(0); ; shift += 7 {
|
|
| 1508 |
+ if shift >= 64 {
|
|
| 1509 |
+ return protohelpers.ErrIntOverflow |
|
| 1510 |
+ } |
|
| 1511 |
+ if iNdEx >= l {
|
|
| 1512 |
+ return io.ErrUnexpectedEOF |
|
| 1513 |
+ } |
|
| 1514 |
+ b := dAtA[iNdEx] |
|
| 1515 |
+ iNdEx++ |
|
| 1516 |
+ m.Version |= int64(b&0x7F) << shift |
|
| 1517 |
+ if b < 0x80 {
|
|
| 1518 |
+ break |
|
| 1519 |
+ } |
|
| 1520 |
+ } |
|
| 1521 |
+ case 2: |
|
| 1522 |
+ if wireType != 2 {
|
|
| 1523 |
+ return fmt.Errorf("proto: wrong wireType = %d for field Feature", wireType)
|
|
| 1524 |
+ } |
|
| 1525 |
+ var stringLen uint64 |
|
| 1526 |
+ for shift := uint(0); ; shift += 7 {
|
|
| 1527 |
+ if shift >= 64 {
|
|
| 1528 |
+ return protohelpers.ErrIntOverflow |
|
| 1529 |
+ } |
|
| 1530 |
+ if iNdEx >= l {
|
|
| 1531 |
+ return io.ErrUnexpectedEOF |
|
| 1532 |
+ } |
|
| 1533 |
+ b := dAtA[iNdEx] |
|
| 1534 |
+ iNdEx++ |
|
| 1535 |
+ stringLen |= uint64(b&0x7F) << shift |
|
| 1536 |
+ if b < 0x80 {
|
|
| 1537 |
+ break |
|
| 1538 |
+ } |
|
| 1539 |
+ } |
|
| 1540 |
+ intStringLen := int(stringLen) |
|
| 1541 |
+ if intStringLen < 0 {
|
|
| 1542 |
+ return protohelpers.ErrInvalidLength |
|
| 1543 |
+ } |
|
| 1544 |
+ postIndex := iNdEx + intStringLen |
|
| 1545 |
+ if postIndex < 0 {
|
|
| 1546 |
+ return protohelpers.ErrInvalidLength |
|
| 1547 |
+ } |
|
| 1548 |
+ if postIndex > l {
|
|
| 1549 |
+ return io.ErrUnexpectedEOF |
|
| 1550 |
+ } |
|
| 1551 |
+ m.Feature = string(dAtA[iNdEx:postIndex]) |
|
| 1552 |
+ iNdEx = postIndex |
|
| 1553 |
+ default: |
|
| 1554 |
+ iNdEx = preIndex |
|
| 1555 |
+ skippy, err := protohelpers.Skip(dAtA[iNdEx:]) |
|
| 1556 |
+ if err != nil {
|
|
| 1557 |
+ return err |
|
| 1558 |
+ } |
|
| 1559 |
+ if (skippy < 0) || (iNdEx+skippy) < 0 {
|
|
| 1560 |
+ return protohelpers.ErrInvalidLength |
|
| 1561 |
+ } |
|
| 1562 |
+ if (iNdEx + skippy) > l {
|
|
| 1563 |
+ return io.ErrUnexpectedEOF |
|
| 1564 |
+ } |
|
| 1565 |
+ m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) |
|
| 1566 |
+ iNdEx += skippy |
|
| 1567 |
+ } |
|
| 1568 |
+ } |
|
| 1569 |
+ |
|
| 1570 |
+ if iNdEx > l {
|
|
| 1571 |
+ return io.ErrUnexpectedEOF |
|
| 1572 |
+ } |
|
| 1573 |
+ return nil |
|
| 1574 |
+} |
|
| 1473 | 1575 |
func (m *Subrequest) UnmarshalVT(dAtA []byte) error {
|
| 1474 | 1576 |
l := len(dAtA) |
| 1475 | 1577 |
iNdEx := 0 |
| ... | ... |
@@ -11,6 +11,7 @@ import ( |
| 11 | 11 |
"github.com/moby/buildkit/identity" |
| 12 | 12 |
"github.com/moby/buildkit/session" |
| 13 | 13 |
"github.com/moby/buildkit/solver/errdefs" |
| 14 |
+ "github.com/moby/buildkit/solver/llbsolver/compat" |
|
| 14 | 15 |
"github.com/moby/buildkit/util/bklog" |
| 15 | 16 |
"github.com/moby/buildkit/util/bkmaps" |
| 16 | 17 |
"github.com/moby/buildkit/util/flightcontrol" |
| ... | ... |
@@ -90,6 +91,34 @@ func (s *state) ResolverCache() ResolverCache {
|
| 90 | 90 |
return s |
| 91 | 91 |
} |
| 92 | 92 |
|
| 93 |
+func (s *state) CompatibilityVersion() (int, error) {
|
|
| 94 |
+ s.mu.Lock() |
|
| 95 |
+ jobs := make([]*Job, 0, len(s.jobs)) |
|
| 96 |
+ for j := range s.jobs {
|
|
| 97 |
+ jobs = append(jobs, j) |
|
| 98 |
+ } |
|
| 99 |
+ s.mu.Unlock() |
|
| 100 |
+ |
|
| 101 |
+ version := 0 |
|
| 102 |
+ for _, j := range jobs {
|
|
| 103 |
+ v, err := j.CompatibilityVersion() |
|
| 104 |
+ if err != nil {
|
|
| 105 |
+ return 0, err |
|
| 106 |
+ } |
|
| 107 |
+ if version == 0 {
|
|
| 108 |
+ version = v |
|
| 109 |
+ continue |
|
| 110 |
+ } |
|
| 111 |
+ if version != v {
|
|
| 112 |
+ return 0, errors.Errorf("conflicting compatibility versions in shared solve state: %d != %d", version, v)
|
|
| 113 |
+ } |
|
| 114 |
+ } |
|
| 115 |
+ if version == 0 {
|
|
| 116 |
+ return compat.CompatibilityVersionCurrent, nil |
|
| 117 |
+ } |
|
| 118 |
+ return version, nil |
|
| 119 |
+} |
|
| 120 |
+ |
|
| 93 | 121 |
func (s *state) Lock(key any) (values []any, release func(any) error, err error) {
|
| 94 | 122 |
var rcs []ResolverCache |
| 95 | 123 |
s.mu.Lock() |
| ... | ... |
@@ -865,6 +894,21 @@ func (j *Job) UniqueID() string {
|
| 865 | 865 |
return j.uniqueID |
| 866 | 866 |
} |
| 867 | 867 |
|
| 868 |
+func (j *Job) CompatibilityVersion() (int, error) {
|
|
| 869 |
+ version := compat.CompatibilityVersionCurrent |
|
| 870 |
+ if err := j.EachValue(context.TODO(), compat.JobValueKey, func(v any) error {
|
|
| 871 |
+ parsed, ok := v.(int) |
|
| 872 |
+ if !ok {
|
|
| 873 |
+ return errors.Errorf("invalid compatibility version %T", v)
|
|
| 874 |
+ } |
|
| 875 |
+ version = parsed |
|
| 876 |
+ return nil |
|
| 877 |
+ }); err != nil {
|
|
| 878 |
+ return 0, err |
|
| 879 |
+ } |
|
| 880 |
+ return version, nil |
|
| 881 |
+} |
|
| 882 |
+ |
|
| 868 | 883 |
func (j *Job) InContext(ctx context.Context, f func(context.Context, JobContext) error) error {
|
| 869 | 884 |
return f(progress.WithProgress(ctx, j.pw), j) |
| 870 | 885 |
} |
| 45 | 46 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,36 @@ |
| 0 |
+package compat |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "slices" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/pkg/errors" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+const ( |
|
| 9 |
+ CompatibilityVersion013 = 10 |
|
| 10 |
+ CompatibilityVersion015 = 20 |
|
| 11 |
+ CompatibilityVersionCurrent = CompatibilityVersion015 |
|
| 12 |
+) |
|
| 13 |
+ |
|
| 14 |
+// JobValueKey is the key used to store the compatibility version on a solver |
|
| 15 |
+// job via Job.SetValue/EachValue. |
|
| 16 |
+const JobValueKey = "llb.compatibilityversion" |
|
| 17 |
+ |
|
| 18 |
+var supportedCompatibilityVersions = []int{
|
|
| 19 |
+ CompatibilityVersion013, |
|
| 20 |
+ CompatibilityVersion015, |
|
| 21 |
+} |
|
| 22 |
+ |
|
| 23 |
+func SupportedCompatibilityVersions() []int {
|
|
| 24 |
+ return slices.Clone(supportedCompatibilityVersions) |
|
| 25 |
+} |
|
| 26 |
+ |
|
| 27 |
+func ValidateCompatibilityVersion(version int) error {
|
|
| 28 |
+ if slices.Contains(supportedCompatibilityVersions, version) {
|
|
| 29 |
+ return nil |
|
| 30 |
+ } |
|
| 31 |
+ if version > CompatibilityVersionCurrent {
|
|
| 32 |
+ return errors.Errorf("unsupported compatibility-version %d: upgrade buildkit (max supported: %d)", version, CompatibilityVersionCurrent)
|
|
| 33 |
+ } |
|
| 34 |
+ return errors.Errorf("unsupported compatibility-version %d (supported: %v)", version, supportedCompatibilityVersions)
|
|
| 35 |
+} |
| ... | ... |
@@ -40,7 +40,7 @@ func (s *Solver) getSessionExporters(ctx context.Context, sessionID string, id i |
| 40 | 40 |
} |
| 41 | 41 |
|
| 42 | 42 |
client := sessionexporter.NewExporterClient(caller.Conn()) |
| 43 |
- |
|
| 43 |
+ ctx = caller.Context(ctx) |
|
| 44 | 44 |
var ids []string |
| 45 | 45 |
if err := inp.EachRef(func(ref cache.ImmutableRef) error {
|
| 46 | 46 |
ids = append(ids, ref.ID()) |
| ... | ... |
@@ -188,11 +188,16 @@ func (s *Solver) runExporters(ctx context.Context, ref string, exporters []expor |
| 188 | 188 |
defer inlineCacheMu.Unlock() |
| 189 | 189 |
return runInlineCacheExporter(ctx, exp, inlineCacheExporter, job, cached) |
| 190 | 190 |
}) |
| 191 |
+ compatibilityVersion, err := job.CompatibilityVersion() |
|
| 192 |
+ if err != nil {
|
|
| 193 |
+ return err |
|
| 194 |
+ } |
|
| 191 | 195 |
|
| 192 | 196 |
resp, finalize, desc, expErr := exp.Export(ctx, inp, exporter.ExportBuildInfo{
|
| 193 |
- Ref: ref, |
|
| 194 |
- SessionID: job.SessionID, |
|
| 195 |
- InlineCache: inlineCache, |
|
| 197 |
+ Ref: ref, |
|
| 198 |
+ SessionID: job.SessionID, |
|
| 199 |
+ InlineCache: inlineCache, |
|
| 200 |
+ CompatibilityVersion: compatibilityVersion, |
|
| 196 | 201 |
}) |
| 197 | 202 |
resps[i], finalizeFuncs[i], descs[i] = resp, finalize, desc |
| 198 | 203 |
if expErr != nil {
|
| ... | ... |
@@ -58,6 +58,7 @@ func (p *policyEvaluator) evaluate(ctx context.Context, op *pb.Op, max int) (boo |
| 58 | 58 |
} |
| 59 | 59 |
|
| 60 | 60 |
verifier := policysession.NewPolicyVerifierClient(caller.Conn()) |
| 61 |
+ ctx = caller.Context(ctx) |
|
| 61 | 62 |
req := &policysession.CheckPolicyRequest{
|
| 62 | 63 |
Platform: op.Platform, |
| 63 | 64 |
Source: &gatewaypb.ResolveSourceMetaResponse{
|
| ... | ... |
@@ -38,12 +38,14 @@ type resultWithBridge struct {
|
| 38 | 38 |
// provenanceBridge provides scoped access to LLBBridge and captures the request it makes for provenance |
| 39 | 39 |
type provenanceBridge struct {
|
| 40 | 40 |
*llbBridge |
| 41 |
- mu sync.Mutex |
|
| 42 |
- req *frontend.SolveRequest |
|
| 43 |
- |
|
| 44 |
- images []provenancetypes.ImageSource |
|
| 45 |
- builds []resultWithBridge |
|
| 46 |
- subBridges []*provenanceBridge |
|
| 41 |
+ mu sync.Mutex |
|
| 42 |
+ req *frontend.SolveRequest |
|
| 43 |
+ rootReq *frontend.SolveRequest |
|
| 44 |
+ |
|
| 45 |
+ images []provenancetypes.ImageSource |
|
| 46 |
+ builds []resultWithBridge |
|
| 47 |
+ subBridges []*provenanceBridge |
|
| 48 |
+ provenanceRefRecordIDs []string |
|
| 47 | 49 |
} |
| 48 | 50 |
|
| 49 | 51 |
func (b *provenanceBridge) eachRef(f func(r solver.ResultProxy) error) error {
|
| ... | ... |
@@ -165,6 +167,7 @@ func (b *provenanceBridge) ResolveSourceMetadata(ctx context.Context, op *pb.Sou |
| 165 | 165 |
} |
| 166 | 166 |
|
| 167 | 167 |
func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid string) (res *frontend.Result, err error) {
|
| 168 |
+ req = req.Clone() |
|
| 168 | 169 |
if req.Definition != nil && req.Definition.Def != nil && req.Frontend != "" {
|
| 169 | 170 |
return nil, errors.New("cannot solve with both Definition and Frontend specified")
|
| 170 | 171 |
} |
| ... | ... |
@@ -180,7 +183,11 @@ func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest, |
| 180 | 180 |
if !ok {
|
| 181 | 181 |
return nil, errors.Errorf("invalid frontend: %s", req.Frontend)
|
| 182 | 182 |
} |
| 183 |
- wb := &provenanceBridge{llbBridge: b.llbBridge, req: &req}
|
|
| 183 |
+ rootReq := b.rootReq |
|
| 184 |
+ if !hasRequestProvenance(rootReq) {
|
|
| 185 |
+ rootReq = b.req |
|
| 186 |
+ } |
|
| 187 |
+ wb := &provenanceBridge{llbBridge: b.llbBridge, req: &req, rootReq: rootReq}
|
|
| 184 | 188 |
res, err = f.Solve(ctx, wb, b.llbBridge, req.FrontendOpt, req.FrontendInputs, sid, b.sm) |
| 185 | 189 |
if err != nil {
|
| 186 | 190 |
fe := errdefs.Frontend{
|
| ... | ... |
@@ -202,6 +209,9 @@ func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest, |
| 202 | 202 |
return err |
| 203 | 203 |
}) |
| 204 | 204 |
} |
| 205 |
+ if err == nil {
|
|
| 206 |
+ err = b.registerProvenanceRefs(res) |
|
| 207 |
+ } |
|
| 205 | 208 |
return |
| 206 | 209 |
} |
| 207 | 210 |
|
| ... | ... |
@@ -397,17 +407,9 @@ func NewProvenanceCreator(ctx context.Context, slsaVersion provenancetypes.Prove |
| 397 | 397 |
|
| 398 | 398 |
switch mode {
|
| 399 | 399 |
case "min": |
| 400 |
- args := make(map[string]string) |
|
| 401 |
- for k, v := range pr.BuildDefinition.ExternalParameters.Request.Args {
|
|
| 402 |
- if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") {
|
|
| 403 |
- pr.RunDetails.Metadata.Completeness.Request = false |
|
| 404 |
- continue |
|
| 405 |
- } |
|
| 406 |
- args[k] = v |
|
| 400 |
+ if scrubMinRequest(&pr.BuildDefinition.ExternalParameters.Request) {
|
|
| 401 |
+ pr.RunDetails.Metadata.Completeness.Request = false |
|
| 407 | 402 |
} |
| 408 |
- pr.BuildDefinition.ExternalParameters.Request.Args = args |
|
| 409 |
- pr.BuildDefinition.ExternalParameters.Request.Secrets = nil |
|
| 410 |
- pr.BuildDefinition.ExternalParameters.Request.SSH = nil |
|
| 411 | 403 |
case "max": |
| 412 | 404 |
dgsts, err := provenance.AddBuildConfig(ctx, pr, cp, res, withUsage) |
| 413 | 405 |
if err != nil {
|
| ... | ... |
@@ -478,6 +480,41 @@ func NewProvenanceCreator(ctx context.Context, slsaVersion provenancetypes.Prove |
| 478 | 478 |
return pc, nil |
| 479 | 479 |
} |
| 480 | 480 |
|
| 481 |
+func scrubMinRequest(req *provenancetypes.Parameters) bool {
|
|
| 482 |
+ if req == nil {
|
|
| 483 |
+ return false |
|
| 484 |
+ } |
|
| 485 |
+ |
|
| 486 |
+ var incomplete bool |
|
| 487 |
+ if len(req.Args) > 0 {
|
|
| 488 |
+ args := make(map[string]string, len(req.Args)) |
|
| 489 |
+ for k, v := range req.Args {
|
|
| 490 |
+ if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") {
|
|
| 491 |
+ incomplete = true |
|
| 492 |
+ continue |
|
| 493 |
+ } |
|
| 494 |
+ args[k] = v |
|
| 495 |
+ } |
|
| 496 |
+ req.Args = args |
|
| 497 |
+ } |
|
| 498 |
+ if len(req.Secrets) > 0 {
|
|
| 499 |
+ req.Secrets = nil |
|
| 500 |
+ } |
|
| 501 |
+ if len(req.SSH) > 0 {
|
|
| 502 |
+ req.SSH = nil |
|
| 503 |
+ } |
|
| 504 |
+ |
|
| 505 |
+ for _, in := range req.Inputs {
|
|
| 506 |
+ if in != nil && scrubMinRequest(in.Request) {
|
|
| 507 |
+ incomplete = true |
|
| 508 |
+ } |
|
| 509 |
+ } |
|
| 510 |
+ if req.Root != nil && scrubMinRequest(req.Root.Request) {
|
|
| 511 |
+ incomplete = true |
|
| 512 |
+ } |
|
| 513 |
+ return incomplete |
|
| 514 |
+} |
|
| 515 |
+ |
|
| 481 | 516 |
func (p *ProvenanceCreator) PredicateType() string {
|
| 482 | 517 |
if p.slsaVersion == provenancetypes.ProvenanceSLSA02 {
|
| 483 | 518 |
return slsa02.PredicateSLSAProvenance |
| ... | ... |
@@ -506,6 +543,12 @@ func (p *ProvenanceCreator) Predicate(ctx context.Context) (any, error) {
|
| 506 | 506 |
p.pr.RunDetails.Metadata.BuildKitMetadata.SysUsage = sysSamples |
| 507 | 507 |
} |
| 508 | 508 |
|
| 509 |
+ compatibilityVersion, err := p.j.CompatibilityVersion() |
|
| 510 |
+ if err != nil {
|
|
| 511 |
+ return nil, err |
|
| 512 |
+ } |
|
| 513 |
+ p.pr.BuildDefinition.ExternalParameters.Request.CompatibilityVersion = compatibilityVersion |
|
| 514 |
+ |
|
| 509 | 515 |
if p.slsaVersion == provenancetypes.ProvenanceSLSA02 {
|
| 510 | 516 |
return p.pr.ConvertToSLSA02(), nil |
| 511 | 517 |
} |
| ... | ... |
@@ -665,14 +708,26 @@ func getRefProvenance(ref solver.ResultProxy, br *provenanceBridge) (*provenance |
| 665 | 665 |
if !ok {
|
| 666 | 666 |
return nil, errors.Errorf("invalid provenance type %T", p)
|
| 667 | 667 |
} |
| 668 |
+ pr = pr.Clone() |
|
| 668 | 669 |
|
| 669 | 670 |
if br.req != nil {
|
| 670 | 671 |
if pr == nil {
|
| 671 | 672 |
return nil, errors.Errorf("missing provenance for %s", ref.ID())
|
| 672 | 673 |
} |
| 673 | 674 |
|
| 674 |
- pr.Frontend = br.req.Frontend |
|
| 675 |
- pr.Args = provenance.FilterArgs(br.req.FrontendOpt) |
|
| 675 |
+ pr.Request.Frontend = br.req.Frontend |
|
| 676 |
+ pr.Request.Args = provenance.FilterArgs(br.req.FrontendOpt) |
|
| 677 |
+ pr.Request.Inputs = br.inputProvenance(br.req.FrontendInputs) |
|
| 678 |
+ if root := br.rootRequestProvenance(pr.Sources); root != nil {
|
|
| 679 |
+ req := provenance.RequestProvenance(pr.Request.Frontend, pr.Request.Args, pr.Sources) |
|
| 680 |
+ if req.Request == nil {
|
|
| 681 |
+ req.Request = &provenancetypes.Parameters{}
|
|
| 682 |
+ } |
|
| 683 |
+ req.Request.Inputs = pr.Request.Inputs |
|
| 684 |
+ if !root.Equal(req) {
|
|
| 685 |
+ pr.Request.Root = root |
|
| 686 |
+ } |
|
| 687 |
+ } |
|
| 676 | 688 |
// TODO: should also save some output options like compression |
| 677 | 689 |
} |
| 678 | 690 |
|
| ... | ... |
@@ -2,6 +2,7 @@ package provenance |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"cmp" |
| 5 |
+ "maps" |
|
| 5 | 6 |
"slices" |
| 6 | 7 |
|
| 7 | 8 |
distreference "github.com/distribution/reference" |
| ... | ... |
@@ -15,16 +16,36 @@ import ( |
| 15 | 15 |
type Result = result.Result[*Capture] |
| 16 | 16 |
|
| 17 | 17 |
type Capture struct {
|
| 18 |
- Frontend string |
|
| 19 |
- Args map[string]string |
|
| 18 |
+ Request provenancetypes.Parameters |
|
| 20 | 19 |
Sources provenancetypes.Sources |
| 21 |
- Secrets []provenancetypes.Secret |
|
| 22 |
- SSH []provenancetypes.SSH |
|
| 23 | 20 |
NetworkAccess bool |
| 24 | 21 |
IncompleteMaterials bool |
| 25 | 22 |
Samples map[digest.Digest]*resourcestypes.Samples |
| 26 | 23 |
} |
| 27 | 24 |
|
| 25 |
+func (c *Capture) Clone() *Capture {
|
|
| 26 |
+ if c == nil {
|
|
| 27 |
+ return nil |
|
| 28 |
+ } |
|
| 29 |
+ out := &Capture{
|
|
| 30 |
+ NetworkAccess: c.NetworkAccess, |
|
| 31 |
+ IncompleteMaterials: c.IncompleteMaterials, |
|
| 32 |
+ } |
|
| 33 |
+ if req := c.Request.Clone(); req != nil {
|
|
| 34 |
+ out.Request = *req |
|
| 35 |
+ } |
|
| 36 |
+ out.Sources.Images = append(out.Sources.Images, c.Sources.Images...) |
|
| 37 |
+ out.Sources.ImageBlobs = append(out.Sources.ImageBlobs, c.Sources.ImageBlobs...) |
|
| 38 |
+ out.Sources.Local = append(out.Sources.Local, c.Sources.Local...) |
|
| 39 |
+ out.Sources.Git = append(out.Sources.Git, c.Sources.Git...) |
|
| 40 |
+ out.Sources.HTTP = append(out.Sources.HTTP, c.Sources.HTTP...) |
|
| 41 |
+ if len(c.Samples) > 0 {
|
|
| 42 |
+ out.Samples = make(map[digest.Digest]*resourcestypes.Samples, len(c.Samples)) |
|
| 43 |
+ maps.Copy(out.Samples, c.Samples) |
|
| 44 |
+ } |
|
| 45 |
+ return out |
|
| 46 |
+} |
|
| 47 |
+ |
|
| 28 | 48 |
func (c *Capture) Merge(c2 *Capture) error {
|
| 29 | 49 |
if c2 == nil {
|
| 30 | 50 |
return nil |
| ... | ... |
@@ -44,11 +65,15 @@ func (c *Capture) Merge(c2 *Capture) error {
|
| 44 | 44 |
for _, h := range c2.Sources.HTTP {
|
| 45 | 45 |
c.AddHTTP(h) |
| 46 | 46 |
} |
| 47 |
- for _, s := range c2.Secrets {
|
|
| 48 |
- c.AddSecret(s) |
|
| 47 |
+ for _, s := range c2.Request.Secrets {
|
|
| 48 |
+ if s != nil {
|
|
| 49 |
+ c.AddSecret(*s) |
|
| 50 |
+ } |
|
| 49 | 51 |
} |
| 50 |
- for _, s := range c2.SSH {
|
|
| 51 |
- c.AddSSH(s) |
|
| 52 |
+ for _, s := range c2.Request.SSH {
|
|
| 53 |
+ if s != nil {
|
|
| 54 |
+ c.AddSSH(*s) |
|
| 55 |
+ } |
|
| 52 | 56 |
} |
| 53 | 57 |
if c2.NetworkAccess {
|
| 54 | 58 |
c.NetworkAccess = true |
| ... | ... |
@@ -75,10 +100,10 @@ func (c *Capture) Sort() {
|
| 75 | 75 |
slices.SortFunc(c.Sources.HTTP, func(a, b provenancetypes.HTTPSource) int {
|
| 76 | 76 |
return cmp.Compare(a.URL, b.URL) |
| 77 | 77 |
}) |
| 78 |
- slices.SortFunc(c.Secrets, func(a, b provenancetypes.Secret) int {
|
|
| 78 |
+ slices.SortFunc(c.Request.Secrets, func(a, b *provenancetypes.Secret) int {
|
|
| 79 | 79 |
return cmp.Compare(a.ID, b.ID) |
| 80 | 80 |
}) |
| 81 |
- slices.SortFunc(c.SSH, func(a, b provenancetypes.SSH) int {
|
|
| 81 |
+ slices.SortFunc(c.Request.SSH, func(a, b *provenancetypes.SSH) int {
|
|
| 82 | 82 |
return cmp.Compare(a.ID, b.ID) |
| 83 | 83 |
}) |
| 84 | 84 |
} |
| ... | ... |
@@ -152,14 +177,33 @@ func (c *Capture) AddLocal(l provenancetypes.LocalSource) {
|
| 152 | 152 |
|
| 153 | 153 |
func (c *Capture) AddGit(g provenancetypes.GitSource) {
|
| 154 | 154 |
g.URL = urlutil.RedactCredentials(g.URL) |
| 155 |
+ // Dedupe on the tuple (URL, Bundle.URL). Two records with the same |
|
| 156 |
+ // URL but different bundle identity (e.g. the same repo referenced |
|
| 157 |
+ // once normally and once through a bundle, or through two different |
|
| 158 |
+ // bundle locators) must both be preserved so neither material is |
|
| 159 |
+ // silently dropped from the provenance. Bundle.URL is the canonical |
|
| 160 |
+ // bundle identity: since scheme/ref/digest are derived from it, |
|
| 161 |
+ // different URLs imply different bundle identity. |
|
| 155 | 162 |
for _, v := range c.Sources.Git {
|
| 156 |
- if v.URL == g.URL {
|
|
| 163 |
+ if v.URL != g.URL {
|
|
| 164 |
+ continue |
|
| 165 |
+ } |
|
| 166 |
+ if bundleKey(v.Bundle) == bundleKey(g.Bundle) {
|
|
| 157 | 167 |
return |
| 158 | 168 |
} |
| 159 | 169 |
} |
| 160 | 170 |
c.Sources.Git = append(c.Sources.Git, g) |
| 161 | 171 |
} |
| 162 | 172 |
|
| 173 |
+// bundleKey returns a comparable identity for dedupe. Nil bundles collapse |
|
| 174 |
+// to the empty key. |
|
| 175 |
+func bundleKey(b *provenancetypes.GitBundle) string {
|
|
| 176 |
+ if b == nil {
|
|
| 177 |
+ return "" |
|
| 178 |
+ } |
|
| 179 |
+ return b.URL |
|
| 180 |
+} |
|
| 181 |
+ |
|
| 163 | 182 |
func (c *Capture) AddHTTP(h provenancetypes.HTTPSource) {
|
| 164 | 183 |
h.URL = urlutil.RedactCredentials(h.URL) |
| 165 | 184 |
for _, v := range c.Sources.HTTP {
|
| ... | ... |
@@ -171,30 +215,30 @@ func (c *Capture) AddHTTP(h provenancetypes.HTTPSource) {
|
| 171 | 171 |
} |
| 172 | 172 |
|
| 173 | 173 |
func (c *Capture) AddSecret(s provenancetypes.Secret) {
|
| 174 |
- for i, v := range c.Secrets {
|
|
| 174 |
+ for i, v := range c.Request.Secrets {
|
|
| 175 | 175 |
if v.ID == s.ID {
|
| 176 | 176 |
if !s.Optional {
|
| 177 |
- c.Secrets[i].Optional = false |
|
| 177 |
+ c.Request.Secrets[i].Optional = false |
|
| 178 | 178 |
} |
| 179 | 179 |
return |
| 180 | 180 |
} |
| 181 | 181 |
} |
| 182 |
- c.Secrets = append(c.Secrets, s) |
|
| 182 |
+ c.Request.Secrets = append(c.Request.Secrets, &s) |
|
| 183 | 183 |
} |
| 184 | 184 |
|
| 185 | 185 |
func (c *Capture) AddSSH(s provenancetypes.SSH) {
|
| 186 | 186 |
if s.ID == "" {
|
| 187 | 187 |
s.ID = "default" |
| 188 | 188 |
} |
| 189 |
- for i, v := range c.SSH {
|
|
| 189 |
+ for i, v := range c.Request.SSH {
|
|
| 190 | 190 |
if v.ID == s.ID {
|
| 191 | 191 |
if !s.Optional {
|
| 192 |
- c.SSH[i].Optional = false |
|
| 192 |
+ c.Request.SSH[i].Optional = false |
|
| 193 | 193 |
} |
| 194 | 194 |
return |
| 195 | 195 |
} |
| 196 | 196 |
} |
| 197 |
- c.SSH = append(c.SSH, s) |
|
| 197 |
+ c.Request.SSH = append(c.Request.SSH, &s) |
|
| 198 | 198 |
} |
| 199 | 199 |
|
| 200 | 200 |
func (c *Capture) AddSamples(dgst digest.Digest, samples *resourcestypes.Samples) {
|
| ... | ... |
@@ -9,8 +9,10 @@ import ( |
| 9 | 9 |
slsa1 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1" |
| 10 | 10 |
"github.com/moby/buildkit/frontend/dockerfile/dfgitutil" |
| 11 | 11 |
provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types" |
| 12 |
+ srctypes "github.com/moby/buildkit/source/types" |
|
| 12 | 13 |
"github.com/moby/buildkit/util/purl" |
| 13 | 14 |
"github.com/moby/buildkit/util/urlutil" |
| 15 |
+ digest "github.com/opencontainers/go-digest" |
|
| 14 | 16 |
"github.com/package-url/packageurl-go" |
| 15 | 17 |
) |
| 16 | 18 |
|
| ... | ... |
@@ -68,10 +70,55 @@ func slsaMaterials(srcs provenancetypes.Sources) ([]slsa.ProvenanceMaterial, err |
| 68 | 68 |
} |
| 69 | 69 |
|
| 70 | 70 |
for _, s := range srcs.Git {
|
| 71 |
+ // Git material URI is always the raw repository URL (same shape |
|
| 72 |
+ // for bundle-backed and normal git sources). |
|
| 71 | 73 |
out = append(out, slsa.ProvenanceMaterial{
|
| 72 | 74 |
URI: s.URL, |
| 73 | 75 |
Digest: digestSetForCommit(s.Commit), |
| 74 | 76 |
}) |
| 77 |
+ if s.Bundle == nil {
|
|
| 78 |
+ continue |
|
| 79 |
+ } |
|
| 80 |
+ |
|
| 81 |
+ // Bundle-backed git source: parse the canonical locator into |
|
| 82 |
+ // its scheme / ref-body / digest components on demand. On parse |
|
| 83 |
+ // failure (shouldn't happen — validateBundleAttrs rejects bad |
|
| 84 |
+ // locators at LLB op construction time) skip bundle material |
|
| 85 |
+ // emission rather than erroring. |
|
| 86 |
+ scheme, refBody, bundleDgst := parseBundleLocatorURL(s.Bundle.URL) |
|
| 87 |
+ if scheme == "" || refBody == "" || bundleDgst == "" {
|
|
| 88 |
+ continue |
|
| 89 |
+ } |
|
| 90 |
+ |
|
| 91 |
+ bundlePurlType := packageurl.TypeDocker |
|
| 92 |
+ if scheme == srctypes.OCIBlobScheme {
|
|
| 93 |
+ bundlePurlType = packageurl.TypeOCI |
|
| 94 |
+ } |
|
| 95 |
+ bundleURI, err := purl.RefToPURL(bundlePurlType, refBody, nil) |
|
| 96 |
+ if err != nil {
|
|
| 97 |
+ return nil, err |
|
| 98 |
+ } |
|
| 99 |
+ bundleURI, err = setPURLQualifier(bundleURI, packageurl.Qualifier{
|
|
| 100 |
+ Key: "ref_type", |
|
| 101 |
+ Value: "bundle", |
|
| 102 |
+ }) |
|
| 103 |
+ if err != nil {
|
|
| 104 |
+ return nil, err |
|
| 105 |
+ } |
|
| 106 |
+ bundleURI, err = setPURLQualifier(bundleURI, packageurl.Qualifier{
|
|
| 107 |
+ Key: "vcs_url", |
|
| 108 |
+ Value: s.URL, |
|
| 109 |
+ }) |
|
| 110 |
+ if err != nil {
|
|
| 111 |
+ return nil, err |
|
| 112 |
+ } |
|
| 113 |
+ |
|
| 114 |
+ out = append(out, slsa.ProvenanceMaterial{
|
|
| 115 |
+ URI: bundleURI, |
|
| 116 |
+ Digest: slsa.DigestSet{
|
|
| 117 |
+ bundleDgst.Algorithm().String(): bundleDgst.Hex(), |
|
| 118 |
+ }, |
|
| 119 |
+ }) |
|
| 75 | 120 |
} |
| 76 | 121 |
|
| 77 | 122 |
for _, s := range srcs.HTTP {
|
| ... | ... |
@@ -86,6 +133,31 @@ func slsaMaterials(srcs provenancetypes.Sources) ([]slsa.ProvenanceMaterial, err |
| 86 | 86 |
return out, nil |
| 87 | 87 |
} |
| 88 | 88 |
|
| 89 |
+// parseBundleLocatorURL parses a "<scheme>://<ref>@<algo>:<hex>" bundle |
|
| 90 |
+// locator into its components. It returns empty values on parse failure; the |
|
| 91 |
+// caller handles that by falling back to the non-bundle emission path. |
|
| 92 |
+// |
|
| 93 |
+// The identifier package has a stricter parseBundleLocator used at LLB op |
|
| 94 |
+// construction time (validateBundleAttrs) that validates the reference and |
|
| 95 |
+// digest algorithm. We intentionally don't import that helper here because |
|
| 96 |
+// the identifier package imports this provenance package. Since the locator |
|
| 97 |
+// has already been validated upstream, the logic here just splits the |
|
| 98 |
+// locator back into its parts for purl emission. |
|
| 99 |
+func parseBundleLocatorURL(raw string) (scheme, refBody string, dgst digest.Digest) {
|
|
| 100 |
+ const sep = "://" |
|
| 101 |
+ i := strings.Index(raw, sep) |
|
| 102 |
+ if i <= 0 {
|
|
| 103 |
+ return "", "", "" |
|
| 104 |
+ } |
|
| 105 |
+ scheme = raw[:i] |
|
| 106 |
+ body := raw[i+len(sep):] |
|
| 107 |
+ at := strings.LastIndex(body, "@") |
|
| 108 |
+ if at <= 0 {
|
|
| 109 |
+ return "", "", "" |
|
| 110 |
+ } |
|
| 111 |
+ return scheme, body[:at], digest.Digest(body[at+1:]) |
|
| 112 |
+} |
|
| 113 |
+ |
|
| 89 | 114 |
func digestSetForCommit(commit string) slsa.DigestSet {
|
| 90 | 115 |
dset := slsa.DigestSet{}
|
| 91 | 116 |
if len(commit) == 64 {
|
| ... | ... |
@@ -112,11 +184,15 @@ func setPURLQualifier(uri string, q packageurl.Qualifier) (string, error) {
|
| 112 | 112 |
} |
| 113 | 113 |
|
| 114 | 114 |
func findMaterial(srcs provenancetypes.Sources, uri string) (*slsa.ProvenanceMaterial, bool) {
|
| 115 |
- uri, _ = dfgitutil.FragmentFormat(uri) |
|
| 115 |
+ configURI := uri |
|
| 116 |
+ if formatted, ok := dfgitutil.FragmentFormat(uri, true); ok {
|
|
| 117 |
+ configURI = formatted |
|
| 118 |
+ } |
|
| 119 |
+ uri, _ = dfgitutil.FragmentFormat(uri, false) |
|
| 116 | 120 |
for _, s := range srcs.Git {
|
| 117 | 121 |
if s.URL == uri {
|
| 118 | 122 |
return &slsa.ProvenanceMaterial{
|
| 119 |
- URI: s.URL, |
|
| 123 |
+ URI: configURI, |
|
| 120 | 124 |
Digest: digestSetForCommit(s.Commit), |
| 121 | 125 |
}, true |
| 122 | 126 |
} |
| ... | ... |
@@ -147,39 +223,25 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) |
| 147 | 147 |
}) |
| 148 | 148 |
} |
| 149 | 149 |
|
| 150 |
- args := maps.Clone(c.Args) |
|
| 151 |
- |
|
| 152 |
- contextKey := "context" |
|
| 153 |
- if v, ok := args["contextkey"]; ok && v != "" {
|
|
| 154 |
- contextKey = v |
|
| 155 |
- } else if v, ok := c.Args["input:context"]; ok && v != "" {
|
|
| 156 |
- contextKey = "input:context" |
|
| 157 |
- } |
|
| 158 |
- |
|
| 159 | 150 |
ext := provenancetypes.ProvenanceExternalParametersSLSA1{}
|
| 160 |
- if v, ok := args[contextKey]; ok && v != "" {
|
|
| 161 |
- if m, ok := findMaterial(c.Sources, v); ok {
|
|
| 162 |
- ext.ConfigSource.URI = m.URI |
|
| 163 |
- ext.ConfigSource.Digest = m.Digest |
|
| 164 |
- } else {
|
|
| 165 |
- ext.ConfigSource.URI = v |
|
| 166 |
- } |
|
| 167 |
- ext.ConfigSource.URI = urlutil.RedactCredentials(ext.ConfigSource.URI) |
|
| 168 |
- delete(args, contextKey) |
|
| 169 |
- } |
|
| 170 |
- |
|
| 171 |
- if v, ok := args["filename"]; ok && v != "" {
|
|
| 172 |
- ext.ConfigSource.Path = v |
|
| 173 |
- delete(args, "filename") |
|
| 174 |
- } |
|
| 151 |
+ reqProv := RequestProvenance(c.Request.Frontend, maps.Clone(c.Request.Args), c.Sources) |
|
| 152 |
+ ext.ConfigSource = reqProv.ConfigSource |
|
| 175 | 153 |
|
| 176 | 154 |
vcs := make(map[string]string) |
| 177 |
- for k, v := range args {
|
|
| 155 |
+ req := c.Request.Clone() |
|
| 156 |
+ if req == nil {
|
|
| 157 |
+ req = &provenancetypes.Parameters{}
|
|
| 158 |
+ } |
|
| 159 |
+ if reqProv.Request != nil {
|
|
| 160 |
+ req.Frontend = reqProv.Request.Frontend |
|
| 161 |
+ req.Args = reqProv.Request.Args |
|
| 162 |
+ } |
|
| 163 |
+ for k, v := range req.Args {
|
|
| 178 | 164 |
if strings.HasPrefix(k, "vcs:") {
|
| 179 | 165 |
if k == "vcs:source" {
|
| 180 | 166 |
v = urlutil.RedactCredentials(v) |
| 181 | 167 |
} |
| 182 |
- delete(args, k) |
|
| 168 |
+ delete(req.Args, k) |
|
| 183 | 169 |
if v != "" {
|
| 184 | 170 |
vcs[strings.TrimPrefix(k, "vcs:")] = v |
| 185 | 171 |
} |
| ... | ... |
@@ -189,27 +251,12 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) |
| 189 | 189 |
internal := provenancetypes.ProvenanceInternalParametersSLSA1{}
|
| 190 | 190 |
internal.BuilderPlatform = platforms.Format(platforms.Normalize(platforms.DefaultSpec())) |
| 191 | 191 |
|
| 192 |
- req := provenancetypes.Parameters{}
|
|
| 193 |
- req.Frontend = c.Frontend |
|
| 194 |
- req.Args = args |
|
| 195 |
- for _, s := range c.Secrets {
|
|
| 196 |
- req.Secrets = append(req.Secrets, &provenancetypes.Secret{
|
|
| 197 |
- ID: s.ID, |
|
| 198 |
- Optional: s.Optional, |
|
| 199 |
- }) |
|
| 200 |
- } |
|
| 201 |
- for _, s := range c.SSH {
|
|
| 202 |
- req.SSH = append(req.SSH, &provenancetypes.SSH{
|
|
| 203 |
- ID: s.ID, |
|
| 204 |
- Optional: s.Optional, |
|
| 205 |
- }) |
|
| 206 |
- } |
|
| 207 | 192 |
for _, s := range c.Sources.Local {
|
| 208 | 193 |
req.Locals = append(req.Locals, &provenancetypes.LocalSource{
|
| 209 | 194 |
Name: s.Name, |
| 210 | 195 |
}) |
| 211 | 196 |
} |
| 212 |
- ext.Request = req |
|
| 197 |
+ ext.Request = *req |
|
| 213 | 198 |
|
| 214 | 199 |
incompleteMaterials := c.IncompleteMaterials |
| 215 | 200 |
if !incompleteMaterials {
|
| ... | ... |
@@ -230,7 +277,7 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) |
| 230 | 230 |
RunDetails: provenancetypes.ProvenanceRunDetailsSLSA1{
|
| 231 | 231 |
Metadata: &provenancetypes.ProvenanceMetadataSLSA1{
|
| 232 | 232 |
Completeness: provenancetypes.BuildKitComplete{
|
| 233 |
- Request: c.Frontend != "", |
|
| 233 |
+ Request: c.Request.Frontend != "", |
|
| 234 | 234 |
ResolvedDependencies: !incompleteMaterials, |
| 235 | 235 |
}, |
| 236 | 236 |
Hermetic: !incompleteMaterials && !c.NetworkAccess, |
| ... | ... |
@@ -245,6 +292,42 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) |
| 245 | 245 |
return pr, nil |
| 246 | 246 |
} |
| 247 | 247 |
|
| 248 |
+func RequestProvenance(frontend string, args map[string]string, srcs provenancetypes.Sources) *provenancetypes.RequestProvenance {
|
|
| 249 |
+ args = maps.Clone(args) |
|
| 250 |
+ contextKey := "context" |
|
| 251 |
+ if v, ok := args["contextkey"]; ok && v != "" {
|
|
| 252 |
+ contextKey = v |
|
| 253 |
+ } else if v, ok := args["input:context"]; ok && v != "" {
|
|
| 254 |
+ contextKey = "input:context" |
|
| 255 |
+ } |
|
| 256 |
+ |
|
| 257 |
+ ext := provenancetypes.RequestProvenance{
|
|
| 258 |
+ Request: &provenancetypes.Parameters{
|
|
| 259 |
+ Frontend: frontend, |
|
| 260 |
+ Args: args, |
|
| 261 |
+ }, |
|
| 262 |
+ } |
|
| 263 |
+ if v, ok := args[contextKey]; ok && v != "" {
|
|
| 264 |
+ if m, ok := findMaterial(srcs, v); ok {
|
|
| 265 |
+ ext.ConfigSource.URI = m.URI |
|
| 266 |
+ ext.ConfigSource.Digest = m.Digest |
|
| 267 |
+ } else {
|
|
| 268 |
+ ext.ConfigSource.URI = v |
|
| 269 |
+ } |
|
| 270 |
+ ext.ConfigSource.URI = urlutil.RedactCredentials(ext.ConfigSource.URI) |
|
| 271 |
+ delete(args, contextKey) |
|
| 272 |
+ } |
|
| 273 |
+ |
|
| 274 |
+ if v, ok := args["filename"]; ok && v != "" {
|
|
| 275 |
+ ext.ConfigSource.Path = v |
|
| 276 |
+ delete(args, "filename") |
|
| 277 |
+ } |
|
| 278 |
+ if len(args) == 0 {
|
|
| 279 |
+ ext.Request.Args = nil |
|
| 280 |
+ } |
|
| 281 |
+ return &ext |
|
| 282 |
+} |
|
| 283 |
+ |
|
| 248 | 284 |
func FilterArgs(m map[string]string) map[string]string {
|
| 249 | 285 |
var hostSpecificArgs = map[string]struct{}{
|
| 250 | 286 |
"cgroup-parent": {},
|
| ... | ... |
@@ -71,6 +71,19 @@ type ImageBlobSource struct {
|
| 71 | 71 |
type GitSource struct {
|
| 72 | 72 |
URL string |
| 73 | 73 |
Commit string |
| 74 |
+ // Bundle, when non-nil, records the bundle blob that the git source |
|
| 75 |
+ // was resolved from. Only present on bundle-backed git sources; nil |
|
| 76 |
+ // for normal remote-backed git sources. |
|
| 77 |
+ Bundle *GitBundle `json:"bundle,omitempty"` |
|
| 78 |
+} |
|
| 79 |
+ |
|
| 80 |
+// GitBundle describes the bundle blob a git source was resolved from. URL |
|
| 81 |
+// is the full locator (e.g. "docker-image+blob://example.com/repo@sha256:...") |
|
| 82 |
+// and is the canonical data: scheme, reference body, and digest are all |
|
| 83 |
+// derivable from it and are parsed on demand by consumers (such as the purl |
|
| 84 |
+// emitter in predicate.go). |
|
| 85 |
+type GitBundle struct {
|
|
| 86 |
+ URL string `json:"url"` |
|
| 74 | 87 |
} |
| 75 | 88 |
|
| 76 | 89 |
type HTTPSource struct {
|
| ... | ... |
@@ -82,16 +95,40 @@ type LocalSource struct {
|
| 82 | 82 |
Name string `json:"name"` |
| 83 | 83 |
} |
| 84 | 84 |
|
| 85 |
+// Equal reports whether the local source matches another local source. |
|
| 86 |
+func (l *LocalSource) Equal(other *LocalSource) bool {
|
|
| 87 |
+ if l == nil || other == nil {
|
|
| 88 |
+ return l == other |
|
| 89 |
+ } |
|
| 90 |
+ return *l == *other |
|
| 91 |
+} |
|
| 92 |
+ |
|
| 85 | 93 |
type Secret struct {
|
| 86 | 94 |
ID string `json:"id"` |
| 87 | 95 |
Optional bool `json:"optional,omitempty"` |
| 88 | 96 |
} |
| 89 | 97 |
|
| 98 |
+// Equal reports whether the secret matches another secret. |
|
| 99 |
+func (s *Secret) Equal(other *Secret) bool {
|
|
| 100 |
+ if s == nil || other == nil {
|
|
| 101 |
+ return s == other |
|
| 102 |
+ } |
|
| 103 |
+ return *s == *other |
|
| 104 |
+} |
|
| 105 |
+ |
|
| 90 | 106 |
type SSH struct {
|
| 91 | 107 |
ID string `json:"id"` |
| 92 | 108 |
Optional bool `json:"optional,omitempty"` |
| 93 | 109 |
} |
| 94 | 110 |
|
| 111 |
+// Equal reports whether the SSH mount matches another SSH mount. |
|
| 112 |
+func (s *SSH) Equal(other *SSH) bool {
|
|
| 113 |
+ if s == nil || other == nil {
|
|
| 114 |
+ return s == other |
|
| 115 |
+ } |
|
| 116 |
+ return *s == *other |
|
| 117 |
+} |
|
| 118 |
+ |
|
| 95 | 119 |
type Sources struct {
|
| 96 | 120 |
Images []ImageSource |
| 97 | 121 |
ImageBlobs []ImageBlobSource |
| ... | ... |
@@ -157,6 +194,17 @@ type ProvenanceConfigSourceSLSA1 struct {
|
| 157 | 157 |
Path string `json:"path,omitempty"` |
| 158 | 158 |
} |
| 159 | 159 |
|
| 160 |
+// Clone returns a deep copy of the config source. |
|
| 161 |
+func (c ProvenanceConfigSourceSLSA1) Clone() ProvenanceConfigSourceSLSA1 {
|
|
| 162 |
+ c.Digest = maps.Clone(c.Digest) |
|
| 163 |
+ return c |
|
| 164 |
+} |
|
| 165 |
+ |
|
| 166 |
+// Equal reports whether the config source matches another config source. |
|
| 167 |
+func (c ProvenanceConfigSourceSLSA1) Equal(other ProvenanceConfigSourceSLSA1) bool {
|
|
| 168 |
+ return c.URI == other.URI && c.Path == other.Path && maps.Equal(c.Digest, other.Digest) |
|
| 169 |
+} |
|
| 170 |
+ |
|
| 160 | 171 |
type ProvenanceInternalParametersSLSA1 struct {
|
| 161 | 172 |
BuildConfig *BuildConfig `json:"buildConfig,omitempty"` |
| 162 | 173 |
BuilderPlatform string `json:"builderPlatform"` |
| ... | ... |
@@ -172,13 +220,121 @@ type ProvenanceMetadataSLSA1 struct {
|
| 172 | 172 |
} |
| 173 | 173 |
|
| 174 | 174 |
type Parameters struct {
|
| 175 |
- Frontend string `json:"frontend,omitempty"` |
|
| 176 |
- Args map[string]string `json:"args,omitempty"` |
|
| 177 |
- Secrets []*Secret `json:"secrets,omitempty"` |
|
| 178 |
- SSH []*SSH `json:"ssh,omitempty"` |
|
| 179 |
- Locals []*LocalSource `json:"locals,omitempty"` |
|
| 175 |
+ Frontend string `json:"frontend,omitempty"` |
|
| 176 |
+ Args map[string]string `json:"args,omitempty"` |
|
| 177 |
+ Secrets []*Secret `json:"secrets,omitempty"` |
|
| 178 |
+ SSH []*SSH `json:"ssh,omitempty"` |
|
| 179 |
+ Locals []*LocalSource `json:"locals,omitempty"` |
|
| 180 |
+ Inputs map[string]*RequestProvenance `json:"inputs,omitempty"` |
|
| 181 |
+ Root *RequestProvenance `json:"root,omitempty"` |
|
| 182 |
+ CompatibilityVersion int `json:"compatibilityVersion,omitempty"` |
|
| 180 | 183 |
// TODO: select export attributes |
| 181 |
- // TODO: frontend inputs |
|
| 184 |
+} |
|
| 185 |
+ |
|
| 186 |
+// Clone returns a deep copy of the request parameters. |
|
| 187 |
+func (p *Parameters) Clone() *Parameters {
|
|
| 188 |
+ if p == nil {
|
|
| 189 |
+ return nil |
|
| 190 |
+ } |
|
| 191 |
+ out := &Parameters{
|
|
| 192 |
+ Frontend: p.Frontend, |
|
| 193 |
+ Args: maps.Clone(p.Args), |
|
| 194 |
+ CompatibilityVersion: p.CompatibilityVersion, |
|
| 195 |
+ } |
|
| 196 |
+ if len(p.Secrets) > 0 {
|
|
| 197 |
+ out.Secrets = slices.Clone(p.Secrets) |
|
| 198 |
+ for i, s := range out.Secrets {
|
|
| 199 |
+ if s != nil {
|
|
| 200 |
+ s2 := *s |
|
| 201 |
+ out.Secrets[i] = &s2 |
|
| 202 |
+ } |
|
| 203 |
+ } |
|
| 204 |
+ } |
|
| 205 |
+ if len(p.SSH) > 0 {
|
|
| 206 |
+ out.SSH = slices.Clone(p.SSH) |
|
| 207 |
+ for i, s := range out.SSH {
|
|
| 208 |
+ if s != nil {
|
|
| 209 |
+ s2 := *s |
|
| 210 |
+ out.SSH[i] = &s2 |
|
| 211 |
+ } |
|
| 212 |
+ } |
|
| 213 |
+ } |
|
| 214 |
+ if len(p.Locals) > 0 {
|
|
| 215 |
+ out.Locals = slices.Clone(p.Locals) |
|
| 216 |
+ for i, l := range out.Locals {
|
|
| 217 |
+ if l != nil {
|
|
| 218 |
+ l2 := *l |
|
| 219 |
+ out.Locals[i] = &l2 |
|
| 220 |
+ } |
|
| 221 |
+ } |
|
| 222 |
+ } |
|
| 223 |
+ if len(p.Inputs) > 0 {
|
|
| 224 |
+ out.Inputs = make(map[string]*RequestProvenance, len(p.Inputs)) |
|
| 225 |
+ for k, in := range p.Inputs {
|
|
| 226 |
+ out.Inputs[k] = in.Clone() |
|
| 227 |
+ } |
|
| 228 |
+ } |
|
| 229 |
+ out.Root = p.Root.Clone() |
|
| 230 |
+ return out |
|
| 231 |
+} |
|
| 232 |
+ |
|
| 233 |
+// Equal reports whether the request parameters match another set of parameters. |
|
| 234 |
+func (p *Parameters) Equal(other *Parameters) bool {
|
|
| 235 |
+ if p == nil || other == nil {
|
|
| 236 |
+ return p == other |
|
| 237 |
+ } |
|
| 238 |
+ if p.Frontend != other.Frontend || p.CompatibilityVersion != other.CompatibilityVersion || !maps.Equal(p.Args, other.Args) {
|
|
| 239 |
+ return false |
|
| 240 |
+ } |
|
| 241 |
+ if !slices.EqualFunc(p.Secrets, other.Secrets, func(a, b *Secret) bool {
|
|
| 242 |
+ return a.Equal(b) |
|
| 243 |
+ }) {
|
|
| 244 |
+ return false |
|
| 245 |
+ } |
|
| 246 |
+ if !slices.EqualFunc(p.SSH, other.SSH, func(a, b *SSH) bool {
|
|
| 247 |
+ return a.Equal(b) |
|
| 248 |
+ }) {
|
|
| 249 |
+ return false |
|
| 250 |
+ } |
|
| 251 |
+ if !slices.EqualFunc(p.Locals, other.Locals, func(a, b *LocalSource) bool {
|
|
| 252 |
+ return a.Equal(b) |
|
| 253 |
+ }) {
|
|
| 254 |
+ return false |
|
| 255 |
+ } |
|
| 256 |
+ if len(p.Inputs) != len(other.Inputs) {
|
|
| 257 |
+ return false |
|
| 258 |
+ } |
|
| 259 |
+ for k, v := range p.Inputs {
|
|
| 260 |
+ v2, ok := other.Inputs[k] |
|
| 261 |
+ if !ok || !v.Equal(v2) {
|
|
| 262 |
+ return false |
|
| 263 |
+ } |
|
| 264 |
+ } |
|
| 265 |
+ return p.Root.Equal(other.Root) |
|
| 266 |
+} |
|
| 267 |
+ |
|
| 268 |
+type RequestProvenance struct {
|
|
| 269 |
+ ConfigSource ProvenanceConfigSourceSLSA1 `json:"configSource"` |
|
| 270 |
+ Request *Parameters `json:"request,omitempty"` |
|
| 271 |
+} |
|
| 272 |
+ |
|
| 273 |
+// Clone returns a deep copy of the request provenance. |
|
| 274 |
+func (r *RequestProvenance) Clone() *RequestProvenance {
|
|
| 275 |
+ if r == nil {
|
|
| 276 |
+ return nil |
|
| 277 |
+ } |
|
| 278 |
+ return &RequestProvenance{
|
|
| 279 |
+ ConfigSource: r.ConfigSource.Clone(), |
|
| 280 |
+ Request: r.Request.Clone(), |
|
| 281 |
+ } |
|
| 282 |
+} |
|
| 283 |
+ |
|
| 284 |
+// Equal reports whether the request provenance matches another request provenance. |
|
| 285 |
+func (r *RequestProvenance) Equal(other *RequestProvenance) bool {
|
|
| 286 |
+ if r == nil || other == nil {
|
|
| 287 |
+ return r == other |
|
| 288 |
+ } |
|
| 289 |
+ return r.ConfigSource.Equal(other.ConfigSource) && r.Request.Equal(other.Request) |
|
| 182 | 290 |
} |
| 183 | 291 |
|
| 184 | 292 |
type Environment struct {
|
| 185 | 293 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,247 @@ |
| 0 |
+package llbsolver |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "slices" |
|
| 4 |
+ "sync" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/moby/buildkit/frontend" |
|
| 7 |
+ "github.com/moby/buildkit/identity" |
|
| 8 |
+ "github.com/moby/buildkit/solver" |
|
| 9 |
+ "github.com/moby/buildkit/solver/llbsolver/provenance" |
|
| 10 |
+ provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types" |
|
| 11 |
+ "github.com/moby/buildkit/solver/pb" |
|
| 12 |
+ digest "github.com/opencontainers/go-digest" |
|
| 13 |
+ "github.com/pkg/errors" |
|
| 14 |
+) |
|
| 15 |
+ |
|
| 16 |
+type provenanceStore struct {
|
|
| 17 |
+ mu sync.Mutex |
|
| 18 |
+ records map[string]*provenanceRecord |
|
| 19 |
+ byDigest map[digest.Digest]map[string]struct{}
|
|
| 20 |
+} |
|
| 21 |
+ |
|
| 22 |
+type provenanceRecord struct {
|
|
| 23 |
+ defDigest digest.Digest |
|
| 24 |
+ request *provenancetypes.RequestProvenance |
|
| 25 |
+} |
|
| 26 |
+ |
|
| 27 |
+func newProvenanceStore() *provenanceStore {
|
|
| 28 |
+ return &provenanceStore{
|
|
| 29 |
+ records: map[string]*provenanceRecord{},
|
|
| 30 |
+ byDigest: map[digest.Digest]map[string]struct{}{},
|
|
| 31 |
+ } |
|
| 32 |
+} |
|
| 33 |
+ |
|
| 34 |
+func (s *provenanceStore) register(def *pb.Definition, req *provenancetypes.RequestProvenance) (string, digest.Digest, error) {
|
|
| 35 |
+ if s == nil || def == nil || req == nil {
|
|
| 36 |
+ return "", "", nil |
|
| 37 |
+ } |
|
| 38 |
+ dgst, err := definitionHeadDigest(def) |
|
| 39 |
+ if err != nil {
|
|
| 40 |
+ return "", "", err |
|
| 41 |
+ } |
|
| 42 |
+ if dgst == "" {
|
|
| 43 |
+ return "", "", nil |
|
| 44 |
+ } |
|
| 45 |
+ recordID := identity.NewID() |
|
| 46 |
+ s.mu.Lock() |
|
| 47 |
+ s.records[recordID] = &provenanceRecord{
|
|
| 48 |
+ defDigest: dgst, |
|
| 49 |
+ request: req.Clone(), |
|
| 50 |
+ } |
|
| 51 |
+ if s.byDigest[dgst] == nil {
|
|
| 52 |
+ s.byDigest[dgst] = map[string]struct{}{}
|
|
| 53 |
+ } |
|
| 54 |
+ s.byDigest[dgst][recordID] = struct{}{}
|
|
| 55 |
+ s.mu.Unlock() |
|
| 56 |
+ return recordID, dgst, nil |
|
| 57 |
+} |
|
| 58 |
+ |
|
| 59 |
+func (s *provenanceStore) unregister(recordIDs []string) {
|
|
| 60 |
+ if s == nil || len(recordIDs) == 0 {
|
|
| 61 |
+ return |
|
| 62 |
+ } |
|
| 63 |
+ s.mu.Lock() |
|
| 64 |
+ for _, recordID := range recordIDs {
|
|
| 65 |
+ rec := s.records[recordID] |
|
| 66 |
+ delete(s.records, recordID) |
|
| 67 |
+ if rec != nil {
|
|
| 68 |
+ delete(s.byDigest[rec.defDigest], recordID) |
|
| 69 |
+ if len(s.byDigest[rec.defDigest]) == 0 {
|
|
| 70 |
+ delete(s.byDigest, rec.defDigest) |
|
| 71 |
+ } |
|
| 72 |
+ } |
|
| 73 |
+ } |
|
| 74 |
+ s.mu.Unlock() |
|
| 75 |
+} |
|
| 76 |
+ |
|
| 77 |
+func (s *provenanceStore) lookup(def *pb.Definition) (*provenancetypes.RequestProvenance, bool) {
|
|
| 78 |
+ if s == nil || def == nil {
|
|
| 79 |
+ return nil, false |
|
| 80 |
+ } |
|
| 81 |
+ dgst, err := definitionHeadDigest(def) |
|
| 82 |
+ if err != nil || dgst == "" {
|
|
| 83 |
+ return nil, false |
|
| 84 |
+ } |
|
| 85 |
+ s.mu.Lock() |
|
| 86 |
+ records := make([]*provenanceRecord, 0, len(s.byDigest[dgst])) |
|
| 87 |
+ for recordID := range s.byDigest[dgst] {
|
|
| 88 |
+ if rec := s.records[recordID]; rec != nil && rec.request != nil {
|
|
| 89 |
+ records = append(records, rec) |
|
| 90 |
+ } |
|
| 91 |
+ } |
|
| 92 |
+ s.mu.Unlock() |
|
| 93 |
+ if len(records) == 0 {
|
|
| 94 |
+ return nil, false |
|
| 95 |
+ } |
|
| 96 |
+ req := records[0].request.Clone() |
|
| 97 |
+ if req.Request != nil {
|
|
| 98 |
+ req.Request.Root = nil |
|
| 99 |
+ } |
|
| 100 |
+ for _, rec := range records[1:] {
|
|
| 101 |
+ req2 := rec.request.Clone() |
|
| 102 |
+ if req2.Request != nil {
|
|
| 103 |
+ req2.Request.Root = nil |
|
| 104 |
+ } |
|
| 105 |
+ if !req.Equal(req2) {
|
|
| 106 |
+ return nil, false |
|
| 107 |
+ } |
|
| 108 |
+ } |
|
| 109 |
+ return req, true |
|
| 110 |
+} |
|
| 111 |
+ |
|
| 112 |
+func (b *provenanceBridge) registerProvenanceRefs(res *frontend.Result) error {
|
|
| 113 |
+ if b == nil || res == nil {
|
|
| 114 |
+ return nil |
|
| 115 |
+ } |
|
| 116 |
+ return res.EachRef(func(ref solver.ResultProxy) error {
|
|
| 117 |
+ if ref == nil {
|
|
| 118 |
+ return nil |
|
| 119 |
+ } |
|
| 120 |
+ return b.registerProvenanceRef(ref.Definition(), ref) |
|
| 121 |
+ }) |
|
| 122 |
+} |
|
| 123 |
+ |
|
| 124 |
+func (b *provenanceBridge) registerProvenanceRef(def *pb.Definition, ref solver.ResultProxy) error {
|
|
| 125 |
+ if def == nil || ref == nil || b.provenanceStore == nil {
|
|
| 126 |
+ return nil |
|
| 127 |
+ } |
|
| 128 |
+ req, ok := b.findByResult(ref) |
|
| 129 |
+ if !ok {
|
|
| 130 |
+ return nil |
|
| 131 |
+ } |
|
| 132 |
+ var srcs provenancetypes.Sources |
|
| 133 |
+ if p := ref.Provenance(); p != nil {
|
|
| 134 |
+ c, ok := p.(*provenance.Capture) |
|
| 135 |
+ if !ok {
|
|
| 136 |
+ return errors.Errorf("invalid provenance type %T", p)
|
|
| 137 |
+ } |
|
| 138 |
+ if c != nil {
|
|
| 139 |
+ srcs = c.Sources |
|
| 140 |
+ } |
|
| 141 |
+ } |
|
| 142 |
+ reqProv := req.bridge.requestProvenance(srcs) |
|
| 143 |
+ recordID, _, err := b.provenanceStore.register(def, reqProv) |
|
| 144 |
+ if err != nil {
|
|
| 145 |
+ return err |
|
| 146 |
+ } |
|
| 147 |
+ if recordID == "" {
|
|
| 148 |
+ return nil |
|
| 149 |
+ } |
|
| 150 |
+ b.addProvenanceRefRecordID(recordID) |
|
| 151 |
+ return nil |
|
| 152 |
+} |
|
| 153 |
+ |
|
| 154 |
+func (b *provenanceBridge) addProvenanceRefRecordID(recordID string) {
|
|
| 155 |
+ if b == nil || recordID == "" {
|
|
| 156 |
+ return |
|
| 157 |
+ } |
|
| 158 |
+ b.mu.Lock() |
|
| 159 |
+ b.provenanceRefRecordIDs = append(b.provenanceRefRecordIDs, recordID) |
|
| 160 |
+ b.mu.Unlock() |
|
| 161 |
+} |
|
| 162 |
+ |
|
| 163 |
+func (b *provenanceBridge) releaseProvenanceRefs() {
|
|
| 164 |
+ if b == nil || b.provenanceStore == nil {
|
|
| 165 |
+ return |
|
| 166 |
+ } |
|
| 167 |
+ for _, sb := range b.subBridges {
|
|
| 168 |
+ sb.releaseProvenanceRefs() |
|
| 169 |
+ } |
|
| 170 |
+ b.mu.Lock() |
|
| 171 |
+ recordIDs := slices.Clone(b.provenanceRefRecordIDs) |
|
| 172 |
+ b.provenanceRefRecordIDs = nil |
|
| 173 |
+ b.mu.Unlock() |
|
| 174 |
+ b.provenanceStore.unregister(recordIDs) |
|
| 175 |
+} |
|
| 176 |
+ |
|
| 177 |
+func (b *provenanceBridge) requestProvenance(srcs provenancetypes.Sources) *provenancetypes.RequestProvenance {
|
|
| 178 |
+ if b == nil || b.req == nil {
|
|
| 179 |
+ return nil |
|
| 180 |
+ } |
|
| 181 |
+ req := provenance.RequestProvenance(b.req.Frontend, provenance.FilterArgs(b.req.FrontendOpt), srcs) |
|
| 182 |
+ if req.Request == nil {
|
|
| 183 |
+ req.Request = &provenancetypes.Parameters{}
|
|
| 184 |
+ } |
|
| 185 |
+ if inputs := b.inputProvenance(b.req.FrontendInputs); len(inputs) > 0 {
|
|
| 186 |
+ req.Request.Inputs = inputs |
|
| 187 |
+ } |
|
| 188 |
+ if root := b.rootRequestProvenance(srcs); root != nil && !root.Equal(req) {
|
|
| 189 |
+ req.Request.Root = root |
|
| 190 |
+ } |
|
| 191 |
+ return req |
|
| 192 |
+} |
|
| 193 |
+ |
|
| 194 |
+func (b *provenanceBridge) inputProvenance(inputs map[string]*pb.Definition) map[string]*provenancetypes.RequestProvenance {
|
|
| 195 |
+ if len(inputs) == 0 || b == nil || b.provenanceStore == nil {
|
|
| 196 |
+ return nil |
|
| 197 |
+ } |
|
| 198 |
+ out := make(map[string]*provenancetypes.RequestProvenance) |
|
| 199 |
+ for name, def := range inputs {
|
|
| 200 |
+ if in, ok := b.provenanceStore.lookup(def); ok {
|
|
| 201 |
+ out[name] = in |
|
| 202 |
+ } |
|
| 203 |
+ } |
|
| 204 |
+ if len(out) == 0 {
|
|
| 205 |
+ return nil |
|
| 206 |
+ } |
|
| 207 |
+ return out |
|
| 208 |
+} |
|
| 209 |
+ |
|
| 210 |
+func (b *provenanceBridge) rootRequestProvenance(srcs provenancetypes.Sources) *provenancetypes.RequestProvenance {
|
|
| 211 |
+ if b == nil || !hasRequestProvenance(b.rootReq) {
|
|
| 212 |
+ return nil |
|
| 213 |
+ } |
|
| 214 |
+ root := provenance.RequestProvenance(b.rootReq.Frontend, provenance.FilterArgs(b.rootReq.FrontendOpt), srcs) |
|
| 215 |
+ if root.Request == nil {
|
|
| 216 |
+ root.Request = &provenancetypes.Parameters{}
|
|
| 217 |
+ } |
|
| 218 |
+ if inputs := b.inputProvenance(b.rootReq.FrontendInputs); len(inputs) > 0 {
|
|
| 219 |
+ root.Request.Inputs = inputs |
|
| 220 |
+ } |
|
| 221 |
+ if root.Request.Frontend == "" && len(root.Request.Args) == 0 && len(root.Request.Inputs) == 0 {
|
|
| 222 |
+ return nil |
|
| 223 |
+ } |
|
| 224 |
+ return root |
|
| 225 |
+} |
|
| 226 |
+ |
|
| 227 |
+func hasRequestProvenance(req *frontend.SolveRequest) bool {
|
|
| 228 |
+ if req == nil {
|
|
| 229 |
+ return false |
|
| 230 |
+ } |
|
| 231 |
+ return req.Frontend != "" || len(req.FrontendOpt) > 0 || len(req.FrontendInputs) > 0 |
|
| 232 |
+} |
|
| 233 |
+ |
|
| 234 |
+func definitionHeadDigest(def *pb.Definition) (digest.Digest, error) {
|
|
| 235 |
+ if def == nil || len(def.Def) == 0 {
|
|
| 236 |
+ return "", nil |
|
| 237 |
+ } |
|
| 238 |
+ var op pb.Op |
|
| 239 |
+ if err := op.UnmarshalVT(def.Def[len(def.Def)-1]); err != nil {
|
|
| 240 |
+ return "", errors.Wrap(err, "failed to parse llb proto op") |
|
| 241 |
+ } |
|
| 242 |
+ if len(op.Inputs) == 0 {
|
|
| 243 |
+ return "", nil |
|
| 244 |
+ } |
|
| 245 |
+ return digest.Digest(op.Inputs[0].Digest), nil |
|
| 246 |
+} |
| ... | ... |
@@ -2,6 +2,7 @@ package llbsolver |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 |
+ "maps" |
|
| 5 | 6 |
"os" |
| 6 | 7 |
"strings" |
| 7 | 8 |
"time" |
| ... | ... |
@@ -19,6 +20,7 @@ import ( |
| 19 | 19 |
"github.com/moby/buildkit/identity" |
| 20 | 20 |
"github.com/moby/buildkit/session" |
| 21 | 21 |
"github.com/moby/buildkit/solver" |
| 22 |
+ "github.com/moby/buildkit/solver/llbsolver/compat" |
|
| 22 | 23 |
"github.com/moby/buildkit/solver/llbsolver/history" |
| 23 | 24 |
"github.com/moby/buildkit/solver/result" |
| 24 | 25 |
spb "github.com/moby/buildkit/sourcepolicy/pb" |
| ... | ... |
@@ -73,6 +75,7 @@ type Solver struct {
|
| 73 | 73 |
history *history.Queue |
| 74 | 74 |
sysSampler *resources.Sampler[*resourcestypes.SysSample] |
| 75 | 75 |
provenanceEnv map[string]any |
| 76 |
+ provenanceStore *provenanceStore |
|
| 76 | 77 |
} |
| 77 | 78 |
|
| 78 | 79 |
// Processor defines a processing function to be applied after solving, but |
| ... | ... |
@@ -103,6 +106,7 @@ func New(opt Opt) (*Solver, error) {
|
| 103 | 103 |
entitlements: opt.Entitlements, |
| 104 | 104 |
history: opt.HistoryQueue, |
| 105 | 105 |
provenanceEnv: opt.ProvenanceEnv, |
| 106 |
+ provenanceStore: newProvenanceStore(), |
|
| 106 | 107 |
} |
| 107 | 108 |
|
| 108 | 109 |
sampler, err := resources.NewSysSampler() |
| ... | ... |
@@ -145,6 +149,7 @@ func (s *Solver) bridge(b solver.Builder) *provenanceBridge {
|
| 145 | 145 |
resolveCacheImporterFuncs: s.resolveCacheImporterFuncs, |
| 146 | 146 |
cms: map[string]solver.CacheManager{},
|
| 147 | 147 |
sm: s.sm, |
| 148 |
+ provenanceStore: s.provenanceStore, |
|
| 148 | 149 |
}} |
| 149 | 150 |
} |
| 150 | 151 |
|
| ... | ... |
@@ -152,7 +157,21 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge {
|
| 152 | 152 |
return s.bridge(b) |
| 153 | 153 |
} |
| 154 | 154 |
|
| 155 |
-func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string) (_ *client.SolveResponse, err error) {
|
|
| 155 |
+func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, compatibilityVersion int, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string) (_ *client.SolveResponse, err error) {
|
|
| 156 |
+ hasNamedDockerfileContext := false |
|
| 157 |
+ for k := range req.FrontendOpt {
|
|
| 158 |
+ if k == "context:dockerfile.v0" || strings.HasPrefix(k, "context:dockerfile.v0::") {
|
|
| 159 |
+ hasNamedDockerfileContext = true |
|
| 160 |
+ break |
|
| 161 |
+ } |
|
| 162 |
+ } |
|
| 163 |
+ if req.Frontend == "gateway.v0" && req.FrontendOpt[frontend.KeySource] == "dockerfile.v0" && !hasNamedDockerfileContext {
|
|
| 164 |
+ frontendOpt := maps.Clone(req.FrontendOpt) |
|
| 165 |
+ delete(frontendOpt, frontend.KeySource) |
|
| 166 |
+ req.Frontend = "dockerfile.v0" |
|
| 167 |
+ req.FrontendOpt = frontendOpt |
|
| 168 |
+ } |
|
| 169 |
+ |
|
| 156 | 170 |
j, err := s.solver.NewJob(id) |
| 157 | 171 |
if err != nil {
|
| 158 | 172 |
return nil, err |
| ... | ... |
@@ -201,10 +220,17 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro |
| 201 | 201 |
if policySession != "" {
|
| 202 | 202 |
j.SetValue(keySourcePolicySession, policySession) |
| 203 | 203 |
} |
| 204 |
+ if compatibilityVersion == 0 {
|
|
| 205 |
+ compatibilityVersion = compat.CompatibilityVersionCurrent |
|
| 206 |
+ } |
|
| 207 |
+ j.SetValue(compat.JobValueKey, compatibilityVersion) |
|
| 204 | 208 |
|
| 205 | 209 |
j.SessionID = sessionID |
| 206 | 210 |
|
| 207 | 211 |
br := s.bridge(j) |
| 212 |
+ defer br.releaseProvenanceRefs() |
|
| 213 |
+ rootReq := req.Clone() |
|
| 214 |
+ br.rootReq = &rootReq |
|
| 208 | 215 |
var fwd gateway.LLBBridgeForwarder |
| 209 | 216 |
if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" {
|
| 210 | 217 |
fwd = gateway.NewBridgeForwarder(ctx, br, br, s.workerController.Infos(), req.FrontendInputs, sessionID, s.sm) |
| ... | ... |
@@ -213,10 +239,8 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro |
| 213 | 213 |
// s.recordBuildHistory can block for several seconds on |
| 214 | 214 |
// LeaseManager calls, and there is a fixed 3s timeout in |
| 215 | 215 |
// GatewayForwarder on build registration. |
| 216 |
- if err := s.gatewayForwarder.RegisterBuild(ctx, id, fwd); err != nil {
|
|
| 217 |
- return nil, err |
|
| 218 |
- } |
|
| 219 |
- defer s.gatewayForwarder.UnregisterBuild(context.WithoutCancel(ctx), id) |
|
| 216 |
+ s.gatewayForwarder.RegisterBuild(ctx, id, fwd) |
|
| 217 |
+ defer s.gatewayForwarder.UnregisterBuild(context.Background(), id) |
|
| 220 | 218 |
} |
| 221 | 219 |
|
| 222 | 220 |
if !internal {
|
| ... | ... |
@@ -9,6 +9,9 @@ const AttrMountSSHSock = "git.mountsshsock" |
| 9 | 9 |
const AttrGitChecksum = "git.checksum" |
| 10 | 10 |
const AttrGitSkipSubmodules = "git.skipsubmodules" |
| 11 | 11 |
const AttrGitMTime = "git.mtime" |
| 12 |
+const AttrGitFetchByCommit = "git.fetchbycommit" |
|
| 13 |
+const AttrGitBundle = "git.bundle" |
|
| 14 |
+const AttrGitCheckoutBundle = "git.checkoutbundle" |
|
| 12 | 15 |
|
| 13 | 16 |
const AttrGitSignatureVerifyPubKey = "git.sig.pubkey" |
| 14 | 17 |
const AttrGitSignatureVerifyRejectExpired = "git.sig.rejectexpired" |
| ... | ... |
@@ -35,6 +35,9 @@ const ( |
| 35 | 35 |
CapSourceGitSkipSubmodules apicaps.CapID = "source.git.skipsubmodules" |
| 36 | 36 |
CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify" |
| 37 | 37 |
CapSourceGitMTime apicaps.CapID = "source.git.mtime" |
| 38 |
+ CapSourceGitFetchByCommit apicaps.CapID = "source.git.fetchbycommit" |
|
| 39 |
+ CapSourceGitBundle apicaps.CapID = "source.git.bundle" |
|
| 40 |
+ CapSourceGitCheckoutBundle apicaps.CapID = "source.git.checkoutbundle" |
|
| 38 | 41 |
|
| 39 | 42 |
CapSourceHTTP apicaps.CapID = "source.http" |
| 40 | 43 |
CapSourceHTTPAuth apicaps.CapID = "source.http.auth" |
| ... | ... |
@@ -263,6 +266,24 @@ func init() {
|
| 263 | 263 |
}) |
| 264 | 264 |
|
| 265 | 265 |
Caps.Init(apicaps.Cap{
|
| 266 |
+ ID: CapSourceGitFetchByCommit, |
|
| 267 |
+ Enabled: true, |
|
| 268 |
+ Status: apicaps.CapStatusExperimental, |
|
| 269 |
+ }) |
|
| 270 |
+ |
|
| 271 |
+ Caps.Init(apicaps.Cap{
|
|
| 272 |
+ ID: CapSourceGitBundle, |
|
| 273 |
+ Enabled: true, |
|
| 274 |
+ Status: apicaps.CapStatusExperimental, |
|
| 275 |
+ }) |
|
| 276 |
+ |
|
| 277 |
+ Caps.Init(apicaps.Cap{
|
|
| 278 |
+ ID: CapSourceGitCheckoutBundle, |
|
| 279 |
+ Enabled: true, |
|
| 280 |
+ Status: apicaps.CapStatusExperimental, |
|
| 281 |
+ }) |
|
| 282 |
+ |
|
| 283 |
+ Caps.Init(apicaps.Cap{
|
|
| 266 | 284 |
ID: CapSourceHTTP, |
| 267 | 285 |
Enabled: true, |
| 268 | 286 |
Status: apicaps.CapStatusExperimental, |
| ... | ... |
@@ -184,6 +184,8 @@ type JobContext interface {
|
| 184 | 184 |
// ResolverCache returns object for memorizing/synchronizing remote resolving decisions during the job. |
| 185 | 185 |
// Steps from same build job will share the same resolver cache. |
| 186 | 186 |
ResolverCache() ResolverCache |
| 187 |
+ // CompatibilityVersion returns the solve-wide compatibility version for the current job. |
|
| 188 |
+ CompatibilityVersion() (int, error) |
|
| 187 | 189 |
} |
| 188 | 190 |
|
| 189 | 191 |
type ResolverCache interface {
|
| 190 | 192 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,128 @@ |
| 0 |
+// Package blobfetch provides a shared helper for resolving a single |
|
| 1 |
+// content-addressable blob from either a docker-image registry or an OCI layout |
|
| 2 |
+// content store, identified by its digest. It is factored out of the |
|
| 3 |
+// containerblob source so other consumers (for example the git-bundle flow on |
|
| 4 |
+// the git source) can reuse the same resolution logic without depending on the |
|
| 5 |
+// source implementation. |
|
| 6 |
+package blobfetch |
|
| 7 |
+ |
|
| 8 |
+import ( |
|
| 9 |
+ "context" |
|
| 10 |
+ "io" |
|
| 11 |
+ "time" |
|
| 12 |
+ |
|
| 13 |
+ "github.com/containerd/containerd/v2/core/remotes" |
|
| 14 |
+ "github.com/containerd/containerd/v2/core/remotes/docker" |
|
| 15 |
+ "github.com/moby/buildkit/session" |
|
| 16 |
+ sessioncontent "github.com/moby/buildkit/session/content" |
|
| 17 |
+ srctypes "github.com/moby/buildkit/source/types" |
|
| 18 |
+ "github.com/moby/buildkit/util/iohelper" |
|
| 19 |
+ "github.com/moby/buildkit/util/resolver" |
|
| 20 |
+ digest "github.com/opencontainers/go-digest" |
|
| 21 |
+ ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
|
| 22 |
+ "github.com/pkg/errors" |
|
| 23 |
+) |
|
| 24 |
+ |
|
| 25 |
+// FetchOpt describes how to resolve a blob. |
|
| 26 |
+type FetchOpt struct {
|
|
| 27 |
+ // Scheme identifies the source kind. Must be one of |
|
| 28 |
+ // srctypes.DockerImageBlobScheme or srctypes.OCIBlobScheme. |
|
| 29 |
+ Scheme string |
|
| 30 |
+ // Ref is the image-ref form of the blob, i.e. "<name>@<digest>" for |
|
| 31 |
+ // registry blobs, or the reference body accepted by the containerd |
|
| 32 |
+ // reference parser for OCI-layout blobs. |
|
| 33 |
+ Ref string |
|
| 34 |
+ // Digest is the blob digest to retrieve. |
|
| 35 |
+ Digest digest.Digest |
|
| 36 |
+ // RegistryHosts is used when Scheme is DockerImageBlobScheme. |
|
| 37 |
+ RegistryHosts docker.RegistryHosts |
|
| 38 |
+ // SessionManager is used to reach the client when Scheme is |
|
| 39 |
+ // OCIBlobScheme. |
|
| 40 |
+ SessionManager *session.Manager |
|
| 41 |
+ // SessionID, if set, pins the OCI-layout fetch to a specific client |
|
| 42 |
+ // session. |
|
| 43 |
+ SessionID string |
|
| 44 |
+ // StoreID is the client-side OCI-layout store name. Required for |
|
| 45 |
+ // Scheme == OCIBlobScheme. |
|
| 46 |
+ StoreID string |
|
| 47 |
+} |
|
| 48 |
+ |
|
| 49 |
+// FetchBlob opens a read stream for the requested blob. The caller owns the |
|
| 50 |
+// returned ReadCloser and must Close it. The returned digest is the blob |
|
| 51 |
+// digest from the locator (echoed for convenience). |
|
| 52 |
+func FetchBlob(ctx context.Context, g session.Group, opt FetchOpt) (io.ReadCloser, digest.Digest, error) {
|
|
| 53 |
+ if err := opt.Digest.Validate(); err != nil {
|
|
| 54 |
+ return nil, "", errors.Wrap(err, "invalid blob digest") |
|
| 55 |
+ } |
|
| 56 |
+ switch opt.Scheme {
|
|
| 57 |
+ case srctypes.OCIBlobScheme: |
|
| 58 |
+ if opt.StoreID == "" {
|
|
| 59 |
+ return nil, "", errors.Errorf("oci-layout blob source requires store id")
|
|
| 60 |
+ } |
|
| 61 |
+ rc, err := fetchFromOCILayoutStore(ctx, g, opt) |
|
| 62 |
+ if err != nil {
|
|
| 63 |
+ return nil, "", err |
|
| 64 |
+ } |
|
| 65 |
+ return rc, opt.Digest, nil |
|
| 66 |
+ case srctypes.DockerImageBlobScheme: |
|
| 67 |
+ r := resolver.DefaultPool.GetResolver(opt.RegistryHosts, opt.Ref, resolver.ScopeType{}, opt.SessionManager, g)
|
|
| 68 |
+ f, err := r.Fetcher(ctx, opt.Ref) |
|
| 69 |
+ if err != nil {
|
|
| 70 |
+ return nil, "", err |
|
| 71 |
+ } |
|
| 72 |
+ fd, ok := f.(remotes.FetcherByDigest) |
|
| 73 |
+ if !ok {
|
|
| 74 |
+ return nil, "", errors.Errorf("invalid blob fetcher: %T", f)
|
|
| 75 |
+ } |
|
| 76 |
+ rc, _, err := fd.FetchByDigest(ctx, opt.Digest) |
|
| 77 |
+ if err != nil {
|
|
| 78 |
+ return nil, "", err |
|
| 79 |
+ } |
|
| 80 |
+ return rc, opt.Digest, nil |
|
| 81 |
+ default: |
|
| 82 |
+ return nil, "", errors.Errorf("unsupported blob scheme %q", opt.Scheme)
|
|
| 83 |
+ } |
|
| 84 |
+} |
|
| 85 |
+ |
|
| 86 |
+func fetchFromOCILayoutStore(ctx context.Context, g session.Group, opt FetchOpt) (io.ReadCloser, error) {
|
|
| 87 |
+ var rc io.ReadCloser |
|
| 88 |
+ err := withOCICaller(ctx, g, opt, func(ctx context.Context, caller session.Caller) error {
|
|
| 89 |
+ store := sessioncontent.NewCallerStore(caller, "oci:"+opt.StoreID) |
|
| 90 |
+ info, err := store.Info(ctx, opt.Digest) |
|
| 91 |
+ if err != nil {
|
|
| 92 |
+ return err |
|
| 93 |
+ } |
|
| 94 |
+ |
|
| 95 |
+ readerAt, err := store.ReaderAt(ctx, ocispecs.Descriptor{
|
|
| 96 |
+ Digest: info.Digest, |
|
| 97 |
+ Size: info.Size, |
|
| 98 |
+ }) |
|
| 99 |
+ if err != nil {
|
|
| 100 |
+ return err |
|
| 101 |
+ } |
|
| 102 |
+ rc = iohelper.ReadCloser(readerAt) |
|
| 103 |
+ return nil |
|
| 104 |
+ }) |
|
| 105 |
+ if err != nil {
|
|
| 106 |
+ return nil, err |
|
| 107 |
+ } |
|
| 108 |
+ return rc, nil |
|
| 109 |
+} |
|
| 110 |
+ |
|
| 111 |
+func withOCICaller(ctx context.Context, g session.Group, opt FetchOpt, f func(context.Context, session.Caller) error) error {
|
|
| 112 |
+ if opt.SessionID != "" {
|
|
| 113 |
+ timeoutCtx, cancel := context.WithCancelCause(ctx) |
|
| 114 |
+ timeoutCtx, _ = context.WithTimeoutCause(timeoutCtx, 5*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet |
|
| 115 |
+ defer func() { cancel(errors.WithStack(context.Canceled)) }()
|
|
| 116 |
+ |
|
| 117 |
+ caller, err := opt.SessionManager.Get(timeoutCtx, opt.SessionID, false) |
|
| 118 |
+ if err != nil {
|
|
| 119 |
+ return err |
|
| 120 |
+ } |
|
| 121 |
+ return f(ctx, caller) |
|
| 122 |
+ } |
|
| 123 |
+ |
|
| 124 |
+ return opt.SessionManager.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error {
|
|
| 125 |
+ return f(ctx, caller) |
|
| 126 |
+ }) |
|
| 127 |
+} |
| ... | ... |
@@ -9,20 +9,15 @@ import ( |
| 9 | 9 |
"os" |
| 10 | 10 |
"time" |
| 11 | 11 |
|
| 12 |
- "github.com/containerd/containerd/v2/core/remotes" |
|
| 13 | 12 |
cerrdefs "github.com/containerd/errdefs" |
| 14 | 13 |
"github.com/moby/buildkit/cache" |
| 15 | 14 |
"github.com/moby/buildkit/session" |
| 16 |
- sessioncontent "github.com/moby/buildkit/session/content" |
|
| 17 | 15 |
"github.com/moby/buildkit/snapshot" |
| 18 | 16 |
"github.com/moby/buildkit/solver" |
| 19 |
- srctypes "github.com/moby/buildkit/source/types" |
|
| 17 |
+ "github.com/moby/buildkit/source/containerblob/blobfetch" |
|
| 20 | 18 |
"github.com/moby/buildkit/source/util/pathutil" |
| 21 | 19 |
"github.com/moby/buildkit/util/contentutil" |
| 22 |
- "github.com/moby/buildkit/util/iohelper" |
|
| 23 |
- "github.com/moby/buildkit/util/resolver" |
|
| 24 | 20 |
digest "github.com/opencontainers/go-digest" |
| 25 |
- ocispecs "github.com/opencontainers/image-spec/specs-go/v1" |
|
| 26 | 21 |
"github.com/pkg/errors" |
| 27 | 22 |
) |
| 28 | 23 |
|
| ... | ... |
@@ -68,31 +63,17 @@ func (p *puller) ensureResolver(ctx context.Context, g session.Group) error {
|
| 68 | 68 |
return errors.Wrap(err, "invalid reference digest") |
| 69 | 69 |
} |
| 70 | 70 |
|
| 71 |
- var ( |
|
| 72 |
- rc io.ReadCloser |
|
| 73 |
- err error |
|
| 74 |
- ) |
|
| 75 |
- if p.id.Scheme() == srctypes.OCIBlobScheme {
|
|
| 76 |
- rc, err = p.fetchFromOCILayoutStore(ctx, g, dgst) |
|
| 77 |
- if err != nil {
|
|
| 78 |
- return err |
|
| 79 |
- } |
|
| 80 |
- } else {
|
|
| 81 |
- r := resolver.DefaultPool.GetResolver(p.src.RegistryHosts, p.id.Reference.String(), resolver.ScopeType{}, p.SessionManager, g)
|
|
| 82 |
- f, err := r.Fetcher(ctx, p.id.Reference.String()) |
|
| 83 |
- if err != nil {
|
|
| 84 |
- return err |
|
| 85 |
- } |
|
| 86 |
- |
|
| 87 |
- fd, ok := f.(remotes.FetcherByDigest) |
|
| 88 |
- if !ok {
|
|
| 89 |
- return errors.Errorf("invalid blob fetcher: %T", f)
|
|
| 90 |
- } |
|
| 91 |
- |
|
| 92 |
- rc, _, err = fd.FetchByDigest(ctx, dgst) |
|
| 93 |
- if err != nil {
|
|
| 94 |
- return err |
|
| 95 |
- } |
|
| 71 |
+ rc, _, err := blobfetch.FetchBlob(ctx, g, blobfetch.FetchOpt{
|
|
| 72 |
+ Scheme: p.id.Scheme(), |
|
| 73 |
+ Ref: p.id.Reference.String(), |
|
| 74 |
+ Digest: dgst, |
|
| 75 |
+ RegistryHosts: p.src.RegistryHosts, |
|
| 76 |
+ SessionManager: p.SessionManager, |
|
| 77 |
+ SessionID: p.id.SessionID, |
|
| 78 |
+ StoreID: p.id.StoreID, |
|
| 79 |
+ }) |
|
| 80 |
+ if err != nil {
|
|
| 81 |
+ return err |
|
| 96 | 82 |
} |
| 97 | 83 |
|
| 98 | 84 |
p.rc = rc |
| ... | ... |
@@ -100,53 +81,6 @@ func (p *puller) ensureResolver(ctx context.Context, g session.Group) error {
|
| 100 | 100 |
return nil |
| 101 | 101 |
} |
| 102 | 102 |
|
| 103 |
-func (p *puller) fetchFromOCILayoutStore(ctx context.Context, g session.Group, dgst digest.Digest) (io.ReadCloser, error) {
|
|
| 104 |
- if p.id.StoreID == "" {
|
|
| 105 |
- return nil, errors.Errorf("oci-layout blob source requires store id")
|
|
| 106 |
- } |
|
| 107 |
- |
|
| 108 |
- var rc io.ReadCloser |
|
| 109 |
- err := p.withOCICaller(ctx, g, func(ctx context.Context, caller session.Caller) error {
|
|
| 110 |
- store := sessioncontent.NewCallerStore(caller, "oci:"+p.id.StoreID) |
|
| 111 |
- info, err := store.Info(ctx, dgst) |
|
| 112 |
- if err != nil {
|
|
| 113 |
- return err |
|
| 114 |
- } |
|
| 115 |
- |
|
| 116 |
- readerAt, err := store.ReaderAt(ctx, ocispecs.Descriptor{
|
|
| 117 |
- Digest: info.Digest, |
|
| 118 |
- Size: info.Size, |
|
| 119 |
- }) |
|
| 120 |
- if err != nil {
|
|
| 121 |
- return err |
|
| 122 |
- } |
|
| 123 |
- rc = iohelper.ReadCloser(readerAt) |
|
| 124 |
- return nil |
|
| 125 |
- }) |
|
| 126 |
- if err != nil {
|
|
| 127 |
- return nil, err |
|
| 128 |
- } |
|
| 129 |
- return rc, nil |
|
| 130 |
-} |
|
| 131 |
- |
|
| 132 |
-func (p *puller) withOCICaller(ctx context.Context, g session.Group, f func(context.Context, session.Caller) error) error {
|
|
| 133 |
- if p.id.SessionID != "" {
|
|
| 134 |
- timeoutCtx, cancel := context.WithCancelCause(ctx) |
|
| 135 |
- timeoutCtx, _ = context.WithTimeoutCause(timeoutCtx, 5*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet |
|
| 136 |
- defer func() { cancel(errors.WithStack(context.Canceled)) }()
|
|
| 137 |
- |
|
| 138 |
- caller, err := p.SessionManager.Get(timeoutCtx, p.id.SessionID, false) |
|
| 139 |
- if err != nil {
|
|
| 140 |
- return err |
|
| 141 |
- } |
|
| 142 |
- return f(ctx, caller) |
|
| 143 |
- } |
|
| 144 |
- |
|
| 145 |
- return p.SessionManager.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error {
|
|
| 146 |
- return f(ctx, caller) |
|
| 147 |
- }) |
|
| 148 |
-} |
|
| 149 |
- |
|
| 150 | 103 |
func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index int) (cacheKey string, imgDigest string, cacheOpts solver.CacheOpts, cacheDone bool, err error) {
|
| 151 | 104 |
dgst := p.id.Reference.Digest() |
| 152 | 105 |
if err := dgst.Validate(); err != nil {
|
| 153 | 106 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,487 @@ |
| 0 |
+package git |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "fmt" |
|
| 5 |
+ "io" |
|
| 6 |
+ "os" |
|
| 7 |
+ "path/filepath" |
|
| 8 |
+ "strings" |
|
| 9 |
+ "sync" |
|
| 10 |
+ |
|
| 11 |
+ "github.com/moby/buildkit/cache" |
|
| 12 |
+ "github.com/moby/buildkit/session" |
|
| 13 |
+ "github.com/moby/buildkit/snapshot" |
|
| 14 |
+ "github.com/moby/buildkit/solver" |
|
| 15 |
+ "github.com/moby/buildkit/source/containerblob/blobfetch" |
|
| 16 |
+ srctypes "github.com/moby/buildkit/source/types" |
|
| 17 |
+ "github.com/moby/buildkit/util/bklog" |
|
| 18 |
+ "github.com/moby/buildkit/util/gitutil" |
|
| 19 |
+ digest "github.com/opencontainers/go-digest" |
|
| 20 |
+ "github.com/pkg/errors" |
|
| 21 |
+) |
|
| 22 |
+ |
|
| 23 |
+const ( |
|
| 24 |
+ // bundleFileName is the name of the bundle file written at the |
|
| 25 |
+ // checkout mount root when git.checkoutbundle is set. Documented on |
|
| 26 |
+ // the GitCheckoutBundle godoc. |
|
| 27 |
+ bundleFileName = "bundle" |
|
| 28 |
+ |
|
| 29 |
+ // bundleImportFileName is the on-disk name of the transient bundle |
|
| 30 |
+ // file streamed in from the blob locator, colocated with the bare |
|
| 31 |
+ // repo and removed once the import completes. |
|
| 32 |
+ bundleImportFileName = "bundle.pack" |
|
| 33 |
+ |
|
| 34 |
+ // bundleFallbackRef is the ref name used when the user's ref is empty |
|
| 35 |
+ // or a commit SHA. Bundle creation needs a named ref so the consumer's |
|
| 36 |
+ // fetch refspec has something to land; consumer-side resolution |
|
| 37 |
+ // short-circuits on SHA when Ref is empty/SHA, so the fallback name |
|
| 38 |
+ // is not user-visible. |
|
| 39 |
+ bundleFallbackRef = "refs/heads/main" |
|
| 40 |
+) |
|
| 41 |
+ |
|
| 42 |
+// bundleTargetRef normalizes the user's ref into a fully-qualified ref name |
|
| 43 |
+// suitable as the tip ref inside the emitted bundle. |
|
| 44 |
+// |
|
| 45 |
+// - Empty / commit SHA: falls back to bundleFallbackRef. The consumer |
|
| 46 |
+// short-circuits on SHA so the chosen branch name is not user-visible, |
|
| 47 |
+// but bundle creation needs a named ref for the fetch refspec to land. |
|
| 48 |
+// - "refs/tags/<name>": preserved as a tag ref so a tag-addressed checkout |
|
| 49 |
+// keeps the tag qualifier on the consumer side. |
|
| 50 |
+// - Anything else (e.g. "master", "refs/heads/main", "v1"): stripped of a |
|
| 51 |
+// leading refs/ or heads/ qualifier and re-prefixed with refs/heads/. |
|
| 52 |
+// |
|
| 53 |
+// Design note: refs/tags/<x> is preserved, but a bare "v1" is mapped to |
|
| 54 |
+// refs/heads/v1 rather than refs/tags/v1 because we cannot tell from a |
|
| 55 |
+// symbolic ref alone whether it originated as a tag or a branch; refs/heads/ |
|
| 56 |
+// is the safer default for tip placement. |
|
| 57 |
+func bundleTargetRef(ref string) string {
|
|
| 58 |
+ if ref == "" || gitutil.IsCommitSHA(ref) {
|
|
| 59 |
+ return bundleFallbackRef |
|
| 60 |
+ } |
|
| 61 |
+ if strings.HasPrefix(ref, "refs/tags/") {
|
|
| 62 |
+ return ref |
|
| 63 |
+ } |
|
| 64 |
+ name := strings.TrimPrefix(ref, "refs/") |
|
| 65 |
+ name = strings.TrimPrefix(name, "heads/") |
|
| 66 |
+ return "refs/heads/" + name |
|
| 67 |
+} |
|
| 68 |
+ |
|
| 69 |
+// detectBundleSHA256 probes a raw git bundle and returns whether it carries |
|
| 70 |
+// sha256 object IDs. git ls-remote reads bundle files directly, so this works |
|
| 71 |
+// before a destination repo exists and lets callers initialize the right object |
|
| 72 |
+// format for a subsequent fetch/import. |
|
| 73 |
+func detectBundleSHA256(ctx context.Context, bundlePath string) (bool, error) {
|
|
| 74 |
+ buf, err := gitCLI().Run(ctx, "ls-remote", "--", bundlePath) |
|
| 75 |
+ if err != nil {
|
|
| 76 |
+ return false, errors.Wrapf(err, "failed to inspect git bundle %s", bundlePath) |
|
| 77 |
+ } |
|
| 78 |
+ for line := range strings.SplitSeq(string(buf), "\n") {
|
|
| 79 |
+ if line == "" {
|
|
| 80 |
+ continue |
|
| 81 |
+ } |
|
| 82 |
+ sha, _, _ := strings.Cut(line, "\t") |
|
| 83 |
+ if !gitutil.IsCommitSHA(sha) {
|
|
| 84 |
+ return false, errors.Errorf("failed to inspect git bundle %s: invalid object ID %q", bundlePath, sha)
|
|
| 85 |
+ } |
|
| 86 |
+ return len(sha) == 64, nil |
|
| 87 |
+ } |
|
| 88 |
+ return false, errors.Errorf("failed to inspect git bundle %s: no refs found", bundlePath)
|
|
| 89 |
+} |
|
| 90 |
+ |
|
| 91 |
+// stageBundle downloads the bundle blob into an isolated temp bare repo, |
|
| 92 |
+// imports it, and validates that the pinned commit is present. The returned |
|
| 93 |
+// URL (file://<tmpRepoDir>) is suitable for use as a git fetch source (for |
|
| 94 |
+// the main fetch flow) or as the remote URL for an ls-remote call (for |
|
| 95 |
+// metadata resolution), so both paths converge on the same git primitives |
|
| 96 |
+// as a normal origin fetch. |
|
| 97 |
+// |
|
| 98 |
+// The returned cleanup removes the temp dir and must be called once the |
|
| 99 |
+// caller is done with the staged URL. |
|
| 100 |
+func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_ string, _ func() error, retErr error) {
|
|
| 101 |
+ scheme, ref, storeID, dgst, err := parseBundleLocator(gs.src.Bundle) |
|
| 102 |
+ if err != nil {
|
|
| 103 |
+ return "", nil, err |
|
| 104 |
+ } |
|
| 105 |
+ |
|
| 106 |
+ tmpDir, err := os.MkdirTemp("", "buildkit-bundle-")
|
|
| 107 |
+ if err != nil {
|
|
| 108 |
+ return "", nil, errors.Wrap(err, "failed to create temp dir for bundle") |
|
| 109 |
+ } |
|
| 110 |
+ cleanup := func() error { return os.RemoveAll(tmpDir) }
|
|
| 111 |
+ defer func() {
|
|
| 112 |
+ if retErr != nil {
|
|
| 113 |
+ cleanup() |
|
| 114 |
+ } |
|
| 115 |
+ }() |
|
| 116 |
+ |
|
| 117 |
+ if err := gs.downloadBundleToFile(ctx, g, scheme, ref, storeID, dgst, tmpDir, bundleImportFileName); err != nil {
|
|
| 118 |
+ return "", nil, err |
|
| 119 |
+ } |
|
| 120 |
+ bundlePath := filepath.Join(tmpDir, bundleImportFileName) |
|
| 121 |
+ sha256, err := detectBundleSHA256(ctx, bundlePath) |
|
| 122 |
+ if err != nil {
|
|
| 123 |
+ return "", nil, err |
|
| 124 |
+ } |
|
| 125 |
+ gs.sha256 = sha256 |
|
| 126 |
+ |
|
| 127 |
+ // Initialize an isolated bare repo in the temp dir and import the |
|
| 128 |
+ // bundle there. This lets us validate the bundle and stage the refs |
|
| 129 |
+ // under their natural names without touching the shared bare repo. |
|
| 130 |
+ tmpRepoDir := filepath.Join(tmpDir, "repo.git") |
|
| 131 |
+ if err := os.Mkdir(tmpRepoDir, 0700); err != nil {
|
|
| 132 |
+ return "", nil, errors.Wrap(err, "failed to create temp bare repo dir") |
|
| 133 |
+ } |
|
| 134 |
+ tmpGit := gitCLI(gitutil.WithGitDir(tmpRepoDir)) |
|
| 135 |
+ initArgs := []string{"-c", "init.defaultBranch=master", "init", "--bare"}
|
|
| 136 |
+ if sha256 {
|
|
| 137 |
+ initArgs = append(initArgs, "--object-format=sha256") |
|
| 138 |
+ } |
|
| 139 |
+ if _, err := tmpGit.Run(ctx, initArgs...); err != nil {
|
|
| 140 |
+ return "", nil, errors.Wrap(err, "failed to init temp bare repo") |
|
| 141 |
+ } |
|
| 142 |
+ |
|
| 143 |
+ if _, err := tmpGit.Run(ctx, "fetch", bundlePath, "+refs/*:refs/*"); err != nil {
|
|
| 144 |
+ return "", nil, errors.Wrapf(err, "failed to import git bundle %s into temp", ref) |
|
| 145 |
+ } |
|
| 146 |
+ |
|
| 147 |
+ // Confirm the pinned commit is actually present in the bundle. The |
|
| 148 |
+ // subsequent fetch in the main flow would surface a generic "failed to |
|
| 149 |
+ // fetch" error; this check gives a clearer message. |
|
| 150 |
+ if _, err := tmpGit.Run(ctx, "cat-file", "-e", gs.src.Checksum+"^{commit}"); err != nil {
|
|
| 151 |
+ return "", nil, errors.Errorf("commit %s not found in bundle %s", gs.src.Checksum, ref)
|
|
| 152 |
+ } |
|
| 153 |
+ |
|
| 154 |
+ return "file://" + tmpRepoDir, cleanup, nil |
|
| 155 |
+} |
|
| 156 |
+ |
|
| 157 |
+// ensureStagedBundle returns the file:// URL of a staged temp bare repo |
|
| 158 |
+// produced from gs.src.Bundle, lazily staging it on first call and reusing |
|
| 159 |
+// the same staging dir on subsequent calls for the lifetime of gs. |
|
| 160 |
+// |
|
| 161 |
+// The call-site contract is: callers may call ensureStagedBundle any number |
|
| 162 |
+// of times and must never run the cleanup themselves. Cleanup is attached to |
|
| 163 |
+// the job via JobContext.Cleanup on the first staging so it fires at |
|
| 164 |
+// end-of-solve regardless of whether Snapshot runs (covering the |
|
| 165 |
+// CacheKey-then-cache-hit path where tryRemoteFetch never executes). If no |
|
| 166 |
+// jobCtx is available (e.g. ResolveMetadata path with a nil jobCtx), the |
|
| 167 |
+// returned cleanup is retained on the handler and teardown must come from a |
|
| 168 |
+// later call that does provide a jobCtx, or from the caller running |
|
| 169 |
+// releaseStagedBundle explicitly. |
|
| 170 |
+// |
|
| 171 |
+// Handler methods run sequentially per solve, so ensureStagedBundle does not |
|
| 172 |
+// guard against concurrent calls. |
|
| 173 |
+func (gs *gitSourceHandler) ensureStagedBundle(ctx context.Context, jobCtx solver.JobContext, g session.Group) (string, error) {
|
|
| 174 |
+ if gs.stagedBundleURL != "" {
|
|
| 175 |
+ return gs.stagedBundleURL, nil |
|
| 176 |
+ } |
|
| 177 |
+ |
|
| 178 |
+ url, cleanup, err := gs.stageBundle(ctx, g) |
|
| 179 |
+ if err != nil {
|
|
| 180 |
+ return "", err |
|
| 181 |
+ } |
|
| 182 |
+ |
|
| 183 |
+ // Make the cleanup idempotent so it is safe to call from both the |
|
| 184 |
+ // job cleanup path and an explicit releaseStagedBundle (or the |
|
| 185 |
+ // CacheKey fallback path below) without double-removing the temp dir. |
|
| 186 |
+ once := sync.OnceValue(cleanup) |
|
| 187 |
+ gs.stagedBundleURL = url |
|
| 188 |
+ gs.stagedBundleCleanup = once |
|
| 189 |
+ |
|
| 190 |
+ if jobCtx != nil {
|
|
| 191 |
+ if err := jobCtx.Cleanup(func() error { return once() }); err != nil {
|
|
| 192 |
+ // Failed to register the job cleanup — run it synchronously |
|
| 193 |
+ // so we do not leak the temp dir. |
|
| 194 |
+ _ = once() |
|
| 195 |
+ gs.stagedBundleURL = "" |
|
| 196 |
+ gs.stagedBundleCleanup = nil |
|
| 197 |
+ return "", err |
|
| 198 |
+ } |
|
| 199 |
+ // The job cleanup now owns the teardown; clearing the |
|
| 200 |
+ // per-handler pointer avoids a second invocation from any |
|
| 201 |
+ // fallback path. The stored url stays set so repeat callers on |
|
| 202 |
+ // this handler get the cached URL for the remainder of the |
|
| 203 |
+ // solve, before the job cleanup fires. |
|
| 204 |
+ gs.stagedBundleCleanup = nil |
|
| 205 |
+ } else {
|
|
| 206 |
+ bklog.G(ctx).Debug("bundle staged without jobCtx; cleanup deferred to handler teardown")
|
|
| 207 |
+ } |
|
| 208 |
+ |
|
| 209 |
+ return url, nil |
|
| 210 |
+} |
|
| 211 |
+ |
|
| 212 |
+// releaseStagedBundle tears down any staged bundle owned by the handler. It |
|
| 213 |
+// is a no-op when the cleanup has already been attached to a JobContext or |
|
| 214 |
+// when nothing was staged. Safe to call multiple times. |
|
| 215 |
+func (gs *gitSourceHandler) releaseStagedBundle() error {
|
|
| 216 |
+ cleanup := gs.stagedBundleCleanup |
|
| 217 |
+ gs.stagedBundleCleanup = nil |
|
| 218 |
+ gs.stagedBundleURL = "" |
|
| 219 |
+ if cleanup == nil {
|
|
| 220 |
+ return nil |
|
| 221 |
+ } |
|
| 222 |
+ return cleanup() |
|
| 223 |
+} |
|
| 224 |
+ |
|
| 225 |
+// downloadBundleToFile streams the bundle blob into a file named bundleName |
|
| 226 |
+// inside bundleDir while verifying its digest against the locator. The file |
|
| 227 |
+// is created via os.Root so a pre-existing symlink at bundleName inside |
|
| 228 |
+// bundleDir cannot redirect the write outside of bundleDir. |
|
| 229 |
+func (gs *gitSourceHandler) downloadBundleToFile(ctx context.Context, g session.Group, scheme, ref, storeID string, expectedDigest digest.Digest, bundleDir, bundleName string) (retErr error) {
|
|
| 230 |
+ rc, err := gs.openBundleBlob(ctx, g, scheme, ref, storeID, expectedDigest) |
|
| 231 |
+ if err != nil {
|
|
| 232 |
+ return errors.Wrapf(err, "failed to fetch git bundle %s", ref) |
|
| 233 |
+ } |
|
| 234 |
+ defer rc.Close() |
|
| 235 |
+ |
|
| 236 |
+ bundleDirRoot, err := os.OpenRoot(bundleDir) |
|
| 237 |
+ if err != nil {
|
|
| 238 |
+ return errors.Wrapf(err, "failed to open bundle dir root %s", bundleDir) |
|
| 239 |
+ } |
|
| 240 |
+ defer bundleDirRoot.Close() |
|
| 241 |
+ |
|
| 242 |
+ f, err := bundleDirRoot.Create(bundleName) |
|
| 243 |
+ if err != nil {
|
|
| 244 |
+ return errors.Wrapf(err, "failed to create bundle file %s", bundleName) |
|
| 245 |
+ } |
|
| 246 |
+ defer func() {
|
|
| 247 |
+ if retErr != nil {
|
|
| 248 |
+ f.Close() |
|
| 249 |
+ bundleDirRoot.Remove(bundleName) |
|
| 250 |
+ } |
|
| 251 |
+ }() |
|
| 252 |
+ |
|
| 253 |
+ d := digest.SHA256.Digester() |
|
| 254 |
+ if _, err := io.Copy(io.MultiWriter(f, d.Hash()), rc); err != nil {
|
|
| 255 |
+ return errors.Wrapf(err, "failed to download git bundle %s", ref) |
|
| 256 |
+ } |
|
| 257 |
+ if err := f.Close(); err != nil {
|
|
| 258 |
+ return errors.Wrapf(err, "failed to close bundle file %s", bundleName) |
|
| 259 |
+ } |
|
| 260 |
+ |
|
| 261 |
+ got := d.Digest() |
|
| 262 |
+ if expectedDigest != "" && got != expectedDigest {
|
|
| 263 |
+ return errors.Errorf("expected checksum to match %s, got %s", expectedDigest, got)
|
|
| 264 |
+ } |
|
| 265 |
+ return nil |
|
| 266 |
+} |
|
| 267 |
+ |
|
| 268 |
+// openBundleBlob resolves the blob using blobfetch. storeID is the pre-derived |
|
| 269 |
+// OCI-layout store id (empty for registry blobs). |
|
| 270 |
+func (gs *gitSourceHandler) openBundleBlob(ctx context.Context, g session.Group, scheme, ref, storeID string, dgst digest.Digest) (io.ReadCloser, error) {
|
|
| 271 |
+ opt := blobfetch.FetchOpt{
|
|
| 272 |
+ Scheme: scheme, |
|
| 273 |
+ Ref: ref, |
|
| 274 |
+ Digest: dgst, |
|
| 275 |
+ RegistryHosts: gs.registryHosts, |
|
| 276 |
+ SessionManager: gs.sm, |
|
| 277 |
+ } |
|
| 278 |
+ if scheme == srctypes.OCIBlobScheme {
|
|
| 279 |
+ opt.SessionID = gs.src.BundleOCISessionID |
|
| 280 |
+ opt.StoreID = storeID |
|
| 281 |
+ if gs.src.BundleOCIStoreID != "" {
|
|
| 282 |
+ opt.StoreID = gs.src.BundleOCIStoreID |
|
| 283 |
+ } |
|
| 284 |
+ } |
|
| 285 |
+ rc, _, err := blobfetch.FetchBlob(ctx, g, opt) |
|
| 286 |
+ return rc, err |
|
| 287 |
+} |
|
| 288 |
+ |
|
| 289 |
+// resolveBundleMetadata stages the bundle in an isolated temp bare repo and |
|
| 290 |
+// delegates to the shared ls-remote code path using that repo's file:// URL. |
|
| 291 |
+// Bundle mode lands refs in refs/heads/* and refs/tags/* just like a normal |
|
| 292 |
+// origin fetch, so the resolution logic is identical once the URL swap is |
|
| 293 |
+// done — and the resulting Metadata.Ref has the same shape as non-bundle |
|
| 294 |
+// mode (e.g. "refs/heads/master" for symbolic ref "master"). |
|
| 295 |
+// |
|
| 296 |
+// For a pinned-commit / empty user ref, short-circuit: the user has already |
|
| 297 |
+// told us the commit, so there is nothing for ls-remote to add and no point |
|
| 298 |
+// downloading the bundle just for metadata resolution. The bundle will still |
|
| 299 |
+// be staged during the Snapshot path if the commit is not already present. |
|
| 300 |
+func (gs *gitSourceHandler) resolveBundleMetadata(ctx context.Context, jobCtx solver.JobContext) (*Metadata, error) {
|
|
| 301 |
+ // Bundle mode requires git.checksum; if Ref is empty or a SHA, the |
|
| 302 |
+ // commit is already pinned and no lookup is needed. Report Ref and |
|
| 303 |
+ // Checksum as the commit SHA — same shape as the commit-SHA short |
|
| 304 |
+ // circuit in non-bundle resolveMetadata. |
|
| 305 |
+ if gs.src.Ref == "" || gitutil.IsCommitSHA(gs.src.Ref) {
|
|
| 306 |
+ ref := gs.src.Ref |
|
| 307 |
+ if ref == "" {
|
|
| 308 |
+ ref = gs.src.Checksum |
|
| 309 |
+ } |
|
| 310 |
+ if gs.src.Checksum != "" && !strings.HasPrefix(ref, gs.src.Checksum) {
|
|
| 311 |
+ return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, ref)
|
|
| 312 |
+ } |
|
| 313 |
+ return &Metadata{Ref: ref, Checksum: ref}, nil
|
|
| 314 |
+ } |
|
| 315 |
+ |
|
| 316 |
+ var g session.Group |
|
| 317 |
+ if jobCtx != nil {
|
|
| 318 |
+ g = jobCtx.Session() |
|
| 319 |
+ } |
|
| 320 |
+ stagedURL, err := gs.ensureStagedBundle(ctx, jobCtx, g) |
|
| 321 |
+ if err != nil {
|
|
| 322 |
+ return nil, err |
|
| 323 |
+ } |
|
| 324 |
+ return gs.resolveMetadataFromURL(ctx, g, stagedURL) |
|
| 325 |
+} |
|
| 326 |
+ |
|
| 327 |
+// checkoutAsBundle writes a single-file git bundle at the checkout mount root |
|
| 328 |
+// instead of a worktree. This is the checkout counterpart of bundle import and |
|
| 329 |
+// is used when GitCheckoutBundle() is set on the identifier. Submodules are |
|
| 330 |
+// not included in the bundle. |
|
| 331 |
+func (gs *gitSourceHandler) checkoutAsBundle(ctx context.Context, repo *gitRepo, g session.Group) (_ cache.ImmutableRef, retErr error) {
|
|
| 332 |
+ ref := gs.src.Ref |
|
| 333 |
+ commit := gs.src.Checksum |
|
| 334 |
+ if commit == "" {
|
|
| 335 |
+ commit = ref |
|
| 336 |
+ } |
|
| 337 |
+ |
|
| 338 |
+ checkoutRef, err := gs.cache.New(ctx, nil, g, cache.CachePolicyRetain, cache.WithDescription(fmt.Sprintf("git bundle checkout for %s#%s", gs.src.Remote, ref)))
|
|
| 339 |
+ if err != nil {
|
|
| 340 |
+ return nil, errors.Wrapf(err, "failed to create new mutable for bundle checkout") |
|
| 341 |
+ } |
|
| 342 |
+ defer func() {
|
|
| 343 |
+ if retErr != nil && checkoutRef != nil {
|
|
| 344 |
+ checkoutRef.Release(context.WithoutCancel(ctx)) |
|
| 345 |
+ } |
|
| 346 |
+ }() |
|
| 347 |
+ |
|
| 348 |
+ mount, err := checkoutRef.Mount(ctx, false, g) |
|
| 349 |
+ if err != nil {
|
|
| 350 |
+ return nil, err |
|
| 351 |
+ } |
|
| 352 |
+ lm := snapshot.LocalMounter(mount) |
|
| 353 |
+ checkoutDir, err := lm.Mount() |
|
| 354 |
+ if err != nil {
|
|
| 355 |
+ return nil, err |
|
| 356 |
+ } |
|
| 357 |
+ defer func() {
|
|
| 358 |
+ if retErr != nil && lm != nil {
|
|
| 359 |
+ lm.Unmount() |
|
| 360 |
+ } |
|
| 361 |
+ }() |
|
| 362 |
+ |
|
| 363 |
+ // The bundle needs to carry a named ref that points at the target |
|
| 364 |
+ // commit, so `git fetch <bundle> +refs/*:refs/*` on the consumer side |
|
| 365 |
+ // has something to land. We preserve the user's ref name (normalized |
|
| 366 |
+ // into refs/heads/* or refs/tags/*) so the bundle reads as if it were |
|
| 367 |
+ // produced by the upstream remote; no fake internal namespace. |
|
| 368 |
+ targetRef := bundleTargetRef(ref) |
|
| 369 |
+ |
|
| 370 |
+ // Stage the bundle in an isolated temp bare repo rather than mutating |
|
| 371 |
+ // the shared bare repo's ref namespace. We fetch the pinned commit |
|
| 372 |
+ // from the shared repo into the temp bare repo and give it the |
|
| 373 |
+ // target-ref name there, so `git bundle create` sees a clean, |
|
| 374 |
+ // self-contained repo with exactly the ref we want in the bundle. |
|
| 375 |
+ tmpDir, err := os.MkdirTemp("", "buildkit-bundle-create-")
|
|
| 376 |
+ if err != nil {
|
|
| 377 |
+ return nil, errors.Wrap(err, "failed to create temp dir for bundle creation") |
|
| 378 |
+ } |
|
| 379 |
+ defer os.RemoveAll(tmpDir) |
|
| 380 |
+ |
|
| 381 |
+ tmpRepoDir := filepath.Join(tmpDir, "repo.git") |
|
| 382 |
+ if err := os.Mkdir(tmpRepoDir, 0700); err != nil {
|
|
| 383 |
+ return nil, errors.Wrap(err, "failed to create temp bare repo dir for bundle creation") |
|
| 384 |
+ } |
|
| 385 |
+ tmpGit := repo.New(gitutil.WithGitDir(tmpRepoDir)) |
|
| 386 |
+ initArgs := []string{"-c", "init.defaultBranch=master", "init", "--bare"}
|
|
| 387 |
+ if gs.sha256 {
|
|
| 388 |
+ initArgs = append(initArgs, "--object-format=sha256") |
|
| 389 |
+ } |
|
| 390 |
+ if _, err := tmpGit.Run(ctx, initArgs...); err != nil {
|
|
| 391 |
+ return nil, errors.Wrap(err, "failed to init temp bare repo for bundle creation") |
|
| 392 |
+ } |
|
| 393 |
+ |
|
| 394 |
+ // Pull the pinned commit from the shared bare repo into temp, then |
|
| 395 |
+ // point the target ref at it. The shared bare repo maps user refs to |
|
| 396 |
+ // refs/tags/<name> via the main-path fetch refspec, so there is no |
|
| 397 |
+ // natural-name ref to pull; update-ref alone places the tip against |
|
| 398 |
+ // the pinned commit under the target name. |
|
| 399 |
+ sharedURL := "file://" + repo.dir |
|
| 400 |
+ if _, err := tmpGit.Run(ctx, "fetch", sharedURL, commit); err != nil {
|
|
| 401 |
+ return nil, errors.Wrapf(err, "failed to fetch commit %s from shared repo for bundle creation", commit) |
|
| 402 |
+ } |
|
| 403 |
+ |
|
| 404 |
+ if _, err := tmpGit.Run(ctx, "update-ref", targetRef, commit); err != nil {
|
|
| 405 |
+ return nil, errors.Wrapf(err, "failed to create target ref for bundle %s", commit) |
|
| 406 |
+ } |
|
| 407 |
+ |
|
| 408 |
+ // Write the bundle under a temp path we fully own, then move it into |
|
| 409 |
+ // the checkout mount through os.OpenRoot so a pre-existing symlink at |
|
| 410 |
+ // <checkoutDir>/bundle cannot redirect the final artifact outside the |
|
| 411 |
+ // mount. git bundle create is an external process that only accepts |
|
| 412 |
+ // path strings, so the only safe landing pattern is: write to a path |
|
| 413 |
+ // outside the mount, then atomically transfer the bytes through an |
|
| 414 |
+ // os.Root-scoped file handle. |
|
| 415 |
+ stagePath := filepath.Join(tmpDir, "out.bundle") |
|
| 416 |
+ if _, err := tmpGit.Run(ctx, "bundle", "create", stagePath, targetRef); err != nil {
|
|
| 417 |
+ return nil, errors.Wrapf(err, "failed to create git bundle for %s", commit) |
|
| 418 |
+ } |
|
| 419 |
+ |
|
| 420 |
+ if err := writeBundleToMount(checkoutDir, bundleFileName, stagePath); err != nil {
|
|
| 421 |
+ return nil, err |
|
| 422 |
+ } |
|
| 423 |
+ |
|
| 424 |
+ outPath := filepath.Join(checkoutDir, bundleFileName) |
|
| 425 |
+ if idmap := mount.IdentityMapping(); idmap != nil {
|
|
| 426 |
+ uid, gid := idmap.RootPair() |
|
| 427 |
+ if err := os.Lchown(outPath, uid, gid); err != nil {
|
|
| 428 |
+ return nil, errors.Wrap(err, "failed to remap git bundle ownership") |
|
| 429 |
+ } |
|
| 430 |
+ } |
|
| 431 |
+ |
|
| 432 |
+ lm.Unmount() |
|
| 433 |
+ lm = nil |
|
| 434 |
+ |
|
| 435 |
+ snap, err := checkoutRef.Commit(ctx) |
|
| 436 |
+ if err != nil {
|
|
| 437 |
+ return nil, err |
|
| 438 |
+ } |
|
| 439 |
+ checkoutRef = nil |
|
| 440 |
+ return snap, nil |
|
| 441 |
+} |
|
| 442 |
+ |
|
| 443 |
+// writeBundleToMount copies the bytes at stagePath into |
|
| 444 |
+// <checkoutDir>/<bundleName> via os.OpenRoot, so a symlink at the destination |
|
| 445 |
+// cannot redirect the write outside checkoutDir. The staging file is expected |
|
| 446 |
+// to be a path the caller fully controls (created under its own temp dir), |
|
| 447 |
+// not a path inside checkoutDir. |
|
| 448 |
+func writeBundleToMount(checkoutDir, bundleName, stagePath string) (retErr error) {
|
|
| 449 |
+ src, err := os.Open(stagePath) |
|
| 450 |
+ if err != nil {
|
|
| 451 |
+ return errors.Wrapf(err, "failed to open staged bundle %s", stagePath) |
|
| 452 |
+ } |
|
| 453 |
+ defer src.Close() |
|
| 454 |
+ |
|
| 455 |
+ checkoutRoot, err := os.OpenRoot(checkoutDir) |
|
| 456 |
+ if err != nil {
|
|
| 457 |
+ return errors.Wrapf(err, "failed to open checkout dir root %s", checkoutDir) |
|
| 458 |
+ } |
|
| 459 |
+ defer checkoutRoot.Close() |
|
| 460 |
+ |
|
| 461 |
+ // Remove any pre-existing entry at bundleName. If it's a symlink, |
|
| 462 |
+ // os.Root.Remove deletes the link itself, not the target, so a symlink |
|
| 463 |
+ // planted in the mount cannot redirect the subsequent create. |
|
| 464 |
+ if err := checkoutRoot.Remove(bundleName); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
| 465 |
+ return errors.Wrapf(err, "failed to clear bundle dest %s", bundleName) |
|
| 466 |
+ } |
|
| 467 |
+ |
|
| 468 |
+ dst, err := checkoutRoot.Create(bundleName) |
|
| 469 |
+ if err != nil {
|
|
| 470 |
+ return errors.Wrapf(err, "failed to create bundle file %s", bundleName) |
|
| 471 |
+ } |
|
| 472 |
+ defer func() {
|
|
| 473 |
+ if retErr != nil {
|
|
| 474 |
+ dst.Close() |
|
| 475 |
+ checkoutRoot.Remove(bundleName) |
|
| 476 |
+ } |
|
| 477 |
+ }() |
|
| 478 |
+ |
|
| 479 |
+ if _, err := io.Copy(dst, src); err != nil {
|
|
| 480 |
+ return errors.Wrapf(err, "failed to write bundle to %s", bundleName) |
|
| 481 |
+ } |
|
| 482 |
+ if err := dst.Close(); err != nil {
|
|
| 483 |
+ return errors.Wrapf(err, "failed to close bundle file %s", bundleName) |
|
| 484 |
+ } |
|
| 485 |
+ return nil |
|
| 486 |
+} |
| ... | ... |
@@ -1,11 +1,16 @@ |
| 1 | 1 |
package git |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "strings" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/containerd/containerd/v2/pkg/reference" |
|
| 4 | 7 |
"github.com/moby/buildkit/solver/llbsolver/provenance" |
| 5 | 8 |
provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types" |
| 6 | 9 |
"github.com/moby/buildkit/source" |
| 7 | 10 |
srctypes "github.com/moby/buildkit/source/types" |
| 8 | 11 |
"github.com/moby/buildkit/util/gitutil" |
| 12 |
+ digest "github.com/opencontainers/go-digest" |
|
| 13 |
+ "github.com/pkg/errors" |
|
| 9 | 14 |
) |
| 10 | 15 |
|
| 11 | 16 |
type GitIdentifier struct {
|
| ... | ... |
@@ -20,6 +25,24 @@ type GitIdentifier struct {
|
| 20 | 20 |
KnownSSHHosts string |
| 21 | 21 |
SkipSubmodules bool |
| 22 | 22 |
MTime string // "checkout" (default) or "commit" |
| 23 |
+ FetchByCommit bool |
|
| 24 |
+ |
|
| 25 |
+ // Bundle, when non-empty, instructs the git source to fetch commits from a |
|
| 26 |
+ // pre-built git bundle stored as a blob instead of fetching from the remote |
|
| 27 |
+ // repository. The locator must use the "docker-image+blob://" or |
|
| 28 |
+ // "oci-layout+blob://" scheme. |
|
| 29 |
+ Bundle string |
|
| 30 |
+ // BundleOCISessionID, when set, pins the OCI-layout bundle fetch to a |
|
| 31 |
+ // specific client session. Only meaningful when Bundle uses the |
|
| 32 |
+ // oci-layout+blob:// scheme. |
|
| 33 |
+ BundleOCISessionID string |
|
| 34 |
+ // BundleOCIStoreID, when set, overrides the OCI-layout store name |
|
| 35 |
+ // derived from the bundle locator. Only meaningful when Bundle uses the |
|
| 36 |
+ // oci-layout+blob:// scheme. |
|
| 37 |
+ BundleOCIStoreID string |
|
| 38 |
+ // CheckoutBundle, when true, produces a single-file git bundle at the |
|
| 39 |
+ // checkout mount root (filename "bundle") instead of a worktree. |
|
| 40 |
+ CheckoutBundle bool |
|
| 23 | 41 |
|
| 24 | 42 |
VerifySignature *GitSignatureVerifyOptions |
| 25 | 43 |
} |
| ... | ... |
@@ -55,14 +78,18 @@ func (GitIdentifier) Scheme() string {
|
| 55 | 55 |
var _ source.Identifier = (*GitIdentifier)(nil) |
| 56 | 56 |
|
| 57 | 57 |
func (id *GitIdentifier) Capture(c *provenance.Capture, pin string) error {
|
| 58 |
- url := id.Remote |
|
| 58 |
+ remoteURL := id.Remote |
|
| 59 | 59 |
if id.Ref != "" {
|
| 60 |
- url += "#" + id.Ref |
|
| 60 |
+ remoteURL += "#" + id.Ref |
|
| 61 | 61 |
} |
| 62 |
- c.AddGit(provenancetypes.GitSource{
|
|
| 63 |
- URL: url, |
|
| 62 |
+ gs := provenancetypes.GitSource{
|
|
| 63 |
+ URL: remoteURL, |
|
| 64 | 64 |
Commit: pin, |
| 65 |
- }) |
|
| 65 |
+ } |
|
| 66 |
+ if id.Bundle != "" {
|
|
| 67 |
+ gs.Bundle = &provenancetypes.GitBundle{URL: id.Bundle}
|
|
| 68 |
+ } |
|
| 69 |
+ c.AddGit(gs) |
|
| 66 | 70 |
if id.AuthTokenSecret != "" {
|
| 67 | 71 |
c.AddSecret(provenancetypes.Secret{
|
| 68 | 72 |
ID: id.AuthTokenSecret, |
| ... | ... |
@@ -83,3 +110,73 @@ func (id *GitIdentifier) Capture(c *provenance.Capture, pin string) error {
|
| 83 | 83 |
} |
| 84 | 84 |
return nil |
| 85 | 85 |
} |
| 86 |
+ |
|
| 87 |
+// validateBundleAttrs enforces the bundle-mode constraints that are statically |
|
| 88 |
+// decidable at identifier parse time. |
|
| 89 |
+func validateBundleAttrs(id *GitIdentifier) error {
|
|
| 90 |
+ if id.Bundle != "" {
|
|
| 91 |
+ if _, _, _, _, err := parseBundleLocator(id.Bundle); err != nil {
|
|
| 92 |
+ return err |
|
| 93 |
+ } |
|
| 94 |
+ if id.Checksum == "" {
|
|
| 95 |
+ return errors.Errorf("git.bundle requires git.checksum to be set")
|
|
| 96 |
+ } |
|
| 97 |
+ // Reject the pathological "Ref=sha1, Checksum=sha2" case: if both |
|
| 98 |
+ // are set and Ref is itself a commit SHA, the two must agree. |
|
| 99 |
+ // Without this check bundle mode silently overwrites Ref with |
|
| 100 |
+ // Checksum in tryRemoteFetch and the mismatch never surfaces. |
|
| 101 |
+ if id.Ref != "" && gitutil.IsCommitSHA(id.Ref) && !strings.HasPrefix(id.Ref, id.Checksum) {
|
|
| 102 |
+ return errors.Errorf("expected checksum to match %s, got %s", id.Checksum, id.Ref)
|
|
| 103 |
+ } |
|
| 104 |
+ } |
|
| 105 |
+ if id.CheckoutBundle {
|
|
| 106 |
+ if id.KeepGitDir {
|
|
| 107 |
+ return errors.Errorf("git.checkoutbundle is incompatible with git.keepgitdir")
|
|
| 108 |
+ } |
|
| 109 |
+ if id.Subdir != "" {
|
|
| 110 |
+ return errors.Errorf("git.checkoutbundle is incompatible with git.subdir")
|
|
| 111 |
+ } |
|
| 112 |
+ } |
|
| 113 |
+ return nil |
|
| 114 |
+} |
|
| 115 |
+ |
|
| 116 |
+// splitBundleLocator splits a locator of the form "<scheme>://<body>" into |
|
| 117 |
+// (scheme, body). It returns ("","") if raw does not contain a "://" separator.
|
|
| 118 |
+// This avoids net/url for schemes like "oci-layout+blob" whose body contains |
|
| 119 |
+// a colon after '@' (the digest) and confuses url.Parse. |
|
| 120 |
+func splitBundleLocator(raw string) (string, string) {
|
|
| 121 |
+ const sep = "://" |
|
| 122 |
+ i := strings.Index(raw, sep) |
|
| 123 |
+ if i < 0 {
|
|
| 124 |
+ return "", "" |
|
| 125 |
+ } |
|
| 126 |
+ return raw[:i], raw[i+len(sep):] |
|
| 127 |
+} |
|
| 128 |
+ |
|
| 129 |
+// parseBundleLocator extracts the scheme, normalized reference string, the |
|
| 130 |
+// reference locator (i.e. the body before '@digest', used as the OCI-layout |
|
| 131 |
+// store id), and the digest from a bundle locator of the form |
|
| 132 |
+// "<scheme>://<ref>@sha256:<hex>". Only sha256 digests are accepted because |
|
| 133 |
+// downloadBundleToFile hashes with sha256 and would fail to match any other |
|
| 134 |
+// algorithm. |
|
| 135 |
+func parseBundleLocator(raw string) (string, string, string, digest.Digest, error) {
|
|
| 136 |
+ scheme, body := splitBundleLocator(raw) |
|
| 137 |
+ if scheme == "" {
|
|
| 138 |
+ return "", "", "", "", errors.Errorf("failed to parse git.bundle locator %q: missing scheme", raw)
|
|
| 139 |
+ } |
|
| 140 |
+ if scheme != srctypes.DockerImageBlobScheme && scheme != srctypes.OCIBlobScheme {
|
|
| 141 |
+ return "", "", "", "", errors.Errorf("git.bundle locator scheme %q is not supported; expected %q or %q", scheme, srctypes.DockerImageBlobScheme, srctypes.OCIBlobScheme)
|
|
| 142 |
+ } |
|
| 143 |
+ ref, err := reference.Parse(body) |
|
| 144 |
+ if err != nil {
|
|
| 145 |
+ return "", "", "", "", errors.Wrapf(err, "failed to parse git.bundle locator %q", raw) |
|
| 146 |
+ } |
|
| 147 |
+ dgst := ref.Digest() |
|
| 148 |
+ if err := dgst.Validate(); err != nil {
|
|
| 149 |
+ return "", "", "", "", errors.Wrapf(err, "failed to parse git.bundle locator %q: invalid digest", raw) |
|
| 150 |
+ } |
|
| 151 |
+ if dgst.Algorithm() != digest.SHA256 {
|
|
| 152 |
+ return "", "", "", "", errors.Errorf("git.bundle locator %q: digest algorithm %q not supported, expected %q", raw, dgst.Algorithm(), digest.SHA256)
|
|
| 153 |
+ } |
|
| 154 |
+ return scheme, ref.String(), ref.Locator, dgst, nil |
|
| 155 |
+} |
| ... | ... |
@@ -16,6 +16,7 @@ import ( |
| 16 | 16 |
"strings" |
| 17 | 17 |
"time" |
| 18 | 18 |
|
| 19 |
+ "github.com/containerd/containerd/v2/core/remotes/docker" |
|
| 19 | 20 |
"github.com/moby/buildkit/cache" |
| 20 | 21 |
"github.com/moby/buildkit/client" |
| 21 | 22 |
"github.com/moby/buildkit/identity" |
| ... | ... |
@@ -24,6 +25,7 @@ import ( |
| 24 | 24 |
"github.com/moby/buildkit/session/sshforward" |
| 25 | 25 |
"github.com/moby/buildkit/snapshot" |
| 26 | 26 |
"github.com/moby/buildkit/solver" |
| 27 |
+ "github.com/moby/buildkit/solver/llbsolver/compat" |
|
| 27 | 28 |
"github.com/moby/buildkit/solver/pb" |
| 28 | 29 |
"github.com/moby/buildkit/source" |
| 29 | 30 |
srctypes "github.com/moby/buildkit/source/types" |
| ... | ... |
@@ -44,11 +46,16 @@ var defaultBranch = regexp.MustCompile(`refs/heads/(\S+)`) |
| 44 | 44 |
|
| 45 | 45 |
type Opt struct {
|
| 46 | 46 |
CacheAccessor cache.Accessor |
| 47 |
+ // RegistryHosts is used to fetch bundle blobs from a docker registry |
|
| 48 |
+ // when a git source uses GitBundleURL("docker-image+blob://...").
|
|
| 49 |
+ // Optional: when unset, bundle mode requires oci-layout+blob. |
|
| 50 |
+ RegistryHosts docker.RegistryHosts |
|
| 47 | 51 |
} |
| 48 | 52 |
|
| 49 | 53 |
type Source struct {
|
| 50 |
- cache cache.Accessor |
|
| 51 |
- locker *locker.Locker |
|
| 54 |
+ cache cache.Accessor |
|
| 55 |
+ locker *locker.Locker |
|
| 56 |
+ registryHosts docker.RegistryHosts |
|
| 52 | 57 |
} |
| 53 | 58 |
|
| 54 | 59 |
type Metadata struct {
|
| ... | ... |
@@ -74,8 +81,9 @@ func Supported() error {
|
| 74 | 74 |
|
| 75 | 75 |
func NewSource(opt Opt) (*Source, error) {
|
| 76 | 76 |
gs := &Source{
|
| 77 |
- cache: opt.CacheAccessor, |
|
| 78 |
- locker: locker.New(), |
|
| 77 |
+ cache: opt.CacheAccessor, |
|
| 78 |
+ locker: locker.New(), |
|
| 79 |
+ registryHosts: opt.RegistryHosts, |
|
| 79 | 80 |
} |
| 80 | 81 |
return gs, nil |
| 81 | 82 |
} |
| ... | ... |
@@ -137,11 +145,24 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo |
| 137 | 137 |
id.VerifySignature.IgnoreSignedTag = v == "true" |
| 138 | 138 |
case pb.AttrGitMTime: |
| 139 | 139 |
id.MTime = v |
| 140 |
+ case pb.AttrGitFetchByCommit: |
|
| 141 |
+ id.FetchByCommit = v == "true" |
|
| 142 |
+ case pb.AttrGitBundle: |
|
| 143 |
+ id.Bundle = v |
|
| 144 |
+ case pb.AttrGitCheckoutBundle: |
|
| 145 |
+ id.CheckoutBundle = v == "true" |
|
| 146 |
+ case pb.AttrOCILayoutSessionID: |
|
| 147 |
+ id.BundleOCISessionID = v |
|
| 148 |
+ case pb.AttrOCILayoutStoreID: |
|
| 149 |
+ id.BundleOCIStoreID = v |
|
| 140 | 150 |
} |
| 141 | 151 |
} |
| 142 | 152 |
if err := validateGitRef(id.Ref); err != nil {
|
| 143 | 153 |
return nil, err |
| 144 | 154 |
} |
| 155 |
+ if err := validateBundleAttrs(id); err != nil {
|
|
| 156 |
+ return nil, err |
|
| 157 |
+ } |
|
| 145 | 158 |
|
| 146 | 159 |
return id, nil |
| 147 | 160 |
} |
| ... | ... |
@@ -254,6 +275,12 @@ type gitSourceHandler struct {
|
| 254 | 254 |
sha256 bool |
| 255 | 255 |
sm *session.Manager |
| 256 | 256 |
authArgs []string |
| 257 |
+ |
|
| 258 |
+ // stagedBundleURL / stagedBundleCleanup cache the temp bundle staging |
|
| 259 |
+ // across resolveBundleMetadata (CacheKey) and tryRemoteFetch (Snapshot) |
|
| 260 |
+ // so the bundle blob is downloaded only once per handler. |
|
| 261 |
+ stagedBundleURL string |
|
| 262 |
+ stagedBundleCleanup func() error |
|
| 257 | 263 |
} |
| 258 | 264 |
|
| 259 | 265 |
func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string {
|
| ... | ... |
@@ -273,6 +300,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string {
|
| 273 | 273 |
if gs.src.MTime != "" && gs.src.MTime != "checkout" {
|
| 274 | 274 |
key += "(mtime=" + gs.src.MTime + ")" |
| 275 | 275 |
} |
| 276 |
+ if gs.src.CheckoutBundle {
|
|
| 277 |
+ key += "(bundle)" |
|
| 278 |
+ } |
|
| 276 | 279 |
return key |
| 277 | 280 |
} |
| 278 | 281 |
|
| ... | ... |
@@ -282,18 +312,28 @@ func (gs *Source) ResolveMetadata(ctx context.Context, id *GitIdentifier, sm *se |
| 282 | 282 |
Source: gs, |
| 283 | 283 |
sm: sm, |
| 284 | 284 |
} |
| 285 |
+ // The handler is scoped to this call, so any staged bundle it owns |
|
| 286 |
+ // must be released before returning. When a jobCtx is present, |
|
| 287 |
+ // ensureStagedBundle hands ownership to the job and this is a no-op; |
|
| 288 |
+ // without a jobCtx (e.g. direct ResolveMetadata callers) this is the |
|
| 289 |
+ // only hook that frees the temp dir. |
|
| 290 |
+ defer func() {
|
|
| 291 |
+ if err := gsh.releaseStagedBundle(); err != nil {
|
|
| 292 |
+ bklog.G(ctx).Warnf("failed to release staged git bundle: %v", err)
|
|
| 293 |
+ } |
|
| 294 |
+ }() |
|
| 285 | 295 |
md, err := gsh.resolveMetadata(ctx, jobCtx) |
| 286 | 296 |
if err != nil {
|
| 287 | 297 |
return nil, err |
| 288 | 298 |
} |
| 289 | 299 |
|
| 300 |
+ gsh.cacheCommit = md.Checksum |
|
| 301 |
+ gsh.sha256 = len(md.Checksum) == 64 |
|
| 302 |
+ |
|
| 290 | 303 |
if !opt.ReturnObject && id.VerifySignature == nil {
|
| 291 | 304 |
return md, nil |
| 292 | 305 |
} |
| 293 | 306 |
|
| 294 |
- gsh.cacheCommit = md.Checksum |
|
| 295 |
- gsh.sha256 = len(md.Checksum) == 64 |
|
| 296 |
- |
|
| 297 | 307 |
if err := gsh.addGitObjectsToMetadata(ctx, jobCtx, md); err != nil {
|
| 298 | 308 |
return nil, err |
| 299 | 309 |
} |
| ... | ... |
@@ -487,10 +527,6 @@ func (gs *gitSourceHandler) remoteKey() string {
|
| 487 | 487 |
} |
| 488 | 488 |
|
| 489 | 489 |
func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.JobContext) (md *Metadata, retErr error) {
|
| 490 |
- remote := gs.src.Remote |
|
| 491 |
- gs.locker.Lock(remote) |
|
| 492 |
- defer gs.locker.Unlock(remote) |
|
| 493 |
- |
|
| 494 | 490 |
if gs.src.Checksum != "" {
|
| 495 | 491 |
matched, err := regexp.MatchString("^[a-fA-F0-9]+$", gs.src.Checksum)
|
| 496 | 492 |
if err != nil || !matched {
|
| ... | ... |
@@ -498,6 +534,19 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J |
| 498 | 498 |
} |
| 499 | 499 |
} |
| 500 | 500 |
|
| 501 |
+ if gs.src.Bundle != "" {
|
|
| 502 |
+ // Bundle mode stages the bundle in a temp bare repo and runs the |
|
| 503 |
+ // shared ls-remote code path against its file:// URL so the |
|
| 504 |
+ // resulting Metadata has the same ref shape as the non-bundle |
|
| 505 |
+ // path. Bundle mode does not need the per-remote lock: the temp |
|
| 506 |
+ // bare repo is private to this call. |
|
| 507 |
+ return gs.resolveBundleMetadata(ctx, jobCtx) |
|
| 508 |
+ } |
|
| 509 |
+ |
|
| 510 |
+ remote := gs.src.Remote |
|
| 511 |
+ gs.locker.Lock(remote) |
|
| 512 |
+ defer gs.locker.Unlock(remote) |
|
| 513 |
+ |
|
| 501 | 514 |
if gitutil.IsCommitSHA(gs.src.Ref) {
|
| 502 | 515 |
if gs.src.Checksum != "" && !strings.HasPrefix(gs.src.Ref, gs.src.Checksum) {
|
| 503 | 516 |
return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, gs.src.Ref)
|
| ... | ... |
@@ -508,6 +557,29 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J |
| 508 | 508 |
}, nil |
| 509 | 509 |
} |
| 510 | 510 |
|
| 511 |
+ if gs.src.FetchByCommit {
|
|
| 512 |
+ if gs.src.Checksum == "" {
|
|
| 513 |
+ return nil, errors.Errorf("fetch-by-commit requires a checksum or a commit SHA ref")
|
|
| 514 |
+ } |
|
| 515 |
+ if !gitutil.IsCommitSHA(gs.src.Checksum) {
|
|
| 516 |
+ return nil, errors.Errorf("fetch-by-commit requires a full commit SHA checksum, got %q", gs.src.Checksum)
|
|
| 517 |
+ } |
|
| 518 |
+ // Canonicalize unqualified refs so that cache keys match those |
|
| 519 |
+ // produced by the normal path's ls-remote-driven normalization. |
|
| 520 |
+ // Unqualified names are treated as branches (git's preferred |
|
| 521 |
+ // resolution); tags must be passed as "refs/tags/<name>". |
|
| 522 |
+ if gs.src.Ref != "" && !strings.HasPrefix(gs.src.Ref, "refs/") {
|
|
| 523 |
+ gs.src.Ref = "refs/heads/" + gs.src.Ref |
|
| 524 |
+ } |
|
| 525 |
+ if gs.src.Ref == "" {
|
|
| 526 |
+ gs.src.Ref = gs.src.Checksum |
|
| 527 |
+ } |
|
| 528 |
+ return &Metadata{
|
|
| 529 |
+ Ref: gs.src.Ref, |
|
| 530 |
+ Checksum: gs.src.Checksum, |
|
| 531 |
+ }, nil |
|
| 532 |
+ } |
|
| 533 |
+ |
|
| 511 | 534 |
var g session.Group |
| 512 | 535 |
if jobCtx != nil {
|
| 513 | 536 |
g = jobCtx.Session() |
| ... | ... |
@@ -544,6 +616,14 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J |
| 544 | 544 |
|
| 545 | 545 |
gs.getAuthToken(ctx, g) |
| 546 | 546 |
|
| 547 |
+ return gs.resolveMetadataFromURL(ctx, g, gs.src.Remote) |
|
| 548 |
+} |
|
| 549 |
+ |
|
| 550 |
+// resolveMetadataFromURL runs ls-remote against the given URL and parses the |
|
| 551 |
+// result into a Metadata. The URL may be the user's configured remote (for |
|
| 552 |
+// non-bundle mode) or a file:// URL for a staged bundle. Auth and lock |
|
| 553 |
+// management are the caller's responsibility. |
|
| 554 |
+func (gs *gitSourceHandler) resolveMetadataFromURL(ctx context.Context, g session.Group, remoteURL string) (*Metadata, error) {
|
|
| 547 | 555 |
tmpGit, cleanup, err := gs.emptyGitCli(ctx, g) |
| 548 | 556 |
if err != nil {
|
| 549 | 557 |
return nil, err |
| ... | ... |
@@ -552,16 +632,16 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J |
| 552 | 552 |
|
| 553 | 553 |
ref := gs.src.Ref |
| 554 | 554 |
if ref == "" {
|
| 555 |
- ref, err = getDefaultBranch(ctx, tmpGit, gs.src.Remote) |
|
| 555 |
+ ref, err = getDefaultBranch(ctx, tmpGit, remoteURL) |
|
| 556 | 556 |
if err != nil {
|
| 557 | 557 |
return nil, err |
| 558 | 558 |
} |
| 559 | 559 |
} |
| 560 | 560 |
// TODO: should we assume that remote tag is immutable? add a timer? |
| 561 | 561 |
|
| 562 |
- buf, err := tmpGit.Run(ctx, "ls-remote", "--", gs.src.Remote, ref, ref+"^{}")
|
|
| 562 |
+ buf, err := tmpGit.Run(ctx, "ls-remote", "--", remoteURL, ref, ref+"^{}")
|
|
| 563 | 563 |
if err != nil {
|
| 564 |
- return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote)) |
|
| 564 |
+ return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remoteURL)) |
|
| 565 | 565 |
} |
| 566 | 566 |
lines := strings.Split(string(buf), "\n") |
| 567 | 567 |
|
| ... | ... |
@@ -616,7 +696,7 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J |
| 616 | 616 |
return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, exp)
|
| 617 | 617 |
} |
| 618 | 618 |
} |
| 619 |
- md = &Metadata{
|
|
| 619 |
+ md := &Metadata{
|
|
| 620 | 620 |
Ref: usedRef, |
| 621 | 621 |
Checksum: sha, |
| 622 | 622 |
} |
| ... | ... |
@@ -725,12 +805,12 @@ func (gs *gitSourceHandler) remoteFetch(ctx context.Context, jobCtx solver.JobCo |
| 725 | 725 |
g = jobCtx.Session() |
| 726 | 726 |
} |
| 727 | 727 |
|
| 728 |
- repo, err := gs.tryRemoteFetch(ctx, g, false) |
|
| 728 |
+ repo, err := gs.tryRemoteFetch(ctx, jobCtx, g, false) |
|
| 729 | 729 |
if err != nil {
|
| 730 | 730 |
var wce *wouldClobberExistingTagError |
| 731 | 731 |
var ulre *unableToUpdateLocalRefError |
| 732 | 732 |
if errors.As(err, &wce) || errors.As(err, &ulre) {
|
| 733 |
- repo, err = gs.tryRemoteFetch(ctx, g, true) |
|
| 733 |
+ repo, err = gs.tryRemoteFetch(ctx, jobCtx, g, true) |
|
| 734 | 734 |
if err != nil {
|
| 735 | 735 |
return nil, err |
| 736 | 736 |
} |
| ... | ... |
@@ -748,6 +828,11 @@ func (gs *gitSourceHandler) remoteFetch(ctx context.Context, jobCtx solver.JobCo |
| 748 | 748 |
}() |
| 749 | 749 |
|
| 750 | 750 |
ref := gs.src.Ref |
| 751 |
+ // With fetch-by-commit, the user-provided ref is not written into the |
|
| 752 |
+ // bare repo, so resolve the checksum directly. |
|
| 753 |
+ if gs.src.FetchByCommit && gs.src.Checksum != "" {
|
|
| 754 |
+ ref = gs.src.Checksum |
|
| 755 |
+ } |
|
| 751 | 756 |
git := repo.GitCLI |
| 752 | 757 |
if gs.src.Checksum != "" {
|
| 753 | 758 |
actualHashBuf, err := repo.Run(ctx, "rev-parse", ref) |
| ... | ... |
@@ -789,6 +874,14 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobConte |
| 789 | 789 |
g = jobCtx.Session() |
| 790 | 790 |
} |
| 791 | 791 |
gs.getAuthToken(ctx, g) |
| 792 |
+ compatibilityVersion := 0 |
|
| 793 |
+ if jobCtx != nil {
|
|
| 794 |
+ var err error |
|
| 795 |
+ compatibilityVersion, err = jobCtx.CompatibilityVersion() |
|
| 796 |
+ if err != nil {
|
|
| 797 |
+ return nil, err |
|
| 798 |
+ } |
|
| 799 |
+ } |
|
| 792 | 800 |
|
| 793 | 801 |
snapshotKey := cacheKey + ":" + gs.src.Subdir |
| 794 | 802 |
gs.locker.Lock(snapshotKey) |
| ... | ... |
@@ -808,7 +901,12 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobConte |
| 808 | 808 |
} |
| 809 | 809 |
defer repo.Release() |
| 810 | 810 |
|
| 811 |
- ref, err := gs.checkout(ctx, repo, g) |
|
| 811 |
+ var ref cache.ImmutableRef |
|
| 812 |
+ if gs.src.CheckoutBundle {
|
|
| 813 |
+ ref, err = gs.checkoutAsBundle(ctx, repo, g) |
|
| 814 |
+ } else {
|
|
| 815 |
+ ref, err = gs.checkout(ctx, repo, g, compatibilityVersion) |
|
| 816 |
+ } |
|
| 812 | 817 |
if err != nil {
|
| 813 | 818 |
return nil, err |
| 814 | 819 |
} |
| ... | ... |
@@ -836,7 +934,7 @@ func (g *gitRepo) Release() error {
|
| 836 | 836 |
return err |
| 837 | 837 |
} |
| 838 | 838 |
|
| 839 |
-func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, reset bool) (_ *gitRepo, retErr error) {
|
|
| 839 |
+func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, jobCtx solver.JobContext, g session.Group, reset bool) (_ *gitRepo, retErr error) {
|
|
| 840 | 840 |
repo := &gitRepo{}
|
| 841 | 841 |
|
| 842 | 842 |
defer func() {
|
| ... | ... |
@@ -846,13 +944,32 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 846 | 846 |
} |
| 847 | 847 |
}() |
| 848 | 848 |
|
| 849 |
+ // Auth args only apply to non-bundle fetches. In bundle mode the |
|
| 850 |
+ // payload comes from a blob fetch, so there is no http remote to |
|
| 851 |
+ // authenticate against. |
|
| 852 |
+ authArgs := gs.authArgs |
|
| 853 |
+ if gs.src.Bundle != "" {
|
|
| 854 |
+ authArgs = nil |
|
| 855 |
+ } |
|
| 856 |
+ |
|
| 849 | 857 |
git, cleanup, err := gs.emptyGitCli(ctx, g) |
| 850 | 858 |
if err != nil {
|
| 851 | 859 |
return nil, err |
| 852 | 860 |
} |
| 853 | 861 |
repo.releasers = append(repo.releasers, cleanup) |
| 854 | 862 |
|
| 855 |
- gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.authArgs, gs.sha256, reset, g) |
|
| 863 |
+ // Bundle mode may need to create the shared bare repo before the main |
|
| 864 |
+ // fetch. Stage the bundle first so stageBundle can probe the bundle's |
|
| 865 |
+ // object format via ls-remote and set gs.sha256 for mountRemote. |
|
| 866 |
+ stagedURL := "" |
|
| 867 |
+ if gs.src.Bundle != "" {
|
|
| 868 |
+ stagedURL, err = gs.ensureStagedBundle(ctx, jobCtx, g) |
|
| 869 |
+ if err != nil {
|
|
| 870 |
+ return nil, err |
|
| 871 |
+ } |
|
| 872 |
+ } |
|
| 873 |
+ |
|
| 874 |
+ gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.sha256, reset, g) |
|
| 856 | 875 |
if err != nil {
|
| 857 | 876 |
return nil, err |
| 858 | 877 |
} |
| ... | ... |
@@ -862,6 +979,40 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 862 | 862 |
git = git.New(gitutil.WithGitDir(gitDir)) |
| 863 | 863 |
repo.GitCLI = git |
| 864 | 864 |
|
| 865 |
+ // fetchSource is the URL used in place of the "origin" remote name for |
|
| 866 |
+ // the main fetch. In bundle mode it points at a staged bundle's file:// |
|
| 867 |
+ // URL; an empty value means fetch from the configured "origin" remote. |
|
| 868 |
+ fetchSource := "" |
|
| 869 |
+ // Bundle mode: stage the bundle into an isolated temp bare repo and |
|
| 870 |
+ // fetch from it as a file:// remote. The rest of this function runs |
|
| 871 |
+ // unchanged — git accepts a URL anywhere a remote name is expected, |
|
| 872 |
+ // so the fetch flow treats the staged repo like a normal origin. |
|
| 873 |
+ if gs.src.Bundle != "" {
|
|
| 874 |
+ // When the user did not supply a symbolic ref, use the pinned |
|
| 875 |
+ // commit as the ref directly. validateBundleAttrs guarantees |
|
| 876 |
+ // Checksum is set. |
|
| 877 |
+ if gs.src.Ref == "" || gitutil.IsCommitSHA(gs.src.Ref) {
|
|
| 878 |
+ gs.src.Ref = gs.src.Checksum |
|
| 879 |
+ } |
|
| 880 |
+ // Bundle mode enters tryRemoteFetch from Snapshot. If CacheKey |
|
| 881 |
+ // has not run yet, prime gs.cacheCommit from the pinned checksum |
|
| 882 |
+ // to make the downstream cacheCommit validation a no-op rather |
|
| 883 |
+ // than a spurious mismatch. |
|
| 884 |
+ if gs.cacheCommit == "" {
|
|
| 885 |
+ gs.cacheCommit = gs.src.Checksum |
|
| 886 |
+ } |
|
| 887 |
+ // If the pinned commit is already present in the shared bare |
|
| 888 |
+ // repo (from a prior origin or bundle fetch), skip staging |
|
| 889 |
+ // entirely. The doFetch check below will no-op the fetch. |
|
| 890 |
+ if _, err := repo.Run(ctx, "cat-file", "-e", gs.src.Checksum+"^{commit}"); err != nil {
|
|
| 891 |
+ // Reuse the staged bundle if CacheKey or the eager bundle-format |
|
| 892 |
+ // probe above already built one. ensureStagedBundle owns the |
|
| 893 |
+ // teardown (wired to jobCtx.Cleanup on first call), so the |
|
| 894 |
+ // cleanup is intentionally not appended to repo.releasers. |
|
| 895 |
+ fetchSource = stagedURL |
|
| 896 |
+ } |
|
| 897 |
+ } |
|
| 898 |
+ |
|
| 865 | 899 |
ref := gs.src.Ref |
| 866 | 900 |
if ref == "" {
|
| 867 | 901 |
ref, err = getDefaultBranch(ctx, git, gs.src.Remote) |
| ... | ... |
@@ -870,10 +1021,16 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 870 | 870 |
} |
| 871 | 871 |
gs.src.Ref = ref |
| 872 | 872 |
} |
| 873 |
+ // fetchRef is the identifier used to fetch from the remote. For fetch-by-commit |
|
| 874 |
+ // mode this is the checksum (commit SHA); otherwise it is the ref itself. |
|
| 875 |
+ fetchRef := ref |
|
| 876 |
+ if gs.src.FetchByCommit && gs.src.Checksum != "" {
|
|
| 877 |
+ fetchRef = gs.src.Checksum |
|
| 878 |
+ } |
|
| 873 | 879 |
doFetch := true |
| 874 |
- if gitutil.IsCommitSHA(ref) {
|
|
| 880 |
+ if gitutil.IsCommitSHA(fetchRef) {
|
|
| 875 | 881 |
// skip fetch if commit already exists |
| 876 |
- if _, err := git.Run(ctx, "cat-file", "-e", "--", ref+"^{commit}"); err == nil {
|
|
| 882 |
+ if _, err := git.Run(ctx, "cat-file", "-e", "--", fetchRef+"^{commit}"); err == nil {
|
|
| 877 | 883 |
doFetch = false |
| 878 | 884 |
} |
| 879 | 885 |
} |
| ... | ... |
@@ -896,8 +1053,16 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 896 | 896 |
// make sure no old lock files have leaked |
| 897 | 897 |
gitDirRoot.RemoveAll("shallow.lock")
|
| 898 | 898 |
|
| 899 |
+ origin := "origin" |
|
| 900 |
+ if fetchSource != "" {
|
|
| 901 |
+ origin = fetchSource |
|
| 902 |
+ } |
|
| 899 | 903 |
args := []string{"fetch"}
|
| 900 |
- if !gitutil.IsCommitSHA(ref) { // TODO: find a branch from ls-remote?
|
|
| 904 |
+ // For fetch-by-commit, assume the server supports |
|
| 905 |
+ // uploadpack.allowReachableSHA1InWant / allowAnySHA1InWant and |
|
| 906 |
+ // fetch only the requested commit, without the tag/unshallow |
|
| 907 |
+ // fallback used by the generic SHA-ref path. |
|
| 908 |
+ if gs.src.FetchByCommit || !gitutil.IsCommitSHA(fetchRef) { // TODO: find a branch from ls-remote?
|
|
| 901 | 909 |
args = append(args, "--depth=1", "--no-tags") |
| 902 | 910 |
} else {
|
| 903 | 911 |
args = append(args, "--tags") |
| ... | ... |
@@ -905,11 +1070,11 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 905 | 905 |
args = append(args, "--unshallow") |
| 906 | 906 |
} |
| 907 | 907 |
} |
| 908 |
- args = append(args, "origin") |
|
| 908 |
+ args = append(args, origin) |
|
| 909 | 909 |
if gitutil.IsCommitSHA(ref) {
|
| 910 | 910 |
args = append(args, ref) |
| 911 | 911 |
} else {
|
| 912 |
- args = append(args, "--force", "--", ref+":"+targetRef) |
|
| 912 |
+ args = append(args, "--force", "--", fetchRef+":"+targetRef) |
|
| 913 | 913 |
} |
| 914 | 914 |
if _, err := git.Run(ctx, args...); err != nil {
|
| 915 | 915 |
err := errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote)) |
| ... | ... |
@@ -928,7 +1093,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 928 | 928 |
} |
| 929 | 929 |
|
| 930 | 930 |
// verify that commit matches the cache key commit |
| 931 |
- dt, err := git.Run(ctx, "rev-parse", ref) |
|
| 931 |
+ dt, err := git.Run(ctx, "rev-parse", fetchRef) |
|
| 932 | 932 |
if err != nil {
|
| 933 | 933 |
return nil, err |
| 934 | 934 |
} |
| ... | ... |
@@ -951,7 +1116,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 951 | 951 |
if _, err := gitDirRoot.Lstat("shallow"); err == nil {
|
| 952 | 952 |
args = append(args, "--unshallow") |
| 953 | 953 |
} |
| 954 |
- args = append(args, "origin", gs.cacheCommit) |
|
| 954 |
+ args = append(args, origin, gs.cacheCommit) |
|
| 955 | 955 |
if _, err := git.Run(ctx, args...); err != nil {
|
| 956 | 956 |
return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote)) |
| 957 | 957 |
} |
| ... | ... |
@@ -975,9 +1140,27 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, |
| 975 | 975 |
return repo, nil |
| 976 | 976 |
} |
| 977 | 977 |
|
| 978 |
-func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g session.Group) (_ cache.ImmutableRef, retErr error) {
|
|
| 978 |
+func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g session.Group, compatibilityVersion int) (_ cache.ImmutableRef, retErr error) {
|
|
| 979 | 979 |
ref := gs.src.Ref |
| 980 |
- checkoutRef, err := gs.cache.New(ctx, nil, g, cache.WithRecordType(client.UsageRecordTypeGitCheckout), cache.WithDescription(fmt.Sprintf("git snapshot for %s#%s", urlutil.RedactCredentials(gs.src.Remote), ref)))
|
|
| 980 |
+ // refOrCommit is the identifier to use against the bare repo. For |
|
| 981 |
+ // fetch-by-commit, the user-provided ref is not written into the bare |
|
| 982 |
+ // repo, so fall back to the commit checksum which is always present. |
|
| 983 |
+ refOrCommit := ref |
|
| 984 |
+ if gs.src.FetchByCommit && gs.src.Checksum != "" {
|
|
| 985 |
+ refOrCommit = gs.src.Checksum |
|
| 986 |
+ } |
|
| 987 |
+ // In bundle mode, if the user did not supply a symbolic Ref (empty or |
|
| 988 |
+ // a SHA), fall back to the pinned checksum so the downstream |
|
| 989 |
+ // `git checkout` / `git fetch` has a valid tip. With a symbolic ref |
|
| 990 |
+ // present, use it as-is: bundle import lands refs in the natural |
|
| 991 |
+ // refs/heads/* / refs/tags/* namespace in the shared bare repo, so |
|
| 992 |
+ // `git checkout <ref>` and `git fetch origin <ref>` resolve it |
|
| 993 |
+ // normally and the resulting .git carries the natural ref name. |
|
| 994 |
+ if gs.src.Bundle != "" && (ref == "" || gitutil.IsCommitSHA(ref)) {
|
|
| 995 |
+ ref = gs.src.Checksum |
|
| 996 |
+ refOrCommit = ref |
|
| 997 |
+ } |
|
| 998 |
+ checkoutRef, err := gs.cache.New(ctx, nil, g, cache.WithRecordType(client.UsageRecordTypeGitCheckout), cache.WithDescription(fmt.Sprintf("git snapshot for %s#%s", urlutil.RedactCredentials(gs.src.Remote), gs.src.Ref)))
|
|
| 981 | 999 |
if err != nil {
|
| 982 | 1000 |
return nil, errors.Wrapf(err, "failed to create new mutable for %s", urlutil.RedactCredentials(gs.src.Remote)) |
| 983 | 1001 |
} |
| ... | ... |
@@ -1034,7 +1217,7 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi |
| 1034 | 1034 |
return nil, err |
| 1035 | 1035 |
} |
| 1036 | 1036 |
|
| 1037 |
- gitCatFileBuf, err := git.Run(ctx, "cat-file", "-t", ref) |
|
| 1037 |
+ gitCatFileBuf, err := git.Run(ctx, "cat-file", "-t", refOrCommit) |
|
| 1038 | 1038 |
if err != nil {
|
| 1039 | 1039 |
return nil, err |
| 1040 | 1040 |
} |
| ... | ... |
@@ -1047,6 +1230,11 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi |
| 1047 | 1047 |
targetRef = "refs/tags/" + pullref |
| 1048 | 1048 |
} |
| 1049 | 1049 |
pullref += ":" + targetRef |
| 1050 |
+ } else if gs.src.FetchByCommit && ref != refOrCommit {
|
|
| 1051 |
+ // Fetch the commit object from the bare repo and save it under the |
|
| 1052 |
+ // user-provided ref name in the checkout clone. The ref does not |
|
| 1053 |
+ // need to exist in the bare repo. |
|
| 1054 |
+ pullref = refOrCommit + ":" + ref |
|
| 1050 | 1055 |
} else if gitutil.IsCommitSHA(ref) {
|
| 1051 | 1056 |
pullref = "refs/buildkit/" + identity.NewID() |
| 1052 | 1057 |
_, err = git.Run(ctx, "update-ref", pullref, ref) |
| ... | ... |
@@ -1084,7 +1272,7 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi |
| 1084 | 1084 |
} |
| 1085 | 1085 |
} |
| 1086 | 1086 |
checkoutGit := git.New(gitutil.WithWorkTree(cd), gitutil.WithGitDir(gitDir)) |
| 1087 |
- _, err = checkoutGit.Run(ctx, "checkout", "--no-overlay", ref, "--", ".") |
|
| 1087 |
+ _, err = checkoutGit.Run(ctx, "checkout", "--no-overlay", refOrCommit, "--", ".") |
|
| 1088 | 1088 |
if err != nil {
|
| 1089 | 1089 |
return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote)) |
| 1090 | 1090 |
} |
| ... | ... |
@@ -1138,8 +1326,14 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi |
| 1138 | 1138 |
} |
| 1139 | 1139 |
} |
| 1140 | 1140 |
|
| 1141 |
+ if compatibilityVersion == compat.CompatibilityVersion013 {
|
|
| 1142 |
+ if err := resetCompatibility014FileModes(checkoutDir); err != nil {
|
|
| 1143 |
+ return nil, errors.Wrapf(err, "failed to normalize compatibility file modes for %s", urlutil.RedactCredentials(gs.src.Remote)) |
|
| 1144 |
+ } |
|
| 1145 |
+ } |
|
| 1146 |
+ |
|
| 1141 | 1147 |
if gs.src.MTime == "commit" {
|
| 1142 |
- commitTime, err := getCommitTime(ctx, git, ref) |
|
| 1148 |
+ commitTime, err := getCommitTime(ctx, git, refOrCommit) |
|
| 1143 | 1149 |
if err != nil {
|
| 1144 | 1150 |
return nil, errors.Wrapf(err, "failed to get commit time for %s", urlutil.RedactCredentials(gs.src.Remote)) |
| 1145 | 1151 |
} |
| ... | ... |
@@ -1220,6 +1414,33 @@ func resetSnapshotMtimes(dir string, t time.Time) error {
|
| 1220 | 1220 |
return nil |
| 1221 | 1221 |
} |
| 1222 | 1222 |
|
| 1223 |
+// resetCompatibility014FileModes restores the pre-v0.15 git checkout file |
|
| 1224 |
+// mode for non-executable regular files, which were stored with group/other |
|
| 1225 |
+// write bits set before the exec-option propagation fix. Executable files are |
|
| 1226 |
+// left untouched: their pre-v0.15 behavior is not covered by the current |
|
| 1227 |
+// compatibility matrix, and blindly adding write bits to 0o755 would be a |
|
| 1228 |
+// guess. |
|
| 1229 |
+func resetCompatibility014FileModes(dir string) error {
|
|
| 1230 |
+ return filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
|
| 1231 |
+ if err != nil {
|
|
| 1232 |
+ return err |
|
| 1233 |
+ } |
|
| 1234 |
+ if d.IsDir() || d.Type()&os.ModeSymlink != 0 {
|
|
| 1235 |
+ return nil |
|
| 1236 |
+ } |
|
| 1237 |
+ |
|
| 1238 |
+ info, err := d.Info() |
|
| 1239 |
+ if err != nil {
|
|
| 1240 |
+ return err |
|
| 1241 |
+ } |
|
| 1242 |
+ mode := info.Mode() |
|
| 1243 |
+ if !mode.IsRegular() || mode&0o111 != 0 {
|
|
| 1244 |
+ return nil |
|
| 1245 |
+ } |
|
| 1246 |
+ return os.Chmod(p, mode|0o222) |
|
| 1247 |
+ }) |
|
| 1248 |
+} |
|
| 1249 |
+ |
|
| 1223 | 1250 |
type wouldClobberExistingTagError struct {
|
| 1224 | 1251 |
error |
| 1225 | 1252 |
} |
| 1226 | 1253 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,48 @@ |
| 0 |
+package appcontext |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "os" |
|
| 5 |
+ "os/signal" |
|
| 6 |
+ "sync" |
|
| 7 |
+ |
|
| 8 |
+ "github.com/moby/buildkit/util/bklog" |
|
| 9 |
+ "github.com/pkg/errors" |
|
| 10 |
+) |
|
| 11 |
+ |
|
| 12 |
+var appContextCache context.Context |
|
| 13 |
+var appContextOnce sync.Once |
|
| 14 |
+ |
|
| 15 |
+// Context returns a static context that reacts to termination signals of the |
|
| 16 |
+// running process. Useful in CLI tools. |
|
| 17 |
+func Context() context.Context {
|
|
| 18 |
+ appContextOnce.Do(func() {
|
|
| 19 |
+ signals := make(chan os.Signal, 2048) |
|
| 20 |
+ signal.Notify(signals, terminationSignals...) |
|
| 21 |
+ |
|
| 22 |
+ const exitLimit = 3 |
|
| 23 |
+ retries := 0 |
|
| 24 |
+ |
|
| 25 |
+ ctx := context.Background() |
|
| 26 |
+ for _, f := range inits {
|
|
| 27 |
+ ctx = f(ctx) //nolint:fatcontext |
|
| 28 |
+ } |
|
| 29 |
+ |
|
| 30 |
+ ctx, cancel := context.WithCancelCause(ctx) |
|
| 31 |
+ appContextCache = ctx //nolint:fatcontext |
|
| 32 |
+ |
|
| 33 |
+ go func() {
|
|
| 34 |
+ for {
|
|
| 35 |
+ <-signals |
|
| 36 |
+ retries++ |
|
| 37 |
+ err := errors.Errorf("got %d SIGTERM/SIGINTs, forcing shutdown", retries)
|
|
| 38 |
+ cancel(err) |
|
| 39 |
+ if retries >= exitLimit {
|
|
| 40 |
+ bklog.G(ctx).Error(err.Error()) |
|
| 41 |
+ os.Exit(1) |
|
| 42 |
+ } |
|
| 43 |
+ } |
|
| 44 |
+ }() |
|
| 45 |
+ }) |
|
| 46 |
+ return appContextCache |
|
| 47 |
+} |
| 0 | 7 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,14 @@ |
| 0 |
+package appcontext |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+) |
|
| 5 |
+ |
|
| 6 |
+type Initializer func(context.Context) context.Context |
|
| 7 |
+ |
|
| 8 |
+var inits []Initializer |
|
| 9 |
+ |
|
| 10 |
+// Register stores a new context initializer that runs on app context creation |
|
| 11 |
+func Register(f Initializer) {
|
|
| 12 |
+ inits = append(inits, f) |
|
| 13 |
+} |
| ... | ... |
@@ -27,6 +27,7 @@ type GitCLI struct {
|
| 27 | 27 |
|
| 28 | 28 |
sshAuthSock string |
| 29 | 29 |
sshKnownHosts string |
| 30 |
+ hostGitConfig bool |
|
| 30 | 31 |
} |
| 31 | 32 |
|
| 32 | 33 |
// Option provides a variadic option for configuring the git client. |
| ... | ... |
@@ -97,6 +98,15 @@ func WithSSHKnownHosts(sshKnownHosts string) Option {
|
| 97 | 97 |
} |
| 98 | 98 |
} |
| 99 | 99 |
|
| 100 |
+// WithHostGitConfig allows git to read the host system and user git config. |
|
| 101 |
+// This is intended for client-side local git inspection. The default remains |
|
| 102 |
+// isolated so daemon-side callers do not leak host configuration into git. |
|
| 103 |
+func WithHostGitConfig() Option {
|
|
| 104 |
+ return func(b *GitCLI) {
|
|
| 105 |
+ b.hostGitConfig = true |
|
| 106 |
+ } |
|
| 107 |
+} |
|
| 108 |
+ |
|
| 100 | 109 |
type StreamFunc func(context.Context) (io.WriteCloser, io.WriteCloser, func()) |
| 101 | 110 |
|
| 102 | 111 |
// WithStreams configures a callback for getting the streams for a command. The |
| ... | ... |
@@ -108,7 +118,7 @@ func WithStreams(streams StreamFunc) Option {
|
| 108 | 108 |
} |
| 109 | 109 |
} |
| 110 | 110 |
|
| 111 |
-// New initializes a new git client |
|
| 111 |
+// NewGitCLI initializes a new git client |
|
| 112 | 112 |
func NewGitCLI(opts ...Option) *GitCLI {
|
| 113 | 113 |
c := &GitCLI{}
|
| 114 | 114 |
for _, opt := range opts {
|
| ... | ... |
@@ -191,9 +201,28 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error |
| 191 | 191 |
"GIT_TERMINAL_PROMPT=0", |
| 192 | 192 |
"GIT_SSH_COMMAND=" + getGitSSHCommand(cli.sshKnownHosts), |
| 193 | 193 |
// "GIT_TRACE=1", |
| 194 |
- "GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig. |
|
| 195 |
- "HOME=/dev/null", // Disable reading from user gitconfig. |
|
| 196 |
- "LC_ALL=C", // Ensure consistent output. |
|
| 194 |
+ "LC_ALL=C", // Ensure consistent output. |
|
| 195 |
+ } |
|
| 196 |
+ if cli.hostGitConfig {
|
|
| 197 |
+ for _, ev := range [...]string{
|
|
| 198 |
+ "HOME", |
|
| 199 |
+ "XDG_CONFIG_HOME", |
|
| 200 |
+ "USERPROFILE", |
|
| 201 |
+ "HOMEDRIVE", |
|
| 202 |
+ "HOMEPATH", |
|
| 203 |
+ "GIT_CONFIG_GLOBAL", |
|
| 204 |
+ "GIT_CONFIG_SYSTEM", |
|
| 205 |
+ } {
|
|
| 206 |
+ if v, ok := os.LookupEnv(ev); ok {
|
|
| 207 |
+ cmd.Env = append(cmd.Env, ev+"="+v) |
|
| 208 |
+ } |
|
| 209 |
+ } |
|
| 210 |
+ } else {
|
|
| 211 |
+ cmd.Env = append(cmd.Env, |
|
| 212 |
+ "GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig. |
|
| 213 |
+ "HOME="+os.DevNull, // Disable reading from user gitconfig. |
|
| 214 |
+ "GIT_CONFIG_GLOBAL="+os.DevNull, // Disable reading from global gitconfig. |
|
| 215 |
+ ) |
|
| 197 | 216 |
} |
| 198 | 217 |
for _, ev := range proxyEnvVars {
|
| 199 | 218 |
if v, ok := os.LookupEnv(ev); ok {
|
| ... | ... |
@@ -244,7 +273,7 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error |
| 244 | 244 |
} |
| 245 | 245 |
|
| 246 | 246 |
func getGitSSHCommand(knownHosts string) string {
|
| 247 |
- gitSSHCommand := "ssh -F /dev/null" |
|
| 247 |
+ gitSSHCommand := "ssh -F " + os.DevNull |
|
| 248 | 248 |
if knownHosts != "" {
|
| 249 | 249 |
gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts |
| 250 | 250 |
} else {
|
| ... | ... |
@@ -1,7 +1,6 @@ |
| 1 | 1 |
package progress |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
- "maps" |
|
| 5 | 4 |
"slices" |
| 6 | 5 |
"sync" |
| 7 | 6 |
"time" |
| ... | ... |
@@ -81,18 +80,7 @@ func (ps *MultiWriter) Write(id string, v any) error {
|
| 81 | 81 |
} |
| 82 | 82 |
|
| 83 | 83 |
func (ps *MultiWriter) WriteRawProgress(p *Progress) error {
|
| 84 |
- meta := p.meta |
|
| 85 |
- if len(ps.meta) > 0 {
|
|
| 86 |
- meta = map[string]any{}
|
|
| 87 |
- maps.Copy(meta, p.meta) |
|
| 88 |
- for k, v := range ps.meta {
|
|
| 89 |
- if _, ok := meta[k]; !ok {
|
|
| 90 |
- meta[k] = v |
|
| 91 |
- } |
|
| 92 |
- } |
|
| 93 |
- } |
|
| 94 |
- p.meta = meta |
|
| 95 |
- return ps.writeRawProgress(p) |
|
| 84 |
+ return ps.writeRawProgress(p.Decorate(ps.meta)) |
|
| 96 | 85 |
} |
| 97 | 86 |
|
| 98 | 87 |
func (ps *MultiWriter) writeRawProgress(p *Progress) error {
|
| ... | ... |
@@ -235,18 +235,7 @@ func (pw *progressWriter) Write(id string, v any) error {
|
| 235 | 235 |
} |
| 236 | 236 |
|
| 237 | 237 |
func (pw *progressWriter) WriteRawProgress(p *Progress) error {
|
| 238 |
- meta := p.meta |
|
| 239 |
- if len(pw.meta) > 0 {
|
|
| 240 |
- meta = map[string]any{}
|
|
| 241 |
- maps.Copy(meta, p.meta) |
|
| 242 |
- for k, v := range pw.meta {
|
|
| 243 |
- if _, ok := meta[k]; !ok {
|
|
| 244 |
- meta[k] = v |
|
| 245 |
- } |
|
| 246 |
- } |
|
| 247 |
- } |
|
| 248 |
- p.meta = meta |
|
| 249 |
- return pw.writeRawProgress(p) |
|
| 238 |
+ return pw.writeRawProgress(p.Decorate(pw.meta)) |
|
| 250 | 239 |
} |
| 251 | 240 |
|
| 252 | 241 |
func (pw *progressWriter) writeRawProgress(p *Progress) error {
|
| ... | ... |
@@ -271,6 +260,21 @@ func (p *Progress) Meta(key string) (any, bool) {
|
| 271 | 271 |
return v, ok |
| 272 | 272 |
} |
| 273 | 273 |
|
| 274 |
+// Decorate merges in missing metadata without overwriting existing keys. |
|
| 275 |
+func (p *Progress) Decorate(meta map[string]any) *Progress {
|
|
| 276 |
+ if len(meta) == 0 {
|
|
| 277 |
+ return p |
|
| 278 |
+ } |
|
| 279 |
+ |
|
| 280 |
+ merged := make(map[string]any, len(p.meta)+len(meta)) |
|
| 281 |
+ maps.Copy(merged, meta) |
|
| 282 |
+ maps.Copy(merged, p.meta) |
|
| 283 |
+ |
|
| 284 |
+ newP := *p |
|
| 285 |
+ newP.meta = merged |
|
| 286 |
+ return &newP |
|
| 287 |
+} |
|
| 288 |
+ |
|
| 274 | 289 |
type noOpWriter struct{}
|
| 275 | 290 |
|
| 276 | 291 |
func (pw *noOpWriter) Write(_ string, _ any) error {
|
| 277 | 292 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,110 @@ |
| 0 |
+package registrar |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "sync" |
|
| 5 |
+ "time" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+type Registrar[K comparable, V any] struct {
|
|
| 9 |
+ mu sync.Mutex |
|
| 10 |
+ values map[K]*registrarValue[V] |
|
| 11 |
+} |
|
| 12 |
+ |
|
| 13 |
+func New[K comparable, V any]() *Registrar[K, V] {
|
|
| 14 |
+ return &Registrar[K, V]{
|
|
| 15 |
+ values: make(map[K]*registrarValue[V]), |
|
| 16 |
+ } |
|
| 17 |
+} |
|
| 18 |
+ |
|
| 19 |
+// Register will register the value with the given id. |
|
| 20 |
+// This value will persist until Discard is called with the same id. |
|
| 21 |
+func (r *Registrar[K, V]) Register(id K, val V) {
|
|
| 22 |
+ reg := r.getOrCreateRegistrar(id, nil) |
|
| 23 |
+ reg.Register(val, nil) |
|
| 24 |
+} |
|
| 25 |
+ |
|
| 26 |
+// Get will retrieve a registered value and will wait a small time period for that |
|
| 27 |
+// value to appear if it hasn't been registered yet. |
|
| 28 |
+func (r *Registrar[K, V]) Get(ctx context.Context, id K) (v V, _ error) {
|
|
| 29 |
+ onCreate := func(reg *registrarValue[V]) {
|
|
| 30 |
+ select {
|
|
| 31 |
+ case <-reg.notifyCh: |
|
| 32 |
+ return |
|
| 33 |
+ case <-time.After(3 * time.Second): |
|
| 34 |
+ r.Discard(id) |
|
| 35 |
+ } |
|
| 36 |
+ } |
|
| 37 |
+ |
|
| 38 |
+ reg := r.getOrCreateRegistrar(id, onCreate) |
|
| 39 |
+ |
|
| 40 |
+ select {
|
|
| 41 |
+ case <-ctx.Done(): |
|
| 42 |
+ return v, context.Cause(ctx) |
|
| 43 |
+ case <-reg.notifyCh: |
|
| 44 |
+ return reg.value, reg.err |
|
| 45 |
+ } |
|
| 46 |
+} |
|
| 47 |
+ |
|
| 48 |
+// Discard will remove the given value from the registrar after it has been registered |
|
| 49 |
+// with Register. |
|
| 50 |
+func (r *Registrar[K, V]) Discard(id K) {
|
|
| 51 |
+ r.mu.Lock() |
|
| 52 |
+ reg, ok := r.values[id] |
|
| 53 |
+ delete(r.values, id) |
|
| 54 |
+ r.mu.Unlock() |
|
| 55 |
+ |
|
| 56 |
+ if ok {
|
|
| 57 |
+ var value V |
|
| 58 |
+ reg.Register(value, context.Canceled) |
|
| 59 |
+ } |
|
| 60 |
+} |
|
| 61 |
+ |
|
| 62 |
+// getOrCreateRegistrar will create a registrar with the given id to be retrieved at a later time. |
|
| 63 |
+// The same id will return the same registrar. |
|
| 64 |
+// |
|
| 65 |
+// If the registrar is newly created, the onCreate function is invoked in a separate goroutine |
|
| 66 |
+// if it is present. If nil, this function is ignored. |
|
| 67 |
+func (r *Registrar[K, V]) getOrCreateRegistrar(id K, onCreate func(*registrarValue[V])) *registrarValue[V] {
|
|
| 68 |
+ r.mu.Lock() |
|
| 69 |
+ defer r.mu.Unlock() |
|
| 70 |
+ |
|
| 71 |
+ reg, ok := r.values[id] |
|
| 72 |
+ if !ok {
|
|
| 73 |
+ reg = ®istrarValue[V]{
|
|
| 74 |
+ notifyCh: make(chan struct{}),
|
|
| 75 |
+ } |
|
| 76 |
+ r.values[id] = reg |
|
| 77 |
+ |
|
| 78 |
+ if onCreate != nil {
|
|
| 79 |
+ go onCreate(reg) |
|
| 80 |
+ } |
|
| 81 |
+ } |
|
| 82 |
+ return reg |
|
| 83 |
+} |
|
| 84 |
+ |
|
| 85 |
+type registrarValue[V any] struct {
|
|
| 86 |
+ // notifyCh is the notification channel that gets closed when |
|
| 87 |
+ // the bridge is registered. |
|
| 88 |
+ notifyCh chan struct{}
|
|
| 89 |
+ |
|
| 90 |
+ value V |
|
| 91 |
+ err error |
|
| 92 |
+ isSet bool |
|
| 93 |
+ |
|
| 94 |
+ mu sync.Mutex |
|
| 95 |
+} |
|
| 96 |
+ |
|
| 97 |
+func (r *registrarValue[V]) Register(value V, err error) {
|
|
| 98 |
+ r.mu.Lock() |
|
| 99 |
+ defer r.mu.Unlock() |
|
| 100 |
+ |
|
| 101 |
+ if r.isSet {
|
|
| 102 |
+ return |
|
| 103 |
+ } |
|
| 104 |
+ |
|
| 105 |
+ r.value = value |
|
| 106 |
+ r.err = err |
|
| 107 |
+ r.isSet = true |
|
| 108 |
+ close(r.notifyCh) |
|
| 109 |
+} |
| ... | ... |
@@ -47,6 +47,7 @@ func newAuthHandlerNS(sm *session.Manager) *authHandlerNS {
|
| 47 | 47 |
} |
| 48 | 48 |
|
| 49 | 49 |
func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manager, g session.Group) *authFetcher {
|
| 50 |
+ hasSession := false |
|
| 50 | 51 |
if g != nil {
|
| 51 | 52 |
if iter := g.SessionIterator(); iter != nil {
|
| 52 | 53 |
for {
|
| ... | ... |
@@ -54,6 +55,7 @@ func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manage |
| 54 | 54 |
if id == "" {
|
| 55 | 55 |
break |
| 56 | 56 |
} |
| 57 |
+ hasSession = true |
|
| 57 | 58 |
h, ok := a.fetchers[host+"/"+id] |
| 58 | 59 |
if ok {
|
| 59 | 60 |
h.lastUsed = time.Now() |
| ... | ... |
@@ -63,6 +65,14 @@ func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manage |
| 63 | 63 |
} |
| 64 | 64 |
} |
| 65 | 65 |
|
| 66 |
+ if !hasSession {
|
|
| 67 |
+ h, ok := a.fetchers[host+"/"] |
|
| 68 |
+ if ok {
|
|
| 69 |
+ h.lastUsed = time.Now() |
|
| 70 |
+ return h |
|
| 71 |
+ } |
|
| 72 |
+ } |
|
| 73 |
+ |
|
| 66 | 74 |
// link existing fetcher |
| 67 | 75 |
for k, h := range a.fetchers {
|
| 68 | 76 |
parts := strings.SplitN(k, "/", 2) |
| ... | ... |
@@ -78,7 +88,7 @@ func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manage |
| 78 | 78 |
return h |
| 79 | 79 |
} |
| 80 | 80 |
} else {
|
| 81 |
- sessionID, username, password, err := sessionauth.CredentialsFunc(sm, g)(host) |
|
| 81 |
+ sessionID, username, password, err := sessionauth.CredentialsFunc(ctx, sm, g)(host) |
|
| 82 | 82 |
if err == nil {
|
| 83 | 83 |
if username == h.common.Username && password == h.common.Secret {
|
| 84 | 84 |
a.fetchers[host+"/"+sessionID] = h |
| ... | ... |
@@ -140,8 +150,8 @@ func (a *dockerAuthorizer) Authorize(ctx context.Context, req *http.Request) err |
| 140 | 140 |
return nil |
| 141 | 141 |
} |
| 142 | 142 |
|
| 143 |
-func (a *dockerAuthorizer) getCredentials(host string) (sessionID, username, secret string, err error) {
|
|
| 144 |
- return sessionauth.CredentialsFunc(a.sm, a.session)(host) |
|
| 143 |
+func (a *dockerAuthorizer) getCredentials(ctx context.Context, host string) (sessionID, username, secret string, err error) {
|
|
| 144 |
+ return sessionauth.CredentialsFunc(ctx, a.sm, a.session)(host) |
|
| 145 | 145 |
} |
| 146 | 146 |
|
| 147 | 147 |
func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.Response) error {
|
| ... | ... |
@@ -185,12 +195,12 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R |
| 185 | 185 |
|
| 186 | 186 |
var username, secret string |
| 187 | 187 |
sessionID, pubKey, err := sessionauth.GetTokenAuthority(ctx, host, a.sm, a.session) |
| 188 |
- if err != nil {
|
|
| 188 |
+ if err != nil && !errors.Is(err, session.ErrNoActiveSessions) {
|
|
| 189 | 189 |
return err |
| 190 | 190 |
} |
| 191 | 191 |
if pubKey == nil {
|
| 192 |
- sessionID, username, secret, err = a.getCredentials(host) |
|
| 193 |
- if err != nil {
|
|
| 192 |
+ sessionID, username, secret, err = a.getCredentials(ctx, host) |
|
| 193 |
+ if err != nil && !errors.Is(err, session.ErrNoActiveSessions) {
|
|
| 194 | 194 |
return err |
| 195 | 195 |
} |
| 196 | 196 |
} |
| ... | ... |
@@ -205,7 +215,7 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R |
| 205 | 205 |
|
| 206 | 206 |
return nil |
| 207 | 207 |
case auth.BasicAuth: |
| 208 |
- sessionID, username, secret, err := a.getCredentials(host) |
|
| 208 |
+ sessionID, username, secret, err := a.getCredentials(ctx, host) |
|
| 209 | 209 |
if err != nil {
|
| 210 | 210 |
return err |
| 211 | 211 |
} |
| 212 | 212 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,43 @@ |
| 0 |
+package childprocess |
|
| 1 |
+ |
|
| 2 |
+const ( |
|
| 3 |
+ // go.opentelemetry.io/otel/propagation doesn't export these as constants. |
|
| 4 |
+ traceparentHeader = "traceparent" |
|
| 5 |
+ tracestateHeader = "tracestate" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+type textMap struct {
|
|
| 9 |
+ parent string |
|
| 10 |
+ state string |
|
| 11 |
+} |
|
| 12 |
+ |
|
| 13 |
+func (tm *textMap) Get(key string) string {
|
|
| 14 |
+ switch key {
|
|
| 15 |
+ case traceparentHeader: |
|
| 16 |
+ return tm.parent |
|
| 17 |
+ case tracestateHeader: |
|
| 18 |
+ return tm.state |
|
| 19 |
+ default: |
|
| 20 |
+ return "" |
|
| 21 |
+ } |
|
| 22 |
+} |
|
| 23 |
+ |
|
| 24 |
+func (tm *textMap) Set(key string, value string) {
|
|
| 25 |
+ switch key {
|
|
| 26 |
+ case traceparentHeader: |
|
| 27 |
+ tm.parent = value |
|
| 28 |
+ case tracestateHeader: |
|
| 29 |
+ tm.state = value |
|
| 30 |
+ } |
|
| 31 |
+} |
|
| 32 |
+ |
|
| 33 |
+func (tm *textMap) Keys() []string {
|
|
| 34 |
+ var k []string |
|
| 35 |
+ if tm.parent != "" {
|
|
| 36 |
+ k = append(k, traceparentHeader) |
|
| 37 |
+ } |
|
| 38 |
+ if tm.state != "" {
|
|
| 39 |
+ k = append(k, tracestateHeader) |
|
| 40 |
+ } |
|
| 41 |
+ return k |
|
| 42 |
+} |
| 0 | 43 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,36 @@ |
| 0 |
+package childprocess |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "os" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/moby/buildkit/util/appcontext" |
|
| 7 |
+ "go.opentelemetry.io/otel/propagation" |
|
| 8 |
+) |
|
| 9 |
+ |
|
| 10 |
+func init() {
|
|
| 11 |
+ appcontext.Register(initContext) |
|
| 12 |
+} |
|
| 13 |
+ |
|
| 14 |
+func initContext(ctx context.Context) context.Context {
|
|
| 15 |
+ // open-telemetry/opentelemetry-specification#740 |
|
| 16 |
+ parent := os.Getenv("TRACEPARENT")
|
|
| 17 |
+ state := os.Getenv("TRACESTATE")
|
|
| 18 |
+ |
|
| 19 |
+ if parent != "" {
|
|
| 20 |
+ tc := propagation.TraceContext{}
|
|
| 21 |
+ return tc.Extract(ctx, &textMap{parent: parent, state: state})
|
|
| 22 |
+ } |
|
| 23 |
+ |
|
| 24 |
+ // deprecated: removed in v0.11.0 |
|
| 25 |
+ // previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15 |
|
| 26 |
+ parent = os.Getenv("OTEL_TRACE_PARENT")
|
|
| 27 |
+ state = os.Getenv("OTEL_TRACE_STATE")
|
|
| 28 |
+ |
|
| 29 |
+ if parent == "" {
|
|
| 30 |
+ return ctx |
|
| 31 |
+ } |
|
| 32 |
+ |
|
| 33 |
+ tc := propagation.TraceContext{}
|
|
| 34 |
+ return tc.Extract(ctx, &textMap{parent: parent, state: state})
|
|
| 35 |
+} |
| 0 | 36 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,36 @@ |
| 0 |
+package childprocess |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ |
|
| 5 |
+ "go.opentelemetry.io/otel/propagation" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+// Environ returns list of environment variables that need to be sent to the child process |
|
| 9 |
+// in order for it to pick up cross-process tracing from same state. |
|
| 10 |
+func Environ(ctx context.Context) []string {
|
|
| 11 |
+ var tm textMap |
|
| 12 |
+ tc := propagation.TraceContext{}
|
|
| 13 |
+ tc.Inject(ctx, &tm) |
|
| 14 |
+ |
|
| 15 |
+ var env []string |
|
| 16 |
+ |
|
| 17 |
+ // deprecated: removed in v0.11.0 |
|
| 18 |
+ // previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15 |
|
| 19 |
+ if tm.parent != "" {
|
|
| 20 |
+ env = append(env, "OTEL_TRACE_PARENT="+tm.parent) |
|
| 21 |
+ } |
|
| 22 |
+ if tm.state != "" {
|
|
| 23 |
+ env = append(env, "OTEL_TRACE_STATE="+tm.state) |
|
| 24 |
+ } |
|
| 25 |
+ |
|
| 26 |
+ // open-telemetry/opentelemetry-specification#740 |
|
| 27 |
+ if tm.parent != "" {
|
|
| 28 |
+ env = append(env, "TRACEPARENT="+tm.parent) |
|
| 29 |
+ } |
|
| 30 |
+ if tm.state != "" {
|
|
| 31 |
+ env = append(env, "TRACESTATE="+tm.state) |
|
| 32 |
+ } |
|
| 33 |
+ |
|
| 34 |
+ return env |
|
| 35 |
+} |
| 0 | 36 |
deleted file mode 100644 |
| ... | ... |
@@ -1,77 +0,0 @@ |
| 1 |
-package detect |
|
| 2 |
- |
|
| 3 |
-import ( |
|
| 4 |
- "context" |
|
| 5 |
- |
|
| 6 |
- "go.opentelemetry.io/otel/propagation" |
|
| 7 |
-) |
|
| 8 |
- |
|
| 9 |
-const ( |
|
| 10 |
- traceparentHeader = "traceparent" |
|
| 11 |
- tracestateHeader = "tracestate" |
|
| 12 |
-) |
|
| 13 |
- |
|
| 14 |
-// Environ returns list of environment variables that need to be sent to the child process |
|
| 15 |
-// in order for it to pick up cross-process tracing from same state. |
|
| 16 |
-func Environ(ctx context.Context) []string {
|
|
| 17 |
- var tm textMap |
|
| 18 |
- tc := propagation.TraceContext{}
|
|
| 19 |
- tc.Inject(ctx, &tm) |
|
| 20 |
- |
|
| 21 |
- var env []string |
|
| 22 |
- |
|
| 23 |
- // deprecated: removed in v0.11.0 |
|
| 24 |
- // previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15 |
|
| 25 |
- if tm.parent != "" {
|
|
| 26 |
- env = append(env, "OTEL_TRACE_PARENT="+tm.parent) |
|
| 27 |
- } |
|
| 28 |
- if tm.state != "" {
|
|
| 29 |
- env = append(env, "OTEL_TRACE_STATE="+tm.state) |
|
| 30 |
- } |
|
| 31 |
- |
|
| 32 |
- // open-telemetry/opentelemetry-specification#740 |
|
| 33 |
- if tm.parent != "" {
|
|
| 34 |
- env = append(env, "TRACEPARENT="+tm.parent) |
|
| 35 |
- } |
|
| 36 |
- if tm.state != "" {
|
|
| 37 |
- env = append(env, "TRACESTATE="+tm.state) |
|
| 38 |
- } |
|
| 39 |
- |
|
| 40 |
- return env |
|
| 41 |
-} |
|
| 42 |
- |
|
| 43 |
-type textMap struct {
|
|
| 44 |
- parent string |
|
| 45 |
- state string |
|
| 46 |
-} |
|
| 47 |
- |
|
| 48 |
-func (tm *textMap) Get(key string) string {
|
|
| 49 |
- switch key {
|
|
| 50 |
- case traceparentHeader: |
|
| 51 |
- return tm.parent |
|
| 52 |
- case tracestateHeader: |
|
| 53 |
- return tm.state |
|
| 54 |
- default: |
|
| 55 |
- return "" |
|
| 56 |
- } |
|
| 57 |
-} |
|
| 58 |
- |
|
| 59 |
-func (tm *textMap) Set(key string, value string) {
|
|
| 60 |
- switch key {
|
|
| 61 |
- case traceparentHeader: |
|
| 62 |
- tm.parent = value |
|
| 63 |
- case tracestateHeader: |
|
| 64 |
- tm.state = value |
|
| 65 |
- } |
|
| 66 |
-} |
|
| 67 |
- |
|
| 68 |
-func (tm *textMap) Keys() []string {
|
|
| 69 |
- var k []string |
|
| 70 |
- if tm.parent != "" {
|
|
| 71 |
- k = append(k, traceparentHeader) |
|
| 72 |
- } |
|
| 73 |
- if tm.state != "" {
|
|
| 74 |
- k = append(k, tracestateHeader) |
|
| 75 |
- } |
|
| 76 |
- return k |
|
| 77 |
-} |
| ... | ... |
@@ -80,7 +80,7 @@ func (c *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc |
| 80 | 80 |
c.lock.Lock() |
| 81 | 81 |
defer c.lock.Unlock() |
| 82 | 82 |
if c.tracesClient == nil {
|
| 83 |
- return errNoClient |
|
| 83 |
+ return errors.New("no client")
|
|
| 84 | 84 |
} |
| 85 | 85 |
|
| 86 | 86 |
_, err := c.tracesClient.Export(ctx, &coltracepb.ExportTraceServiceRequest{
|
| ... | ... |
@@ -159,6 +159,7 @@ func NewWorker(ctx context.Context, opt WorkerOpt) (*Worker, error) {
|
| 159 | 159 |
if err := git.Supported(); err == nil {
|
| 160 | 160 |
gs, err := git.NewSource(git.Opt{
|
| 161 | 161 |
CacheAccessor: cm, |
| 162 |
+ RegistryHosts: opt.RegistryHosts, |
|
| 162 | 163 |
}) |
| 163 | 164 |
if err != nil {
|
| 164 | 165 |
return nil, err |
| ... | ... |
@@ -238,6 +238,8 @@ type CommonLanguageSettings struct {
|
| 238 | 238 |
// The destination where API teams want this client library to be published. |
| 239 | 239 |
Destinations []ClientLibraryDestination `protobuf:"varint,2,rep,packed,name=destinations,proto3,enum=google.api.ClientLibraryDestination" json:"destinations,omitempty"` |
| 240 | 240 |
// Configuration for which RPCs should be generated in the GAPIC client. |
| 241 |
+ // |
|
| 242 |
+ // Note: This field should not be used in most cases. |
|
| 241 | 243 |
SelectiveGapicGeneration *SelectiveGapicGeneration `protobuf:"bytes,3,opt,name=selective_gapic_generation,json=selectiveGapicGeneration,proto3" json:"selective_gapic_generation,omitempty"` |
| 242 | 244 |
} |
| 243 | 245 |
|
| ... | ... |
@@ -1249,6 +1251,8 @@ func (x *MethodSettings) GetBatching() *BatchingConfigProto {
|
| 1249 | 1249 |
|
| 1250 | 1250 |
// This message is used to configure the generation of a subset of the RPCs in |
| 1251 | 1251 |
// a service for client libraries. |
| 1252 |
+// |
|
| 1253 |
+// Note: This feature should not be used in most cases. |
|
| 1252 | 1254 |
type SelectiveGapicGeneration struct {
|
| 1253 | 1255 |
state protoimpl.MessageState |
| 1254 | 1256 |
sizeCache protoimpl.SizeCache |
| ... | ... |
@@ -950,7 +950,7 @@ github.com/ishidawataru/sctp |
| 950 | 950 |
# github.com/jmoiron/sqlx v1.4.0 |
| 951 | 951 |
## explicit; go 1.10 |
| 952 | 952 |
github.com/jmoiron/sqlx/types |
| 953 |
-# github.com/klauspost/compress v1.18.5 |
|
| 953 |
+# github.com/klauspost/compress v1.18.6 |
|
| 954 | 954 |
## explicit; go 1.24 |
| 955 | 955 |
github.com/klauspost/compress |
| 956 | 956 |
github.com/klauspost/compress/fse |
| ... | ... |
@@ -978,7 +978,7 @@ github.com/mitchellh/hashstructure/v2 |
| 978 | 978 |
# github.com/mitchellh/reflectwalk v1.0.2 |
| 979 | 979 |
## explicit |
| 980 | 980 |
github.com/mitchellh/reflectwalk |
| 981 |
-# github.com/moby/buildkit v0.29.0 |
|
| 981 |
+# github.com/moby/buildkit v0.30.0-rc1 |
|
| 982 | 982 |
## explicit; go 1.25.5 |
| 983 | 983 |
github.com/moby/buildkit/api/services/control |
| 984 | 984 |
github.com/moby/buildkit/api/types |
| ... | ... |
@@ -1065,6 +1065,7 @@ github.com/moby/buildkit/solver/errdefs |
| 1065 | 1065 |
github.com/moby/buildkit/solver/internal/pipe |
| 1066 | 1066 |
github.com/moby/buildkit/solver/llbsolver |
| 1067 | 1067 |
github.com/moby/buildkit/solver/llbsolver/cdidevices |
| 1068 |
+github.com/moby/buildkit/solver/llbsolver/compat |
|
| 1068 | 1069 |
github.com/moby/buildkit/solver/llbsolver/errdefs |
| 1069 | 1070 |
github.com/moby/buildkit/solver/llbsolver/file |
| 1070 | 1071 |
github.com/moby/buildkit/solver/llbsolver/history |
| ... | ... |
@@ -1079,6 +1080,7 @@ github.com/moby/buildkit/solver/pb |
| 1079 | 1079 |
github.com/moby/buildkit/solver/result |
| 1080 | 1080 |
github.com/moby/buildkit/source |
| 1081 | 1081 |
github.com/moby/buildkit/source/containerblob |
| 1082 |
+github.com/moby/buildkit/source/containerblob/blobfetch |
|
| 1082 | 1083 |
github.com/moby/buildkit/source/containerimage |
| 1083 | 1084 |
github.com/moby/buildkit/source/git |
| 1084 | 1085 |
github.com/moby/buildkit/source/http |
| ... | ... |
@@ -1090,6 +1092,7 @@ github.com/moby/buildkit/sourcepolicy/pb |
| 1090 | 1090 |
github.com/moby/buildkit/sourcepolicy/policysession |
| 1091 | 1091 |
github.com/moby/buildkit/util/apicaps |
| 1092 | 1092 |
github.com/moby/buildkit/util/apicaps/pb |
| 1093 |
+github.com/moby/buildkit/util/appcontext |
|
| 1093 | 1094 |
github.com/moby/buildkit/util/appdefaults |
| 1094 | 1095 |
github.com/moby/buildkit/util/archutil |
| 1095 | 1096 |
github.com/moby/buildkit/util/attestation |
| ... | ... |
@@ -1132,6 +1135,7 @@ github.com/moby/buildkit/util/pull |
| 1132 | 1132 |
github.com/moby/buildkit/util/pull/pullprogress |
| 1133 | 1133 |
github.com/moby/buildkit/util/purl |
| 1134 | 1134 |
github.com/moby/buildkit/util/push |
| 1135 |
+github.com/moby/buildkit/util/registrar |
|
| 1135 | 1136 |
github.com/moby/buildkit/util/resolvconf |
| 1136 | 1137 |
github.com/moby/buildkit/util/resolver |
| 1137 | 1138 |
github.com/moby/buildkit/util/resolver/config |
| ... | ... |
@@ -1146,8 +1150,8 @@ github.com/moby/buildkit/util/suggest |
| 1146 | 1146 |
github.com/moby/buildkit/util/system |
| 1147 | 1147 |
github.com/moby/buildkit/util/throttle |
| 1148 | 1148 |
github.com/moby/buildkit/util/tracing |
| 1149 |
+github.com/moby/buildkit/util/tracing/childprocess |
|
| 1149 | 1150 |
github.com/moby/buildkit/util/tracing/detect |
| 1150 |
-github.com/moby/buildkit/util/tracing/exec |
|
| 1151 | 1151 |
github.com/moby/buildkit/util/tracing/otlptracegrpc |
| 1152 | 1152 |
github.com/moby/buildkit/util/tracing/transform |
| 1153 | 1153 |
github.com/moby/buildkit/util/urlutil |
| ... | ... |
@@ -1970,7 +1974,7 @@ google.golang.org/api/transport/http |
| 1970 | 1970 |
# google.golang.org/genproto v0.0.0-20260401024825-9d38bb4040a9 |
| 1971 | 1971 |
## explicit; go 1.25.0 |
| 1972 | 1972 |
google.golang.org/genproto/googleapis/logging/type |
| 1973 |
-# google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 |
|
| 1973 |
+# google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d |
|
| 1974 | 1974 |
## explicit; go 1.25.0 |
| 1975 | 1975 |
google.golang.org/genproto/googleapis/api |
| 1976 | 1976 |
google.golang.org/genproto/googleapis/api/annotations |