Browse code

vendor: update buildkit to v0.30.0-rc1

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Tonis Tiigi authored on 2026/05/07 08:27:44
Showing 85 changed files
... ...
@@ -440,6 +440,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt
440 440
 		ImageSource:       src,
441 441
 		DownloadManager:   dist.DownloadManager,
442 442
 		V2MetadataService: dist.V2MetadataService,
443
+		RegistryHosts:     opt.RegistryHosts,
443 444
 		Exporter:          exp,
444 445
 		Transport:         rt,
445 446
 		Layers:            layers,
... ...
@@ -10,6 +10,7 @@ import (
10 10
 
11 11
 	"github.com/containerd/containerd/v2/core/content"
12 12
 	c8dimages "github.com/containerd/containerd/v2/core/images"
13
+	"github.com/containerd/containerd/v2/core/remotes/docker"
13 14
 	"github.com/containerd/containerd/v2/pkg/gc"
14 15
 	"github.com/containerd/containerd/v2/pkg/rootfs"
15 16
 	cerrdefs "github.com/containerd/errdefs"
... ...
@@ -84,6 +85,7 @@ type Opt struct {
84 84
 
85 85
 	DownloadManager   *xfer.LayerDownloadManager
86 86
 	V2MetadataService distmetadata.V2MetadataService
87
+	RegistryHosts     docker.RegistryHosts
87 88
 	Transport         nethttp.RoundTripper
88 89
 	Exporter          exporter.Exporter
89 90
 	Layers            LayerAccess
... ...
@@ -116,6 +118,7 @@ func NewWorker(opt Opt) (*Worker, error) {
116 116
 
117 117
 	gs, err := git.NewSource(git.Opt{
118 118
 		CacheAccessor: cm,
119
+		RegistryHosts: opt.RegistryHosts,
119 120
 	})
120 121
 	if err == nil {
121 122
 		sm.Register(gs)
... ...
@@ -55,7 +55,7 @@ require (
55 55
 	github.com/miekg/dns v1.1.72
56 56
 	github.com/mistifyio/go-zfs/v4 v4.0.0
57 57
 	github.com/mitchellh/copystructure v1.2.0
58
-	github.com/moby/buildkit v0.29.0
58
+	github.com/moby/buildkit v0.30.0-rc1
59 59
 	github.com/moby/docker-image-spec v1.3.1
60 60
 	github.com/moby/go-archive v0.2.0
61 61
 	github.com/moby/ipvs v1.1.0
... ...
@@ -108,7 +108,7 @@ require (
108 108
 	golang.org/x/sys v0.43.0
109 109
 	golang.org/x/text v0.36.0
110 110
 	golang.org/x/time v0.15.0
111
-	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9
111
+	google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d
112 112
 	google.golang.org/grpc v1.80.0
113 113
 	google.golang.org/protobuf v1.36.11
114 114
 	gotest.tools/v3 v3.5.2
... ...
@@ -222,7 +222,7 @@ require (
222 222
 	github.com/in-toto/attestation v1.1.2 // indirect
223 223
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
224 224
 	github.com/jmoiron/sqlx v1.4.0 // indirect
225
-	github.com/klauspost/compress v1.18.5 // indirect
225
+	github.com/klauspost/compress v1.18.6 // indirect
226 226
 	github.com/knqyf263/go-plugin v0.9.0 // indirect
227 227
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
228 228
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
... ...
@@ -226,8 +226,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi
226 226
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
227 227
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
228 228
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
229
-github.com/docker/cli v29.3.1+incompatible h1:M04FDj2TRehDacrosh7Vlkgc7AuQoWloQkf1PA5hmoI=
230
-github.com/docker/cli v29.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
229
+github.com/docker/cli v29.4.2+incompatible h1:nhxMY4v7wB0QMMc5ppeqV6FBMwzqv0n4t2gogu/R2DQ=
230
+github.com/docker/cli v29.4.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
231 231
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
232 232
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
233 233
 github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY=
... ...
@@ -505,8 +505,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
505 505
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
506 506
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
507 507
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
508
-github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
509
-github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
508
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
509
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
510 510
 github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI=
511 511
 github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q=
512 512
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
... ...
@@ -549,8 +549,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
549 549
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
550 550
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
551 551
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
552
-github.com/moby/buildkit v0.29.0 h1:wxLEFbCOJntEDjSNNN2YWd8zxltZxT5muDQ0LzpbtpU=
553
-github.com/moby/buildkit v0.29.0/go.mod h1:Dmv2FeDe34t75QuzeU87rBoZpAAkcpT5zeu4hXzmASc=
552
+github.com/moby/buildkit v0.30.0-rc1 h1:meXm1eqqweZ7EPnTbRiU8c4BjgjDKs7rUczHUsU0nJo=
553
+github.com/moby/buildkit v0.30.0-rc1/go.mod h1:4ha5jioeEAREXOpmpEUSQ4bKZON1yEc/co9I57+zCZs=
554 554
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
555 555
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
556 556
 github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
... ...
@@ -1043,8 +1043,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
1043 1043
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
1044 1044
 google.golang.org/genproto v0.0.0-20260401024825-9d38bb4040a9 h1:w8JYjr7zHemS95YA5FFwk+fUv5tdQU4I8twN9bFdxVU=
1045 1045
 google.golang.org/genproto v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:YCEC8W7HTtK7iBv+pI7g7hGAi7qdGB6bQXw3BIYAusM=
1046
-google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
1047
-google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
1046
+google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d h1:/aDRtSZJjyLQzm75d+a1wOJaqyKBMvIAfeQmoa3ORiI=
1047
+google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:etfGUgejTiadZAUaEP14NP97xi1RGeawqkjDARA/UOs=
1048 1048
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA=
1049 1049
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
1050 1050
 google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
... ...
@@ -1090,10 +1090,10 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
1090 1090
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
1091 1091
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
1092 1092
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
1093
-kernel.org/pub/linux/libs/security/libcap/cap v1.2.77 h1:iQtQTjFUOcTT19fI8sTCzYXsjeVs56et3D8AbKS2Uks=
1094
-kernel.org/pub/linux/libs/security/libcap/cap v1.2.77/go.mod h1:oV+IO8kGh0B7TxErbydDe2+BRmi9g/W0CkpVV+QBTJU=
1095
-kernel.org/pub/linux/libs/security/libcap/psx v1.2.77 h1:Z06sMOzc0GNCwp6efaVrIrz4ywGJ1v+DP0pjVkOfDuA=
1096
-kernel.org/pub/linux/libs/security/libcap/psx v1.2.77/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
1093
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.78 h1:jgqg4gyu2BaYW9L6uzEtGLf8GNREwk/z4UFdwt5F3pE=
1094
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.78/go.mod h1:VjuVda6m2qGkpCVfrFkpTGyvkdlZ2N5/yfo89tujlg8=
1095
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.78 h1:PC3yNs51cX5LZ7U57a7xielBcoXB3xnV+rXD8V0H0DQ=
1096
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.78/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
1097 1097
 pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc=
1098 1098
 pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk=
1099 1099
 resenje.org/singleflight v0.4.3 h1:l7foFYg8X/VEHPxWs1K/Pw77807RMVzvXgWGb0J1sdM=
... ...
@@ -1,2 +1,3 @@
1 1
 * -text
2 2
 *.bin -text -diff
3
+*.md text eol=lf
... ...
@@ -1,700 +1,700 @@
1
-# compress
2
-
3
-This package provides various compression algorithms.
4
-
5
-* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go.
6
-* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy.
7
-* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
8
-* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams.
9
-* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
10
-* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently.
11
-* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
12
-
13
-[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
14
-[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml)
15
-[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge)
16
-
17
-# package usage
18
-
19
-Use `go get github.com/klauspost/compress@latest` to add it to your project.
20
-
21
-This package will support the current Go version and 2 versions back.
22
-
23
-* Use the `nounsafe` tag to disable all use of the "unsafe" package.
24
-* Use the `noasm` tag to disable all assembly across packages.
25
-
26
-Use the links above for more information on each.
27
-
28
-# changelog
29
-
30
-* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4)
31
-	* gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121
32
-	* zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122
33
-	* gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116
34
-
35
-* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3)
36
-	* Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102).
37
-
38
-* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2)
39
-	* flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115
40
-	* flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106
41
-
42
-* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED
43
-  * zstd: Add simple zstd EncodeTo/DecodeTo functions  https://github.com/klauspost/compress/pull/1079
44
-  * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059
45
-  * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080
46
-  * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086
47
-  * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090
48
-  * flate: Faster load+store https://github.com/klauspost/compress/pull/1104
49
-  * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101
50
-  * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103
51
-
52
-* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0)
53
-  * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036
54
-  * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028
55
-  * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043
56
-  * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045
57
-  * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048
58
-  * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049
59
-  * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050
60
-
61
-<details>
62
-	<summary>See changes to v1.17.x</summary>
63
-
64
-* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11)
65
-  * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017
66
-  * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014
67
-  * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011
68
-  * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013
69
-
70
-* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)
71
-	* gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978
72
-	* gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002
73
-	* s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982
74
-	* zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007
75
-	* flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996
76
-
77
-* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9)
78
-	* s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949
79
-	* flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963
80
-	* Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971
81
-	* zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951
82
-
83
-* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8)
84
-	* zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885
85
-	* zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938
86
-
87
-* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7)
88
-	* s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927
89
-	* s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930
90
-  
91
-* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6)
92
-	* zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923
93
-	* s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925
94
-  
95
-* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5)
96
-	* flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912
97
-	* zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908
98
-	* zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913
99
-	* zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910
100
-	* s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917
101
-https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918
102
-
103
-* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4)
104
-	* huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887
105
-	* huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886
106
-	* gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892
107
-	* gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890
108
-	* gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891
109
-
110
-* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3)
111
-	* fse: Fix max header size https://github.com/klauspost/compress/pull/881
112
-	* zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877
113
-	* gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883
114
-
115
-* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2)
116
-	* zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876
117
-
118
-* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1)
119
-	* s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871
120
-	* flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869
121
-	* s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867
122
-
123
-* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
124
-	* Add experimental dictionary builder  https://github.com/klauspost/compress/pull/853
125
-	* Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838
126
-	* flate: Add limited window compression https://github.com/klauspost/compress/pull/843
127
-	* s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
128
-	* flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
129
-	* gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
130
-   
131
-</details>
132
-<details>
133
-	<summary>See changes to v1.16.x</summary>
134
-
135
-   
136
-* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
137
-	* zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
138
-	* s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832
139
-
140
-* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6)
141
-	* zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806
142
-	* zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824
143
-	* gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815
144
-	* s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663
145
-
146
-* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5)
147
-	* zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802
148
-	* gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804
149
-
150
-* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4)
151
-	* zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784
152
-	* zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792
153
-	* zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785
154
-	* zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795
155
-	* s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779
156
-	* s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780
157
-	* gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799
158
-
159
-* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1)
160
-	* zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776
161
-	* gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767
162
-	* s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766
163
-	* zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773
164
-	* huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774
165
-
166
-* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0)
167
-	* s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support.  https://github.com/klauspost/compress/pull/685
168
-	* s2: Add Compression Size Estimate.  https://github.com/klauspost/compress/pull/752
169
-	* s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755
170
-	* s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748
171
-	* s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747
172
-	* s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746
173
-</details>
174
-
175
-<details>
176
-	<summary>See changes to v1.15.x</summary>
177
-	
178
-* Jan 21st, 2023 (v1.15.15)
179
-	* deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739
180
-	* zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728
181
-	* zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745
182
-	* gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740
183
-
184
-* Jan 3rd, 2023 (v1.15.14)
185
-
186
-	* flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718
187
-	* zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720
188
-	* export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722
189
-	* s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723
190
-
191
-* Dec 11, 2022 (v1.15.13)
192
-	* zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder  https://github.com/klauspost/compress/pull/691
193
-	* zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708
194
-
195
-* Oct 26, 2022 (v1.15.12)
196
-
197
-	* zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680
198
-	* gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683
199
-
200
-* Sept 26, 2022 (v1.15.11)
201
-
202
-	* flate: Improve level 1-3 compression  https://github.com/klauspost/compress/pull/678
203
-	* zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677
204
-	* zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668
205
-	* zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667
206
-
207
-* Sept 16, 2022 (v1.15.10)
208
-
209
-	* zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649
210
-	* Add Go 1.19 - deprecate Go 1.16  https://github.com/klauspost/compress/pull/651
211
-	* flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656
212
-	* zstd: Improve "better" compression  https://github.com/klauspost/compress/pull/657
213
-	* s2: Improve "best" compression https://github.com/klauspost/compress/pull/658
214
-	* s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635
215
-	* s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646
216
-	* Use arrays for constant size copies https://github.com/klauspost/compress/pull/659
217
-
218
-* July 21, 2022 (v1.15.9)
219
-
220
-	* zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645
221
-	* zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644
222
-	* zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643
223
-
224
-* July 13, 2022 (v1.15.8)
225
-
226
-	* gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641
227
-	* s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638
228
-	* zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636
229
-	* zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637
230
-	* huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634
231
-	* zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640
232
-	* gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639
233
-
234
-* June 29, 2022 (v1.15.7)
235
-
236
-	* s2: Fix absolute forward seeks  https://github.com/klauspost/compress/pull/633
237
-	* zip: Merge upstream  https://github.com/klauspost/compress/pull/631
238
-	* zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624
239
-	* zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598
240
-	* flate: Faster histograms  https://github.com/klauspost/compress/pull/620
241
-	* deflate: Use compound hcode  https://github.com/klauspost/compress/pull/622
242
-
243
-* June 3, 2022 (v1.15.6)
244
-	* s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613
245
-	* s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611
246
-	* zstd: Always use configured block size https://github.com/klauspost/compress/pull/605
247
-	* zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606
248
-	* zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608
249
-	* gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612
250
-	* s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609
251
-	* s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607
252
-	* snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614
253
-	* s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610
254
-
255
-* May 25, 2022 (v1.15.5)
256
-	* s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602
257
-	* s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601
258
-	* huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596
259
-	* zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588
260
-	* zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592
261
-	* zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599
262
-	* zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593
263
-	* huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586
264
-	* flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590
265
-
266
-
267
-* May 11, 2022 (v1.15.4)
268
-	* huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577)
269
-	* inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581)
270
-	* zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583)
271
-	* zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580)
272
-
273
-* May 5, 2022 (v1.15.3)
274
-	* zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572)
275
-	* s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575)
276
-
277
-* Apr 26, 2022 (v1.15.2)
278
-	* zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster.  [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537)
279
-	* zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539)
280
-	* s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555)
281
-	* Minimum version is Go 1.16, added CI test on 1.18.
282
-
283
-* Mar 11, 2022 (v1.15.1)
284
-	* huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512)
285
-	* zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514)
286
-	* zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520)
287
-	* zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521)
288
-	* zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523)
289
-
290
-* Mar 3, 2022 (v1.15.0)
291
-	* zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498)
292
-	* zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505)
293
-	* huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507)
294
-	* flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509)
295
-	* gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400)
296
-	* gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510)
297
-
298
-Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines.
299
-
300
-Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected.
301
-
302
-While the release has been extensively tested, it is recommended to testing when upgrading.
303
-
304
-</details>
305
-
306
-<details>
307
-	<summary>See changes to v1.14.x</summary>
308
-	
309
-* Feb 22, 2022 (v1.14.4)
310
-	* flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503)
311
-	* zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502)
312
-	* zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501)  #501
313
-	* huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500)
314
-
315
-* Feb 17, 2022 (v1.14.3)
316
-	* flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494)  [#478](https://github.com/klauspost/compress/pull/478)
317
-	* flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483)
318
-	* s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486)
319
-
320
-* Jan 25, 2022 (v1.14.2)
321
-	* zstd: improve header decoder by @dsnet  [#476](https://github.com/klauspost/compress/pull/476)
322
-	* zstd: Add bigger default blocks  [#469](https://github.com/klauspost/compress/pull/469)
323
-	* zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470)
324
-	* zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472)
325
-	* flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473)
326
-	* zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475)
327
-
328
-* Jan 11, 2022 (v1.14.1)
329
-	* s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462)
330
-	* flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458)
331
-	* zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468)
332
-	* zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464)
333
-	* Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445)
334
-</details>
335
-
336
-<details>
337
-	<summary>See changes to v1.13.x</summary>
338
-	
339
-* Aug 30, 2021 (v1.13.5)
340
-	* gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425)
341
-	* s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413)
342
-	* zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426)
343
-	* Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421)
344
-
345
-* Aug 12, 2021 (v1.13.4)
346
-	* Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy).
347
-	* zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415)
348
-
349
-* Aug 3, 2021 (v1.13.3) 
350
-	* zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404)
351
-	* zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411)
352
-	* gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406)
353
-	* s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399)
354
-	* zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401)
355
-	* zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410)
356
-
357
-* Jun 14, 2021 (v1.13.1)
358
-	* s2: Add full Snappy output support  [#396](https://github.com/klauspost/compress/pull/396)
359
-	* zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394)
360
-	* gzhttp: Add header to skip compression  [#389](https://github.com/klauspost/compress/pull/389)
361
-	* s2: Improve speed with bigger output margin  [#395](https://github.com/klauspost/compress/pull/395)
362
-
363
-* Jun 3, 2021 (v1.13.0)
364
-	* Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors.
365
-	* zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)
366
-	* zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)
367
-</details>
368
-
369
-
370
-<details>
371
-	<summary>See changes to v1.12.x</summary>
372
-	
373
-* May 25, 2021 (v1.12.3)
374
-	* deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374)
375
-	* deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375)
376
-	* zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) 
377
-
378
-* Apr 27, 2021 (v1.12.2)
379
-	* zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365)
380
-	* zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363)
381
-	* deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367)
382
-	* s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358)
383
-	* s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362)
384
-	* s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) 
385
-
386
-* Apr 14, 2021 (v1.12.1)
387
-	* snappy package removed. Upstream added as dependency.
388
-	* s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353)
389
-	* s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352)
390
-	* s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348)
391
-	* s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352)
392
-	* zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346)
393
-	* s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349)
394
-</details>
395
-
396
-<details>
397
-	<summary>See changes to v1.11.x</summary>
398
-	
399
-* Mar 26, 2021 (v1.11.13)
400
-	* zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345)
401
-	* zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336)
402
-	* deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338)
403
-	* s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341)
404
-
405
-* Mar 5, 2021 (v1.11.12)
406
-	* s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives).
407
-	* s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328)
408
-
409
-* Mar 1, 2021 (v1.11.9)
410
-	* s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324)
411
-	* s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325)
412
-	* s2: Fix binaries.
413
-
414
-* Feb 25, 2021 (v1.11.8)
415
-	* s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended.
416
-	* s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315)
417
-	* s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322)
418
-	* zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314)
419
-	* zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313)
420
-  
421
-* Jan 14, 2021 (v1.11.7)
422
-	* Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309)
423
-	* s2: Add 'best' compression option.  [#310](https://github.com/klauspost/compress/pull/310)
424
-	* s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311)
425
-	* s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308)
426
-	* s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312)
427
-
428
-* Jan 7, 2021 (v1.11.6)
429
-	* zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306)
430
-	* zstd: Free Decoder resources when Reset is called with a nil io.Reader  [#305](https://github.com/klauspost/compress/pull/305)
431
-
432
-* Dec 20, 2020 (v1.11.4)
433
-	* zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304)
434
-	* Add header decoder [#299](https://github.com/klauspost/compress/pull/299)
435
-	* s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297)
436
-	* Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300)
437
-	* zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303)
438
-
439
-* Nov 15, 2020 (v1.11.3)
440
-	* inflate: 10-15% faster decompression  [#293](https://github.com/klauspost/compress/pull/293)
441
-	* zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295)
442
-
443
-* Oct 11, 2020 (v1.11.2)
444
-	* s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291)
445
-
446
-* Oct 1, 2020 (v1.11.1)
447
-	* zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286)
448
-
449
-* Sept 8, 2020 (v1.11.0)
450
-	* zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281)
451
-	* zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282)
452
-	* inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274)
453
-</details>
454
-
455
-<details>
456
-	<summary>See changes to v1.10.x</summary>
457
- 
458
-* July 8, 2020 (v1.10.11) 
459
-	* zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278)
460
-	* huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275)
461
-	
462
-* June 23, 2020 (v1.10.10) 
463
-	* zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270)
464
-	
465
-* June 16, 2020 (v1.10.9): 
466
-	* zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268)
467
-	* zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266)
468
-	* Fuzzit tests removed. The service has been purchased and is no longer available.
469
-	
470
-* June 5, 2020 (v1.10.8): 
471
-	* 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265)
472
-	
473
-* June 1, 2020 (v1.10.7): 
474
-	* Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries)
475
-	* Increase zstd decompression speed up to 1.19x.  [#259](https://github.com/klauspost/compress/pull/259)
476
-	* Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263)
477
-	
478
-* May 21, 2020: (v1.10.6) 
479
-	* zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252)
480
-	* zstd: Stricter decompression checks.
481
-	
482
-* April 12, 2020: (v1.10.5)
483
-	* s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239)
484
-	
485
-* Apr 8, 2020: (v1.10.4) 
486
-	* zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251),  [#250](https://github.com/klauspost/compress/pull/250),  [#249](https://github.com/klauspost/compress/pull/249),  [#247](https://github.com/klauspost/compress/pull/247)
487
-* Mar 11, 2020: (v1.10.3) 
488
-	* s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245)
489
-	* s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244)
490
-	* zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240)
491
-	* zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241)
492
-	* zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238)
493
-	
494
-* Feb 27, 2020: (v1.10.2) 
495
-	* Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232)
496
-	* Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227)
497
-	
498
-* Feb 18, 2020: (v1.10.1)
499
-	* Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226)
500
-	* deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224)
501
-	* Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224)
502
-	
503
-* Feb 4, 2020: (v1.10.0) 
504
-	* Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216)
505
-	* Fix buffer overflow on repeated small block deflate.  [#218](https://github.com/klauspost/compress/pull/218)
506
-	* Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214)
507
-	* Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s.  [#186](https://github.com/klauspost/compress/pull/186)
508
-
509
-</details>
510
-
511
-<details>
512
-	<summary>See changes prior to v1.10.0</summary>
513
-
514
-* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056),  [#206](https://github.com/klauspost/compress/pull/206).
515
-* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) 
516
-* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed.
517
-* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases.
518
-* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192)
519
-* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder.
520
-* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199)
521
-* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features
522
-* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197)
523
-* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198)
524
-* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit.
525
-* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191)
526
-* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188)
527
-* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187)
528
-* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines.
529
-* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate.
530
-* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184)
531
-* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate.
532
-* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180)
533
-* Nov 11, 2019: Set default  [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB.
534
-* Nov 11, 2019: Reduce inflate memory use by 1KB.
535
-* Nov 10, 2019: Less allocations in deflate bit writer.
536
-* Nov 10, 2019: Fix inconsistent error returned by zstd decoder.
537
-* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174)
538
-* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173)
539
-* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) 
540
-* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105)
541
-
542
-</details>
543
-
544
-<details>
545
-	<summary>See changes prior to v1.9.0</summary>
546
-
547
-* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169)
548
-* Oct 3, 2019: Fix inconsistent results on broken zstd streams.
549
-* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools)
550
-* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools).
551
-* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip).
552
-* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes).
553
-* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option.
554
-* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables.
555
-* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode.
556
-* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding.
557
-* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. 
558
-* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing.
559
-* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing.
560
-* Aug 14, 2019: zstd: Skip incompressible data 2x faster.  [#147](https://github.com/klauspost/compress/pull/147)
561
-* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146)
562
-* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144)
563
-* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142)
564
-* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder.
565
-* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder.
566
-* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content.
567
-* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix.
568
-* June 17, 2019: zstd decompression bugfix.
569
-* June 17, 2019: fix 32 bit builds.
570
-* June 17, 2019: Easier use in modules (less dependencies).
571
-* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio.
572
-* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression.
573
-* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels.
574
-* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression!
575
-* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels.
576
-* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added.
577
-* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression).
578
-* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below.
579
-* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0).
580
-* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change.
581
-* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change.
582
-* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function.
583
-* May 28, 2017: Reduce allocations when resetting decoder.
584
-* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7.
585
-* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625).
586
-* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before.
587
-* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update.
588
-* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. 
589
-* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression.
590
-* Mar 24, 2016: Small speedup for level 1-3.
591
-* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster.
592
-* Feb 19, 2016: Handle small payloads faster in level 1-3.
593
-* Feb 19, 2016: Added faster level 2 + 3 compression modes.
594
-* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5.
595
-* Feb 14, 2016: Snappy: Merge upstream changes. 
596
-* Feb 14, 2016: Snappy: Fix aggressive skipping.
597
-* Feb 14, 2016: Snappy: Update benchmark.
598
-* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression.
599
-* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%.
600
-* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content.
601
-* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup.
602
-* Jan 16, 2016: Optimization on deflate level 1,2,3 compression.
603
-* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives.
604
-* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs.
605
-* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms.
606
-* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update!
607
-* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet).
608
-* Nov 20 2015: Small optimization to bit writer on 64 bit systems.
609
-* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15).
610
-* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate.
611
-* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file
612
-* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x.
613
-
614
-</details>
615
-
616
-# deflate usage
617
-
618
-The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them:
619
-
620
-Typical speed is about 2x of the standard library packages.
621
-
622
-| old import       | new import                            | Documentation                                                           |
623
-|------------------|---------------------------------------|-------------------------------------------------------------------------|
624
-| `compress/gzip`  | `github.com/klauspost/compress/gzip`  | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc)   |
625
-| `compress/zlib`  | `github.com/klauspost/compress/zlib`  | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc)   |
626
-| `archive/zip`    | `github.com/klauspost/compress/zip`   | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc)     |
627
-| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) |
628
-
629
-You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
630
-
631
-The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/).
632
-
633
-Currently there is only minor speedup on decompression (mostly CRC32 calculation).
634
-
635
-Memory usage is typically 1MB for a Writer. stdlib is in the same range. 
636
-If you expect to have a lot of concurrently allocated Writers consider using 
637
-the stateless compression described below.
638
-
639
-For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing).
640
-
641
-To disable all assembly add `-tags=noasm`. This works across all packages.
642
-
643
-# Stateless compression
644
-
645
-This package offers stateless compression as a special option for gzip/deflate. 
646
-It will do compression but without maintaining any state between Write calls.
647
-
648
-This means there will be no memory kept between Write calls, but compression and speed will be suboptimal.
649
-
650
-This is only relevant in cases where you expect to run many thousands of compressors concurrently, 
651
-but with very little activity. This is *not* intended for regular web servers serving individual requests.  
652
-
653
-Because of this, the size of actual Write calls will affect output size.
654
-
655
-In gzip, specify level `-3` / `gzip.StatelessCompression` to enable.
656
-
657
-For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter)
658
-
659
-A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer:
660
-
661
-```go
662
-	// replace 'ioutil.Discard' with your output.
663
-	gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression)
664
-	if err != nil {
665
-		return err
666
-	}
667
-	defer gzw.Close()
668
-
669
-	w := bufio.NewWriterSize(gzw, 4096)
670
-	defer w.Flush()
671
-	
672
-	// Write to 'w' 
673
-```
674
-
675
-This will only use up to 4KB in memory when the writer is idle. 
676
-
677
-Compression is almost always worse than the fastest compression level 
678
-and each write will allocate (a little) memory. 
679
-
680
-
681
-# Other packages
682
-
683
-Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code):
684
-
685
-* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression.
686
-* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression.
687
-* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer.
688
-* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression.
689
-* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression.
690
-* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index.
691
-* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor.
692
-
693
-# license
694
-
695
-This code is licensed under the same conditions as the original Go code. See LICENSE file.
696
-
697
-
698
-
699
-
700
-
1
+# compress
2
+
3
+This package provides various compression algorithms.
4
+
5
+* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go.
6
+* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy.
7
+* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
8
+* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams.
9
+* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
10
+* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently.
11
+* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
12
+
13
+[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
14
+[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml)
15
+[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge)
16
+
17
+# package usage
18
+
19
+Use `go get github.com/klauspost/compress@latest` to add it to your project.
20
+
21
+This package will support the current Go version and 2 versions back.
22
+
23
+* Use the `nounsafe` tag to disable all use of the "unsafe" package.
24
+* Use the `noasm` tag to disable all assembly across packages.
25
+
26
+Use the links above for more information on each.
27
+
28
+# changelog
29
+
30
+* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4)
31
+	* gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121
32
+	* zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122
33
+	* gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116
34
+
35
+* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3)
36
+	* Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102).
37
+
38
+* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2)
39
+	* flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115
40
+	* flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106
41
+
42
+* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED
43
+  * zstd: Add simple zstd EncodeTo/DecodeTo functions  https://github.com/klauspost/compress/pull/1079
44
+  * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059
45
+  * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080
46
+  * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086
47
+  * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090
48
+  * flate: Faster load+store https://github.com/klauspost/compress/pull/1104
49
+  * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101
50
+  * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103
51
+
52
+* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0)
53
+  * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036
54
+  * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028
55
+  * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043
56
+  * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045
57
+  * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048
58
+  * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049
59
+  * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050
60
+
61
+<details>
62
+	<summary>See changes to v1.17.x</summary>
63
+
64
+* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11)
65
+  * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017
66
+  * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014
67
+  * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011
68
+  * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013
69
+
70
+* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)
71
+	* gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978
72
+	* gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002
73
+	* s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982
74
+	* zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007
75
+	* flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996
76
+
77
+* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9)
78
+	* s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949
79
+	* flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963
80
+	* Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971
81
+	* zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951
82
+
83
+* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8)
84
+	* zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885
85
+	* zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938
86
+
87
+* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7)
88
+	* s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927
89
+	* s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930
90
+  
91
+* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6)
92
+	* zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923
93
+	* s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925
94
+  
95
+* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5)
96
+	* flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912
97
+	* zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908
98
+	* zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913
99
+	* zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910
100
+	* s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917
101
+https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918
102
+
103
+* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4)
104
+	* huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887
105
+	* huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886
106
+	* gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892
107
+	* gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890
108
+	* gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891
109
+
110
+* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3)
111
+	* fse: Fix max header size https://github.com/klauspost/compress/pull/881
112
+	* zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877
113
+	* gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883
114
+
115
+* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2)
116
+	* zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876
117
+
118
+* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1)
119
+	* s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871
120
+	* flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869
121
+	* s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867
122
+
123
+* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
124
+	* Add experimental dictionary builder  https://github.com/klauspost/compress/pull/853
125
+	* Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838
126
+	* flate: Add limited window compression https://github.com/klauspost/compress/pull/843
127
+	* s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
128
+	* flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
129
+	* gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
130
+   
131
+</details>
132
+<details>
133
+	<summary>See changes to v1.16.x</summary>
134
+
135
+   
136
+* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
137
+	* zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
138
+	* s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832
139
+
140
+* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6)
141
+	* zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806
142
+	* zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824
143
+	* gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815
144
+	* s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663
145
+
146
+* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5)
147
+	* zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802
148
+	* gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804
149
+
150
+* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4)
151
+	* zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784
152
+	* zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792
153
+	* zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785
154
+	* zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795
155
+	* s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779
156
+	* s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780
157
+	* gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799
158
+
159
+* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1)
160
+	* zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776
161
+	* gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767
162
+	* s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766
163
+	* zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773
164
+	* huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774
165
+
166
+* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0)
167
+	* s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support.  https://github.com/klauspost/compress/pull/685
168
+	* s2: Add Compression Size Estimate.  https://github.com/klauspost/compress/pull/752
169
+	* s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755
170
+	* s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748
171
+	* s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747
172
+	* s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746
173
+</details>
174
+
175
+<details>
176
+	<summary>See changes to v1.15.x</summary>
177
+	
178
+* Jan 21st, 2023 (v1.15.15)
179
+	* deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739
180
+	* zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728
181
+	* zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745
182
+	* gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740
183
+
184
+* Jan 3rd, 2023 (v1.15.14)
185
+
186
+	* flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718
187
+	* zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720
188
+	* export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722
189
+	* s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723
190
+
191
+* Dec 11, 2022 (v1.15.13)
192
+	* zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder  https://github.com/klauspost/compress/pull/691
193
+	* zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708
194
+
195
+* Oct 26, 2022 (v1.15.12)
196
+
197
+	* zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680
198
+	* gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683
199
+
200
+* Sept 26, 2022 (v1.15.11)
201
+
202
+	* flate: Improve level 1-3 compression  https://github.com/klauspost/compress/pull/678
203
+	* zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677
204
+	* zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668
205
+	* zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667
206
+
207
+* Sept 16, 2022 (v1.15.10)
208
+
209
+	* zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649
210
+	* Add Go 1.19 - deprecate Go 1.16  https://github.com/klauspost/compress/pull/651
211
+	* flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656
212
+	* zstd: Improve "better" compression  https://github.com/klauspost/compress/pull/657
213
+	* s2: Improve "best" compression https://github.com/klauspost/compress/pull/658
214
+	* s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635
215
+	* s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646
216
+	* Use arrays for constant size copies https://github.com/klauspost/compress/pull/659
217
+
218
+* July 21, 2022 (v1.15.9)
219
+
220
+	* zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645
221
+	* zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644
222
+	* zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643
223
+
224
+* July 13, 2022 (v1.15.8)
225
+
226
+	* gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641
227
+	* s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638
228
+	* zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636
229
+	* zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637
230
+	* huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634
231
+	* zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640
232
+	* gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639
233
+
234
+* June 29, 2022 (v1.15.7)
235
+
236
+	* s2: Fix absolute forward seeks  https://github.com/klauspost/compress/pull/633
237
+	* zip: Merge upstream  https://github.com/klauspost/compress/pull/631
238
+	* zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624
239
+	* zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598
240
+	* flate: Faster histograms  https://github.com/klauspost/compress/pull/620
241
+	* deflate: Use compound hcode  https://github.com/klauspost/compress/pull/622
242
+
243
+* June 3, 2022 (v1.15.6)
244
+	* s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613
245
+	* s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611
246
+	* zstd: Always use configured block size https://github.com/klauspost/compress/pull/605
247
+	* zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606
248
+	* zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608
249
+	* gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612
250
+	* s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609
251
+	* s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607
252
+	* snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614
253
+	* s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610
254
+
255
+* May 25, 2022 (v1.15.5)
256
+	* s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602
257
+	* s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601
258
+	* huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596
259
+	* zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588
260
+	* zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592
261
+	* zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599
262
+	* zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593
263
+	* huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586
264
+	* flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590
265
+
266
+
267
+* May 11, 2022 (v1.15.4)
268
+	* huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577)
269
+	* inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581)
270
+	* zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583)
271
+	* zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580)
272
+
273
+* May 5, 2022 (v1.15.3)
274
+	* zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572)
275
+	* s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575)
276
+
277
+* Apr 26, 2022 (v1.15.2)
278
+	* zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster.  [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537)
279
+	* zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539)
280
+	* s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555)
281
+	* Minimum version is Go 1.16, added CI test on 1.18.
282
+
283
+* Mar 11, 2022 (v1.15.1)
284
+	* huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512)
285
+	* zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514)
286
+	* zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520)
287
+	* zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521)
288
+	* zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523)
289
+
290
+* Mar 3, 2022 (v1.15.0)
291
+	* zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498)
292
+	* zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505)
293
+	* huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507)
294
+	* flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509)
295
+	* gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400)
296
+	* gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510)
297
+
298
+Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines.
299
+
300
+Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected.
301
+
302
+While the release has been extensively tested, it is recommended to testing when upgrading.
303
+
304
+</details>
305
+
306
+<details>
307
+	<summary>See changes to v1.14.x</summary>
308
+	
309
+* Feb 22, 2022 (v1.14.4)
310
+	* flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503)
311
+	* zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502)
312
+	* zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501)  #501
313
+	* huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500)
314
+
315
+* Feb 17, 2022 (v1.14.3)
316
+	* flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494)  [#478](https://github.com/klauspost/compress/pull/478)
317
+	* flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483)
318
+	* s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486)
319
+
320
+* Jan 25, 2022 (v1.14.2)
321
+	* zstd: improve header decoder by @dsnet  [#476](https://github.com/klauspost/compress/pull/476)
322
+	* zstd: Add bigger default blocks  [#469](https://github.com/klauspost/compress/pull/469)
323
+	* zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470)
324
+	* zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472)
325
+	* flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473)
326
+	* zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475)
327
+
328
+* Jan 11, 2022 (v1.14.1)
329
+	* s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462)
330
+	* flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458)
331
+	* zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468)
332
+	* zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464)
333
+	* Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445)
334
+</details>
335
+
336
+<details>
337
+	<summary>See changes to v1.13.x</summary>
338
+	
339
+* Aug 30, 2021 (v1.13.5)
340
+	* gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425)
341
+	* s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413)
342
+	* zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426)
343
+	* Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421)
344
+
345
+* Aug 12, 2021 (v1.13.4)
346
+	* Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy).
347
+	* zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415)
348
+
349
+* Aug 3, 2021 (v1.13.3) 
350
+	* zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404)
351
+	* zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411)
352
+	* gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406)
353
+	* s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399)
354
+	* zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401)
355
+	* zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410)
356
+
357
+* Jun 14, 2021 (v1.13.1)
358
+	* s2: Add full Snappy output support  [#396](https://github.com/klauspost/compress/pull/396)
359
+	* zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394)
360
+	* gzhttp: Add header to skip compression  [#389](https://github.com/klauspost/compress/pull/389)
361
+	* s2: Improve speed with bigger output margin  [#395](https://github.com/klauspost/compress/pull/395)
362
+
363
+* Jun 3, 2021 (v1.13.0)
364
+	* Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors.
365
+	* zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)
366
+	* zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)
367
+</details>
368
+
369
+
370
+<details>
371
+	<summary>See changes to v1.12.x</summary>
372
+	
373
+* May 25, 2021 (v1.12.3)
374
+	* deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374)
375
+	* deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375)
376
+	* zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) 
377
+
378
+* Apr 27, 2021 (v1.12.2)
379
+	* zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365)
380
+	* zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363)
381
+	* deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367)
382
+	* s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358)
383
+	* s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362)
384
+	* s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) 
385
+
386
+* Apr 14, 2021 (v1.12.1)
387
+	* snappy package removed. Upstream added as dependency.
388
+	* s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353)
389
+	* s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352)
390
+	* s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348)
391
+	* s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352)
392
+	* zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346)
393
+	* s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349)
394
+</details>
395
+
396
+<details>
397
+	<summary>See changes to v1.11.x</summary>
398
+	
399
+* Mar 26, 2021 (v1.11.13)
400
+	* zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345)
401
+	* zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336)
402
+	* deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338)
403
+	* s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341)
404
+
405
+* Mar 5, 2021 (v1.11.12)
406
+	* s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives).
407
+	* s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328)
408
+
409
+* Mar 1, 2021 (v1.11.9)
410
+	* s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324)
411
+	* s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325)
412
+	* s2: Fix binaries.
413
+
414
+* Feb 25, 2021 (v1.11.8)
415
+	* s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended.
416
+	* s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315)
417
+	* s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322)
418
+	* zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314)
419
+	* zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313)
420
+  
421
+* Jan 14, 2021 (v1.11.7)
422
+	* Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309)
423
+	* s2: Add 'best' compression option.  [#310](https://github.com/klauspost/compress/pull/310)
424
+	* s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311)
425
+	* s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308)
426
+	* s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312)
427
+
428
+* Jan 7, 2021 (v1.11.6)
429
+	* zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306)
430
+	* zstd: Free Decoder resources when Reset is called with a nil io.Reader  [#305](https://github.com/klauspost/compress/pull/305)
431
+
432
+* Dec 20, 2020 (v1.11.4)
433
+	* zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304)
434
+	* Add header decoder [#299](https://github.com/klauspost/compress/pull/299)
435
+	* s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297)
436
+	* Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300)
437
+	* zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303)
438
+
439
+* Nov 15, 2020 (v1.11.3)
440
+	* inflate: 10-15% faster decompression  [#293](https://github.com/klauspost/compress/pull/293)
441
+	* zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295)
442
+
443
+* Oct 11, 2020 (v1.11.2)
444
+	* s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291)
445
+
446
+* Oct 1, 2020 (v1.11.1)
447
+	* zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286)
448
+
449
+* Sept 8, 2020 (v1.11.0)
450
+	* zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281)
451
+	* zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282)
452
+	* inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274)
453
+</details>
454
+
455
+<details>
456
+	<summary>See changes to v1.10.x</summary>
457
+ 
458
+* July 8, 2020 (v1.10.11) 
459
+	* zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278)
460
+	* huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275)
461
+	
462
+* June 23, 2020 (v1.10.10) 
463
+	* zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270)
464
+	
465
+* June 16, 2020 (v1.10.9): 
466
+	* zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268)
467
+	* zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266)
468
+	* Fuzzit tests removed. The service has been purchased and is no longer available.
469
+	
470
+* June 5, 2020 (v1.10.8): 
471
+	* 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265)
472
+	
473
+* June 1, 2020 (v1.10.7): 
474
+	* Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries)
475
+	* Increase zstd decompression speed up to 1.19x.  [#259](https://github.com/klauspost/compress/pull/259)
476
+	* Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263)
477
+	
478
+* May 21, 2020: (v1.10.6) 
479
+	* zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252)
480
+	* zstd: Stricter decompression checks.
481
+	
482
+* April 12, 2020: (v1.10.5)
483
+	* s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239)
484
+	
485
+* Apr 8, 2020: (v1.10.4) 
486
+	* zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251),  [#250](https://github.com/klauspost/compress/pull/250),  [#249](https://github.com/klauspost/compress/pull/249),  [#247](https://github.com/klauspost/compress/pull/247)
487
+* Mar 11, 2020: (v1.10.3) 
488
+	* s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245)
489
+	* s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244)
490
+	* zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240)
491
+	* zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241)
492
+	* zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238)
493
+	
494
+* Feb 27, 2020: (v1.10.2) 
495
+	* Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232)
496
+	* Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227)
497
+	
498
+* Feb 18, 2020: (v1.10.1)
499
+	* Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226)
500
+	* deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224)
501
+	* Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224)
502
+	
503
+* Feb 4, 2020: (v1.10.0) 
504
+	* Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216)
505
+	* Fix buffer overflow on repeated small block deflate.  [#218](https://github.com/klauspost/compress/pull/218)
506
+	* Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214)
507
+	* Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s.  [#186](https://github.com/klauspost/compress/pull/186)
508
+
509
+</details>
510
+
511
+<details>
512
+	<summary>See changes prior to v1.10.0</summary>
513
+
514
+* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056),  [#206](https://github.com/klauspost/compress/pull/206).
515
+* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) 
516
+* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed.
517
+* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases.
518
+* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192)
519
+* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder.
520
+* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199)
521
+* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features
522
+* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197)
523
+* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198)
524
+* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit.
525
+* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191)
526
+* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188)
527
+* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187)
528
+* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines.
529
+* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate.
530
+* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184)
531
+* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate.
532
+* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180)
533
+* Nov 11, 2019: Set default  [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB.
534
+* Nov 11, 2019: Reduce inflate memory use by 1KB.
535
+* Nov 10, 2019: Less allocations in deflate bit writer.
536
+* Nov 10, 2019: Fix inconsistent error returned by zstd decoder.
537
+* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174)
538
+* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173)
539
+* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) 
540
+* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105)
541
+
542
+</details>
543
+
544
+<details>
545
+	<summary>See changes prior to v1.9.0</summary>
546
+
547
+* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169)
548
+* Oct 3, 2019: Fix inconsistent results on broken zstd streams.
549
+* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools)
550
+* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools).
551
+* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip).
552
+* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes).
553
+* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option.
554
+* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables.
555
+* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode.
556
+* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding.
557
+* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. 
558
+* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing.
559
+* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing.
560
+* Aug 14, 2019: zstd: Skip incompressible data 2x faster.  [#147](https://github.com/klauspost/compress/pull/147)
561
+* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146)
562
+* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144)
563
+* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142)
564
+* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder.
565
+* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder.
566
+* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content.
567
+* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix.
568
+* June 17, 2019: zstd decompression bugfix.
569
+* June 17, 2019: fix 32 bit builds.
570
+* June 17, 2019: Easier use in modules (less dependencies).
571
+* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio.
572
+* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression.
573
+* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels.
574
+* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression!
575
+* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels.
576
+* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added.
577
+* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression).
578
+* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below.
579
+* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0).
580
+* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change.
581
+* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change.
582
+* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function.
583
+* May 28, 2017: Reduce allocations when resetting decoder.
584
+* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7.
585
+* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625).
586
+* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before.
587
+* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update.
588
+* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. 
589
+* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression.
590
+* Mar 24, 2016: Small speedup for level 1-3.
591
+* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster.
592
+* Feb 19, 2016: Handle small payloads faster in level 1-3.
593
+* Feb 19, 2016: Added faster level 2 + 3 compression modes.
594
+* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5.
595
+* Feb 14, 2016: Snappy: Merge upstream changes. 
596
+* Feb 14, 2016: Snappy: Fix aggressive skipping.
597
+* Feb 14, 2016: Snappy: Update benchmark.
598
+* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression.
599
+* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%.
600
+* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content.
601
+* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup.
602
+* Jan 16, 2016: Optimization on deflate level 1,2,3 compression.
603
+* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives.
604
+* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs.
605
+* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms.
606
+* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update!
607
+* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet).
608
+* Nov 20 2015: Small optimization to bit writer on 64 bit systems.
609
+* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15).
610
+* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate.
611
+* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file
612
+* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x.
613
+
614
+</details>
615
+
616
+# deflate usage
617
+
618
+The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them:
619
+
620
+Typical speed is about 2x of the standard library packages.
621
+
622
+| old import       | new import                            | Documentation                                                           |
623
+|------------------|---------------------------------------|-------------------------------------------------------------------------|
624
+| `compress/gzip`  | `github.com/klauspost/compress/gzip`  | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc)   |
625
+| `compress/zlib`  | `github.com/klauspost/compress/zlib`  | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc)   |
626
+| `archive/zip`    | `github.com/klauspost/compress/zip`   | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc)     |
627
+| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) |
628
+
629
+You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
630
+
631
+The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/).
632
+
633
+Currently there is only minor speedup on decompression (mostly CRC32 calculation).
634
+
635
+Memory usage is typically 1MB for a Writer. stdlib is in the same range. 
636
+If you expect to have a lot of concurrently allocated Writers consider using 
637
+the stateless compression described below.
638
+
639
+For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing).
640
+
641
+To disable all assembly add `-tags=noasm`. This works across all packages.
642
+
643
+# Stateless compression
644
+
645
+This package offers stateless compression as a special option for gzip/deflate. 
646
+It will do compression but without maintaining any state between Write calls.
647
+
648
+This means there will be no memory kept between Write calls, but compression and speed will be suboptimal.
649
+
650
+This is only relevant in cases where you expect to run many thousands of compressors concurrently, 
651
+but with very little activity. This is *not* intended for regular web servers serving individual requests.  
652
+
653
+Because of this, the size of actual Write calls will affect output size.
654
+
655
+In gzip, specify level `-3` / `gzip.StatelessCompression` to enable.
656
+
657
+For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter)
658
+
659
+A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer:
660
+
661
+```go
662
+	// replace 'ioutil.Discard' with your output.
663
+	gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression)
664
+	if err != nil {
665
+		return err
666
+	}
667
+	defer gzw.Close()
668
+
669
+	w := bufio.NewWriterSize(gzw, 4096)
670
+	defer w.Flush()
671
+	
672
+	// Write to 'w' 
673
+```
674
+
675
+This will only use up to 4KB in memory when the writer is idle. 
676
+
677
+Compression is almost always worse than the fastest compression level 
678
+and each write will allocate (a little) memory. 
679
+
680
+
681
+# Other packages
682
+
683
+Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code):
684
+
685
+* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression.
686
+* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression.
687
+* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer.
688
+* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression.
689
+* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression.
690
+* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index.
691
+* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor.
692
+
693
+# license
694
+
695
+This code is licensed under the same conditions as the original Go code. See LICENSE file.
696
+
697
+
698
+
699
+
700
+
... ...
@@ -1,79 +1,79 @@
1
-# Finite State Entropy
2
-
3
-This package provides Finite State Entropy encoding and decoding.
4
-            
5
-Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) 
6
-encoding provides a fast near-optimal symbol encoding/decoding
7
-for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
8
-
9
-This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
10
-This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
11
-but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. 
12
-
13
-* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
14
-
15
-## News
16
-
17
- * Feb 2018: First implementation released. Consider this beta software for now.
18
-
19
-# Usage
20
-
21
-This package provides a low level interface that allows to compress single independent blocks. 
22
-
23
-Each block is separate, and there is no built in integrity checks. 
24
-This means that the caller should keep track of block sizes and also do checksums if needed.  
25
-
26
-Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
27
-You must provide input and will receive the output and maybe an error.
28
-
29
-These error values can be returned:
30
-
31
-| Error               | Description                                                                 |
32
-|---------------------|-----------------------------------------------------------------------------|
33
-| `<nil>`             | Everything ok, output is returned                                           |
34
-| `ErrIncompressible` | Returned when input is judged to be too hard to compress                    |
35
-| `ErrUseRLE`         | Returned from the compressor when the input is a single byte value repeated |
36
-| `(error)`           | An internal error occurred.                                                 |
37
-
38
-As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
39
-
40
-To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object 
41
-that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same 
42
-object can be used for both.   
43
-
44
-Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
45
-you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
46
-
47
-Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
48
-You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
49
-your input was likely corrupted. 
50
-
51
-It is important to note that a successful decoding does *not* mean your output matches your original input. 
52
-There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
53
-
54
-For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
55
-
56
-# Performance
57
-
58
-A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.  
59
-All compression functions are currently only running on the calling goroutine so only one core will be used per block.  
60
-
61
-The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
62
-is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be 
63
-beneficial to transpose all your input values down by 64.   
64
-
65
-With moderate block sizes around 64k speed are typically 200MB/s per core for compression and 
66
-around 300MB/s decompression speed. 
67
-
68
-The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. 
69
-
70
-# Plans
71
-
72
-At one point, more internals will be exposed to facilitate more "expert" usage of the components. 
73
-
74
-A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).  
75
-
76
-# Contributing
77
-
78
-Contributions are always welcome. Be aware that adding public functions will require good justification and breaking 
1
+# Finite State Entropy
2
+
3
+This package provides Finite State Entropy encoding and decoding.
4
+            
5
+Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) 
6
+encoding provides a fast near-optimal symbol encoding/decoding
7
+for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd).
8
+
9
+This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
10
+This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
11
+but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. 
12
+
13
+* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse)
14
+
15
+## News
16
+
17
+ * Feb 2018: First implementation released. Consider this beta software for now.
18
+
19
+# Usage
20
+
21
+This package provides a low level interface that allows to compress single independent blocks. 
22
+
23
+Each block is separate, and there is no built in integrity checks. 
24
+This means that the caller should keep track of block sizes and also do checksums if needed.  
25
+
26
+Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function.
27
+You must provide input and will receive the output and maybe an error.
28
+
29
+These error values can be returned:
30
+
31
+| Error               | Description                                                                 |
32
+|---------------------|-----------------------------------------------------------------------------|
33
+| `<nil>`             | Everything ok, output is returned                                           |
34
+| `ErrIncompressible` | Returned when input is judged to be too hard to compress                    |
35
+| `ErrUseRLE`         | Returned from the compressor when the input is a single byte value repeated |
36
+| `(error)`           | An internal error occurred.                                                 |
37
+
38
+As can be seen above there are errors that will be returned even under normal operation so it is important to handle these.
39
+
40
+To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object 
41
+that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same 
42
+object can be used for both.   
43
+
44
+Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
45
+you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
46
+
47
+Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function.
48
+You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
49
+your input was likely corrupted. 
50
+
51
+It is important to note that a successful decoding does *not* mean your output matches your original input. 
52
+There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
53
+
54
+For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples).
55
+
56
+# Performance
57
+
58
+A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors.  
59
+All compression functions are currently only running on the calling goroutine so only one core will be used per block.  
60
+
61
+The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input
62
+is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be 
63
+beneficial to transpose all your input values down by 64.   
64
+
65
+With moderate block sizes around 64k speed are typically 200MB/s per core for compression and 
66
+around 300MB/s decompression speed. 
67
+
68
+The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. 
69
+
70
+# Plans
71
+
72
+At one point, more internals will be exposed to facilitate more "expert" usage of the components. 
73
+
74
+A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261).  
75
+
76
+# Contributing
77
+
78
+Contributions are always welcome. Be aware that adding public functions will require good justification and breaking 
79 79
 changes will likely not be accepted. If in doubt open an issue before writing the PR.  
80 80
\ No newline at end of file
... ...
@@ -1,89 +1,89 @@
1
-# Huff0 entropy compression
2
-
3
-This package provides Huff0 encoding and decoding as used in zstd.
4
-            
5
-[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), 
6
-a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU 
7
-(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
8
-
9
-This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
10
-This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
11
-but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. 
12
-
13
-* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
14
-
15
-## News
16
-
17
-This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
18
-
19
-This ensures that most functionality is well tested.
20
-
21
-# Usage
22
-
23
-This package provides a low level interface that allows to compress single independent blocks. 
24
-
25
-Each block is separate, and there is no built in integrity checks. 
26
-This means that the caller should keep track of block sizes and also do checksums if needed.  
27
-
28
-Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and 
29
-[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
30
-You must provide input and will receive the output and maybe an error.
31
-
32
-These error values can be returned:
33
-
34
-| Error               | Description                                                                 |
35
-|---------------------|-----------------------------------------------------------------------------|
36
-| `<nil>`             | Everything ok, output is returned                                           |
37
-| `ErrIncompressible` | Returned when input is judged to be too hard to compress                    |
38
-| `ErrUseRLE`         | Returned from the compressor when the input is a single byte value repeated |
39
-| `ErrTooBig`         | Returned if the input block exceeds the maximum allowed size (128 Kib)      |
40
-| `(error)`           | An internal error occurred.                                                 |
41
-
42
-
43
-As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
44
-
45
-To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object 
46
-that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same 
47
-object can be used for both.   
48
-
49
-Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
50
-you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
51
-
52
-The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.  
53
-
54
-## Tables and re-use
55
-
56
-Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. 
57
-
58
-The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) 
59
-that controls this behaviour. See the documentation for details. This can be altered between each block.
60
-
61
-Do however note that this information is *not* stored in the output block and it is up to the users of the package to
62
-record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
63
-based on the boolean reported back from the CompressXX call. 
64
-
65
-If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the 
66
-[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
67
-
68
-## Decompressing
69
-
70
-The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
71
-This will initialize the decoding tables. 
72
-You can supply the complete block to `ReadTable` and it will return the data part of the block 
73
-which can be given to the decompressor. 
74
-
75
-Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) 
76
-or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
77
-
78
-For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
79
-
80
-You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
81
-your input was likely corrupted. 
82
-
83
-It is important to note that a successful decoding does *not* mean your output matches your original input. 
84
-There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
85
-
86
-# Contributing
87
-
88
-Contributions are always welcome. Be aware that adding public functions will require good justification and breaking 
89
-changes will likely not be accepted. If in doubt open an issue before writing the PR.
1
+# Huff0 entropy compression
2
+
3
+This package provides Huff0 encoding and decoding as used in zstd.
4
+            
5
+[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), 
6
+a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU 
7
+(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds.
8
+
9
+This can be used for compressing input with a lot of similar input values to the smallest number of bytes.
10
+This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders,
11
+but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. 
12
+
13
+* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0)
14
+
15
+## News
16
+
17
+This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package.
18
+
19
+This ensures that most functionality is well tested.
20
+
21
+# Usage
22
+
23
+This package provides a low level interface that allows to compress single independent blocks. 
24
+
25
+Each block is separate, and there is no built in integrity checks. 
26
+This means that the caller should keep track of block sizes and also do checksums if needed.  
27
+
28
+Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and 
29
+[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions.
30
+You must provide input and will receive the output and maybe an error.
31
+
32
+These error values can be returned:
33
+
34
+| Error               | Description                                                                 |
35
+|---------------------|-----------------------------------------------------------------------------|
36
+| `<nil>`             | Everything ok, output is returned                                           |
37
+| `ErrIncompressible` | Returned when input is judged to be too hard to compress                    |
38
+| `ErrUseRLE`         | Returned from the compressor when the input is a single byte value repeated |
39
+| `ErrTooBig`         | Returned if the input block exceeds the maximum allowed size (128 Kib)      |
40
+| `(error)`           | An internal error occurred.                                                 |
41
+
42
+
43
+As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these.
44
+
45
+To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object 
46
+that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same 
47
+object can be used for both.   
48
+
49
+Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this
50
+you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output.
51
+
52
+The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding.  
53
+
54
+## Tables and re-use
55
+
56
+Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. 
57
+
58
+The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) 
59
+that controls this behaviour. See the documentation for details. This can be altered between each block.
60
+
61
+Do however note that this information is *not* stored in the output block and it is up to the users of the package to
62
+record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called,
63
+based on the boolean reported back from the CompressXX call. 
64
+
65
+If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the 
66
+[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object.
67
+
68
+## Decompressing
69
+
70
+The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable).
71
+This will initialize the decoding tables. 
72
+You can supply the complete block to `ReadTable` and it will return the data part of the block 
73
+which can be given to the decompressor. 
74
+
75
+Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) 
76
+or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function.
77
+
78
+For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size.
79
+
80
+You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back
81
+your input was likely corrupted. 
82
+
83
+It is important to note that a successful decoding does *not* mean your output matches your original input. 
84
+There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid.
85
+
86
+# Contributing
87
+
88
+Contributions are always welcome. Be aware that adding public functions will require good justification and breaking 
89
+changes will likely not be accepted. If in doubt open an issue before writing the PR.
... ...
@@ -409,6 +409,7 @@ type SolveRequest struct {
409 409
 	Exporters               []*Exporter               `protobuf:"bytes,13,rep,name=Exporters,proto3" json:"Exporters,omitempty"`
410 410
 	EnableSessionExporter   bool                      `protobuf:"varint,14,opt,name=EnableSessionExporter,proto3" json:"EnableSessionExporter,omitempty"`
411 411
 	SourcePolicySession     string                    `protobuf:"bytes,15,opt,name=SourcePolicySession,proto3" json:"SourcePolicySession,omitempty"`
412
+	CompatibilityVersion    int64                     `protobuf:"varint,16,opt,name=CompatibilityVersion,proto3" json:"CompatibilityVersion,omitempty"`
412 413
 	unknownFields           protoimpl.UnknownFields
413 414
 	sizeCache               protoimpl.SizeCache
414 415
 }
... ...
@@ -548,6 +549,13 @@ func (x *SolveRequest) GetSourcePolicySession() string {
548 548
 	return ""
549 549
 }
550 550
 
551
+func (x *SolveRequest) GetCompatibilityVersion() int64 {
552
+	if x != nil {
553
+		return x.CompatibilityVersion
554
+	}
555
+	return 0
556
+}
557
+
551 558
 type CacheOptions struct {
552 559
 	state protoimpl.MessageState `protogen:"open.v1"`
553 560
 	// ExportRefDeprecated is deprecated in favor or the new Exports since BuildKit v0.4.0.
... ...
@@ -2058,7 +2066,7 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc =
2058 2058
 	" \x01(\tR\n" +
2059 2059
 	"RecordType\x12\x16\n" +
2060 2060
 	"\x06Shared\x18\v \x01(\bR\x06Shared\x12\x18\n" +
2061
-	"\aParents\x18\f \x03(\tR\aParents\"\xa6\b\n" +
2061
+	"\aParents\x18\f \x03(\tR\aParents\"\xda\b\n" +
2062 2062
 	"\fSolveRequest\x12\x10\n" +
2063 2063
 	"\x03Ref\x18\x01 \x01(\tR\x03Ref\x12.\n" +
2064 2064
 	"\n" +
... ...
@@ -2077,7 +2085,8 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc =
2077 2077
 	"\fSourcePolicy\x18\f \x01(\v2%.moby.buildkit.v1.sourcepolicy.PolicyR\fSourcePolicy\x128\n" +
2078 2078
 	"\tExporters\x18\r \x03(\v2\x1a.moby.buildkit.v1.ExporterR\tExporters\x124\n" +
2079 2079
 	"\x15EnableSessionExporter\x18\x0e \x01(\bR\x15EnableSessionExporter\x120\n" +
2080
-	"\x13SourcePolicySession\x18\x0f \x01(\tR\x13SourcePolicySession\x1aJ\n" +
2080
+	"\x13SourcePolicySession\x18\x0f \x01(\tR\x13SourcePolicySession\x122\n" +
2081
+	"\x14CompatibilityVersion\x18\x10 \x01(\x03R\x14CompatibilityVersion\x1aJ\n" +
2081 2082
 	"\x1cExporterAttrsDeprecatedEntry\x12\x10\n" +
2082 2083
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
2083 2084
 	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a@\n" +
... ...
@@ -77,6 +77,7 @@ message SolveRequest {
77 77
 	repeated Exporter Exporters = 13;
78 78
 	bool EnableSessionExporter = 14;
79 79
 	string SourcePolicySession = 15;
80
+	int64 CompatibilityVersion = 16;
80 81
 }
81 82
 
82 83
 message CacheOptions {
... ...
@@ -143,6 +143,7 @@ func (m *SolveRequest) CloneVT() *SolveRequest {
143 143
 	r.SourcePolicy = m.SourcePolicy.CloneVT()
144 144
 	r.EnableSessionExporter = m.EnableSessionExporter
145 145
 	r.SourcePolicySession = m.SourcePolicySession
146
+	r.CompatibilityVersion = m.CompatibilityVersion
146 147
 	if rhs := m.ExporterAttrsDeprecated; rhs != nil {
147 148
 		tmpContainer := make(map[string]string, len(rhs))
148 149
 		for k, v := range rhs {
... ...
@@ -1043,6 +1044,9 @@ func (this *SolveRequest) EqualVT(that *SolveRequest) bool {
1043 1043
 	if this.SourcePolicySession != that.SourcePolicySession {
1044 1044
 		return false
1045 1045
 	}
1046
+	if this.CompatibilityVersion != that.CompatibilityVersion {
1047
+		return false
1048
+	}
1046 1049
 	return string(this.unknownFields) == string(that.unknownFields)
1047 1050
 }
1048 1051
 
... ...
@@ -2249,6 +2253,13 @@ func (m *SolveRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
2249 2249
 		i -= len(m.unknownFields)
2250 2250
 		copy(dAtA[i:], m.unknownFields)
2251 2251
 	}
2252
+	if m.CompatibilityVersion != 0 {
2253
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CompatibilityVersion))
2254
+		i--
2255
+		dAtA[i] = 0x1
2256
+		i--
2257
+		dAtA[i] = 0x80
2258
+	}
2252 2259
 	if len(m.SourcePolicySession) > 0 {
2253 2260
 		i -= len(m.SourcePolicySession)
2254 2261
 		copy(dAtA[i:], m.SourcePolicySession)
... ...
@@ -4174,6 +4185,9 @@ func (m *SolveRequest) SizeVT() (n int) {
4174 4174
 	if l > 0 {
4175 4175
 		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
4176 4176
 	}
4177
+	if m.CompatibilityVersion != 0 {
4178
+		n += 2 + protohelpers.SizeOfVarint(uint64(m.CompatibilityVersion))
4179
+	}
4177 4180
 	n += len(m.unknownFields)
4178 4181
 	return n
4179 4182
 }
... ...
@@ -6326,6 +6340,25 @@ func (m *SolveRequest) UnmarshalVT(dAtA []byte) error {
6326 6326
 			}
6327 6327
 			m.SourcePolicySession = string(dAtA[iNdEx:postIndex])
6328 6328
 			iNdEx = postIndex
6329
+		case 16:
6330
+			if wireType != 0 {
6331
+				return fmt.Errorf("proto: wrong wireType = %d for field CompatibilityVersion", wireType)
6332
+			}
6333
+			m.CompatibilityVersion = 0
6334
+			for shift := uint(0); ; shift += 7 {
6335
+				if shift >= 64 {
6336
+					return protohelpers.ErrIntOverflow
6337
+				}
6338
+				if iNdEx >= l {
6339
+					return io.ErrUnexpectedEOF
6340
+				}
6341
+				b := dAtA[iNdEx]
6342
+				iNdEx++
6343
+				m.CompatibilityVersion |= int64(b&0x7F) << shift
6344
+				if b < 0x80 {
6345
+					break
6346
+				}
6347
+			}
6329 6348
 		default:
6330 6349
 			iNdEx = preIndex
6331 6350
 			skippy, err := protohelpers.Skip(dAtA[iNdEx:])
... ...
@@ -483,6 +483,26 @@ func Git(url, fragment string, opts ...GitOption) State {
483 483
 		addCap(&gi.Constraints, pb.CapSourceGitMTime)
484 484
 	}
485 485
 
486
+	if gi.FetchByCommit {
487
+		attrs[pb.AttrGitFetchByCommit] = "true"
488
+		addCap(&gi.Constraints, pb.CapSourceGitFetchByCommit)
489
+	}
490
+	if gi.Bundle != "" {
491
+		attrs[pb.AttrGitBundle] = gi.Bundle
492
+		addCap(&gi.Constraints, pb.CapSourceGitBundle)
493
+	}
494
+	if gi.BundleOCISessionID != "" {
495
+		attrs[pb.AttrOCILayoutSessionID] = gi.BundleOCISessionID
496
+	}
497
+	if gi.BundleOCIStoreID != "" {
498
+		attrs[pb.AttrOCILayoutStoreID] = gi.BundleOCIStoreID
499
+	}
500
+
501
+	if gi.CheckoutBundle {
502
+		attrs[pb.AttrGitCheckoutBundle] = "true"
503
+		addCap(&gi.Constraints, pb.CapSourceGitCheckoutBundle)
504
+	}
505
+
486 506
 	addCap(&gi.Constraints, pb.CapSourceGit)
487 507
 
488 508
 	source := NewSource("git://"+id, attrs, gi.Constraints)
... ...
@@ -500,17 +520,22 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) {
500 500
 
501 501
 type GitInfo struct {
502 502
 	constraintsWrapper
503
-	KeepGitDir       bool
504
-	AuthTokenSecret  string
505
-	AuthHeaderSecret string
506
-	addAuthCap       bool
507
-	KnownSSHHosts    string
508
-	MountSSHSock     string
509
-	Checksum         string
510
-	Ref              string
511
-	SubDir           string
512
-	SkipSubmodules   bool
513
-	MTime            string
503
+	KeepGitDir         bool
504
+	AuthTokenSecret    string
505
+	AuthHeaderSecret   string
506
+	addAuthCap         bool
507
+	KnownSSHHosts      string
508
+	MountSSHSock       string
509
+	Checksum           string
510
+	Ref                string
511
+	SubDir             string
512
+	SkipSubmodules     bool
513
+	MTime              string
514
+	Bundle             string
515
+	BundleOCISessionID string
516
+	BundleOCIStoreID   string
517
+	CheckoutBundle     bool
518
+	FetchByCommit      bool
514 519
 }
515 520
 
516 521
 func GitRef(v string) GitOption {
... ...
@@ -577,6 +602,99 @@ func GitChecksum(v string) GitOption {
577 577
 	})
578 578
 }
579 579
 
580
+// GitFetchByCommit makes the git source trust the provided checksum as the
581
+// commit to fetch, without resolving the ref against the remote. The ref, if
582
+// set, is applied locally after the commit is fetched so that cache keys
583
+// still depend on it. This is useful when the remote ref may have moved
584
+// since the original resolution.
585
+//
586
+// Unqualified ref names are canonicalized to "refs/heads/<name>" to match
587
+// the normal path's cache keys for branches. Tags must be passed fully
588
+// qualified ("refs/tags/<name>").
589
+func GitFetchByCommit() GitOption {
590
+	return gitOptionFunc(func(gi *GitInfo) {
591
+		gi.FetchByCommit = true
592
+	})
593
+}
594
+
595
+// GitBundleInfo carries scheme-specific configuration for [GitBundleURL].
596
+// It is populated by [GitBundleOption] values and consumed by GitBundleURL.
597
+type GitBundleInfo struct {
598
+	// OCISessionID pins the OCI-layout bundle fetch to a specific client
599
+	// session. Only meaningful when the locator uses the
600
+	// "oci-layout+blob://" scheme.
601
+	OCISessionID string
602
+	// OCIStoreID overrides the OCI-layout store name derived from the
603
+	// bundle locator body. Only meaningful when the locator uses the
604
+	// "oci-layout+blob://" scheme.
605
+	OCIStoreID string
606
+}
607
+
608
+// GitBundleOption configures bundle-specific behavior for [GitBundleURL].
609
+type GitBundleOption interface {
610
+	SetGitBundleOption(*GitBundleInfo)
611
+}
612
+
613
+type gitBundleOptionFunc func(*GitBundleInfo)
614
+
615
+func (fn gitBundleOptionFunc) SetGitBundleOption(bi *GitBundleInfo) {
616
+	fn(bi)
617
+}
618
+
619
+// GitBundleURL configures the git source to import the commit history from a
620
+// pre-built git bundle instead of fetching from the upstream Git remote. The
621
+// locator must use one of the following schemes:
622
+//
623
+//	docker-image+blob://<registry-ref>@sha256:<digest>
624
+//	oci-layout+blob://<ref>@sha256:<digest>
625
+//
626
+// [GitChecksum] is required when GitBundleURL is used and the commit it
627
+// identifies must be reachable from a ref inside the bundle (i.e. the bundle
628
+// must contain that commit; BuildKit validates this after import). Auth and
629
+// SSH options are ignored in bundle mode. Submodules, if present, still fetch
630
+// from their own remotes. See [GitCheckoutBundle] for the checkout side; it
631
+// is mutually exclusive with [KeepGitDir] and [GitSubDir].
632
+//
633
+// Scheme-specific behavior can be tuned via [GitBundleOption] values, e.g.
634
+// [GitBundleOCIStore] for the "oci-layout+blob://" scheme.
635
+func GitBundleURL(locator string, opts ...GitBundleOption) GitOption {
636
+	bi := &GitBundleInfo{}
637
+	for _, o := range opts {
638
+		o.SetGitBundleOption(bi)
639
+	}
640
+	return gitOptionFunc(func(gi *GitInfo) {
641
+		gi.Bundle = locator
642
+		gi.BundleOCISessionID = bi.OCISessionID
643
+		gi.BundleOCIStoreID = bi.OCIStoreID
644
+	})
645
+}
646
+
647
+// GitBundleOCIStore configures the OCI-layout session and store id used to
648
+// resolve a bundle locator with the "oci-layout+blob://" scheme. When unset,
649
+// the locator body (the part before "@digest") is used as the store id and no
650
+// session is pinned. This mirrors [ImageBlobOCIStore] for regular OCI-layout
651
+// blobs.
652
+//
653
+// Only meaningful when passed to [GitBundleURL] together with a locator that
654
+// uses the "oci-layout+blob://" scheme.
655
+func GitBundleOCIStore(sessionID, storeID string) GitBundleOption {
656
+	return gitBundleOptionFunc(func(bi *GitBundleInfo) {
657
+		bi.OCISessionID = sessionID
658
+		bi.OCIStoreID = storeID
659
+	})
660
+}
661
+
662
+// GitCheckoutBundle produces a single-file git bundle at the checkout mount
663
+// root (filename "bundle") containing the resolved commit, instead of a
664
+// worktree. Mutually exclusive with [KeepGitDir] and [GitSubDir]. Submodules
665
+// are not included in the bundle. Pair with [GitBundleURL] to re-import the
666
+// bundle back into a later build.
667
+func GitCheckoutBundle() GitOption {
668
+	return gitOptionFunc(func(gi *GitInfo) {
669
+		gi.CheckoutBundle = true
670
+	})
671
+}
672
+
580 673
 // AuthOption can be used with either HTTP or Git sources.
581 674
 type AuthOption interface {
582 675
 	GitOption
... ...
@@ -36,8 +36,8 @@ import (
36 36
 
37 37
 type SolveOpt struct {
38 38
 	Exports               []ExportEntry
39
+	CompatibilityVersion  int
39 40
 	EnableSessionExporter bool
40
-	LocalDirs             map[string]string // Deprecated: use LocalMounts
41 41
 	LocalMounts           map[string]fsutil.FS
42 42
 	OCIStores             map[string]content.Store
43 43
 	SharedKey             string
... ...
@@ -95,11 +95,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
95 95
 		return nil, errors.New("invalid with def and cb")
96 96
 	}
97 97
 
98
-	mounts, err := prepareMounts(&opt)
99
-	if err != nil {
100
-		return nil, err
101
-	}
102
-	syncedDirs, err := prepareSyncedFiles(def, mounts)
98
+	syncedDirs, err := prepareSyncedFiles(def, opt.LocalMounts)
103 99
 	if err != nil {
104 100
 		return nil, err
105 101
 	}
... ...
@@ -311,6 +307,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
311 311
 			Cache:                   &cacheOpt.options,
312 312
 			Entitlements:            slices.Clone(opt.AllowedEntitlements),
313 313
 			Internal:                opt.Internal,
314
+			CompatibilityVersion:    int64(opt.CompatibilityVersion),
314 315
 			SourcePolicy:            opt.SourcePolicy,
315 316
 		}
316 317
 		if opt.SourcePolicyProvider != nil {
... ...
@@ -586,20 +583,3 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
586 586
 	}
587 587
 	return &res, nil
588 588
 }
589
-
590
-func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
591
-	// merge local mounts and fallback local directories together
592
-	mounts := make(map[string]fsutil.FS)
593
-	maps.Copy(mounts, opt.LocalMounts)
594
-	for k, dir := range opt.LocalDirs {
595
-		mount, err := fsutil.NewFS(dir)
596
-		if err != nil {
597
-			return nil, err
598
-		}
599
-		if _, ok := mounts[k]; ok {
600
-			return nil, errors.Errorf("local mount %s already exists", k)
601
-		}
602
-		mounts[k] = mount
603
-	}
604
-	return mounts, nil
605
-}
... ...
@@ -7,7 +7,10 @@ import (
7 7
 
8 8
 // Config provides containerd configuration data for the server
9 9
 type Config struct {
10
+	// Deprecated: Use Log.Level with "debug" set instead.
10 11
 	Debug bool `toml:"debug"`
12
+
13
+	// Deprecated: Use Log.Level with "trace" set instead.
11 14
 	Trace bool `toml:"trace"`
12 15
 
13 16
 	// Root is the path to a directory where buildkit will store persistent data
... ...
@@ -63,6 +66,7 @@ type SystemConfig struct {
63 63
 
64 64
 type LogConfig struct {
65 65
 	Format string `toml:"format"`
66
+	Level  string `toml:"level"`
66 67
 }
67 68
 
68 69
 type GRPCConfig struct {
... ...
@@ -34,6 +34,7 @@ import (
34 34
 	"github.com/moby/buildkit/solver/bboltcachestorage"
35 35
 	"github.com/moby/buildkit/solver/llbsolver"
36 36
 	"github.com/moby/buildkit/solver/llbsolver/cdidevices"
37
+	"github.com/moby/buildkit/solver/llbsolver/compat"
37 38
 	"github.com/moby/buildkit/solver/llbsolver/history"
38 39
 	"github.com/moby/buildkit/solver/llbsolver/proc"
39 40
 	provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
... ...
@@ -389,6 +390,14 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
389 389
 	}
390 390
 	translateLegacySolveRequest(req)
391 391
 
392
+	compatibilityVersion := int(req.CompatibilityVersion)
393
+	if compatibilityVersion == 0 {
394
+		compatibilityVersion = compat.CompatibilityVersionCurrent
395
+	}
396
+	if err := compat.ValidateCompatibilityVersion(compatibilityVersion); err != nil {
397
+		return nil, err
398
+	}
399
+
392 400
 	defer func() {
393 401
 		time.AfterFunc(time.Second, c.throttledGC)
394 402
 	}()
... ...
@@ -538,7 +547,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
538 538
 		FrontendOpt:    req.FrontendAttrs,
539 539
 		FrontendInputs: req.FrontendInputs,
540 540
 		CacheImports:   cacheImports,
541
-	}, llbsolver.ExporterRequest{
541
+	}, compatibilityVersion, llbsolver.ExporterRequest{
542 542
 		Exporters:             expis,
543 543
 		CacheExporters:        cacheExporters,
544 544
 		EnableSessionExporter: req.EnableSessionExporter,
... ...
@@ -2,55 +2,36 @@ package gateway
2 2
 
3 3
 import (
4 4
 	"context"
5
-	"sync"
6
-	"time"
7 5
 
8 6
 	"github.com/moby/buildkit/client/buildid"
9 7
 	"github.com/moby/buildkit/frontend/gateway"
10 8
 	gwapi "github.com/moby/buildkit/frontend/gateway/pb"
11 9
 	"github.com/moby/buildkit/solver/errdefs"
10
+	"github.com/moby/buildkit/util/registrar"
12 11
 	"github.com/pkg/errors"
13 12
 	"google.golang.org/grpc"
14 13
 )
15 14
 
16 15
 type GatewayForwarder struct {
17
-	mu         sync.RWMutex
18
-	updateCond *sync.Cond
19
-	builds     map[string]gateway.LLBBridgeForwarder
16
+	registrar *registrar.Registrar[string, gateway.LLBBridgeForwarder]
20 17
 }
21 18
 
22 19
 func NewGatewayForwarder() *GatewayForwarder {
23
-	gwf := &GatewayForwarder{
24
-		builds: map[string]gateway.LLBBridgeForwarder{},
20
+	return &GatewayForwarder{
21
+		registrar: registrar.New[string, gateway.LLBBridgeForwarder](),
25 22
 	}
26
-	gwf.updateCond = sync.NewCond(gwf.mu.RLocker())
27
-	return gwf
28 23
 }
29 24
 
30 25
 func (gwf *GatewayForwarder) Register(server *grpc.Server) {
31 26
 	gwapi.RegisterLLBBridgeServer(server, gwf)
32 27
 }
33 28
 
34
-func (gwf *GatewayForwarder) RegisterBuild(ctx context.Context, id string, bridge gateway.LLBBridgeForwarder) error {
35
-	gwf.mu.Lock()
36
-	defer gwf.mu.Unlock()
37
-
38
-	if _, ok := gwf.builds[id]; ok {
39
-		return errors.Errorf("build ID %s exists", id)
40
-	}
41
-
42
-	gwf.builds[id] = bridge
43
-	gwf.updateCond.Broadcast()
44
-
45
-	return nil
29
+func (gwf *GatewayForwarder) RegisterBuild(ctx context.Context, id string, bridge gateway.LLBBridgeForwarder) {
30
+	gwf.registrar.Register(id, bridge)
46 31
 }
47 32
 
48 33
 func (gwf *GatewayForwarder) UnregisterBuild(ctx context.Context, id string) {
49
-	gwf.mu.Lock()
50
-	defer gwf.mu.Unlock()
51
-
52
-	delete(gwf.builds, id)
53
-	gwf.updateCond.Broadcast()
34
+	gwf.registrar.Discard(id)
54 35
 }
55 36
 
56 37
 func (gwf *GatewayForwarder) lookupForwarder(ctx context.Context) (gateway.LLBBridgeForwarder, error) {
... ...
@@ -59,32 +40,14 @@ func (gwf *GatewayForwarder) lookupForwarder(ctx context.Context) (gateway.LLBBr
59 59
 		return nil, errors.New("no buildid found in context")
60 60
 	}
61 61
 
62
-	ctx, cancel := context.WithCancelCause(ctx)
63
-	ctx, _ = context.WithTimeoutCause(ctx, 3*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
64
-	defer func() { cancel(errors.WithStack(context.Canceled)) }()
65
-
66
-	go func() {
67
-		<-ctx.Done()
68
-		gwf.mu.Lock()
69
-		gwf.updateCond.Broadcast()
70
-		gwf.mu.Unlock()
71
-	}()
72
-
73
-	gwf.mu.RLock()
74
-	defer gwf.mu.RUnlock()
75
-	for {
76
-		select {
77
-		case <-ctx.Done():
78
-			return nil, errdefs.NewUnknownJobError(bid)
79
-		default:
80
-		}
81
-		fwd, ok := gwf.builds[bid]
82
-		if !ok {
83
-			gwf.updateCond.Wait()
84
-			continue
62
+	fwd, err := gwf.registrar.Get(ctx, bid)
63
+	if err != nil {
64
+		if errors.Is(err, context.Canceled) {
65
+			return nil, errors.WithStack(errdefs.NewUnknownJobError(bid))
85 66
 		}
86
-		return fwd, nil
67
+		return nil, err
87 68
 	}
69
+	return fwd, nil
88 70
 }
89 71
 
90 72
 func (gwf *GatewayForwarder) ResolveImageConfig(ctx context.Context, req *gwapi.ResolveImageConfigRequest) (*gwapi.ResolveImageConfigResponse, error) {
... ...
@@ -20,7 +20,7 @@ import (
20 20
 	"github.com/moby/buildkit/util/network"
21 21
 	rootlessmountopts "github.com/moby/buildkit/util/rootless/mountopts"
22 22
 	"github.com/moby/buildkit/util/system"
23
-	traceexec "github.com/moby/buildkit/util/tracing/exec"
23
+	"github.com/moby/buildkit/util/tracing/childprocess"
24 24
 	"github.com/moby/sys/user"
25 25
 	"github.com/moby/sys/userns"
26 26
 	specs "github.com/opencontainers/runtime-spec/specs-go"
... ...
@@ -120,7 +120,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
120 120
 	if tracingSocket != "" {
121 121
 		// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
122 122
 		meta.Env = append(meta.Env, tracingEnvVars...)
123
-		meta.Env = append(meta.Env, traceexec.Environ(ctx)...)
123
+		meta.Env = append(meta.Env, childprocess.Environ(ctx)...)
124 124
 	}
125 125
 
126 126
 	opts = append(opts,
... ...
@@ -246,7 +246,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
246 246
 		}
247 247
 	}()
248 248
 
249
-	desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts)
249
+	desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts, buildInfo.CompatibilityVersion, e.Type())
250 250
 	if err != nil {
251 251
 		return nil, nil, nil, err
252 252
 	}
... ...
@@ -25,6 +25,8 @@ import (
25 25
 	"github.com/moby/buildkit/session"
26 26
 	"github.com/moby/buildkit/snapshot"
27 27
 	"github.com/moby/buildkit/solver"
28
+	solvererrdefs "github.com/moby/buildkit/solver/errdefs"
29
+	"github.com/moby/buildkit/solver/llbsolver/compat"
28 30
 	"github.com/moby/buildkit/solver/result"
29 31
 	attestationTypes "github.com/moby/buildkit/util/attestation"
30 32
 	"github.com/moby/buildkit/util/bklog"
... ...
@@ -63,10 +65,14 @@ type ImageWriter struct {
63 63
 	opt WriterOpt
64 64
 }
65 65
 
66
-func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, sessionID string, inlineCache exptypes.InlineCache, opts *ImageCommitOpts) (*ocispecs.Descriptor, error) {
66
+func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, sessionID string, inlineCache exptypes.InlineCache, opts *ImageCommitOpts, compatibilityVersion int, exporterType string) (*ocispecs.Descriptor, error) {
67 67
 	if _, ok := inp.Metadata[exptypes.ExporterPlatformsKey]; len(inp.Refs) > 0 && !ok {
68 68
 		return nil, errors.Errorf("unable to export multiple refs, missing platforms mapping")
69 69
 	}
70
+	if compatibilityVersion == compat.CompatibilityVersion013 && opts.RefCfg.Compression.Type == compression.Zstd {
71
+		feature := fmt.Sprintf("%s exporter compression=%s", exporterType, opts.RefCfg.Compression.Type.String())
72
+		return nil, solvererrdefs.NewUnsupportedCompatibilityFeatureError(compatibilityVersion, feature)
73
+	}
70 74
 
71 75
 	isMap := len(inp.Refs) > 0
72 76
 
... ...
@@ -50,9 +50,10 @@ type ExporterInstance interface {
50 50
 }
51 51
 
52 52
 type ExportBuildInfo struct {
53
-	Ref         string
54
-	InlineCache exptypes.InlineCache
55
-	SessionID   string
53
+	Ref                  string
54
+	InlineCache          exptypes.InlineCache
55
+	SessionID            string
56
+	CompatibilityVersion int
56 57
 }
57 58
 
58 59
 type DescriptorReference interface {
... ...
@@ -159,7 +159,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source
159 159
 		}
160 160
 	}()
161 161
 
162
-	desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts)
162
+	desc, err := e.opt.ImageWriter.Commit(ctx, src, buildInfo.SessionID, buildInfo.InlineCache, &opts, buildInfo.CompatibilityVersion, e.Type())
163 163
 	if err != nil {
164 164
 		return nil, nil, nil, err
165 165
 	}
... ...
@@ -21,7 +21,13 @@ type Epoch struct {
21 21
 
22 22
 func ParseBuildArgs(opt map[string]string) (string, bool) {
23 23
 	v, ok := opt[frontendSourceDateEpochArg]
24
-	return v, ok
24
+	if !ok {
25
+		return "", false
26
+	}
27
+	if _, err := parseTime(frontendSourceDateEpochArg, v); err != nil {
28
+		return "", false
29
+	}
30
+	return v, true
25 31
 }
26 32
 
27 33
 func ParseExporterAttrs(opt map[string]string) (*Epoch, map[string]string, error) {
... ...
@@ -2,6 +2,7 @@ package builder
2 2
 
3 3
 import (
4 4
 	"context"
5
+	"maps"
5 6
 	"strings"
6 7
 
7 8
 	"github.com/containerd/platforms"
... ...
@@ -131,6 +132,7 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
131 131
 
132 132
 	rb, err := bc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (*dockerui.BuildResult, error) {
133 133
 		opt := convertOpt
134
+		opt.BuildArgs = maps.Clone(opt.BuildArgs)
134 135
 		opt.TargetPlatform = platform
135 136
 		if idx != 0 {
136 137
 			opt.Warn = nil
... ...
@@ -3,6 +3,7 @@ package dfgitutil
3 3
 
4 4
 import (
5 5
 	"net/url"
6
+	"path"
6 7
 	"strconv"
7 8
 	"strings"
8 9
 
... ...
@@ -59,6 +60,11 @@ type GitRef struct {
59 59
 
60 60
 	// MTime controls file modification time policy: "checkout" (default) or "commit".
61 61
 	MTime string
62
+
63
+	// FetchByCommit, when true, trusts Checksum as the commit and skips comparing
64
+	// it against the remote ref. The commit is fetched directly and the ref name
65
+	// (if any) is applied locally.
66
+	FetchByCommit bool
62 67
 }
63 68
 
64 69
 // ParseGitRef parses a git ref.
... ...
@@ -134,7 +140,7 @@ func (gf *GitRef) loadQuery(query url.Values) error {
134 134
 		case 0, 1:
135 135
 			if len(v) == 0 || v[0] == "" {
136 136
 				switch k {
137
-				case "submodules", "keep-git-dir":
137
+				case "submodules", "keep-git-dir", "fetch-by-commit":
138 138
 					v = nil
139 139
 				default:
140 140
 					return errors.Errorf("query %q has no value", k)
... ...
@@ -192,6 +198,18 @@ func (gf *GitRef) loadQuery(query url.Values) error {
192 192
 			default:
193 193
 				return errors.Errorf("invalid mtime value: %q (must be \"checkout\" or \"commit\")", v[0])
194 194
 			}
195
+		case "fetch-by-commit":
196
+			var vv bool
197
+			if len(v) == 0 {
198
+				vv = true
199
+			} else {
200
+				var err error
201
+				vv, err = strconv.ParseBool(v[0])
202
+				if err != nil {
203
+					return errors.Errorf("invalid fetch-by-commit value: %q", v[0])
204
+				}
205
+			}
206
+			gf.FetchByCommit = vv
195 207
 		default:
196 208
 			return errors.Errorf("unexpected query %q", k)
197 209
 		}
... ...
@@ -223,16 +241,23 @@ func (gf *GitRef) loadQuery(query url.Values) error {
223 223
 	return nil
224 224
 }
225 225
 
226
-// FragmentFormat returns a simplified git URL in fragment format with only ref.
226
+// FragmentFormat returns a simplified git URL in fragment format.
227 227
 // If the URL cannot be parsed, the original string is returned with false.
228
-func FragmentFormat(remote string) (string, bool) {
228
+func FragmentFormat(remote string, withSubdir bool) (string, bool) {
229 229
 	gitRef, _, err := ParseGitRef(remote)
230 230
 	if err != nil || gitRef == nil {
231 231
 		return remote, false
232 232
 	}
233 233
 	u := gitRef.Remote
234
-	if gitRef.Ref != "" {
234
+	subdir := ""
235
+	if withSubdir {
236
+		subdir = strings.TrimPrefix(path.Join("/", gitRef.SubDir), "/")
237
+	}
238
+	if gitRef.Ref != "" || subdir != "" {
235 239
 		u += "#" + gitRef.Ref
240
+		if subdir != "" {
241
+			u += ":" + subdir
242
+		}
236 243
 	}
237 244
 	return u, true
238 245
 }
... ...
@@ -208,6 +208,7 @@ type dispatchContext struct {
208 208
 	opt               ConvertOpt
209 209
 	platformOpt       *platformOpt
210 210
 	globalArgs        *llb.EnvList
211
+	epoch             *time.Time
211 212
 	shlex             *shell.Lex
212 213
 	outline           outlineCapture
213 214
 	lint              *linter.Linter
... ...
@@ -300,9 +301,13 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
300 300
 		return nil, err
301 301
 	}
302 302
 
303
-	opt.Epoch, err = resolveSourceDateEpoch(opt.Epoch, globalArgs)
304
-	if err != nil {
305
-		return nil, err
303
+	var resolvedEpoch *time.Time
304
+	if sourceDateEpoch, ok := getBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH"); ok {
305
+		resolvedEpoch, err = resolveSourceDateEpochValue(ctx, sourceDateEpoch, opt, stages, globalArgs, shlex)
306
+		if err != nil {
307
+			return nil, err
308
+		}
309
+		globalArgs = setBuildArgValue(opt.BuildArgs, globalArgs, "SOURCE_DATE_EPOCH", formatSourceDateEpochValue(resolvedEpoch))
306 310
 	}
307 311
 
308 312
 	metaResolver := opt.MetaResolver
... ...
@@ -314,6 +319,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
314 314
 		opt:               opt,
315 315
 		platformOpt:       platformOpt,
316 316
 		globalArgs:        globalArgs,
317
+		epoch:             resolvedEpoch,
317 318
 		shlex:             shlex,
318 319
 		outline:           outline,
319 320
 		lint:              lint,
... ...
@@ -353,25 +359,39 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
353 353
 	return target, nil
354 354
 }
355 355
 
356
-func resolveSourceDateEpoch(explicit *time.Time, globalArgs *llb.EnvList) (*time.Time, error) {
357
-	if explicit != nil {
358
-		return explicit, nil
356
+func getBuildArgValue(buildArgs map[string]string, globalArgs *llb.EnvList, key string) (string, bool) {
357
+	if v, ok := buildArgs[key]; ok {
358
+		return v, true
359 359
 	}
360 360
 	if globalArgs == nil {
361
-		return nil, nil
361
+		return "", false
362 362
 	}
363
-
364
-	v, ok := globalArgs.Get("SOURCE_DATE_EPOCH")
363
+	v, ok := globalArgs.Get(key)
365 364
 	if !ok || v == "" {
366
-		return nil, nil
365
+		return "", false
367 366
 	}
367
+	return v, true
368
+}
368 369
 
369
-	sde, err := strconv.ParseInt(v, 10, 64)
370
-	if err != nil {
371
-		return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v)
370
+func setBuildArgValue(buildArgs map[string]string, globalArgs *llb.EnvList, key, value string) *llb.EnvList {
371
+	if _, ok := buildArgs[key]; ok {
372
+		if value == "" {
373
+			delete(buildArgs, key)
374
+		} else {
375
+			buildArgs[key] = value
376
+		}
377
+	}
378
+	if globalArgs != nil {
379
+		if _, ok := globalArgs.Get(key); ok {
380
+			if value == "" {
381
+				updated := globalArgs.Delete(key)
382
+				globalArgs = &updated
383
+			} else {
384
+				globalArgs = globalArgs.AddOrReplace(key, value)
385
+			}
386
+		}
372 387
 	}
373
-	tm := time.Unix(sde, 0).UTC()
374
-	return &tm, nil
388
+	return globalArgs
375 389
 }
376 390
 
377 391
 func (dctx *dispatchContext) buildDispatchStates(stages []instructions.Stage) error {
... ...
@@ -402,7 +422,7 @@ func (dctx *dispatchContext) buildDispatchStates(stages []instructions.Stage) er
402 402
 			stageName:      st.Name,
403 403
 			prefixPlatform: dctx.opt.MultiPlatformRequested,
404 404
 			outline:        dctx.outline.clone(),
405
-			epoch:          dctx.opt.Epoch,
405
+			epoch:          dctx.epoch,
406 406
 		}
407 407
 
408 408
 		if v := st.Platform; v != "" {
... ...
@@ -1598,7 +1618,7 @@ func dispatchHealthcheck(d *dispatchState, c *instructions.HealthCheckCommand, l
1598 1598
 		StartInterval: c.Health.StartInterval,
1599 1599
 		Retries:       c.Health.Retries,
1600 1600
 	}
1601
-	return commitToHistory(&d.image, fmt.Sprintf("HEALTHCHECK %q", d.image.Config.Healthcheck), false, nil, d.epoch)
1601
+	return commitToHistory(&d.image, fmt.Sprintf("HEALTHCHECK %+v", *d.image.Config.Healthcheck), false, nil, d.epoch)
1602 1602
 }
1603 1603
 
1604 1604
 func dispatchUser(d *dispatchState, c *instructions.UserCommand, commit bool) error {
... ...
@@ -167,6 +167,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
167 167
 			if gitRef.Submodules != nil && !*gitRef.Submodules {
168 168
 				gitOptions = append(gitOptions, llb.GitSkipSubmodules())
169 169
 			}
170
+			if gitRef.FetchByCommit {
171
+				gitOptions = append(gitOptions, llb.GitFetchByCommit())
172
+			}
170 173
 
171 174
 			st := llb.Git(gitRef.Remote, "", gitOptions...)
172 175
 			opts := append([]llb.CopyOption{&llb.CopyInfo{
173 176
new file mode 100644
... ...
@@ -0,0 +1,381 @@
0
+package dockerfile2llb
1
+
2
+import (
3
+	"archive/tar"
4
+	"bytes"
5
+	"context"
6
+	"io"
7
+	"maps"
8
+	"net/url"
9
+	"path"
10
+	"strconv"
11
+	"strings"
12
+	"time"
13
+
14
+	"github.com/moby/buildkit/client/llb"
15
+	"github.com/moby/buildkit/client/llb/sourceresolver"
16
+	"github.com/moby/buildkit/frontend/dockerfile/dfgitutil"
17
+	"github.com/moby/buildkit/frontend/dockerfile/instructions"
18
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
19
+	"github.com/moby/buildkit/frontend/dockerfile/shell"
20
+	"github.com/moby/buildkit/frontend/dockerui"
21
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
22
+	"github.com/moby/buildkit/solver/pb"
23
+	"github.com/moby/buildkit/util/gitutil/gitobject"
24
+	archivecompression "github.com/moby/go-archive/compression"
25
+	digest "github.com/opencontainers/go-digest"
26
+	"github.com/pkg/errors"
27
+)
28
+
29
+type sourceDateEpochStateOpt struct {
30
+	LogName string
31
+}
32
+
33
+func resolveSourceDateEpochValue(ctx context.Context, v string, opt ConvertOpt, stages []instructions.Stage, globalArgs *llb.EnvList, shlex *shell.Lex) (*time.Time, error) {
34
+	if v == "" {
35
+		return nil, nil
36
+	}
37
+	if sde, err := strconv.ParseInt(v, 10, 64); err == nil {
38
+		tm := time.Unix(sde, 0).UTC()
39
+		return &tm, nil
40
+	}
41
+
42
+	state, stateOpt, err := resolveSourceDateEpochState(ctx, v, opt, stages, globalArgs, shlex)
43
+	if err != nil {
44
+		return nil, err
45
+	}
46
+	if state == nil || opt.Client == nil {
47
+		return nil, nil
48
+	}
49
+	return resolveSourceDateEpochFromState(ctx, *state, opt.Client, stateOpt)
50
+}
51
+
52
+func formatSourceDateEpochValue(tm *time.Time) string {
53
+	if tm == nil {
54
+		return ""
55
+	}
56
+	return strconv.FormatInt(tm.Unix(), 10)
57
+}
58
+
59
+func resolveSourceDateEpochState(ctx context.Context, value string, opt ConvertOpt, stages []instructions.Stage, globalArgs *llb.EnvList, shlex *shell.Lex) (*llb.State, sourceDateEpochStateOpt, error) {
60
+	if value == "context" {
61
+		if opt.Client == nil {
62
+			return nil, sourceDateEpochStateOpt{}, nil
63
+		}
64
+		mainContextState, err := opt.Client.MainContext(ctx)
65
+		if err != nil {
66
+			return nil, sourceDateEpochStateOpt{}, err
67
+		}
68
+		return mainContextState, sourceDateEpochStateOpt{
69
+			LogName: "[internal] resolve main build context metadata",
70
+		}, nil
71
+	}
72
+
73
+	if opt.Client != nil {
74
+		nc, err := opt.Client.NamedContext(value, dockerui.ContextOpt{})
75
+		if err != nil {
76
+			return nil, sourceDateEpochStateOpt{}, err
77
+		}
78
+		if nc != nil {
79
+			st, _, err := nc.Load(ctx)
80
+			if err != nil {
81
+				return nil, sourceDateEpochStateOpt{}, err
82
+			}
83
+			return st, sourceDateEpochStateOpt{
84
+				LogName: "[internal] resolve SOURCE_DATE_EPOCH named context " + value,
85
+			}, nil
86
+		}
87
+	}
88
+
89
+	for i := range stages {
90
+		if !strings.EqualFold(stages[i].Name, value) {
91
+			continue
92
+		}
93
+
94
+		args := globalArgs
95
+		if globalArgs != nil {
96
+			updated := globalArgs.Delete("SOURCE_DATE_EPOCH")
97
+			args = &updated
98
+		}
99
+
100
+		sourceState, err := sourceDateEpochStageSource(stages[i], opt.BuildArgs, args, shlex)
101
+		if err != nil {
102
+			return nil, sourceDateEpochStateOpt{}, parser.WithLocation(err, stages[i].Location)
103
+		}
104
+
105
+		return sourceState, sourceDateEpochStateOpt{
106
+			LogName: "[internal] resolve SOURCE_DATE_EPOCH source stage " + stages[i].Name,
107
+		}, nil
108
+	}
109
+	return nil, sourceDateEpochStateOpt{}, errors.Errorf("invalid SOURCE_DATE_EPOCH: %s", value)
110
+}
111
+
112
+func sourceDateEpochStageSource(stage instructions.Stage, buildArgs map[string]string, globalArgs *llb.EnvList, shlex *shell.Lex) (*llb.State, error) {
113
+	stageBaseName, _, err := shlex.ProcessWord(stage.BaseName, globalArgs)
114
+	if err != nil {
115
+		return nil, errors.Wrapf(err, "failed to process source stage base name %q", stage.BaseName)
116
+	}
117
+	if stageBaseName != emptyImageName {
118
+		return nil, errors.New("SOURCE_DATE_EPOCH stage must use FROM scratch")
119
+	}
120
+
121
+	env := globalArgs
122
+	var sourceState *llb.State
123
+
124
+	for _, cmd := range stage.Commands {
125
+		switch c := cmd.(type) {
126
+		case *instructions.ArgCommand:
127
+			env, err = applySourceDateEpochStageArgs(c.Args, env, buildArgs, shlex)
128
+			if err != nil {
129
+				return nil, err
130
+			}
131
+		case *instructions.AddCommand:
132
+			if sourceState != nil {
133
+				return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD")
134
+			}
135
+			sourceState, err = sourceDateEpochAddSource(c, env, shlex)
136
+			if err != nil {
137
+				return nil, err
138
+			}
139
+		default:
140
+			return nil, errors.Errorf("SOURCE_DATE_EPOCH stage does not meet source-only requirements: unsupported %s instruction", cmd.Name())
141
+		}
142
+	}
143
+
144
+	if sourceState == nil {
145
+		return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD")
146
+	}
147
+
148
+	return sourceState, nil
149
+}
150
+
151
+func applySourceDateEpochStageArgs(args []instructions.KeyValuePairOptional, env *llb.EnvList, buildArgs map[string]string, shlex *shell.Lex) (*llb.EnvList, error) {
152
+	for _, arg := range args {
153
+		if v, ok := buildArgs[arg.Key]; ok {
154
+			env = env.AddOrReplace(arg.Key, v)
155
+			continue
156
+		}
157
+		if arg.Value == nil {
158
+			continue
159
+		}
160
+		v, _, err := shlex.ProcessWord(*arg.Value, env)
161
+		if err != nil {
162
+			return nil, err
163
+		}
164
+		env = env.AddOrReplace(arg.Key, v)
165
+	}
166
+	return env, nil
167
+}
168
+
169
+func sourceDateEpochAddSource(cmd *instructions.AddCommand, env *llb.EnvList, shlex *shell.Lex) (*llb.State, error) {
170
+	if len(cmd.SourceContents) != 0 || len(cmd.SourcePaths) != 1 {
171
+		return nil, errors.New("SOURCE_DATE_EPOCH stage must contain exactly one remote ADD source")
172
+	}
173
+
174
+	src, _, err := shlex.ProcessWord(cmd.SourcePaths[0], env)
175
+	if err != nil {
176
+		return nil, err
177
+	}
178
+
179
+	if isHTTPSource(src) {
180
+		var checksum digest.Digest
181
+		if cmd.Checksum != "" {
182
+			expandedChecksum, _, err := shlex.ProcessWord(cmd.Checksum, env)
183
+			if err != nil {
184
+				return nil, err
185
+			}
186
+			checksum, err = digest.Parse(expandedChecksum)
187
+			if err != nil {
188
+				return nil, err
189
+			}
190
+		}
191
+		st := llb.HTTP(src, llb.Filename(sourceDateEpochHTTPFilename(src)), llb.Checksum(checksum))
192
+		return &st, nil
193
+	}
194
+
195
+	gitRef, isGit, gitRefErr := dfgitutil.ParseGitRef(src)
196
+	if gitRefErr != nil && isGit {
197
+		return nil, gitRefErr
198
+	}
199
+	if gitRefErr == nil && !gitRef.IndistinguishableFromLocal {
200
+		gitOptions := []llb.GitOption{
201
+			llb.GitRef(gitRef.Ref),
202
+		}
203
+		if cmd.KeepGitDir != nil && *cmd.KeepGitDir {
204
+			gitOptions = append(gitOptions, llb.KeepGitDir())
205
+		}
206
+		if gitRef.KeepGitDir != nil && *gitRef.KeepGitDir {
207
+			gitOptions = append(gitOptions, llb.KeepGitDir())
208
+		}
209
+		if cmd.Checksum != "" {
210
+			expandedChecksum, _, err := shlex.ProcessWord(cmd.Checksum, env)
211
+			if err != nil {
212
+				return nil, err
213
+			}
214
+			gitOptions = append(gitOptions, llb.GitChecksum(expandedChecksum))
215
+		} else if gitRef.Checksum != "" {
216
+			gitOptions = append(gitOptions, llb.GitChecksum(gitRef.Checksum))
217
+		}
218
+		if gitRef.SubDir != "" {
219
+			gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir))
220
+		}
221
+		if gitRef.Submodules != nil && !*gitRef.Submodules {
222
+			gitOptions = append(gitOptions, llb.GitSkipSubmodules())
223
+		}
224
+		st := llb.Git(gitRef.Remote, "", gitOptions...)
225
+		return &st, nil
226
+	}
227
+
228
+	return nil, errors.New("SOURCE_DATE_EPOCH stage source must be a single HTTP(S) or Git ADD")
229
+}
230
+
231
+func sourceDateEpochHTTPFilename(src string) string {
232
+	u, err := url.Parse(src)
233
+	if err == nil {
234
+		if base := path.Base(u.Path); base != "." && base != "/" {
235
+			return base
236
+		}
237
+	}
238
+	return "__unnamed__"
239
+}
240
+
241
+func resolveSourceDateEpochFromState(ctx context.Context, st llb.State, client *dockerui.Client, opt sourceDateEpochStateOpt) (*time.Time, error) {
242
+	sourceOp, err := sourceOpFromState(ctx, &st, llb.WithCaps(client.BuildOpts().Caps))
243
+	if err != nil {
244
+		return nil, err
245
+	}
246
+	if sourceOp == nil {
247
+		return nil, nil
248
+	}
249
+
250
+	metaOpt := sourceresolver.Opt{
251
+		LogName: opt.LogName,
252
+	}
253
+	if strings.HasPrefix(sourceOp.Identifier, "git://") {
254
+		metaOpt.GitOpt = &sourceresolver.ResolveGitOpt{ReturnObject: true}
255
+	}
256
+	isHTTP := strings.HasPrefix(sourceOp.Identifier, "http://") || strings.HasPrefix(sourceOp.Identifier, "https://")
257
+	md, err := client.GatewayClient().ResolveSourceMetadata(ctx, sourceOp, metaOpt)
258
+	if err != nil {
259
+		return nil, err
260
+	}
261
+	if tm, ok, err := sourceDateEpochFromMetadata(md); ok || err != nil {
262
+		return tm, err
263
+	}
264
+	filename := sourceOp.Attrs[pb.AttrHTTPFilename]
265
+	if !isHTTP || filename == "" {
266
+		return nil, nil
267
+	}
268
+
269
+	sourceState := llb.NewState(llb.NewSource(sourceOp.Identifier, maps.Clone(sourceOp.Attrs), llb.Constraints{}).Output())
270
+	def, err := sourceState.Marshal(ctx, llb.WithCaps(client.BuildOpts().Caps))
271
+	if err != nil {
272
+		return nil, err
273
+	}
274
+	res, err := client.GatewayClient().Solve(ctx, gwclient.SolveRequest{
275
+		Definition: def.ToPB(),
276
+	})
277
+	if err != nil {
278
+		return nil, err
279
+	}
280
+	ref, err := res.SingleRef()
281
+	if err != nil {
282
+		return nil, err
283
+	}
284
+
285
+	return archiveMaxTimeFromRef(ctx, ref, filename, true)
286
+}
287
+
288
+func sourceOpFromState(ctx context.Context, st *llb.State, opts ...llb.ConstraintsOpt) (*pb.SourceOp, error) {
289
+	if st == nil {
290
+		return nil, nil
291
+	}
292
+	def, err := st.Marshal(ctx, opts...)
293
+	if err != nil {
294
+		return nil, err
295
+	}
296
+	dt := def.ToPB().Def
297
+	var src *pb.SourceOp
298
+	for _, d := range dt {
299
+		var op pb.Op
300
+		if err := op.Unmarshal(d); err != nil {
301
+			return nil, err
302
+		}
303
+		opSrc := op.GetSource()
304
+		if opSrc == nil {
305
+			continue
306
+		}
307
+		if src != nil {
308
+			return nil, nil
309
+		}
310
+		src = opSrc
311
+	}
312
+	return cloneSourceOp(src), nil
313
+}
314
+
315
+func cloneSourceOp(op *pb.SourceOp) *pb.SourceOp {
316
+	if op == nil {
317
+		return nil
318
+	}
319
+	return &pb.SourceOp{
320
+		Identifier: op.Identifier,
321
+		Attrs:      maps.Clone(op.Attrs),
322
+	}
323
+}
324
+
325
+func sourceDateEpochFromMetadata(md *sourceresolver.MetaResponse) (*time.Time, bool, error) {
326
+	if md.Git != nil && len(md.Git.CommitObject) > 0 {
327
+		obj, err := gitobject.Parse(md.Git.CommitObject)
328
+		if err != nil {
329
+			return nil, false, err
330
+		}
331
+		commit, err := obj.ToCommit()
332
+		if err != nil {
333
+			return nil, false, err
334
+		}
335
+		return commit.Committer.When, true, nil
336
+	}
337
+	if md.HTTP != nil && md.HTTP.LastModified != nil {
338
+		return md.HTTP.LastModified, true, nil
339
+	}
340
+	return nil, false, nil
341
+}
342
+
343
+func archiveMaxTimeFromRef(ctx context.Context, ref gwclient.Reference, filename string, allowNonArchive bool) (*time.Time, error) {
344
+	dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
345
+		Filename: filename,
346
+	})
347
+	if err != nil {
348
+		return nil, err
349
+	}
350
+	rc, err := archivecompression.DecompressStream(bytes.NewReader(dt))
351
+	if err != nil {
352
+		if allowNonArchive {
353
+			return nil, nil
354
+		}
355
+		return nil, err
356
+	}
357
+	defer rc.Close()
358
+
359
+	tr := tar.NewReader(rc)
360
+	var maxTime *time.Time
361
+	for {
362
+		hdr, err := tr.Next()
363
+		if err != nil {
364
+			if errors.Is(err, io.EOF) {
365
+				return maxTime, nil
366
+			}
367
+			if allowNonArchive {
368
+				return nil, nil
369
+			}
370
+			return nil, err
371
+		}
372
+		if !hdr.FileInfo().Mode().IsRegular() {
373
+			continue
374
+		}
375
+		tm := hdr.ModTime.UTC()
376
+		if maxTime == nil || tm.After(*maxTime) {
377
+			maxTime = &tm
378
+		}
379
+	}
380
+}
... ...
@@ -1,6 +1,7 @@
1 1
 package dockerfile2llb
2 2
 
3 3
 import (
4
+	"maps"
4 5
 	"slices"
5 6
 
6 7
 	"github.com/moby/buildkit/util/system"
... ...
@@ -15,6 +16,16 @@ func clone(src dockerspec.DockerOCIImage) dockerspec.DockerOCIImage {
15 15
 	img.Config.Cmd = slices.Clone(src.Config.Cmd)
16 16
 	img.Config.Entrypoint = slices.Clone(src.Config.Entrypoint)
17 17
 	img.Config.OnBuild = slices.Clone(src.Config.OnBuild)
18
+	img.Config.ExposedPorts = maps.Clone(src.Config.ExposedPorts)
19
+	img.Config.Volumes = maps.Clone(src.Config.Volumes)
20
+	img.Config.Labels = maps.Clone(src.Config.Labels)
21
+	img.Config.Shell = slices.Clone(src.Config.Shell)
22
+	if src.Config.Healthcheck != nil {
23
+		hc := *src.Config.Healthcheck
24
+		hc.Test = slices.Clone(src.Config.Healthcheck.Test)
25
+		img.Config.Healthcheck = &hc
26
+	}
27
+	img.History = slices.Clone(src.History)
18 28
 	return img
19 29
 }
20 30
 
... ...
@@ -4,7 +4,6 @@ import (
4 4
 	"net"
5 5
 	"strconv"
6 6
 	"strings"
7
-	"time"
8 7
 
9 8
 	"github.com/containerd/platforms"
10 9
 	"github.com/docker/go-units"
... ...
@@ -113,18 +112,6 @@ func parseNetMode(v string) (pb.NetMode, error) {
113 113
 	}
114 114
 }
115 115
 
116
-func parseSourceDateEpoch(v string) (*time.Time, error) {
117
-	if v == "" {
118
-		return nil, nil
119
-	}
120
-	sde, err := strconv.ParseInt(v, 10, 64)
121
-	if err != nil {
122
-		return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v)
123
-	}
124
-	tm := time.Unix(sde, 0).UTC()
125
-	return &tm, nil
126
-}
127
-
128 116
 func parseLocalSessionIDs(opt map[string]string) map[string]string {
129 117
 	m := map[string]string{}
130 118
 	for k, v := range opt {
... ...
@@ -8,7 +8,6 @@ import (
8 8
 	"strconv"
9 9
 	"strings"
10 10
 	"sync"
11
-	"time"
12 11
 
13 12
 	"github.com/containerd/platforms"
14 13
 	"github.com/distribution/reference"
... ...
@@ -51,14 +50,12 @@ const (
51 51
 	keyHostnameArg          = "build-arg:BUILDKIT_SANDBOX_HOSTNAME"
52 52
 	keyDockerfileLintArg    = "build-arg:BUILDKIT_DOCKERFILE_CHECK"
53 53
 	keyContextKeepGitDirArg = "build-arg:BUILDKIT_CONTEXT_KEEP_GIT_DIR"
54
-	keySourceDateEpoch      = "build-arg:SOURCE_DATE_EPOCH"
55 54
 )
56 55
 
57 56
 type Config struct {
58 57
 	BuildArgs        map[string]string
59 58
 	CacheIDNamespace string
60 59
 	CgroupParent     string
61
-	Epoch            *time.Time
62 60
 	ExtraHosts       []llb.HostIP
63 61
 	Hostname         string
64 62
 	ImageResolveMode llb.ResolveMode
... ...
@@ -147,6 +144,10 @@ func (bc *Client) BuildOpts() client.BuildOpts {
147 147
 	return bc.bopts
148 148
 }
149 149
 
150
+func (bc *Client) GatewayClient() client.Client {
151
+	return bc.client
152
+}
153
+
150 154
 func (bc *Client) init() error {
151 155
 	opts := bc.bopts.Opts
152 156
 
... ...
@@ -251,12 +252,6 @@ func (bc *Client) init() error {
251 251
 	}
252 252
 	bc.CacheImports = cacheImports
253 253
 
254
-	epoch, err := parseSourceDateEpoch(opts[keySourceDateEpoch])
255
-	if err != nil {
256
-		return err
257
-	}
258
-	bc.Epoch = epoch
259
-
260 254
 	attests, err := attestations.Parse(opts)
261 255
 	if err != nil {
262 256
 		return err
... ...
@@ -4,15 +4,23 @@ import (
4 4
 	"archive/tar"
5 5
 	"bytes"
6 6
 	"context"
7
+	"io"
8
+	"maps"
7 9
 	"path/filepath"
8 10
 	"regexp"
9 11
 	"slices"
10 12
 	"strconv"
13
+	"strings"
14
+	"time"
11 15
 
12 16
 	"github.com/moby/buildkit/client/llb"
17
+	"github.com/moby/buildkit/client/llb/sourceresolver"
13 18
 	"github.com/moby/buildkit/frontend/dockerfile/dfgitutil"
14 19
 	"github.com/moby/buildkit/frontend/gateway/client"
15 20
 	gwpb "github.com/moby/buildkit/frontend/gateway/pb"
21
+	"github.com/moby/buildkit/solver/pb"
22
+	"github.com/moby/buildkit/util/gitutil/gitobject"
23
+	archivecompression "github.com/moby/go-archive/compression"
16 24
 	"github.com/pkg/errors"
17 25
 )
18 26
 
... ...
@@ -36,10 +44,14 @@ var httpPrefix = regexp.MustCompile(`^https?://`)
36 36
 type buildContext struct {
37 37
 	context              *llb.State // set if not local
38 38
 	dockerfile           *llb.State // override remoteContext if set
39
+	contextRef           client.Reference
39 40
 	contextLocalName     string
40 41
 	dockerfileLocalName  string
41 42
 	filename             string
42 43
 	forceLocalDockerfile bool
44
+	sourceOp             *pb.SourceOp
45
+	httpContextIsArchive bool
46
+	httpContextFilename  string
43 47
 }
44 48
 
45 49
 func (bc *Client) marshalOpts() []llb.ConstraintsOpt {
... ...
@@ -75,16 +87,25 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
75 75
 		keepGit = &v
76 76
 	}
77 77
 	var extraGitOpts []llb.GitOption
78
-	if opts[keySourceDateEpoch] != "" {
78
+	if opts[buildArgPrefix+"SOURCE_DATE_EPOCH"] != "" {
79 79
 		extraGitOpts = append(extraGitOpts, llb.GitMTimeCommit())
80 80
 	}
81 81
 	if st, ok, err := DetectGitContext(opts[localNameContext], keepGit, extraGitOpts...); ok {
82 82
 		if err != nil {
83 83
 			return nil, err
84 84
 		}
85
+		sourceOp, err := sourceOpFromState(ctx, st, bc.marshalOpts()...)
86
+		if err != nil {
87
+			return nil, errors.Wrapf(err, "failed to derive git source op")
88
+		}
85 89
 		bctx.context = st
86 90
 		bctx.dockerfile = st
91
+		bctx.sourceOp = sourceOp
87 92
 	} else if st, filename, ok := DetectHTTPContext(opts[localNameContext]); ok {
93
+		sourceOp, err := sourceOpFromState(ctx, st, bc.marshalOpts()...)
94
+		if err != nil {
95
+			return nil, errors.Wrapf(err, "failed to derive http source op")
96
+		}
88 97
 		def, err := st.Marshal(ctx, bc.marshalOpts()...)
89 98
 		if err != nil {
90 99
 			return nil, errors.Wrapf(err, "failed to marshal httpcontext")
... ...
@@ -115,11 +136,15 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
115 115
 				AttemptUnpack: true,
116 116
 			}))
117 117
 			bctx.context = &bc
118
+			bctx.httpContextIsArchive = true
118 119
 		} else {
119 120
 			bctx.filename = filename
120 121
 			bctx.context = st
121 122
 		}
123
+		bctx.contextRef = ref
122 124
 		bctx.dockerfile = bctx.context
125
+		bctx.sourceOp = sourceOp
126
+		bctx.httpContextFilename = filename
123 127
 	} else if (&gwcaps).Supports(gwpb.CapFrontendInputs) == nil {
124 128
 		inputs, err := bc.client.Inputs(ctx)
125 129
 		if err != nil {
... ...
@@ -148,6 +173,123 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) {
148 148
 	return bctx, nil
149 149
 }
150 150
 
151
+func (bc *Client) ResolveMainContextSourceDateEpoch(ctx context.Context) (*time.Time, error) {
152
+	bctx, err := bc.buildContext(ctx)
153
+	if err != nil {
154
+		return nil, err
155
+	}
156
+	if bctx.sourceOp == nil {
157
+		return nil, nil
158
+	}
159
+
160
+	opt := sourceresolver.Opt{
161
+		LogName: "[internal] resolve main build context metadata",
162
+	}
163
+	if strings.HasPrefix(bctx.sourceOp.Identifier, "git://") {
164
+		opt.GitOpt = &sourceresolver.ResolveGitOpt{ReturnObject: true}
165
+	}
166
+	md, err := bc.client.ResolveSourceMetadata(ctx, cloneSourceOp(bctx.sourceOp), opt)
167
+	if err != nil {
168
+		return nil, err
169
+	}
170
+	if md.Git != nil && len(md.Git.CommitObject) > 0 {
171
+		obj, err := gitobject.Parse(md.Git.CommitObject)
172
+		if err != nil {
173
+			return nil, err
174
+		}
175
+		commit, err := obj.ToCommit()
176
+		if err != nil {
177
+			return nil, err
178
+		}
179
+		return commit.Committer.When, nil
180
+	}
181
+	if md.HTTP != nil {
182
+		if md.HTTP.LastModified != nil {
183
+			return md.HTTP.LastModified, nil
184
+		}
185
+		if bctx.httpContextIsArchive {
186
+			return archiveMaxTimeFromHTTPArchive(ctx, bctx)
187
+		}
188
+	}
189
+	return nil, nil
190
+}
191
+
192
+func archiveMaxTimeFromHTTPArchive(ctx context.Context, bctx *buildContext) (*time.Time, error) {
193
+	if bctx.contextRef == nil || bctx.httpContextFilename == "" {
194
+		return nil, nil
195
+	}
196
+	dt, err := bctx.contextRef.ReadFile(ctx, client.ReadRequest{
197
+		Filename: bctx.httpContextFilename,
198
+	})
199
+	if err != nil {
200
+		return nil, err
201
+	}
202
+	rc, err := archivecompression.DecompressStream(bytes.NewReader(dt))
203
+	if err != nil {
204
+		return nil, err
205
+	}
206
+	defer rc.Close()
207
+
208
+	tr := tar.NewReader(rc)
209
+	var maxTime *time.Time
210
+	for {
211
+		hdr, err := tr.Next()
212
+		if err != nil {
213
+			if errors.Is(err, io.EOF) {
214
+				return maxTime, nil
215
+			}
216
+			return nil, err
217
+		}
218
+		if !hdr.FileInfo().Mode().IsRegular() {
219
+			continue
220
+		}
221
+		tm := hdr.ModTime.UTC()
222
+		if maxTime == nil || tm.After(*maxTime) {
223
+			maxTime = &tm
224
+		}
225
+	}
226
+}
227
+
228
+func cloneSourceOp(op *pb.SourceOp) *pb.SourceOp {
229
+	if op == nil {
230
+		return nil
231
+	}
232
+	return &pb.SourceOp{
233
+		Identifier: op.Identifier,
234
+		Attrs:      maps.Clone(op.Attrs),
235
+	}
236
+}
237
+
238
+func sourceOpFromState(ctx context.Context, st *llb.State, opts ...llb.ConstraintsOpt) (*pb.SourceOp, error) {
239
+	if st == nil {
240
+		return nil, nil
241
+	}
242
+	def, err := st.Marshal(ctx, opts...)
243
+	if err != nil {
244
+		return nil, err
245
+	}
246
+	dt := def.ToPB().Def
247
+	var src *pb.SourceOp
248
+	for _, d := range dt {
249
+		var op pb.Op
250
+		if err := op.Unmarshal(d); err != nil {
251
+			return nil, err
252
+		}
253
+		opSrc := op.GetSource()
254
+		if opSrc == nil {
255
+			continue
256
+		}
257
+		if src != nil {
258
+			return nil, errors.New("state marshaled to multiple source ops")
259
+		}
260
+		src = opSrc
261
+	}
262
+	if src == nil {
263
+		return nil, errors.New("state did not marshal to a source op")
264
+	}
265
+	return cloneSourceOp(src), nil
266
+}
267
+
151 268
 func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.State, bool, error) {
152 269
 	g, isGit, err := dfgitutil.ParseGitRef(ref)
153 270
 	if err != nil {
... ...
@@ -175,6 +317,9 @@ func DetectGitContext(ref string, keepGit *bool, opts ...llb.GitOption) (*llb.St
175 175
 	if g.MTime != "" {
176 176
 		gitOpts = append(gitOpts, llb.GitMTime(g.MTime))
177 177
 	}
178
+	if g.FetchByCommit {
179
+		gitOpts = append(gitOpts, llb.GitFetchByCommit())
180
+	}
178 181
 
179 182
 	st := llb.Git(g.Remote, "", gitOpts...)
180 183
 	return &st, true, nil
... ...
@@ -151,14 +151,14 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke
151 151
 		return st, nil, nil
152 152
 	case "http", "https":
153 153
 		st, ok, err := DetectGitContext(nc.input, nil)
154
-		if !ok {
155
-			httpst := llb.HTTP(nc.input, llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input))
156
-			st = &httpst
157
-		}
158
-		if err != nil {
159
-			return nil, nil, err
154
+		if ok {
155
+			if err != nil {
156
+				return nil, nil, err
157
+			}
158
+			return st, nil, nil
160 159
 		}
161
-		return st, nil, nil
160
+		httpst := llb.HTTP(nc.input, llb.Filename("context"), llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input))
161
+		return &httpst, nil, nil
162 162
 	case "oci-layout":
163 163
 		refSpec := strings.TrimPrefix(vv[1], "//")
164 164
 		ref, err := reference.Parse(refSpec)
... ...
@@ -3,6 +3,8 @@ package client
3 3
 import (
4 4
 	"context"
5 5
 	"io"
6
+	"maps"
7
+	"slices"
6 8
 	"syscall"
7 9
 
8 10
 	"github.com/moby/buildkit/client/llb"
... ...
@@ -154,6 +156,40 @@ type SolveRequest struct {
154 154
 	SourcePolicies []*spb.Policy
155 155
 }
156 156
 
157
+// Clone returns a deep copy of the solve request.
158
+func (r SolveRequest) Clone() SolveRequest {
159
+	if r.Definition != nil {
160
+		r.Definition = r.Definition.CloneVT()
161
+	}
162
+	r.FrontendOpt = maps.Clone(r.FrontendOpt)
163
+	if len(r.FrontendInputs) > 0 {
164
+		inputs := r.FrontendInputs
165
+		r.FrontendInputs = make(map[string]*pb.Definition, len(inputs))
166
+		for k, v := range inputs {
167
+			if v != nil {
168
+				v = v.CloneVT()
169
+			}
170
+			r.FrontendInputs[k] = v
171
+		}
172
+	}
173
+	if len(r.CacheImports) > 0 {
174
+		r.CacheImports = slices.Clone(r.CacheImports)
175
+		for i, ci := range r.CacheImports {
176
+			ci.Attrs = maps.Clone(ci.Attrs)
177
+			r.CacheImports[i] = ci
178
+		}
179
+	}
180
+	if len(r.SourcePolicies) > 0 {
181
+		r.SourcePolicies = slices.Clone(r.SourcePolicies)
182
+		for i, p := range r.SourcePolicies {
183
+			if p != nil {
184
+				r.SourcePolicies[i] = p.CloneVT()
185
+			}
186
+		}
187
+	}
188
+	return r
189
+}
190
+
157 191
 type CacheOptionsEntry struct {
158 192
 	Type  string
159 193
 	Attrs map[string]string
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	"crypto/rand"
6 6
 	"crypto/subtle"
7 7
 	"sync"
8
+	"time"
8 9
 
9 10
 	"github.com/moby/buildkit/session"
10 11
 	"github.com/moby/buildkit/util/grpcerrors"
... ...
@@ -13,6 +14,8 @@ import (
13 13
 	"google.golang.org/grpc/codes"
14 14
 )
15 15
 
16
+const sessionAuthTimeout = 60 * time.Second
17
+
16 18
 var salt []byte
17 19
 var saltOnce sync.Once
18 20
 
... ...
@@ -25,10 +28,12 @@ func getSalt() []byte {
25 25
 	return salt
26 26
 }
27 27
 
28
-func CredentialsFunc(sm *session.Manager, g session.Group) func(string) (session, username, secret string, err error) {
28
+func CredentialsFunc(ctx context.Context, sm *session.Manager, g session.Group) func(string) (session, username, secret string, err error) {
29 29
 	return func(host string) (string, string, string, error) {
30
+		ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "resolving credentials from session"))
31
+		defer cancel()
30 32
 		var sessionID, user, secret string
31
-		err := sm.Any(context.TODO(), g, func(ctx context.Context, id string, c session.Caller) error {
33
+		err := sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
32 34
 			client := NewAuthClient(c.Conn())
33 35
 
34 36
 			resp, err := client.Credentials(ctx, &CredentialsRequest{
... ...
@@ -53,6 +58,8 @@ func CredentialsFunc(sm *session.Manager, g session.Group) func(string) (session
53 53
 }
54 54
 
55 55
 func FetchToken(ctx context.Context, req *FetchTokenRequest, sm *session.Manager, g session.Group) (resp *FetchTokenResponse, err error) {
56
+	ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "fetching auth token from session"))
57
+	defer cancel()
56 58
 	err = sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
57 59
 		client := NewAuthClient(c.Conn())
58 60
 
... ...
@@ -69,6 +76,8 @@ func FetchToken(ctx context.Context, req *FetchTokenRequest, sm *session.Manager
69 69
 }
70 70
 
71 71
 func VerifyTokenAuthority(ctx context.Context, host string, pubKey *[32]byte, sm *session.Manager, g session.Group) (sessionID string, ok bool, err error) {
72
+	ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "verifying token authority from session"))
73
+	defer cancel()
72 74
 	var verified bool
73 75
 	err = sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
74 76
 		client := NewAuthClient(c.Conn())
... ...
@@ -101,6 +110,8 @@ func VerifyTokenAuthority(ctx context.Context, host string, pubKey *[32]byte, sm
101 101
 }
102 102
 
103 103
 func GetTokenAuthority(ctx context.Context, host string, sm *session.Manager, g session.Group) (sessionID string, pubKey *[32]byte, err error) {
104
+	ctx, cancel := context.WithTimeoutCause(ctx, sessionAuthTimeout, errors.Wrap(context.DeadlineExceeded, "getting token authority from session"))
105
+	defer cancel()
104 106
 	err = sm.Any(ctx, g, func(ctx context.Context, id string, c session.Caller) error {
105 107
 		client := NewAuthClient(c.Conn())
106 108
 
... ...
@@ -16,9 +16,11 @@ import (
16 16
 type callerContentStore struct {
17 17
 	store   content.Store
18 18
 	storeID string
19
+	caller  session.Caller
19 20
 }
20 21
 
21 22
 func (cs *callerContentStore) choose(ctx context.Context) context.Context {
23
+	ctx = cs.caller.Context(ctx)
22 24
 	nsheader := metadata.Pairs(GRPCHeaderID, cs.storeID)
23 25
 	md, ok := metadata.FromOutgoingContext(ctx) // merge with outgoing context.
24 26
 	if !ok {
... ...
@@ -87,5 +89,6 @@ func NewCallerStore(c session.Caller, storeID string) content.Store {
87 87
 	return &callerContentStore{
88 88
 		store:   proxy.NewContentStore(client),
89 89
 		storeID: storeID,
90
+		caller:  c,
90 91
 	}
91 92
 }
92 93
new file mode 100644
... ...
@@ -0,0 +1,17 @@
0
+package session
1
+
2
+import "context"
3
+
4
+// contextWithCaller returns a context that is canceled when either the request
5
+// context is done or the session context is closed.
6
+func contextWithCaller(ctx context.Context, callerCtx context.Context) context.Context {
7
+	ctx, cancel := context.WithCancelCause(ctx)
8
+	context.AfterFunc(callerCtx, func() {
9
+		cause := context.Cause(callerCtx)
10
+		if cause == nil {
11
+			cause = context.Canceled
12
+		}
13
+		cancel(cause)
14
+	})
15
+	return ctx
16
+}
... ...
@@ -207,6 +207,7 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error {
207 207
 
208 208
 	opts[keyDirName] = []string{opt.Name}
209 209
 
210
+	ctx = c.Context(ctx)
210 211
 	ctx, cancel := context.WithCancelCause(ctx)
211 212
 	defer func() { cancel(errors.WithStack(context.Canceled)) }()
212 213
 
... ...
@@ -362,6 +363,8 @@ func CopyToCaller(ctx context.Context, fs fsutil.FS, id int, c session.Caller, p
362 362
 		return errors.Errorf("method %s not supported by the client", method)
363 363
 	}
364 364
 
365
+	ctx = c.Context(ctx)
366
+
365 367
 	client := NewFileSendClient(c.Conn())
366 368
 
367 369
 	opts, ok := metadata.FromOutgoingContext(ctx)
... ...
@@ -388,6 +391,7 @@ func CopyFileWriter(ctx context.Context, md map[string]string, id int, c session
388 388
 		return nil, errors.Errorf("method %s not supported by the client", method)
389 389
 	}
390 390
 
391
+	ctx = c.Context(ctx)
391 392
 	client := NewFileSendClient(c.Conn())
392 393
 
393 394
 	opts, ok := metadata.FromOutgoingContext(ctx)
... ...
@@ -7,6 +7,10 @@ import (
7 7
 	"github.com/pkg/errors"
8 8
 )
9 9
 
10
+// ErrNoActiveSessions is returned when a session group does not contain any
11
+// active session IDs.
12
+var ErrNoActiveSessions = errors.New("no active sessions")
13
+
10 14
 type Group interface {
11 15
 	SessionIterator() Iterator
12 16
 }
... ...
@@ -69,7 +73,7 @@ func (sm *Manager) Any(ctx context.Context, g Group, f func(context.Context, str
69 69
 			if lastErr != nil {
70 70
 				return lastErr
71 71
 			}
72
-			return errors.Errorf("no active sessions")
72
+			return ErrNoActiveSessions
73 73
 		}
74 74
 
75 75
 		timeoutCtx, cancel := context.WithCancelCause(ctx)
... ...
@@ -80,7 +84,7 @@ func (sm *Manager) Any(ctx context.Context, g Group, f func(context.Context, str
80 80
 			lastErr = err
81 81
 			continue
82 82
 		}
83
-		if err := f(ctx, id, c); err != nil {
83
+		if err := f(c.Context(ctx), id, c); err != nil {
84 84
 			lastErr = err
85 85
 			continue
86 86
 		}
... ...
@@ -4,6 +4,8 @@ import (
4 4
 	"context"
5 5
 	"math"
6 6
 	"net"
7
+	"net/http"
8
+	"strconv"
7 9
 	"sync/atomic"
8 10
 	"time"
9 11
 
... ...
@@ -21,6 +23,40 @@ import (
21 21
 	"google.golang.org/grpc/health/grpc_health_v1"
22 22
 )
23 23
 
24
+type healthCheckConfig struct {
25
+	interval              time.Duration
26
+	defaultTimeout        time.Duration
27
+	failureThreshold      int
28
+	successResetThreshold int
29
+}
30
+
31
+var defaultHealthCheckConfig = healthCheckConfig{
32
+	interval:              5 * time.Second,
33
+	defaultTimeout:        15 * time.Second,
34
+	failureThreshold:      2,
35
+	successResetThreshold: 1,
36
+}
37
+
38
+const headerSessionHealthCustomTimeout = "X-Buildkit-Session-Health-Custom-Timeout"
39
+
40
+func healthCheckConfigFromHeaders(h http.Header) healthCheckConfig {
41
+	cfg := defaultHealthCheckConfig
42
+
43
+	if v := h.Get(headerSessionHealthCustomTimeout); v != "" {
44
+		if ms, err := strconv.Atoi(v); err == nil && ms > 0 {
45
+			d := time.Duration(ms) * time.Millisecond
46
+			// Avoid test overrides that are unrealistically aggressive for slower machines.
47
+			d = max(d, time.Second)
48
+			cfg.interval = d
49
+			cfg.defaultTimeout = d
50
+			cfg.failureThreshold = 1
51
+			cfg.successResetThreshold = 1
52
+		}
53
+	}
54
+
55
+	return cfg
56
+}
57
+
24 58
 func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) {
25 59
 	go func() {
26 60
 		<-ctx.Done()
... ...
@@ -30,7 +66,7 @@ func serve(ctx context.Context, grpcServer *grpc.Server, conn net.Conn) {
30 30
 	(&http2.Server{}).ServeConn(conn, &http2.ServeConnOpts{Handler: grpcServer})
31 31
 }
32 32
 
33
-func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc.ClientConn, error) {
33
+func grpcClientConn(ctx context.Context, conn net.Conn, opts map[string][]string) (context.Context, *grpc.ClientConn, error) {
34 34
 	var dialCount int64
35 35
 	dialer := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
36 36
 		if c := atomic.AddInt64(&dialCount, 1); c > 1 {
... ...
@@ -63,22 +99,29 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc.
63 63
 	}
64 64
 
65 65
 	ctx, cancel := context.WithCancelCause(ctx)
66
-	go monitorHealth(ctx, cc, cancel)
66
+	go monitorHealth(ctx, conn, cc, cancel, healthCheckConfigFromHeaders(http.Header(opts)))
67 67
 
68 68
 	return ctx, cc, nil
69 69
 }
70 70
 
71
-func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(error)) {
72
-	defer cancelConn(errors.WithStack(context.Canceled))
73
-	defer cc.Close()
74
-
75
-	ticker := time.NewTicker(5 * time.Second)
71
+func monitorHealth(ctx context.Context, conn net.Conn, cc *grpc.ClientConn, cancelConn func(error), cfg healthCheckConfig) {
72
+	closed := false
73
+	closeConn := func(err error) {
74
+		if closed {
75
+			return
76
+		}
77
+		closed = true
78
+		cancelConn(err)
79
+		cc.Close()
80
+		go conn.Close()
81
+	}
82
+	defer closeConn(errors.WithStack(context.Canceled))
83
+	ticker := time.NewTicker(cfg.interval)
76 84
 	defer ticker.Stop()
77 85
 	healthClient := grpc_health_v1.NewHealthClient(cc)
78 86
 
79
-	failedBefore := false
87
+	consecutiveFailures := 0
80 88
 	consecutiveSuccessful := 0
81
-	defaultHealthcheckDuration := 30 * time.Second
82 89
 	lastHealthcheckDuration := time.Duration(0)
83 90
 
84 91
 	for {
... ...
@@ -90,13 +133,24 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(err
90 90
 			// So, this healthcheck is purposely long, and can tolerate some failures on purpose.
91 91
 
92 92
 			healthcheckStart := time.Now()
93
-
94
-			timeout := time.Duration(math.Max(float64(defaultHealthcheckDuration), float64(lastHealthcheckDuration)*1.5))
95
-
96
-			ctx, cancel := context.WithCancelCause(ctx)
97
-			ctx, _ = context.WithTimeoutCause(ctx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
98
-			_, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
99
-			cancel(errors.WithStack(context.Canceled))
93
+			timeout := time.Duration(math.Max(float64(cfg.defaultTimeout), float64(lastHealthcheckDuration)*1.5))
94
+			resultCh := make(chan error, 1)
95
+			go func() {
96
+				checkCtx, cancel := context.WithCancelCause(ctx)
97
+				checkCtx, _ = context.WithTimeoutCause(checkCtx, timeout, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
98
+				_, err := healthClient.Check(checkCtx, &grpc_health_v1.HealthCheckRequest{})
99
+				cancel(errors.WithStack(context.Canceled))
100
+				resultCh <- err
101
+			}()
102
+
103
+			var err error
104
+			select {
105
+			case <-ctx.Done():
106
+				return
107
+			case err = <-resultCh:
108
+			case <-time.After(timeout):
109
+				err = errors.WithStack(context.DeadlineExceeded)
110
+			}
100 111
 
101 112
 			lastHealthcheckDuration = time.Since(healthcheckStart)
102 113
 			logFields := logrus.Fields{
... ...
@@ -110,19 +164,21 @@ func monitorHealth(ctx context.Context, cc *grpc.ClientConn, cancelConn func(err
110 110
 					return
111 111
 				default:
112 112
 				}
113
-				if failedBefore {
114
-					bklog.G(ctx).Error("healthcheck failed fatally")
113
+				consecutiveFailures++
114
+				consecutiveSuccessful = 0
115
+				if consecutiveFailures >= cfg.failureThreshold {
116
+					err = errors.Wrap(err, "session healthcheck failed fatally")
117
+					bklog.G(ctx).WithError(err).Error("healthcheck failed fatally")
118
+					closeConn(err)
115 119
 					return
116 120
 				}
117 121
 
118
-				failedBefore = true
119
-				consecutiveSuccessful = 0
120
-				bklog.G(ctx).WithFields(logFields).Warn("healthcheck failed")
122
+				bklog.G(ctx).WithError(err).WithFields(logFields).Warn("healthcheck failed")
121 123
 			} else {
122 124
 				consecutiveSuccessful++
125
+				consecutiveFailures = 0
123 126
 
124
-				if consecutiveSuccessful >= 5 && failedBefore {
125
-					failedBefore = false
127
+				if consecutiveSuccessful >= cfg.successResetThreshold {
126 128
 					bklog.G(ctx).WithFields(logFields).Debug("reset healthcheck failure")
127 129
 				}
128 130
 			}
... ...
@@ -13,7 +13,7 @@ import (
13 13
 
14 14
 // Caller can invoke requests on the session
15 15
 type Caller interface {
16
-	Context() context.Context
16
+	Context(context.Context) context.Context
17 17
 	Supports(method string) bool
18 18
 	Conn() *grpc.ClientConn
19 19
 	SharedKey() string
... ...
@@ -107,7 +107,7 @@ func (sm *Manager) handleConn(ctx context.Context, conn net.Conn, opts map[strin
107 107
 	id := h.Get(headerSessionID)
108 108
 	sharedKey := h.Get(headerSessionSharedKey)
109 109
 
110
-	ctx, cc, err := grpcClientConn(ctx, conn)
110
+	ctx, cc, err := grpcClientConn(ctx, conn, opts)
111 111
 	if err != nil {
112 112
 		sm.mu.Unlock()
113 113
 		return err
... ...
@@ -190,8 +190,8 @@ func (sm *Manager) Get(ctx context.Context, id string, noWait bool) (Caller, err
190 190
 	return c, nil
191 191
 }
192 192
 
193
-func (c *client) Context() context.Context {
194
-	return c.context()
193
+func (c *client) Context(ctx context.Context) context.Context {
194
+	return contextWithCaller(ctx, c.context())
195 195
 }
196 196
 
197 197
 func (c *client) SharedKey() string {
... ...
@@ -16,6 +16,7 @@ type SecretStore interface {
16 16
 var ErrNotFound = errors.Errorf("not found")
17 17
 
18 18
 func GetSecret(ctx context.Context, c session.Caller, id string) ([]byte, error) {
19
+	ctx = c.Context(ctx)
19 20
 	client := NewSecretsClient(c.Conn())
20 21
 	resp, err := client.GetSecret(ctx, &GetSecretRequest{
21 22
 		ID: id,
... ...
@@ -37,18 +37,19 @@ func (s *server) run(ctx context.Context, l net.Listener, id string) error {
37 37
 			}
38 38
 
39 39
 			client := NewSSHClient(s.caller.Conn())
40
+			rpcCtx := s.caller.Context(ctx)
40 41
 
41 42
 			opts := make(map[string][]string)
42 43
 			opts[KeySSHID] = []string{id}
43
-			ctx := metadata.NewOutgoingContext(ctx, opts)
44
+			rpcCtx = metadata.NewOutgoingContext(rpcCtx, opts)
44 45
 
45
-			stream, err := client.ForwardAgent(ctx)
46
+			stream, err := client.ForwardAgent(rpcCtx)
46 47
 			if err != nil {
47 48
 				conn.Close()
48 49
 				return err
49 50
 			}
50 51
 
51
-			go Copy(ctx, conn, stream, stream.CloseSend)
52
+			go Copy(rpcCtx, conn, stream, stream.CloseSend)
52 53
 		}
53 54
 	})
54 55
 
... ...
@@ -112,6 +113,7 @@ func MountSSHSocket(ctx context.Context, c session.Caller, opt SocketOpt) (sockP
112 112
 }
113 113
 
114 114
 func CheckSSHID(ctx context.Context, c session.Caller, id string) error {
115
+	ctx = c.Context(ctx)
115 116
 	client := NewSSHClient(c.Conn())
116 117
 	_, err := client.CheckAgent(ctx, &CheckAgentRequest{ID: id})
117 118
 	return errors.WithStack(err)
... ...
@@ -16,6 +16,7 @@ const (
16 16
 )
17 17
 
18 18
 func New(ctx context.Context, c session.Caller, url *url.URL) (*Upload, error) {
19
+	ctx = c.Context(ctx)
19 20
 	opts := map[string][]string{
20 21
 		keyPath: {url.Path},
21 22
 		keyHost: {url.Host},
22 23
new file mode 100644
... ...
@@ -0,0 +1,43 @@
0
+package errdefs
1
+
2
+import (
3
+	fmt "fmt"
4
+
5
+	"github.com/containerd/typeurl/v2"
6
+	"github.com/moby/buildkit/util/grpcerrors"
7
+)
8
+
9
+func init() {
10
+	typeurl.Register((*CompatibilityFeature)(nil), "github.com/moby/buildkit", "errdefs.CompatibilityFeature+json")
11
+}
12
+
13
+type UnsupportedCompatibilityFeatureError struct {
14
+	*CompatibilityFeature
15
+	error
16
+}
17
+
18
+func (e *UnsupportedCompatibilityFeatureError) Error() string {
19
+	msg := fmt.Sprintf("unsupported compatibility-version %d feature %s", e.Version, e.Feature)
20
+	if e.error != nil {
21
+		msg += ": " + e.error.Error()
22
+	}
23
+	return msg
24
+}
25
+
26
+func (e *UnsupportedCompatibilityFeatureError) Unwrap() error {
27
+	return e.error
28
+}
29
+
30
+func (e *UnsupportedCompatibilityFeatureError) ToProto() grpcerrors.TypedErrorProto {
31
+	return e.CompatibilityFeature
32
+}
33
+
34
+func NewUnsupportedCompatibilityFeatureError(version int, feature string) error {
35
+	return &UnsupportedCompatibilityFeatureError{
36
+		CompatibilityFeature: &CompatibilityFeature{Version: int64(version), Feature: feature},
37
+	}
38
+}
39
+
40
+func (v *CompatibilityFeature) WrapError(err error) error {
41
+	return &UnsupportedCompatibilityFeatureError{error: err, CompatibilityFeature: v}
42
+}
... ...
@@ -214,6 +214,58 @@ func (x *FrontendCap) GetName() string {
214 214
 	return ""
215 215
 }
216 216
 
217
+type CompatibilityFeature struct {
218
+	state         protoimpl.MessageState `protogen:"open.v1"`
219
+	Version       int64                  `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"`
220
+	Feature       string                 `protobuf:"bytes,2,opt,name=feature,proto3" json:"feature,omitempty"`
221
+	unknownFields protoimpl.UnknownFields
222
+	sizeCache     protoimpl.SizeCache
223
+}
224
+
225
+func (x *CompatibilityFeature) Reset() {
226
+	*x = CompatibilityFeature{}
227
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4]
228
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
229
+	ms.StoreMessageInfo(mi)
230
+}
231
+
232
+func (x *CompatibilityFeature) String() string {
233
+	return protoimpl.X.MessageStringOf(x)
234
+}
235
+
236
+func (*CompatibilityFeature) ProtoMessage() {}
237
+
238
+func (x *CompatibilityFeature) ProtoReflect() protoreflect.Message {
239
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4]
240
+	if x != nil {
241
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
242
+		if ms.LoadMessageInfo() == nil {
243
+			ms.StoreMessageInfo(mi)
244
+		}
245
+		return ms
246
+	}
247
+	return mi.MessageOf(x)
248
+}
249
+
250
+// Deprecated: Use CompatibilityFeature.ProtoReflect.Descriptor instead.
251
+func (*CompatibilityFeature) Descriptor() ([]byte, []int) {
252
+	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{4}
253
+}
254
+
255
+func (x *CompatibilityFeature) GetVersion() int64 {
256
+	if x != nil {
257
+		return x.Version
258
+	}
259
+	return 0
260
+}
261
+
262
+func (x *CompatibilityFeature) GetFeature() string {
263
+	if x != nil {
264
+		return x.Feature
265
+	}
266
+	return ""
267
+}
268
+
217 269
 type Subrequest struct {
218 270
 	state         protoimpl.MessageState `protogen:"open.v1"`
219 271
 	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
... ...
@@ -223,7 +275,7 @@ type Subrequest struct {
223 223
 
224 224
 func (x *Subrequest) Reset() {
225 225
 	*x = Subrequest{}
226
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4]
226
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5]
227 227
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
228 228
 	ms.StoreMessageInfo(mi)
229 229
 }
... ...
@@ -235,7 +287,7 @@ func (x *Subrequest) String() string {
235 235
 func (*Subrequest) ProtoMessage() {}
236 236
 
237 237
 func (x *Subrequest) ProtoReflect() protoreflect.Message {
238
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[4]
238
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5]
239 239
 	if x != nil {
240 240
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
241 241
 		if ms.LoadMessageInfo() == nil {
... ...
@@ -248,7 +300,7 @@ func (x *Subrequest) ProtoReflect() protoreflect.Message {
248 248
 
249 249
 // Deprecated: Use Subrequest.ProtoReflect.Descriptor instead.
250 250
 func (*Subrequest) Descriptor() ([]byte, []int) {
251
-	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{4}
251
+	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{5}
252 252
 }
253 253
 
254 254
 func (x *Subrequest) GetName() string {
... ...
@@ -275,7 +327,7 @@ type Solve struct {
275 275
 
276 276
 func (x *Solve) Reset() {
277 277
 	*x = Solve{}
278
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5]
278
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6]
279 279
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
280 280
 	ms.StoreMessageInfo(mi)
281 281
 }
... ...
@@ -287,7 +339,7 @@ func (x *Solve) String() string {
287 287
 func (*Solve) ProtoMessage() {}
288 288
 
289 289
 func (x *Solve) ProtoReflect() protoreflect.Message {
290
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5]
290
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6]
291 291
 	if x != nil {
292 292
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
293 293
 		if ms.LoadMessageInfo() == nil {
... ...
@@ -300,7 +352,7 @@ func (x *Solve) ProtoReflect() protoreflect.Message {
300 300
 
301 301
 // Deprecated: Use Solve.ProtoReflect.Descriptor instead.
302 302
 func (*Solve) Descriptor() ([]byte, []int) {
303
-	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{5}
303
+	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{6}
304 304
 }
305 305
 
306 306
 func (x *Solve) GetInputIDs() []string {
... ...
@@ -382,7 +434,7 @@ type FileAction struct {
382 382
 
383 383
 func (x *FileAction) Reset() {
384 384
 	*x = FileAction{}
385
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6]
385
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7]
386 386
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
387 387
 	ms.StoreMessageInfo(mi)
388 388
 }
... ...
@@ -394,7 +446,7 @@ func (x *FileAction) String() string {
394 394
 func (*FileAction) ProtoMessage() {}
395 395
 
396 396
 func (x *FileAction) ProtoReflect() protoreflect.Message {
397
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6]
397
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7]
398 398
 	if x != nil {
399 399
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
400 400
 		if ms.LoadMessageInfo() == nil {
... ...
@@ -407,7 +459,7 @@ func (x *FileAction) ProtoReflect() protoreflect.Message {
407 407
 
408 408
 // Deprecated: Use FileAction.ProtoReflect.Descriptor instead.
409 409
 func (*FileAction) Descriptor() ([]byte, []int) {
410
-	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{6}
410
+	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{7}
411 411
 }
412 412
 
413 413
 func (x *FileAction) GetIndex() int64 {
... ...
@@ -427,7 +479,7 @@ type ContentCache struct {
427 427
 
428 428
 func (x *ContentCache) Reset() {
429 429
 	*x = ContentCache{}
430
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7]
430
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[8]
431 431
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
432 432
 	ms.StoreMessageInfo(mi)
433 433
 }
... ...
@@ -439,7 +491,7 @@ func (x *ContentCache) String() string {
439 439
 func (*ContentCache) ProtoMessage() {}
440 440
 
441 441
 func (x *ContentCache) ProtoReflect() protoreflect.Message {
442
-	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[7]
442
+	mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[8]
443 443
 	if x != nil {
444 444
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
445 445
 		if ms.LoadMessageInfo() == nil {
... ...
@@ -452,7 +504,7 @@ func (x *ContentCache) ProtoReflect() protoreflect.Message {
452 452
 
453 453
 // Deprecated: Use ContentCache.ProtoReflect.Descriptor instead.
454 454
 func (*ContentCache) Descriptor() ([]byte, []int) {
455
-	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{7}
455
+	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{8}
456 456
 }
457 457
 
458 458
 func (x *ContentCache) GetIndex() int64 {
... ...
@@ -476,7 +528,10 @@ const file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc = "" +
476 476
 	"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
477 477
 	"\x06source\x18\x02 \x01(\tR\x06source\"!\n" +
478 478
 	"\vFrontendCap\x12\x12\n" +
479
-	"\x04name\x18\x01 \x01(\tR\x04name\" \n" +
479
+	"\x04name\x18\x01 \x01(\tR\x04name\"J\n" +
480
+	"\x14CompatibilityFeature\x12\x18\n" +
481
+	"\aversion\x18\x01 \x01(\x03R\aversion\x12\x18\n" +
482
+	"\afeature\x18\x02 \x01(\tR\afeature\" \n" +
480 483
 	"\n" +
481 484
 	"Subrequest\x12\x12\n" +
482 485
 	"\x04name\x18\x01 \x01(\tR\x04name\"\xbf\x02\n" +
... ...
@@ -509,28 +564,29 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP() []
509 509
 	return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescData
510 510
 }
511 511
 
512
-var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
512
+var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
513 513
 var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_goTypes = []any{
514
-	(*Vertex)(nil),        // 0: errdefs.Vertex
515
-	(*Source)(nil),        // 1: errdefs.Source
516
-	(*Frontend)(nil),      // 2: errdefs.Frontend
517
-	(*FrontendCap)(nil),   // 3: errdefs.FrontendCap
518
-	(*Subrequest)(nil),    // 4: errdefs.Subrequest
519
-	(*Solve)(nil),         // 5: errdefs.Solve
520
-	(*FileAction)(nil),    // 6: errdefs.FileAction
521
-	(*ContentCache)(nil),  // 7: errdefs.ContentCache
522
-	nil,                   // 8: errdefs.Solve.DescriptionEntry
523
-	(*pb.SourceInfo)(nil), // 9: pb.SourceInfo
524
-	(*pb.Range)(nil),      // 10: pb.Range
525
-	(*pb.Op)(nil),         // 11: pb.Op
514
+	(*Vertex)(nil),               // 0: errdefs.Vertex
515
+	(*Source)(nil),               // 1: errdefs.Source
516
+	(*Frontend)(nil),             // 2: errdefs.Frontend
517
+	(*FrontendCap)(nil),          // 3: errdefs.FrontendCap
518
+	(*CompatibilityFeature)(nil), // 4: errdefs.CompatibilityFeature
519
+	(*Subrequest)(nil),           // 5: errdefs.Subrequest
520
+	(*Solve)(nil),                // 6: errdefs.Solve
521
+	(*FileAction)(nil),           // 7: errdefs.FileAction
522
+	(*ContentCache)(nil),         // 8: errdefs.ContentCache
523
+	nil,                          // 9: errdefs.Solve.DescriptionEntry
524
+	(*pb.SourceInfo)(nil),        // 10: pb.SourceInfo
525
+	(*pb.Range)(nil),             // 11: pb.Range
526
+	(*pb.Op)(nil),                // 12: pb.Op
526 527
 }
527 528
 var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_depIdxs = []int32{
528
-	9,  // 0: errdefs.Source.info:type_name -> pb.SourceInfo
529
-	10, // 1: errdefs.Source.ranges:type_name -> pb.Range
530
-	11, // 2: errdefs.Solve.op:type_name -> pb.Op
531
-	6,  // 3: errdefs.Solve.file:type_name -> errdefs.FileAction
532
-	7,  // 4: errdefs.Solve.cache:type_name -> errdefs.ContentCache
533
-	8,  // 5: errdefs.Solve.description:type_name -> errdefs.Solve.DescriptionEntry
529
+	10, // 0: errdefs.Source.info:type_name -> pb.SourceInfo
530
+	11, // 1: errdefs.Source.ranges:type_name -> pb.Range
531
+	12, // 2: errdefs.Solve.op:type_name -> pb.Op
532
+	7,  // 3: errdefs.Solve.file:type_name -> errdefs.FileAction
533
+	8,  // 4: errdefs.Solve.cache:type_name -> errdefs.ContentCache
534
+	9,  // 5: errdefs.Solve.description:type_name -> errdefs.Solve.DescriptionEntry
534 535
 	6,  // [6:6] is the sub-list for method output_type
535 536
 	6,  // [6:6] is the sub-list for method input_type
536 537
 	6,  // [6:6] is the sub-list for extension type_name
... ...
@@ -543,7 +599,7 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_init() {
543 543
 	if File_github_com_moby_buildkit_solver_errdefs_errdefs_proto != nil {
544 544
 		return
545 545
 	}
546
-	file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[5].OneofWrappers = []any{
546
+	file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[6].OneofWrappers = []any{
547 547
 		(*Solve_File)(nil),
548 548
 		(*Solve_Cache)(nil),
549 549
 	}
... ...
@@ -553,7 +609,7 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_init() {
553 553
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
554 554
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc), len(file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc)),
555 555
 			NumEnums:      0,
556
-			NumMessages:   9,
556
+			NumMessages:   10,
557 557
 			NumExtensions: 0,
558 558
 			NumServices:   0,
559 559
 		},
... ...
@@ -24,6 +24,11 @@ message FrontendCap {
24 24
 	string name = 1;
25 25
 }
26 26
 
27
+message CompatibilityFeature {
28
+	int64 version = 1;
29
+	string feature = 2;
30
+}
31
+
27 32
 message Subrequest {
28 33
 	string name = 1;
29 34
 }
... ...
@@ -96,6 +96,24 @@ func (m *FrontendCap) CloneMessageVT() proto.Message {
96 96
 	return m.CloneVT()
97 97
 }
98 98
 
99
+func (m *CompatibilityFeature) CloneVT() *CompatibilityFeature {
100
+	if m == nil {
101
+		return (*CompatibilityFeature)(nil)
102
+	}
103
+	r := new(CompatibilityFeature)
104
+	r.Version = m.Version
105
+	r.Feature = m.Feature
106
+	if len(m.unknownFields) > 0 {
107
+		r.unknownFields = make([]byte, len(m.unknownFields))
108
+		copy(r.unknownFields, m.unknownFields)
109
+	}
110
+	return r
111
+}
112
+
113
+func (m *CompatibilityFeature) CloneMessageVT() proto.Message {
114
+	return m.CloneVT()
115
+}
116
+
99 117
 func (m *Subrequest) CloneVT() *Subrequest {
100 118
 	if m == nil {
101 119
 		return (*Subrequest)(nil)
... ...
@@ -298,6 +316,28 @@ func (this *FrontendCap) EqualMessageVT(thatMsg proto.Message) bool {
298 298
 	}
299 299
 	return this.EqualVT(that)
300 300
 }
301
+func (this *CompatibilityFeature) EqualVT(that *CompatibilityFeature) bool {
302
+	if this == that {
303
+		return true
304
+	} else if this == nil || that == nil {
305
+		return false
306
+	}
307
+	if this.Version != that.Version {
308
+		return false
309
+	}
310
+	if this.Feature != that.Feature {
311
+		return false
312
+	}
313
+	return string(this.unknownFields) == string(that.unknownFields)
314
+}
315
+
316
+func (this *CompatibilityFeature) EqualMessageVT(thatMsg proto.Message) bool {
317
+	that, ok := thatMsg.(*CompatibilityFeature)
318
+	if !ok {
319
+		return false
320
+	}
321
+	return this.EqualVT(that)
322
+}
301 323
 func (this *Subrequest) EqualVT(that *Subrequest) bool {
302 324
 	if this == that {
303 325
 		return true
... ...
@@ -646,6 +686,51 @@ func (m *FrontendCap) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
646 646
 	return len(dAtA) - i, nil
647 647
 }
648 648
 
649
+func (m *CompatibilityFeature) MarshalVT() (dAtA []byte, err error) {
650
+	if m == nil {
651
+		return nil, nil
652
+	}
653
+	size := m.SizeVT()
654
+	dAtA = make([]byte, size)
655
+	n, err := m.MarshalToSizedBufferVT(dAtA[:size])
656
+	if err != nil {
657
+		return nil, err
658
+	}
659
+	return dAtA[:n], nil
660
+}
661
+
662
+func (m *CompatibilityFeature) MarshalToVT(dAtA []byte) (int, error) {
663
+	size := m.SizeVT()
664
+	return m.MarshalToSizedBufferVT(dAtA[:size])
665
+}
666
+
667
+func (m *CompatibilityFeature) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
668
+	if m == nil {
669
+		return 0, nil
670
+	}
671
+	i := len(dAtA)
672
+	_ = i
673
+	var l int
674
+	_ = l
675
+	if m.unknownFields != nil {
676
+		i -= len(m.unknownFields)
677
+		copy(dAtA[i:], m.unknownFields)
678
+	}
679
+	if len(m.Feature) > 0 {
680
+		i -= len(m.Feature)
681
+		copy(dAtA[i:], m.Feature)
682
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Feature)))
683
+		i--
684
+		dAtA[i] = 0x12
685
+	}
686
+	if m.Version != 0 {
687
+		i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Version))
688
+		i--
689
+		dAtA[i] = 0x8
690
+	}
691
+	return len(dAtA) - i, nil
692
+}
693
+
649 694
 func (m *Subrequest) MarshalVT() (dAtA []byte, err error) {
650 695
 	if m == nil {
651 696
 		return nil, nil
... ...
@@ -963,6 +1048,23 @@ func (m *FrontendCap) SizeVT() (n int) {
963 963
 	return n
964 964
 }
965 965
 
966
+func (m *CompatibilityFeature) SizeVT() (n int) {
967
+	if m == nil {
968
+		return 0
969
+	}
970
+	var l int
971
+	_ = l
972
+	if m.Version != 0 {
973
+		n += 1 + protohelpers.SizeOfVarint(uint64(m.Version))
974
+	}
975
+	l = len(m.Feature)
976
+	if l > 0 {
977
+		n += 1 + l + protohelpers.SizeOfVarint(uint64(l))
978
+	}
979
+	n += len(m.unknownFields)
980
+	return n
981
+}
982
+
966 983
 func (m *Subrequest) SizeVT() (n int) {
967 984
 	if m == nil {
968 985
 		return 0
... ...
@@ -1470,6 +1572,108 @@ func (m *FrontendCap) UnmarshalVT(dAtA []byte) error {
1470 1470
 	}
1471 1471
 	return nil
1472 1472
 }
1473
+func (m *CompatibilityFeature) UnmarshalVT(dAtA []byte) error {
1474
+	l := len(dAtA)
1475
+	iNdEx := 0
1476
+	for iNdEx < l {
1477
+		preIndex := iNdEx
1478
+		var wire uint64
1479
+		for shift := uint(0); ; shift += 7 {
1480
+			if shift >= 64 {
1481
+				return protohelpers.ErrIntOverflow
1482
+			}
1483
+			if iNdEx >= l {
1484
+				return io.ErrUnexpectedEOF
1485
+			}
1486
+			b := dAtA[iNdEx]
1487
+			iNdEx++
1488
+			wire |= uint64(b&0x7F) << shift
1489
+			if b < 0x80 {
1490
+				break
1491
+			}
1492
+		}
1493
+		fieldNum := int32(wire >> 3)
1494
+		wireType := int(wire & 0x7)
1495
+		if wireType == 4 {
1496
+			return fmt.Errorf("proto: CompatibilityFeature: wiretype end group for non-group")
1497
+		}
1498
+		if fieldNum <= 0 {
1499
+			return fmt.Errorf("proto: CompatibilityFeature: illegal tag %d (wire type %d)", fieldNum, wire)
1500
+		}
1501
+		switch fieldNum {
1502
+		case 1:
1503
+			if wireType != 0 {
1504
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
1505
+			}
1506
+			m.Version = 0
1507
+			for shift := uint(0); ; shift += 7 {
1508
+				if shift >= 64 {
1509
+					return protohelpers.ErrIntOverflow
1510
+				}
1511
+				if iNdEx >= l {
1512
+					return io.ErrUnexpectedEOF
1513
+				}
1514
+				b := dAtA[iNdEx]
1515
+				iNdEx++
1516
+				m.Version |= int64(b&0x7F) << shift
1517
+				if b < 0x80 {
1518
+					break
1519
+				}
1520
+			}
1521
+		case 2:
1522
+			if wireType != 2 {
1523
+				return fmt.Errorf("proto: wrong wireType = %d for field Feature", wireType)
1524
+			}
1525
+			var stringLen uint64
1526
+			for shift := uint(0); ; shift += 7 {
1527
+				if shift >= 64 {
1528
+					return protohelpers.ErrIntOverflow
1529
+				}
1530
+				if iNdEx >= l {
1531
+					return io.ErrUnexpectedEOF
1532
+				}
1533
+				b := dAtA[iNdEx]
1534
+				iNdEx++
1535
+				stringLen |= uint64(b&0x7F) << shift
1536
+				if b < 0x80 {
1537
+					break
1538
+				}
1539
+			}
1540
+			intStringLen := int(stringLen)
1541
+			if intStringLen < 0 {
1542
+				return protohelpers.ErrInvalidLength
1543
+			}
1544
+			postIndex := iNdEx + intStringLen
1545
+			if postIndex < 0 {
1546
+				return protohelpers.ErrInvalidLength
1547
+			}
1548
+			if postIndex > l {
1549
+				return io.ErrUnexpectedEOF
1550
+			}
1551
+			m.Feature = string(dAtA[iNdEx:postIndex])
1552
+			iNdEx = postIndex
1553
+		default:
1554
+			iNdEx = preIndex
1555
+			skippy, err := protohelpers.Skip(dAtA[iNdEx:])
1556
+			if err != nil {
1557
+				return err
1558
+			}
1559
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
1560
+				return protohelpers.ErrInvalidLength
1561
+			}
1562
+			if (iNdEx + skippy) > l {
1563
+				return io.ErrUnexpectedEOF
1564
+			}
1565
+			m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...)
1566
+			iNdEx += skippy
1567
+		}
1568
+	}
1569
+
1570
+	if iNdEx > l {
1571
+		return io.ErrUnexpectedEOF
1572
+	}
1573
+	return nil
1574
+}
1473 1575
 func (m *Subrequest) UnmarshalVT(dAtA []byte) error {
1474 1576
 	l := len(dAtA)
1475 1577
 	iNdEx := 0
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	"github.com/moby/buildkit/identity"
12 12
 	"github.com/moby/buildkit/session"
13 13
 	"github.com/moby/buildkit/solver/errdefs"
14
+	"github.com/moby/buildkit/solver/llbsolver/compat"
14 15
 	"github.com/moby/buildkit/util/bklog"
15 16
 	"github.com/moby/buildkit/util/bkmaps"
16 17
 	"github.com/moby/buildkit/util/flightcontrol"
... ...
@@ -90,6 +91,34 @@ func (s *state) ResolverCache() ResolverCache {
90 90
 	return s
91 91
 }
92 92
 
93
+func (s *state) CompatibilityVersion() (int, error) {
94
+	s.mu.Lock()
95
+	jobs := make([]*Job, 0, len(s.jobs))
96
+	for j := range s.jobs {
97
+		jobs = append(jobs, j)
98
+	}
99
+	s.mu.Unlock()
100
+
101
+	version := 0
102
+	for _, j := range jobs {
103
+		v, err := j.CompatibilityVersion()
104
+		if err != nil {
105
+			return 0, err
106
+		}
107
+		if version == 0 {
108
+			version = v
109
+			continue
110
+		}
111
+		if version != v {
112
+			return 0, errors.Errorf("conflicting compatibility versions in shared solve state: %d != %d", version, v)
113
+		}
114
+	}
115
+	if version == 0 {
116
+		return compat.CompatibilityVersionCurrent, nil
117
+	}
118
+	return version, nil
119
+}
120
+
93 121
 func (s *state) Lock(key any) (values []any, release func(any) error, err error) {
94 122
 	var rcs []ResolverCache
95 123
 	s.mu.Lock()
... ...
@@ -865,6 +894,21 @@ func (j *Job) UniqueID() string {
865 865
 	return j.uniqueID
866 866
 }
867 867
 
868
+func (j *Job) CompatibilityVersion() (int, error) {
869
+	version := compat.CompatibilityVersionCurrent
870
+	if err := j.EachValue(context.TODO(), compat.JobValueKey, func(v any) error {
871
+		parsed, ok := v.(int)
872
+		if !ok {
873
+			return errors.Errorf("invalid compatibility version %T", v)
874
+		}
875
+		version = parsed
876
+		return nil
877
+	}); err != nil {
878
+		return 0, err
879
+	}
880
+	return version, nil
881
+}
882
+
868 883
 func (j *Job) InContext(ctx context.Context, f func(context.Context, JobContext) error) error {
869 884
 	return f(progress.WithProgress(ctx, j.pw), j)
870 885
 }
... ...
@@ -39,6 +39,7 @@ type llbBridge struct {
39 39
 	cms                       map[string]solver.CacheManager
40 40
 	cmsMu                     sync.Mutex
41 41
 	sm                        *session.Manager
42
+	provenanceStore           *provenanceStore
42 43
 
43 44
 	executorOnce sync.Once
44 45
 	executorErr  error
45 46
new file mode 100644
... ...
@@ -0,0 +1,36 @@
0
+package compat
1
+
2
+import (
3
+	"slices"
4
+
5
+	"github.com/pkg/errors"
6
+)
7
+
8
+const (
9
+	CompatibilityVersion013     = 10
10
+	CompatibilityVersion015     = 20
11
+	CompatibilityVersionCurrent = CompatibilityVersion015
12
+)
13
+
14
+// JobValueKey is the key used to store the compatibility version on a solver
15
+// job via Job.SetValue/EachValue.
16
+const JobValueKey = "llb.compatibilityversion"
17
+
18
+var supportedCompatibilityVersions = []int{
19
+	CompatibilityVersion013,
20
+	CompatibilityVersion015,
21
+}
22
+
23
+func SupportedCompatibilityVersions() []int {
24
+	return slices.Clone(supportedCompatibilityVersions)
25
+}
26
+
27
+func ValidateCompatibilityVersion(version int) error {
28
+	if slices.Contains(supportedCompatibilityVersions, version) {
29
+		return nil
30
+	}
31
+	if version > CompatibilityVersionCurrent {
32
+		return errors.Errorf("unsupported compatibility-version %d: upgrade buildkit (max supported: %d)", version, CompatibilityVersionCurrent)
33
+	}
34
+	return errors.Errorf("unsupported compatibility-version %d (supported: %v)", version, supportedCompatibilityVersions)
35
+}
... ...
@@ -40,7 +40,7 @@ func (s *Solver) getSessionExporters(ctx context.Context, sessionID string, id i
40 40
 	}
41 41
 
42 42
 	client := sessionexporter.NewExporterClient(caller.Conn())
43
-
43
+	ctx = caller.Context(ctx)
44 44
 	var ids []string
45 45
 	if err := inp.EachRef(func(ref cache.ImmutableRef) error {
46 46
 		ids = append(ids, ref.ID())
... ...
@@ -188,11 +188,16 @@ func (s *Solver) runExporters(ctx context.Context, ref string, exporters []expor
188 188
 					defer inlineCacheMu.Unlock()
189 189
 					return runInlineCacheExporter(ctx, exp, inlineCacheExporter, job, cached)
190 190
 				})
191
+				compatibilityVersion, err := job.CompatibilityVersion()
192
+				if err != nil {
193
+					return err
194
+				}
191 195
 
192 196
 				resp, finalize, desc, expErr := exp.Export(ctx, inp, exporter.ExportBuildInfo{
193
-					Ref:         ref,
194
-					SessionID:   job.SessionID,
195
-					InlineCache: inlineCache,
197
+					Ref:                  ref,
198
+					SessionID:            job.SessionID,
199
+					InlineCache:          inlineCache,
200
+					CompatibilityVersion: compatibilityVersion,
196 201
 				})
197 202
 				resps[i], finalizeFuncs[i], descs[i] = resp, finalize, desc
198 203
 				if expErr != nil {
... ...
@@ -58,6 +58,7 @@ func (p *policyEvaluator) evaluate(ctx context.Context, op *pb.Op, max int) (boo
58 58
 	}
59 59
 
60 60
 	verifier := policysession.NewPolicyVerifierClient(caller.Conn())
61
+	ctx = caller.Context(ctx)
61 62
 	req := &policysession.CheckPolicyRequest{
62 63
 		Platform: op.Platform,
63 64
 		Source: &gatewaypb.ResolveSourceMetaResponse{
... ...
@@ -71,7 +71,6 @@ func ProvenanceProcessor(slsaVersion provenancetypes.ProvenanceSLSA, attrs map[s
71 71
 					if err != nil {
72 72
 						return nil, err
73 73
 					}
74
-
75 74
 					return json.MarshalIndent(pr, "", "  ")
76 75
 				},
77 76
 			})
... ...
@@ -38,12 +38,14 @@ type resultWithBridge struct {
38 38
 // provenanceBridge provides scoped access to LLBBridge and captures the request it makes for provenance
39 39
 type provenanceBridge struct {
40 40
 	*llbBridge
41
-	mu  sync.Mutex
42
-	req *frontend.SolveRequest
43
-
44
-	images     []provenancetypes.ImageSource
45
-	builds     []resultWithBridge
46
-	subBridges []*provenanceBridge
41
+	mu      sync.Mutex
42
+	req     *frontend.SolveRequest
43
+	rootReq *frontend.SolveRequest
44
+
45
+	images                 []provenancetypes.ImageSource
46
+	builds                 []resultWithBridge
47
+	subBridges             []*provenanceBridge
48
+	provenanceRefRecordIDs []string
47 49
 }
48 50
 
49 51
 func (b *provenanceBridge) eachRef(f func(r solver.ResultProxy) error) error {
... ...
@@ -165,6 +167,7 @@ func (b *provenanceBridge) ResolveSourceMetadata(ctx context.Context, op *pb.Sou
165 165
 }
166 166
 
167 167
 func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid string) (res *frontend.Result, err error) {
168
+	req = req.Clone()
168 169
 	if req.Definition != nil && req.Definition.Def != nil && req.Frontend != "" {
169 170
 		return nil, errors.New("cannot solve with both Definition and Frontend specified")
170 171
 	}
... ...
@@ -180,7 +183,11 @@ func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest,
180 180
 		if !ok {
181 181
 			return nil, errors.Errorf("invalid frontend: %s", req.Frontend)
182 182
 		}
183
-		wb := &provenanceBridge{llbBridge: b.llbBridge, req: &req}
183
+		rootReq := b.rootReq
184
+		if !hasRequestProvenance(rootReq) {
185
+			rootReq = b.req
186
+		}
187
+		wb := &provenanceBridge{llbBridge: b.llbBridge, req: &req, rootReq: rootReq}
184 188
 		res, err = f.Solve(ctx, wb, b.llbBridge, req.FrontendOpt, req.FrontendInputs, sid, b.sm)
185 189
 		if err != nil {
186 190
 			fe := errdefs.Frontend{
... ...
@@ -202,6 +209,9 @@ func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest,
202 202
 			return err
203 203
 		})
204 204
 	}
205
+	if err == nil {
206
+		err = b.registerProvenanceRefs(res)
207
+	}
205 208
 	return
206 209
 }
207 210
 
... ...
@@ -397,17 +407,9 @@ func NewProvenanceCreator(ctx context.Context, slsaVersion provenancetypes.Prove
397 397
 
398 398
 	switch mode {
399 399
 	case "min":
400
-		args := make(map[string]string)
401
-		for k, v := range pr.BuildDefinition.ExternalParameters.Request.Args {
402
-			if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") {
403
-				pr.RunDetails.Metadata.Completeness.Request = false
404
-				continue
405
-			}
406
-			args[k] = v
400
+		if scrubMinRequest(&pr.BuildDefinition.ExternalParameters.Request) {
401
+			pr.RunDetails.Metadata.Completeness.Request = false
407 402
 		}
408
-		pr.BuildDefinition.ExternalParameters.Request.Args = args
409
-		pr.BuildDefinition.ExternalParameters.Request.Secrets = nil
410
-		pr.BuildDefinition.ExternalParameters.Request.SSH = nil
411 403
 	case "max":
412 404
 		dgsts, err := provenance.AddBuildConfig(ctx, pr, cp, res, withUsage)
413 405
 		if err != nil {
... ...
@@ -478,6 +480,41 @@ func NewProvenanceCreator(ctx context.Context, slsaVersion provenancetypes.Prove
478 478
 	return pc, nil
479 479
 }
480 480
 
481
+func scrubMinRequest(req *provenancetypes.Parameters) bool {
482
+	if req == nil {
483
+		return false
484
+	}
485
+
486
+	var incomplete bool
487
+	if len(req.Args) > 0 {
488
+		args := make(map[string]string, len(req.Args))
489
+		for k, v := range req.Args {
490
+			if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") {
491
+				incomplete = true
492
+				continue
493
+			}
494
+			args[k] = v
495
+		}
496
+		req.Args = args
497
+	}
498
+	if len(req.Secrets) > 0 {
499
+		req.Secrets = nil
500
+	}
501
+	if len(req.SSH) > 0 {
502
+		req.SSH = nil
503
+	}
504
+
505
+	for _, in := range req.Inputs {
506
+		if in != nil && scrubMinRequest(in.Request) {
507
+			incomplete = true
508
+		}
509
+	}
510
+	if req.Root != nil && scrubMinRequest(req.Root.Request) {
511
+		incomplete = true
512
+	}
513
+	return incomplete
514
+}
515
+
481 516
 func (p *ProvenanceCreator) PredicateType() string {
482 517
 	if p.slsaVersion == provenancetypes.ProvenanceSLSA02 {
483 518
 		return slsa02.PredicateSLSAProvenance
... ...
@@ -506,6 +543,12 @@ func (p *ProvenanceCreator) Predicate(ctx context.Context) (any, error) {
506 506
 		p.pr.RunDetails.Metadata.BuildKitMetadata.SysUsage = sysSamples
507 507
 	}
508 508
 
509
+	compatibilityVersion, err := p.j.CompatibilityVersion()
510
+	if err != nil {
511
+		return nil, err
512
+	}
513
+	p.pr.BuildDefinition.ExternalParameters.Request.CompatibilityVersion = compatibilityVersion
514
+
509 515
 	if p.slsaVersion == provenancetypes.ProvenanceSLSA02 {
510 516
 		return p.pr.ConvertToSLSA02(), nil
511 517
 	}
... ...
@@ -665,14 +708,26 @@ func getRefProvenance(ref solver.ResultProxy, br *provenanceBridge) (*provenance
665 665
 	if !ok {
666 666
 		return nil, errors.Errorf("invalid provenance type %T", p)
667 667
 	}
668
+	pr = pr.Clone()
668 669
 
669 670
 	if br.req != nil {
670 671
 		if pr == nil {
671 672
 			return nil, errors.Errorf("missing provenance for %s", ref.ID())
672 673
 		}
673 674
 
674
-		pr.Frontend = br.req.Frontend
675
-		pr.Args = provenance.FilterArgs(br.req.FrontendOpt)
675
+		pr.Request.Frontend = br.req.Frontend
676
+		pr.Request.Args = provenance.FilterArgs(br.req.FrontendOpt)
677
+		pr.Request.Inputs = br.inputProvenance(br.req.FrontendInputs)
678
+		if root := br.rootRequestProvenance(pr.Sources); root != nil {
679
+			req := provenance.RequestProvenance(pr.Request.Frontend, pr.Request.Args, pr.Sources)
680
+			if req.Request == nil {
681
+				req.Request = &provenancetypes.Parameters{}
682
+			}
683
+			req.Request.Inputs = pr.Request.Inputs
684
+			if !root.Equal(req) {
685
+				pr.Request.Root = root
686
+			}
687
+		}
676 688
 		// TODO: should also save some output options like compression
677 689
 	}
678 690
 
... ...
@@ -2,6 +2,7 @@ package provenance
2 2
 
3 3
 import (
4 4
 	"cmp"
5
+	"maps"
5 6
 	"slices"
6 7
 
7 8
 	distreference "github.com/distribution/reference"
... ...
@@ -15,16 +16,36 @@ import (
15 15
 type Result = result.Result[*Capture]
16 16
 
17 17
 type Capture struct {
18
-	Frontend            string
19
-	Args                map[string]string
18
+	Request             provenancetypes.Parameters
20 19
 	Sources             provenancetypes.Sources
21
-	Secrets             []provenancetypes.Secret
22
-	SSH                 []provenancetypes.SSH
23 20
 	NetworkAccess       bool
24 21
 	IncompleteMaterials bool
25 22
 	Samples             map[digest.Digest]*resourcestypes.Samples
26 23
 }
27 24
 
25
+func (c *Capture) Clone() *Capture {
26
+	if c == nil {
27
+		return nil
28
+	}
29
+	out := &Capture{
30
+		NetworkAccess:       c.NetworkAccess,
31
+		IncompleteMaterials: c.IncompleteMaterials,
32
+	}
33
+	if req := c.Request.Clone(); req != nil {
34
+		out.Request = *req
35
+	}
36
+	out.Sources.Images = append(out.Sources.Images, c.Sources.Images...)
37
+	out.Sources.ImageBlobs = append(out.Sources.ImageBlobs, c.Sources.ImageBlobs...)
38
+	out.Sources.Local = append(out.Sources.Local, c.Sources.Local...)
39
+	out.Sources.Git = append(out.Sources.Git, c.Sources.Git...)
40
+	out.Sources.HTTP = append(out.Sources.HTTP, c.Sources.HTTP...)
41
+	if len(c.Samples) > 0 {
42
+		out.Samples = make(map[digest.Digest]*resourcestypes.Samples, len(c.Samples))
43
+		maps.Copy(out.Samples, c.Samples)
44
+	}
45
+	return out
46
+}
47
+
28 48
 func (c *Capture) Merge(c2 *Capture) error {
29 49
 	if c2 == nil {
30 50
 		return nil
... ...
@@ -44,11 +65,15 @@ func (c *Capture) Merge(c2 *Capture) error {
44 44
 	for _, h := range c2.Sources.HTTP {
45 45
 		c.AddHTTP(h)
46 46
 	}
47
-	for _, s := range c2.Secrets {
48
-		c.AddSecret(s)
47
+	for _, s := range c2.Request.Secrets {
48
+		if s != nil {
49
+			c.AddSecret(*s)
50
+		}
49 51
 	}
50
-	for _, s := range c2.SSH {
51
-		c.AddSSH(s)
52
+	for _, s := range c2.Request.SSH {
53
+		if s != nil {
54
+			c.AddSSH(*s)
55
+		}
52 56
 	}
53 57
 	if c2.NetworkAccess {
54 58
 		c.NetworkAccess = true
... ...
@@ -75,10 +100,10 @@ func (c *Capture) Sort() {
75 75
 	slices.SortFunc(c.Sources.HTTP, func(a, b provenancetypes.HTTPSource) int {
76 76
 		return cmp.Compare(a.URL, b.URL)
77 77
 	})
78
-	slices.SortFunc(c.Secrets, func(a, b provenancetypes.Secret) int {
78
+	slices.SortFunc(c.Request.Secrets, func(a, b *provenancetypes.Secret) int {
79 79
 		return cmp.Compare(a.ID, b.ID)
80 80
 	})
81
-	slices.SortFunc(c.SSH, func(a, b provenancetypes.SSH) int {
81
+	slices.SortFunc(c.Request.SSH, func(a, b *provenancetypes.SSH) int {
82 82
 		return cmp.Compare(a.ID, b.ID)
83 83
 	})
84 84
 }
... ...
@@ -152,14 +177,33 @@ func (c *Capture) AddLocal(l provenancetypes.LocalSource) {
152 152
 
153 153
 func (c *Capture) AddGit(g provenancetypes.GitSource) {
154 154
 	g.URL = urlutil.RedactCredentials(g.URL)
155
+	// Dedupe on the tuple (URL, Bundle.URL). Two records with the same
156
+	// URL but different bundle identity (e.g. the same repo referenced
157
+	// once normally and once through a bundle, or through two different
158
+	// bundle locators) must both be preserved so neither material is
159
+	// silently dropped from the provenance. Bundle.URL is the canonical
160
+	// bundle identity: since scheme/ref/digest are derived from it,
161
+	// different URLs imply different bundle identity.
155 162
 	for _, v := range c.Sources.Git {
156
-		if v.URL == g.URL {
163
+		if v.URL != g.URL {
164
+			continue
165
+		}
166
+		if bundleKey(v.Bundle) == bundleKey(g.Bundle) {
157 167
 			return
158 168
 		}
159 169
 	}
160 170
 	c.Sources.Git = append(c.Sources.Git, g)
161 171
 }
162 172
 
173
+// bundleKey returns a comparable identity for dedupe. Nil bundles collapse
174
+// to the empty key.
175
+func bundleKey(b *provenancetypes.GitBundle) string {
176
+	if b == nil {
177
+		return ""
178
+	}
179
+	return b.URL
180
+}
181
+
163 182
 func (c *Capture) AddHTTP(h provenancetypes.HTTPSource) {
164 183
 	h.URL = urlutil.RedactCredentials(h.URL)
165 184
 	for _, v := range c.Sources.HTTP {
... ...
@@ -171,30 +215,30 @@ func (c *Capture) AddHTTP(h provenancetypes.HTTPSource) {
171 171
 }
172 172
 
173 173
 func (c *Capture) AddSecret(s provenancetypes.Secret) {
174
-	for i, v := range c.Secrets {
174
+	for i, v := range c.Request.Secrets {
175 175
 		if v.ID == s.ID {
176 176
 			if !s.Optional {
177
-				c.Secrets[i].Optional = false
177
+				c.Request.Secrets[i].Optional = false
178 178
 			}
179 179
 			return
180 180
 		}
181 181
 	}
182
-	c.Secrets = append(c.Secrets, s)
182
+	c.Request.Secrets = append(c.Request.Secrets, &s)
183 183
 }
184 184
 
185 185
 func (c *Capture) AddSSH(s provenancetypes.SSH) {
186 186
 	if s.ID == "" {
187 187
 		s.ID = "default"
188 188
 	}
189
-	for i, v := range c.SSH {
189
+	for i, v := range c.Request.SSH {
190 190
 		if v.ID == s.ID {
191 191
 			if !s.Optional {
192
-				c.SSH[i].Optional = false
192
+				c.Request.SSH[i].Optional = false
193 193
 			}
194 194
 			return
195 195
 		}
196 196
 	}
197
-	c.SSH = append(c.SSH, s)
197
+	c.Request.SSH = append(c.Request.SSH, &s)
198 198
 }
199 199
 
200 200
 func (c *Capture) AddSamples(dgst digest.Digest, samples *resourcestypes.Samples) {
... ...
@@ -9,8 +9,10 @@ import (
9 9
 	slsa1 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1"
10 10
 	"github.com/moby/buildkit/frontend/dockerfile/dfgitutil"
11 11
 	provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
12
+	srctypes "github.com/moby/buildkit/source/types"
12 13
 	"github.com/moby/buildkit/util/purl"
13 14
 	"github.com/moby/buildkit/util/urlutil"
15
+	digest "github.com/opencontainers/go-digest"
14 16
 	"github.com/package-url/packageurl-go"
15 17
 )
16 18
 
... ...
@@ -68,10 +70,55 @@ func slsaMaterials(srcs provenancetypes.Sources) ([]slsa.ProvenanceMaterial, err
68 68
 	}
69 69
 
70 70
 	for _, s := range srcs.Git {
71
+		// Git material URI is always the raw repository URL (same shape
72
+		// for bundle-backed and normal git sources).
71 73
 		out = append(out, slsa.ProvenanceMaterial{
72 74
 			URI:    s.URL,
73 75
 			Digest: digestSetForCommit(s.Commit),
74 76
 		})
77
+		if s.Bundle == nil {
78
+			continue
79
+		}
80
+
81
+		// Bundle-backed git source: parse the canonical locator into
82
+		// its scheme / ref-body / digest components on demand. On parse
83
+		// failure (shouldn't happen — validateBundleAttrs rejects bad
84
+		// locators at LLB op construction time) skip bundle material
85
+		// emission rather than erroring.
86
+		scheme, refBody, bundleDgst := parseBundleLocatorURL(s.Bundle.URL)
87
+		if scheme == "" || refBody == "" || bundleDgst == "" {
88
+			continue
89
+		}
90
+
91
+		bundlePurlType := packageurl.TypeDocker
92
+		if scheme == srctypes.OCIBlobScheme {
93
+			bundlePurlType = packageurl.TypeOCI
94
+		}
95
+		bundleURI, err := purl.RefToPURL(bundlePurlType, refBody, nil)
96
+		if err != nil {
97
+			return nil, err
98
+		}
99
+		bundleURI, err = setPURLQualifier(bundleURI, packageurl.Qualifier{
100
+			Key:   "ref_type",
101
+			Value: "bundle",
102
+		})
103
+		if err != nil {
104
+			return nil, err
105
+		}
106
+		bundleURI, err = setPURLQualifier(bundleURI, packageurl.Qualifier{
107
+			Key:   "vcs_url",
108
+			Value: s.URL,
109
+		})
110
+		if err != nil {
111
+			return nil, err
112
+		}
113
+
114
+		out = append(out, slsa.ProvenanceMaterial{
115
+			URI: bundleURI,
116
+			Digest: slsa.DigestSet{
117
+				bundleDgst.Algorithm().String(): bundleDgst.Hex(),
118
+			},
119
+		})
75 120
 	}
76 121
 
77 122
 	for _, s := range srcs.HTTP {
... ...
@@ -86,6 +133,31 @@ func slsaMaterials(srcs provenancetypes.Sources) ([]slsa.ProvenanceMaterial, err
86 86
 	return out, nil
87 87
 }
88 88
 
89
+// parseBundleLocatorURL parses a "<scheme>://<ref>@<algo>:<hex>" bundle
90
+// locator into its components. It returns empty values on parse failure; the
91
+// caller handles that by falling back to the non-bundle emission path.
92
+//
93
+// The identifier package has a stricter parseBundleLocator used at LLB op
94
+// construction time (validateBundleAttrs) that validates the reference and
95
+// digest algorithm. We intentionally don't import that helper here because
96
+// the identifier package imports this provenance package. Since the locator
97
+// has already been validated upstream, the logic here just splits the
98
+// locator back into its parts for purl emission.
99
+func parseBundleLocatorURL(raw string) (scheme, refBody string, dgst digest.Digest) {
100
+	const sep = "://"
101
+	i := strings.Index(raw, sep)
102
+	if i <= 0 {
103
+		return "", "", ""
104
+	}
105
+	scheme = raw[:i]
106
+	body := raw[i+len(sep):]
107
+	at := strings.LastIndex(body, "@")
108
+	if at <= 0 {
109
+		return "", "", ""
110
+	}
111
+	return scheme, body[:at], digest.Digest(body[at+1:])
112
+}
113
+
89 114
 func digestSetForCommit(commit string) slsa.DigestSet {
90 115
 	dset := slsa.DigestSet{}
91 116
 	if len(commit) == 64 {
... ...
@@ -112,11 +184,15 @@ func setPURLQualifier(uri string, q packageurl.Qualifier) (string, error) {
112 112
 }
113 113
 
114 114
 func findMaterial(srcs provenancetypes.Sources, uri string) (*slsa.ProvenanceMaterial, bool) {
115
-	uri, _ = dfgitutil.FragmentFormat(uri)
115
+	configURI := uri
116
+	if formatted, ok := dfgitutil.FragmentFormat(uri, true); ok {
117
+		configURI = formatted
118
+	}
119
+	uri, _ = dfgitutil.FragmentFormat(uri, false)
116 120
 	for _, s := range srcs.Git {
117 121
 		if s.URL == uri {
118 122
 			return &slsa.ProvenanceMaterial{
119
-				URI:    s.URL,
123
+				URI:    configURI,
120 124
 				Digest: digestSetForCommit(s.Commit),
121 125
 			}, true
122 126
 		}
... ...
@@ -147,39 +223,25 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error)
147 147
 		})
148 148
 	}
149 149
 
150
-	args := maps.Clone(c.Args)
151
-
152
-	contextKey := "context"
153
-	if v, ok := args["contextkey"]; ok && v != "" {
154
-		contextKey = v
155
-	} else if v, ok := c.Args["input:context"]; ok && v != "" {
156
-		contextKey = "input:context"
157
-	}
158
-
159 150
 	ext := provenancetypes.ProvenanceExternalParametersSLSA1{}
160
-	if v, ok := args[contextKey]; ok && v != "" {
161
-		if m, ok := findMaterial(c.Sources, v); ok {
162
-			ext.ConfigSource.URI = m.URI
163
-			ext.ConfigSource.Digest = m.Digest
164
-		} else {
165
-			ext.ConfigSource.URI = v
166
-		}
167
-		ext.ConfigSource.URI = urlutil.RedactCredentials(ext.ConfigSource.URI)
168
-		delete(args, contextKey)
169
-	}
170
-
171
-	if v, ok := args["filename"]; ok && v != "" {
172
-		ext.ConfigSource.Path = v
173
-		delete(args, "filename")
174
-	}
151
+	reqProv := RequestProvenance(c.Request.Frontend, maps.Clone(c.Request.Args), c.Sources)
152
+	ext.ConfigSource = reqProv.ConfigSource
175 153
 
176 154
 	vcs := make(map[string]string)
177
-	for k, v := range args {
155
+	req := c.Request.Clone()
156
+	if req == nil {
157
+		req = &provenancetypes.Parameters{}
158
+	}
159
+	if reqProv.Request != nil {
160
+		req.Frontend = reqProv.Request.Frontend
161
+		req.Args = reqProv.Request.Args
162
+	}
163
+	for k, v := range req.Args {
178 164
 		if strings.HasPrefix(k, "vcs:") {
179 165
 			if k == "vcs:source" {
180 166
 				v = urlutil.RedactCredentials(v)
181 167
 			}
182
-			delete(args, k)
168
+			delete(req.Args, k)
183 169
 			if v != "" {
184 170
 				vcs[strings.TrimPrefix(k, "vcs:")] = v
185 171
 			}
... ...
@@ -189,27 +251,12 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error)
189 189
 	internal := provenancetypes.ProvenanceInternalParametersSLSA1{}
190 190
 	internal.BuilderPlatform = platforms.Format(platforms.Normalize(platforms.DefaultSpec()))
191 191
 
192
-	req := provenancetypes.Parameters{}
193
-	req.Frontend = c.Frontend
194
-	req.Args = args
195
-	for _, s := range c.Secrets {
196
-		req.Secrets = append(req.Secrets, &provenancetypes.Secret{
197
-			ID:       s.ID,
198
-			Optional: s.Optional,
199
-		})
200
-	}
201
-	for _, s := range c.SSH {
202
-		req.SSH = append(req.SSH, &provenancetypes.SSH{
203
-			ID:       s.ID,
204
-			Optional: s.Optional,
205
-		})
206
-	}
207 192
 	for _, s := range c.Sources.Local {
208 193
 		req.Locals = append(req.Locals, &provenancetypes.LocalSource{
209 194
 			Name: s.Name,
210 195
 		})
211 196
 	}
212
-	ext.Request = req
197
+	ext.Request = *req
213 198
 
214 199
 	incompleteMaterials := c.IncompleteMaterials
215 200
 	if !incompleteMaterials {
... ...
@@ -230,7 +277,7 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error)
230 230
 		RunDetails: provenancetypes.ProvenanceRunDetailsSLSA1{
231 231
 			Metadata: &provenancetypes.ProvenanceMetadataSLSA1{
232 232
 				Completeness: provenancetypes.BuildKitComplete{
233
-					Request:              c.Frontend != "",
233
+					Request:              c.Request.Frontend != "",
234 234
 					ResolvedDependencies: !incompleteMaterials,
235 235
 				},
236 236
 				Hermetic: !incompleteMaterials && !c.NetworkAccess,
... ...
@@ -245,6 +292,42 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error)
245 245
 	return pr, nil
246 246
 }
247 247
 
248
+func RequestProvenance(frontend string, args map[string]string, srcs provenancetypes.Sources) *provenancetypes.RequestProvenance {
249
+	args = maps.Clone(args)
250
+	contextKey := "context"
251
+	if v, ok := args["contextkey"]; ok && v != "" {
252
+		contextKey = v
253
+	} else if v, ok := args["input:context"]; ok && v != "" {
254
+		contextKey = "input:context"
255
+	}
256
+
257
+	ext := provenancetypes.RequestProvenance{
258
+		Request: &provenancetypes.Parameters{
259
+			Frontend: frontend,
260
+			Args:     args,
261
+		},
262
+	}
263
+	if v, ok := args[contextKey]; ok && v != "" {
264
+		if m, ok := findMaterial(srcs, v); ok {
265
+			ext.ConfigSource.URI = m.URI
266
+			ext.ConfigSource.Digest = m.Digest
267
+		} else {
268
+			ext.ConfigSource.URI = v
269
+		}
270
+		ext.ConfigSource.URI = urlutil.RedactCredentials(ext.ConfigSource.URI)
271
+		delete(args, contextKey)
272
+	}
273
+
274
+	if v, ok := args["filename"]; ok && v != "" {
275
+		ext.ConfigSource.Path = v
276
+		delete(args, "filename")
277
+	}
278
+	if len(args) == 0 {
279
+		ext.Request.Args = nil
280
+	}
281
+	return &ext
282
+}
283
+
248 284
 func FilterArgs(m map[string]string) map[string]string {
249 285
 	var hostSpecificArgs = map[string]struct{}{
250 286
 		"cgroup-parent":      {},
... ...
@@ -71,6 +71,19 @@ type ImageBlobSource struct {
71 71
 type GitSource struct {
72 72
 	URL    string
73 73
 	Commit string
74
+	// Bundle, when non-nil, records the bundle blob that the git source
75
+	// was resolved from. Only present on bundle-backed git sources; nil
76
+	// for normal remote-backed git sources.
77
+	Bundle *GitBundle `json:"bundle,omitempty"`
78
+}
79
+
80
+// GitBundle describes the bundle blob a git source was resolved from. URL
81
+// is the full locator (e.g. "docker-image+blob://example.com/repo@sha256:...")
82
+// and is the canonical data: scheme, reference body, and digest are all
83
+// derivable from it and are parsed on demand by consumers (such as the purl
84
+// emitter in predicate.go).
85
+type GitBundle struct {
86
+	URL string `json:"url"`
74 87
 }
75 88
 
76 89
 type HTTPSource struct {
... ...
@@ -82,16 +95,40 @@ type LocalSource struct {
82 82
 	Name string `json:"name"`
83 83
 }
84 84
 
85
+// Equal reports whether the local source matches another local source.
86
+func (l *LocalSource) Equal(other *LocalSource) bool {
87
+	if l == nil || other == nil {
88
+		return l == other
89
+	}
90
+	return *l == *other
91
+}
92
+
85 93
 type Secret struct {
86 94
 	ID       string `json:"id"`
87 95
 	Optional bool   `json:"optional,omitempty"`
88 96
 }
89 97
 
98
+// Equal reports whether the secret matches another secret.
99
+func (s *Secret) Equal(other *Secret) bool {
100
+	if s == nil || other == nil {
101
+		return s == other
102
+	}
103
+	return *s == *other
104
+}
105
+
90 106
 type SSH struct {
91 107
 	ID       string `json:"id"`
92 108
 	Optional bool   `json:"optional,omitempty"`
93 109
 }
94 110
 
111
+// Equal reports whether the SSH mount matches another SSH mount.
112
+func (s *SSH) Equal(other *SSH) bool {
113
+	if s == nil || other == nil {
114
+		return s == other
115
+	}
116
+	return *s == *other
117
+}
118
+
95 119
 type Sources struct {
96 120
 	Images     []ImageSource
97 121
 	ImageBlobs []ImageBlobSource
... ...
@@ -157,6 +194,17 @@ type ProvenanceConfigSourceSLSA1 struct {
157 157
 	Path   string         `json:"path,omitempty"`
158 158
 }
159 159
 
160
+// Clone returns a deep copy of the config source.
161
+func (c ProvenanceConfigSourceSLSA1) Clone() ProvenanceConfigSourceSLSA1 {
162
+	c.Digest = maps.Clone(c.Digest)
163
+	return c
164
+}
165
+
166
+// Equal reports whether the config source matches another config source.
167
+func (c ProvenanceConfigSourceSLSA1) Equal(other ProvenanceConfigSourceSLSA1) bool {
168
+	return c.URI == other.URI && c.Path == other.Path && maps.Equal(c.Digest, other.Digest)
169
+}
170
+
160 171
 type ProvenanceInternalParametersSLSA1 struct {
161 172
 	BuildConfig     *BuildConfig `json:"buildConfig,omitempty"`
162 173
 	BuilderPlatform string       `json:"builderPlatform"`
... ...
@@ -172,13 +220,121 @@ type ProvenanceMetadataSLSA1 struct {
172 172
 }
173 173
 
174 174
 type Parameters struct {
175
-	Frontend string            `json:"frontend,omitempty"`
176
-	Args     map[string]string `json:"args,omitempty"`
177
-	Secrets  []*Secret         `json:"secrets,omitempty"`
178
-	SSH      []*SSH            `json:"ssh,omitempty"`
179
-	Locals   []*LocalSource    `json:"locals,omitempty"`
175
+	Frontend             string                        `json:"frontend,omitempty"`
176
+	Args                 map[string]string             `json:"args,omitempty"`
177
+	Secrets              []*Secret                     `json:"secrets,omitempty"`
178
+	SSH                  []*SSH                        `json:"ssh,omitempty"`
179
+	Locals               []*LocalSource                `json:"locals,omitempty"`
180
+	Inputs               map[string]*RequestProvenance `json:"inputs,omitempty"`
181
+	Root                 *RequestProvenance            `json:"root,omitempty"`
182
+	CompatibilityVersion int                           `json:"compatibilityVersion,omitempty"`
180 183
 	// TODO: select export attributes
181
-	// TODO: frontend inputs
184
+}
185
+
186
+// Clone returns a deep copy of the request parameters.
187
+func (p *Parameters) Clone() *Parameters {
188
+	if p == nil {
189
+		return nil
190
+	}
191
+	out := &Parameters{
192
+		Frontend:             p.Frontend,
193
+		Args:                 maps.Clone(p.Args),
194
+		CompatibilityVersion: p.CompatibilityVersion,
195
+	}
196
+	if len(p.Secrets) > 0 {
197
+		out.Secrets = slices.Clone(p.Secrets)
198
+		for i, s := range out.Secrets {
199
+			if s != nil {
200
+				s2 := *s
201
+				out.Secrets[i] = &s2
202
+			}
203
+		}
204
+	}
205
+	if len(p.SSH) > 0 {
206
+		out.SSH = slices.Clone(p.SSH)
207
+		for i, s := range out.SSH {
208
+			if s != nil {
209
+				s2 := *s
210
+				out.SSH[i] = &s2
211
+			}
212
+		}
213
+	}
214
+	if len(p.Locals) > 0 {
215
+		out.Locals = slices.Clone(p.Locals)
216
+		for i, l := range out.Locals {
217
+			if l != nil {
218
+				l2 := *l
219
+				out.Locals[i] = &l2
220
+			}
221
+		}
222
+	}
223
+	if len(p.Inputs) > 0 {
224
+		out.Inputs = make(map[string]*RequestProvenance, len(p.Inputs))
225
+		for k, in := range p.Inputs {
226
+			out.Inputs[k] = in.Clone()
227
+		}
228
+	}
229
+	out.Root = p.Root.Clone()
230
+	return out
231
+}
232
+
233
+// Equal reports whether the request parameters match another set of parameters.
234
+func (p *Parameters) Equal(other *Parameters) bool {
235
+	if p == nil || other == nil {
236
+		return p == other
237
+	}
238
+	if p.Frontend != other.Frontend || p.CompatibilityVersion != other.CompatibilityVersion || !maps.Equal(p.Args, other.Args) {
239
+		return false
240
+	}
241
+	if !slices.EqualFunc(p.Secrets, other.Secrets, func(a, b *Secret) bool {
242
+		return a.Equal(b)
243
+	}) {
244
+		return false
245
+	}
246
+	if !slices.EqualFunc(p.SSH, other.SSH, func(a, b *SSH) bool {
247
+		return a.Equal(b)
248
+	}) {
249
+		return false
250
+	}
251
+	if !slices.EqualFunc(p.Locals, other.Locals, func(a, b *LocalSource) bool {
252
+		return a.Equal(b)
253
+	}) {
254
+		return false
255
+	}
256
+	if len(p.Inputs) != len(other.Inputs) {
257
+		return false
258
+	}
259
+	for k, v := range p.Inputs {
260
+		v2, ok := other.Inputs[k]
261
+		if !ok || !v.Equal(v2) {
262
+			return false
263
+		}
264
+	}
265
+	return p.Root.Equal(other.Root)
266
+}
267
+
268
+type RequestProvenance struct {
269
+	ConfigSource ProvenanceConfigSourceSLSA1 `json:"configSource"`
270
+	Request      *Parameters                 `json:"request,omitempty"`
271
+}
272
+
273
+// Clone returns a deep copy of the request provenance.
274
+func (r *RequestProvenance) Clone() *RequestProvenance {
275
+	if r == nil {
276
+		return nil
277
+	}
278
+	return &RequestProvenance{
279
+		ConfigSource: r.ConfigSource.Clone(),
280
+		Request:      r.Request.Clone(),
281
+	}
282
+}
283
+
284
+// Equal reports whether the request provenance matches another request provenance.
285
+func (r *RequestProvenance) Equal(other *RequestProvenance) bool {
286
+	if r == nil || other == nil {
287
+		return r == other
288
+	}
289
+	return r.ConfigSource.Equal(other.ConfigSource) && r.Request.Equal(other.Request)
182 290
 }
183 291
 
184 292
 type Environment struct {
185 293
new file mode 100644
... ...
@@ -0,0 +1,247 @@
0
+package llbsolver
1
+
2
+import (
3
+	"slices"
4
+	"sync"
5
+
6
+	"github.com/moby/buildkit/frontend"
7
+	"github.com/moby/buildkit/identity"
8
+	"github.com/moby/buildkit/solver"
9
+	"github.com/moby/buildkit/solver/llbsolver/provenance"
10
+	provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
11
+	"github.com/moby/buildkit/solver/pb"
12
+	digest "github.com/opencontainers/go-digest"
13
+	"github.com/pkg/errors"
14
+)
15
+
16
+type provenanceStore struct {
17
+	mu       sync.Mutex
18
+	records  map[string]*provenanceRecord
19
+	byDigest map[digest.Digest]map[string]struct{}
20
+}
21
+
22
+type provenanceRecord struct {
23
+	defDigest digest.Digest
24
+	request   *provenancetypes.RequestProvenance
25
+}
26
+
27
+func newProvenanceStore() *provenanceStore {
28
+	return &provenanceStore{
29
+		records:  map[string]*provenanceRecord{},
30
+		byDigest: map[digest.Digest]map[string]struct{}{},
31
+	}
32
+}
33
+
34
+func (s *provenanceStore) register(def *pb.Definition, req *provenancetypes.RequestProvenance) (string, digest.Digest, error) {
35
+	if s == nil || def == nil || req == nil {
36
+		return "", "", nil
37
+	}
38
+	dgst, err := definitionHeadDigest(def)
39
+	if err != nil {
40
+		return "", "", err
41
+	}
42
+	if dgst == "" {
43
+		return "", "", nil
44
+	}
45
+	recordID := identity.NewID()
46
+	s.mu.Lock()
47
+	s.records[recordID] = &provenanceRecord{
48
+		defDigest: dgst,
49
+		request:   req.Clone(),
50
+	}
51
+	if s.byDigest[dgst] == nil {
52
+		s.byDigest[dgst] = map[string]struct{}{}
53
+	}
54
+	s.byDigest[dgst][recordID] = struct{}{}
55
+	s.mu.Unlock()
56
+	return recordID, dgst, nil
57
+}
58
+
59
+func (s *provenanceStore) unregister(recordIDs []string) {
60
+	if s == nil || len(recordIDs) == 0 {
61
+		return
62
+	}
63
+	s.mu.Lock()
64
+	for _, recordID := range recordIDs {
65
+		rec := s.records[recordID]
66
+		delete(s.records, recordID)
67
+		if rec != nil {
68
+			delete(s.byDigest[rec.defDigest], recordID)
69
+			if len(s.byDigest[rec.defDigest]) == 0 {
70
+				delete(s.byDigest, rec.defDigest)
71
+			}
72
+		}
73
+	}
74
+	s.mu.Unlock()
75
+}
76
+
77
+func (s *provenanceStore) lookup(def *pb.Definition) (*provenancetypes.RequestProvenance, bool) {
78
+	if s == nil || def == nil {
79
+		return nil, false
80
+	}
81
+	dgst, err := definitionHeadDigest(def)
82
+	if err != nil || dgst == "" {
83
+		return nil, false
84
+	}
85
+	s.mu.Lock()
86
+	records := make([]*provenanceRecord, 0, len(s.byDigest[dgst]))
87
+	for recordID := range s.byDigest[dgst] {
88
+		if rec := s.records[recordID]; rec != nil && rec.request != nil {
89
+			records = append(records, rec)
90
+		}
91
+	}
92
+	s.mu.Unlock()
93
+	if len(records) == 0 {
94
+		return nil, false
95
+	}
96
+	req := records[0].request.Clone()
97
+	if req.Request != nil {
98
+		req.Request.Root = nil
99
+	}
100
+	for _, rec := range records[1:] {
101
+		req2 := rec.request.Clone()
102
+		if req2.Request != nil {
103
+			req2.Request.Root = nil
104
+		}
105
+		if !req.Equal(req2) {
106
+			return nil, false
107
+		}
108
+	}
109
+	return req, true
110
+}
111
+
112
+func (b *provenanceBridge) registerProvenanceRefs(res *frontend.Result) error {
113
+	if b == nil || res == nil {
114
+		return nil
115
+	}
116
+	return res.EachRef(func(ref solver.ResultProxy) error {
117
+		if ref == nil {
118
+			return nil
119
+		}
120
+		return b.registerProvenanceRef(ref.Definition(), ref)
121
+	})
122
+}
123
+
124
+func (b *provenanceBridge) registerProvenanceRef(def *pb.Definition, ref solver.ResultProxy) error {
125
+	if def == nil || ref == nil || b.provenanceStore == nil {
126
+		return nil
127
+	}
128
+	req, ok := b.findByResult(ref)
129
+	if !ok {
130
+		return nil
131
+	}
132
+	var srcs provenancetypes.Sources
133
+	if p := ref.Provenance(); p != nil {
134
+		c, ok := p.(*provenance.Capture)
135
+		if !ok {
136
+			return errors.Errorf("invalid provenance type %T", p)
137
+		}
138
+		if c != nil {
139
+			srcs = c.Sources
140
+		}
141
+	}
142
+	reqProv := req.bridge.requestProvenance(srcs)
143
+	recordID, _, err := b.provenanceStore.register(def, reqProv)
144
+	if err != nil {
145
+		return err
146
+	}
147
+	if recordID == "" {
148
+		return nil
149
+	}
150
+	b.addProvenanceRefRecordID(recordID)
151
+	return nil
152
+}
153
+
154
+func (b *provenanceBridge) addProvenanceRefRecordID(recordID string) {
155
+	if b == nil || recordID == "" {
156
+		return
157
+	}
158
+	b.mu.Lock()
159
+	b.provenanceRefRecordIDs = append(b.provenanceRefRecordIDs, recordID)
160
+	b.mu.Unlock()
161
+}
162
+
163
+func (b *provenanceBridge) releaseProvenanceRefs() {
164
+	if b == nil || b.provenanceStore == nil {
165
+		return
166
+	}
167
+	for _, sb := range b.subBridges {
168
+		sb.releaseProvenanceRefs()
169
+	}
170
+	b.mu.Lock()
171
+	recordIDs := slices.Clone(b.provenanceRefRecordIDs)
172
+	b.provenanceRefRecordIDs = nil
173
+	b.mu.Unlock()
174
+	b.provenanceStore.unregister(recordIDs)
175
+}
176
+
177
+func (b *provenanceBridge) requestProvenance(srcs provenancetypes.Sources) *provenancetypes.RequestProvenance {
178
+	if b == nil || b.req == nil {
179
+		return nil
180
+	}
181
+	req := provenance.RequestProvenance(b.req.Frontend, provenance.FilterArgs(b.req.FrontendOpt), srcs)
182
+	if req.Request == nil {
183
+		req.Request = &provenancetypes.Parameters{}
184
+	}
185
+	if inputs := b.inputProvenance(b.req.FrontendInputs); len(inputs) > 0 {
186
+		req.Request.Inputs = inputs
187
+	}
188
+	if root := b.rootRequestProvenance(srcs); root != nil && !root.Equal(req) {
189
+		req.Request.Root = root
190
+	}
191
+	return req
192
+}
193
+
194
+func (b *provenanceBridge) inputProvenance(inputs map[string]*pb.Definition) map[string]*provenancetypes.RequestProvenance {
195
+	if len(inputs) == 0 || b == nil || b.provenanceStore == nil {
196
+		return nil
197
+	}
198
+	out := make(map[string]*provenancetypes.RequestProvenance)
199
+	for name, def := range inputs {
200
+		if in, ok := b.provenanceStore.lookup(def); ok {
201
+			out[name] = in
202
+		}
203
+	}
204
+	if len(out) == 0 {
205
+		return nil
206
+	}
207
+	return out
208
+}
209
+
210
+func (b *provenanceBridge) rootRequestProvenance(srcs provenancetypes.Sources) *provenancetypes.RequestProvenance {
211
+	if b == nil || !hasRequestProvenance(b.rootReq) {
212
+		return nil
213
+	}
214
+	root := provenance.RequestProvenance(b.rootReq.Frontend, provenance.FilterArgs(b.rootReq.FrontendOpt), srcs)
215
+	if root.Request == nil {
216
+		root.Request = &provenancetypes.Parameters{}
217
+	}
218
+	if inputs := b.inputProvenance(b.rootReq.FrontendInputs); len(inputs) > 0 {
219
+		root.Request.Inputs = inputs
220
+	}
221
+	if root.Request.Frontend == "" && len(root.Request.Args) == 0 && len(root.Request.Inputs) == 0 {
222
+		return nil
223
+	}
224
+	return root
225
+}
226
+
227
+func hasRequestProvenance(req *frontend.SolveRequest) bool {
228
+	if req == nil {
229
+		return false
230
+	}
231
+	return req.Frontend != "" || len(req.FrontendOpt) > 0 || len(req.FrontendInputs) > 0
232
+}
233
+
234
+func definitionHeadDigest(def *pb.Definition) (digest.Digest, error) {
235
+	if def == nil || len(def.Def) == 0 {
236
+		return "", nil
237
+	}
238
+	var op pb.Op
239
+	if err := op.UnmarshalVT(def.Def[len(def.Def)-1]); err != nil {
240
+		return "", errors.Wrap(err, "failed to parse llb proto op")
241
+	}
242
+	if len(op.Inputs) == 0 {
243
+		return "", nil
244
+	}
245
+	return digest.Digest(op.Inputs[0].Digest), nil
246
+}
... ...
@@ -2,6 +2,7 @@ package llbsolver
2 2
 
3 3
 import (
4 4
 	"context"
5
+	"maps"
5 6
 	"os"
6 7
 	"strings"
7 8
 	"time"
... ...
@@ -19,6 +20,7 @@ import (
19 19
 	"github.com/moby/buildkit/identity"
20 20
 	"github.com/moby/buildkit/session"
21 21
 	"github.com/moby/buildkit/solver"
22
+	"github.com/moby/buildkit/solver/llbsolver/compat"
22 23
 	"github.com/moby/buildkit/solver/llbsolver/history"
23 24
 	"github.com/moby/buildkit/solver/result"
24 25
 	spb "github.com/moby/buildkit/sourcepolicy/pb"
... ...
@@ -73,6 +75,7 @@ type Solver struct {
73 73
 	history                   *history.Queue
74 74
 	sysSampler                *resources.Sampler[*resourcestypes.SysSample]
75 75
 	provenanceEnv             map[string]any
76
+	provenanceStore           *provenanceStore
76 77
 }
77 78
 
78 79
 // Processor defines a processing function to be applied after solving, but
... ...
@@ -103,6 +106,7 @@ func New(opt Opt) (*Solver, error) {
103 103
 		entitlements:              opt.Entitlements,
104 104
 		history:                   opt.HistoryQueue,
105 105
 		provenanceEnv:             opt.ProvenanceEnv,
106
+		provenanceStore:           newProvenanceStore(),
106 107
 	}
107 108
 
108 109
 	sampler, err := resources.NewSysSampler()
... ...
@@ -145,6 +149,7 @@ func (s *Solver) bridge(b solver.Builder) *provenanceBridge {
145 145
 		resolveCacheImporterFuncs: s.resolveCacheImporterFuncs,
146 146
 		cms:                       map[string]solver.CacheManager{},
147 147
 		sm:                        s.sm,
148
+		provenanceStore:           s.provenanceStore,
148 149
 	}}
149 150
 }
150 151
 
... ...
@@ -152,7 +157,21 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge {
152 152
 	return s.bridge(b)
153 153
 }
154 154
 
155
-func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string) (_ *client.SolveResponse, err error) {
155
+func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, compatibilityVersion int, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string) (_ *client.SolveResponse, err error) {
156
+	hasNamedDockerfileContext := false
157
+	for k := range req.FrontendOpt {
158
+		if k == "context:dockerfile.v0" || strings.HasPrefix(k, "context:dockerfile.v0::") {
159
+			hasNamedDockerfileContext = true
160
+			break
161
+		}
162
+	}
163
+	if req.Frontend == "gateway.v0" && req.FrontendOpt[frontend.KeySource] == "dockerfile.v0" && !hasNamedDockerfileContext {
164
+		frontendOpt := maps.Clone(req.FrontendOpt)
165
+		delete(frontendOpt, frontend.KeySource)
166
+		req.Frontend = "dockerfile.v0"
167
+		req.FrontendOpt = frontendOpt
168
+	}
169
+
156 170
 	j, err := s.solver.NewJob(id)
157 171
 	if err != nil {
158 172
 		return nil, err
... ...
@@ -201,10 +220,17 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
201 201
 	if policySession != "" {
202 202
 		j.SetValue(keySourcePolicySession, policySession)
203 203
 	}
204
+	if compatibilityVersion == 0 {
205
+		compatibilityVersion = compat.CompatibilityVersionCurrent
206
+	}
207
+	j.SetValue(compat.JobValueKey, compatibilityVersion)
204 208
 
205 209
 	j.SessionID = sessionID
206 210
 
207 211
 	br := s.bridge(j)
212
+	defer br.releaseProvenanceRefs()
213
+	rootReq := req.Clone()
214
+	br.rootReq = &rootReq
208 215
 	var fwd gateway.LLBBridgeForwarder
209 216
 	if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" {
210 217
 		fwd = gateway.NewBridgeForwarder(ctx, br, br, s.workerController.Infos(), req.FrontendInputs, sessionID, s.sm)
... ...
@@ -213,10 +239,8 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro
213 213
 		// s.recordBuildHistory can block for several seconds on
214 214
 		// LeaseManager calls, and there is a fixed 3s timeout in
215 215
 		// GatewayForwarder on build registration.
216
-		if err := s.gatewayForwarder.RegisterBuild(ctx, id, fwd); err != nil {
217
-			return nil, err
218
-		}
219
-		defer s.gatewayForwarder.UnregisterBuild(context.WithoutCancel(ctx), id)
216
+		s.gatewayForwarder.RegisterBuild(ctx, id, fwd)
217
+		defer s.gatewayForwarder.UnregisterBuild(context.Background(), id)
220 218
 	}
221 219
 
222 220
 	if !internal {
... ...
@@ -9,6 +9,9 @@ const AttrMountSSHSock = "git.mountsshsock"
9 9
 const AttrGitChecksum = "git.checksum"
10 10
 const AttrGitSkipSubmodules = "git.skipsubmodules"
11 11
 const AttrGitMTime = "git.mtime"
12
+const AttrGitFetchByCommit = "git.fetchbycommit"
13
+const AttrGitBundle = "git.bundle"
14
+const AttrGitCheckoutBundle = "git.checkoutbundle"
12 15
 
13 16
 const AttrGitSignatureVerifyPubKey = "git.sig.pubkey"
14 17
 const AttrGitSignatureVerifyRejectExpired = "git.sig.rejectexpired"
... ...
@@ -35,6 +35,9 @@ const (
35 35
 	CapSourceGitSkipSubmodules  apicaps.CapID = "source.git.skipsubmodules"
36 36
 	CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify"
37 37
 	CapSourceGitMTime           apicaps.CapID = "source.git.mtime"
38
+	CapSourceGitFetchByCommit   apicaps.CapID = "source.git.fetchbycommit"
39
+	CapSourceGitBundle          apicaps.CapID = "source.git.bundle"
40
+	CapSourceGitCheckoutBundle  apicaps.CapID = "source.git.checkoutbundle"
38 41
 
39 42
 	CapSourceHTTP         apicaps.CapID = "source.http"
40 43
 	CapSourceHTTPAuth     apicaps.CapID = "source.http.auth"
... ...
@@ -263,6 +266,24 @@ func init() {
263 263
 	})
264 264
 
265 265
 	Caps.Init(apicaps.Cap{
266
+		ID:      CapSourceGitFetchByCommit,
267
+		Enabled: true,
268
+		Status:  apicaps.CapStatusExperimental,
269
+	})
270
+
271
+	Caps.Init(apicaps.Cap{
272
+		ID:      CapSourceGitBundle,
273
+		Enabled: true,
274
+		Status:  apicaps.CapStatusExperimental,
275
+	})
276
+
277
+	Caps.Init(apicaps.Cap{
278
+		ID:      CapSourceGitCheckoutBundle,
279
+		Enabled: true,
280
+		Status:  apicaps.CapStatusExperimental,
281
+	})
282
+
283
+	Caps.Init(apicaps.Cap{
266 284
 		ID:      CapSourceHTTP,
267 285
 		Enabled: true,
268 286
 		Status:  apicaps.CapStatusExperimental,
... ...
@@ -184,6 +184,8 @@ type JobContext interface {
184 184
 	// ResolverCache returns object for memorizing/synchronizing remote resolving decisions during the job.
185 185
 	// Steps from same build job will share the same resolver cache.
186 186
 	ResolverCache() ResolverCache
187
+	// CompatibilityVersion returns the solve-wide compatibility version for the current job.
188
+	CompatibilityVersion() (int, error)
187 189
 }
188 190
 
189 191
 type ResolverCache interface {
190 192
new file mode 100644
... ...
@@ -0,0 +1,128 @@
0
+// Package blobfetch provides a shared helper for resolving a single
1
+// content-addressable blob from either a docker-image registry or an OCI layout
2
+// content store, identified by its digest. It is factored out of the
3
+// containerblob source so other consumers (for example the git-bundle flow on
4
+// the git source) can reuse the same resolution logic without depending on the
5
+// source implementation.
6
+package blobfetch
7
+
8
+import (
9
+	"context"
10
+	"io"
11
+	"time"
12
+
13
+	"github.com/containerd/containerd/v2/core/remotes"
14
+	"github.com/containerd/containerd/v2/core/remotes/docker"
15
+	"github.com/moby/buildkit/session"
16
+	sessioncontent "github.com/moby/buildkit/session/content"
17
+	srctypes "github.com/moby/buildkit/source/types"
18
+	"github.com/moby/buildkit/util/iohelper"
19
+	"github.com/moby/buildkit/util/resolver"
20
+	digest "github.com/opencontainers/go-digest"
21
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
22
+	"github.com/pkg/errors"
23
+)
24
+
25
+// FetchOpt describes how to resolve a blob.
26
+type FetchOpt struct {
27
+	// Scheme identifies the source kind. Must be one of
28
+	// srctypes.DockerImageBlobScheme or srctypes.OCIBlobScheme.
29
+	Scheme string
30
+	// Ref is the image-ref form of the blob, i.e. "<name>@<digest>" for
31
+	// registry blobs, or the reference body accepted by the containerd
32
+	// reference parser for OCI-layout blobs.
33
+	Ref string
34
+	// Digest is the blob digest to retrieve.
35
+	Digest digest.Digest
36
+	// RegistryHosts is used when Scheme is DockerImageBlobScheme.
37
+	RegistryHosts docker.RegistryHosts
38
+	// SessionManager is used to reach the client when Scheme is
39
+	// OCIBlobScheme.
40
+	SessionManager *session.Manager
41
+	// SessionID, if set, pins the OCI-layout fetch to a specific client
42
+	// session.
43
+	SessionID string
44
+	// StoreID is the client-side OCI-layout store name. Required for
45
+	// Scheme == OCIBlobScheme.
46
+	StoreID string
47
+}
48
+
49
+// FetchBlob opens a read stream for the requested blob. The caller owns the
50
+// returned ReadCloser and must Close it. The returned digest is the blob
51
+// digest from the locator (echoed for convenience).
52
+func FetchBlob(ctx context.Context, g session.Group, opt FetchOpt) (io.ReadCloser, digest.Digest, error) {
53
+	if err := opt.Digest.Validate(); err != nil {
54
+		return nil, "", errors.Wrap(err, "invalid blob digest")
55
+	}
56
+	switch opt.Scheme {
57
+	case srctypes.OCIBlobScheme:
58
+		if opt.StoreID == "" {
59
+			return nil, "", errors.Errorf("oci-layout blob source requires store id")
60
+		}
61
+		rc, err := fetchFromOCILayoutStore(ctx, g, opt)
62
+		if err != nil {
63
+			return nil, "", err
64
+		}
65
+		return rc, opt.Digest, nil
66
+	case srctypes.DockerImageBlobScheme:
67
+		r := resolver.DefaultPool.GetResolver(opt.RegistryHosts, opt.Ref, resolver.ScopeType{}, opt.SessionManager, g)
68
+		f, err := r.Fetcher(ctx, opt.Ref)
69
+		if err != nil {
70
+			return nil, "", err
71
+		}
72
+		fd, ok := f.(remotes.FetcherByDigest)
73
+		if !ok {
74
+			return nil, "", errors.Errorf("invalid blob fetcher: %T", f)
75
+		}
76
+		rc, _, err := fd.FetchByDigest(ctx, opt.Digest)
77
+		if err != nil {
78
+			return nil, "", err
79
+		}
80
+		return rc, opt.Digest, nil
81
+	default:
82
+		return nil, "", errors.Errorf("unsupported blob scheme %q", opt.Scheme)
83
+	}
84
+}
85
+
86
+func fetchFromOCILayoutStore(ctx context.Context, g session.Group, opt FetchOpt) (io.ReadCloser, error) {
87
+	var rc io.ReadCloser
88
+	err := withOCICaller(ctx, g, opt, func(ctx context.Context, caller session.Caller) error {
89
+		store := sessioncontent.NewCallerStore(caller, "oci:"+opt.StoreID)
90
+		info, err := store.Info(ctx, opt.Digest)
91
+		if err != nil {
92
+			return err
93
+		}
94
+
95
+		readerAt, err := store.ReaderAt(ctx, ocispecs.Descriptor{
96
+			Digest: info.Digest,
97
+			Size:   info.Size,
98
+		})
99
+		if err != nil {
100
+			return err
101
+		}
102
+		rc = iohelper.ReadCloser(readerAt)
103
+		return nil
104
+	})
105
+	if err != nil {
106
+		return nil, err
107
+	}
108
+	return rc, nil
109
+}
110
+
111
+func withOCICaller(ctx context.Context, g session.Group, opt FetchOpt, f func(context.Context, session.Caller) error) error {
112
+	if opt.SessionID != "" {
113
+		timeoutCtx, cancel := context.WithCancelCause(ctx)
114
+		timeoutCtx, _ = context.WithTimeoutCause(timeoutCtx, 5*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
115
+		defer func() { cancel(errors.WithStack(context.Canceled)) }()
116
+
117
+		caller, err := opt.SessionManager.Get(timeoutCtx, opt.SessionID, false)
118
+		if err != nil {
119
+			return err
120
+		}
121
+		return f(ctx, caller)
122
+	}
123
+
124
+	return opt.SessionManager.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error {
125
+		return f(ctx, caller)
126
+	})
127
+}
... ...
@@ -9,20 +9,15 @@ import (
9 9
 	"os"
10 10
 	"time"
11 11
 
12
-	"github.com/containerd/containerd/v2/core/remotes"
13 12
 	cerrdefs "github.com/containerd/errdefs"
14 13
 	"github.com/moby/buildkit/cache"
15 14
 	"github.com/moby/buildkit/session"
16
-	sessioncontent "github.com/moby/buildkit/session/content"
17 15
 	"github.com/moby/buildkit/snapshot"
18 16
 	"github.com/moby/buildkit/solver"
19
-	srctypes "github.com/moby/buildkit/source/types"
17
+	"github.com/moby/buildkit/source/containerblob/blobfetch"
20 18
 	"github.com/moby/buildkit/source/util/pathutil"
21 19
 	"github.com/moby/buildkit/util/contentutil"
22
-	"github.com/moby/buildkit/util/iohelper"
23
-	"github.com/moby/buildkit/util/resolver"
24 20
 	digest "github.com/opencontainers/go-digest"
25
-	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
26 21
 	"github.com/pkg/errors"
27 22
 )
28 23
 
... ...
@@ -68,31 +63,17 @@ func (p *puller) ensureResolver(ctx context.Context, g session.Group) error {
68 68
 		return errors.Wrap(err, "invalid reference digest")
69 69
 	}
70 70
 
71
-	var (
72
-		rc  io.ReadCloser
73
-		err error
74
-	)
75
-	if p.id.Scheme() == srctypes.OCIBlobScheme {
76
-		rc, err = p.fetchFromOCILayoutStore(ctx, g, dgst)
77
-		if err != nil {
78
-			return err
79
-		}
80
-	} else {
81
-		r := resolver.DefaultPool.GetResolver(p.src.RegistryHosts, p.id.Reference.String(), resolver.ScopeType{}, p.SessionManager, g)
82
-		f, err := r.Fetcher(ctx, p.id.Reference.String())
83
-		if err != nil {
84
-			return err
85
-		}
86
-
87
-		fd, ok := f.(remotes.FetcherByDigest)
88
-		if !ok {
89
-			return errors.Errorf("invalid blob fetcher: %T", f)
90
-		}
91
-
92
-		rc, _, err = fd.FetchByDigest(ctx, dgst)
93
-		if err != nil {
94
-			return err
95
-		}
71
+	rc, _, err := blobfetch.FetchBlob(ctx, g, blobfetch.FetchOpt{
72
+		Scheme:         p.id.Scheme(),
73
+		Ref:            p.id.Reference.String(),
74
+		Digest:         dgst,
75
+		RegistryHosts:  p.src.RegistryHosts,
76
+		SessionManager: p.SessionManager,
77
+		SessionID:      p.id.SessionID,
78
+		StoreID:        p.id.StoreID,
79
+	})
80
+	if err != nil {
81
+		return err
96 82
 	}
97 83
 
98 84
 	p.rc = rc
... ...
@@ -100,53 +81,6 @@ func (p *puller) ensureResolver(ctx context.Context, g session.Group) error {
100 100
 	return nil
101 101
 }
102 102
 
103
-func (p *puller) fetchFromOCILayoutStore(ctx context.Context, g session.Group, dgst digest.Digest) (io.ReadCloser, error) {
104
-	if p.id.StoreID == "" {
105
-		return nil, errors.Errorf("oci-layout blob source requires store id")
106
-	}
107
-
108
-	var rc io.ReadCloser
109
-	err := p.withOCICaller(ctx, g, func(ctx context.Context, caller session.Caller) error {
110
-		store := sessioncontent.NewCallerStore(caller, "oci:"+p.id.StoreID)
111
-		info, err := store.Info(ctx, dgst)
112
-		if err != nil {
113
-			return err
114
-		}
115
-
116
-		readerAt, err := store.ReaderAt(ctx, ocispecs.Descriptor{
117
-			Digest: info.Digest,
118
-			Size:   info.Size,
119
-		})
120
-		if err != nil {
121
-			return err
122
-		}
123
-		rc = iohelper.ReadCloser(readerAt)
124
-		return nil
125
-	})
126
-	if err != nil {
127
-		return nil, err
128
-	}
129
-	return rc, nil
130
-}
131
-
132
-func (p *puller) withOCICaller(ctx context.Context, g session.Group, f func(context.Context, session.Caller) error) error {
133
-	if p.id.SessionID != "" {
134
-		timeoutCtx, cancel := context.WithCancelCause(ctx)
135
-		timeoutCtx, _ = context.WithTimeoutCause(timeoutCtx, 5*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet
136
-		defer func() { cancel(errors.WithStack(context.Canceled)) }()
137
-
138
-		caller, err := p.SessionManager.Get(timeoutCtx, p.id.SessionID, false)
139
-		if err != nil {
140
-			return err
141
-		}
142
-		return f(ctx, caller)
143
-	}
144
-
145
-	return p.SessionManager.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error {
146
-		return f(ctx, caller)
147
-	})
148
-}
149
-
150 103
 func (p *puller) CacheKey(ctx context.Context, jobCtx solver.JobContext, index int) (cacheKey string, imgDigest string, cacheOpts solver.CacheOpts, cacheDone bool, err error) {
151 104
 	dgst := p.id.Reference.Digest()
152 105
 	if err := dgst.Validate(); err != nil {
153 106
new file mode 100644
... ...
@@ -0,0 +1,487 @@
0
+package git
1
+
2
+import (
3
+	"context"
4
+	"fmt"
5
+	"io"
6
+	"os"
7
+	"path/filepath"
8
+	"strings"
9
+	"sync"
10
+
11
+	"github.com/moby/buildkit/cache"
12
+	"github.com/moby/buildkit/session"
13
+	"github.com/moby/buildkit/snapshot"
14
+	"github.com/moby/buildkit/solver"
15
+	"github.com/moby/buildkit/source/containerblob/blobfetch"
16
+	srctypes "github.com/moby/buildkit/source/types"
17
+	"github.com/moby/buildkit/util/bklog"
18
+	"github.com/moby/buildkit/util/gitutil"
19
+	digest "github.com/opencontainers/go-digest"
20
+	"github.com/pkg/errors"
21
+)
22
+
23
+const (
24
+	// bundleFileName is the name of the bundle file written at the
25
+	// checkout mount root when git.checkoutbundle is set. Documented on
26
+	// the GitCheckoutBundle godoc.
27
+	bundleFileName = "bundle"
28
+
29
+	// bundleImportFileName is the on-disk name of the transient bundle
30
+	// file streamed in from the blob locator, colocated with the bare
31
+	// repo and removed once the import completes.
32
+	bundleImportFileName = "bundle.pack"
33
+
34
+	// bundleFallbackRef is the ref name used when the user's ref is empty
35
+	// or a commit SHA. Bundle creation needs a named ref so the consumer's
36
+	// fetch refspec has something to land; consumer-side resolution
37
+	// short-circuits on SHA when Ref is empty/SHA, so the fallback name
38
+	// is not user-visible.
39
+	bundleFallbackRef = "refs/heads/main"
40
+)
41
+
42
+// bundleTargetRef normalizes the user's ref into a fully-qualified ref name
43
+// suitable as the tip ref inside the emitted bundle.
44
+//
45
+//   - Empty / commit SHA: falls back to bundleFallbackRef. The consumer
46
+//     short-circuits on SHA so the chosen branch name is not user-visible,
47
+//     but bundle creation needs a named ref for the fetch refspec to land.
48
+//   - "refs/tags/<name>": preserved as a tag ref so a tag-addressed checkout
49
+//     keeps the tag qualifier on the consumer side.
50
+//   - Anything else (e.g. "master", "refs/heads/main", "v1"): stripped of a
51
+//     leading refs/ or heads/ qualifier and re-prefixed with refs/heads/.
52
+//
53
+// Design note: refs/tags/<x> is preserved, but a bare "v1" is mapped to
54
+// refs/heads/v1 rather than refs/tags/v1 because we cannot tell from a
55
+// symbolic ref alone whether it originated as a tag or a branch; refs/heads/
56
+// is the safer default for tip placement.
57
+func bundleTargetRef(ref string) string {
58
+	if ref == "" || gitutil.IsCommitSHA(ref) {
59
+		return bundleFallbackRef
60
+	}
61
+	if strings.HasPrefix(ref, "refs/tags/") {
62
+		return ref
63
+	}
64
+	name := strings.TrimPrefix(ref, "refs/")
65
+	name = strings.TrimPrefix(name, "heads/")
66
+	return "refs/heads/" + name
67
+}
68
+
69
+// detectBundleSHA256 probes a raw git bundle and returns whether it carries
70
+// sha256 object IDs. git ls-remote reads bundle files directly, so this works
71
+// before a destination repo exists and lets callers initialize the right object
72
+// format for a subsequent fetch/import.
73
+func detectBundleSHA256(ctx context.Context, bundlePath string) (bool, error) {
74
+	buf, err := gitCLI().Run(ctx, "ls-remote", "--", bundlePath)
75
+	if err != nil {
76
+		return false, errors.Wrapf(err, "failed to inspect git bundle %s", bundlePath)
77
+	}
78
+	for line := range strings.SplitSeq(string(buf), "\n") {
79
+		if line == "" {
80
+			continue
81
+		}
82
+		sha, _, _ := strings.Cut(line, "\t")
83
+		if !gitutil.IsCommitSHA(sha) {
84
+			return false, errors.Errorf("failed to inspect git bundle %s: invalid object ID %q", bundlePath, sha)
85
+		}
86
+		return len(sha) == 64, nil
87
+	}
88
+	return false, errors.Errorf("failed to inspect git bundle %s: no refs found", bundlePath)
89
+}
90
+
91
+// stageBundle downloads the bundle blob into an isolated temp bare repo,
92
+// imports it, and validates that the pinned commit is present. The returned
93
+// URL (file://<tmpRepoDir>) is suitable for use as a git fetch source (for
94
+// the main fetch flow) or as the remote URL for an ls-remote call (for
95
+// metadata resolution), so both paths converge on the same git primitives
96
+// as a normal origin fetch.
97
+//
98
+// The returned cleanup removes the temp dir and must be called once the
99
+// caller is done with the staged URL.
100
+func (gs *gitSourceHandler) stageBundle(ctx context.Context, g session.Group) (_ string, _ func() error, retErr error) {
101
+	scheme, ref, storeID, dgst, err := parseBundleLocator(gs.src.Bundle)
102
+	if err != nil {
103
+		return "", nil, err
104
+	}
105
+
106
+	tmpDir, err := os.MkdirTemp("", "buildkit-bundle-")
107
+	if err != nil {
108
+		return "", nil, errors.Wrap(err, "failed to create temp dir for bundle")
109
+	}
110
+	cleanup := func() error { return os.RemoveAll(tmpDir) }
111
+	defer func() {
112
+		if retErr != nil {
113
+			cleanup()
114
+		}
115
+	}()
116
+
117
+	if err := gs.downloadBundleToFile(ctx, g, scheme, ref, storeID, dgst, tmpDir, bundleImportFileName); err != nil {
118
+		return "", nil, err
119
+	}
120
+	bundlePath := filepath.Join(tmpDir, bundleImportFileName)
121
+	sha256, err := detectBundleSHA256(ctx, bundlePath)
122
+	if err != nil {
123
+		return "", nil, err
124
+	}
125
+	gs.sha256 = sha256
126
+
127
+	// Initialize an isolated bare repo in the temp dir and import the
128
+	// bundle there. This lets us validate the bundle and stage the refs
129
+	// under their natural names without touching the shared bare repo.
130
+	tmpRepoDir := filepath.Join(tmpDir, "repo.git")
131
+	if err := os.Mkdir(tmpRepoDir, 0700); err != nil {
132
+		return "", nil, errors.Wrap(err, "failed to create temp bare repo dir")
133
+	}
134
+	tmpGit := gitCLI(gitutil.WithGitDir(tmpRepoDir))
135
+	initArgs := []string{"-c", "init.defaultBranch=master", "init", "--bare"}
136
+	if sha256 {
137
+		initArgs = append(initArgs, "--object-format=sha256")
138
+	}
139
+	if _, err := tmpGit.Run(ctx, initArgs...); err != nil {
140
+		return "", nil, errors.Wrap(err, "failed to init temp bare repo")
141
+	}
142
+
143
+	if _, err := tmpGit.Run(ctx, "fetch", bundlePath, "+refs/*:refs/*"); err != nil {
144
+		return "", nil, errors.Wrapf(err, "failed to import git bundle %s into temp", ref)
145
+	}
146
+
147
+	// Confirm the pinned commit is actually present in the bundle. The
148
+	// subsequent fetch in the main flow would surface a generic "failed to
149
+	// fetch" error; this check gives a clearer message.
150
+	if _, err := tmpGit.Run(ctx, "cat-file", "-e", gs.src.Checksum+"^{commit}"); err != nil {
151
+		return "", nil, errors.Errorf("commit %s not found in bundle %s", gs.src.Checksum, ref)
152
+	}
153
+
154
+	return "file://" + tmpRepoDir, cleanup, nil
155
+}
156
+
157
+// ensureStagedBundle returns the file:// URL of a staged temp bare repo
158
+// produced from gs.src.Bundle, lazily staging it on first call and reusing
159
+// the same staging dir on subsequent calls for the lifetime of gs.
160
+//
161
+// The call-site contract is: callers may call ensureStagedBundle any number
162
+// of times and must never run the cleanup themselves. Cleanup is attached to
163
+// the job via JobContext.Cleanup on the first staging so it fires at
164
+// end-of-solve regardless of whether Snapshot runs (covering the
165
+// CacheKey-then-cache-hit path where tryRemoteFetch never executes). If no
166
+// jobCtx is available (e.g. ResolveMetadata path with a nil jobCtx), the
167
+// returned cleanup is retained on the handler and teardown must come from a
168
+// later call that does provide a jobCtx, or from the caller running
169
+// releaseStagedBundle explicitly.
170
+//
171
+// Handler methods run sequentially per solve, so ensureStagedBundle does not
172
+// guard against concurrent calls.
173
+func (gs *gitSourceHandler) ensureStagedBundle(ctx context.Context, jobCtx solver.JobContext, g session.Group) (string, error) {
174
+	if gs.stagedBundleURL != "" {
175
+		return gs.stagedBundleURL, nil
176
+	}
177
+
178
+	url, cleanup, err := gs.stageBundle(ctx, g)
179
+	if err != nil {
180
+		return "", err
181
+	}
182
+
183
+	// Make the cleanup idempotent so it is safe to call from both the
184
+	// job cleanup path and an explicit releaseStagedBundle (or the
185
+	// CacheKey fallback path below) without double-removing the temp dir.
186
+	once := sync.OnceValue(cleanup)
187
+	gs.stagedBundleURL = url
188
+	gs.stagedBundleCleanup = once
189
+
190
+	if jobCtx != nil {
191
+		if err := jobCtx.Cleanup(func() error { return once() }); err != nil {
192
+			// Failed to register the job cleanup — run it synchronously
193
+			// so we do not leak the temp dir.
194
+			_ = once()
195
+			gs.stagedBundleURL = ""
196
+			gs.stagedBundleCleanup = nil
197
+			return "", err
198
+		}
199
+		// The job cleanup now owns the teardown; clearing the
200
+		// per-handler pointer avoids a second invocation from any
201
+		// fallback path. The stored url stays set so repeat callers on
202
+		// this handler get the cached URL for the remainder of the
203
+		// solve, before the job cleanup fires.
204
+		gs.stagedBundleCleanup = nil
205
+	} else {
206
+		bklog.G(ctx).Debug("bundle staged without jobCtx; cleanup deferred to handler teardown")
207
+	}
208
+
209
+	return url, nil
210
+}
211
+
212
+// releaseStagedBundle tears down any staged bundle owned by the handler. It
213
+// is a no-op when the cleanup has already been attached to a JobContext or
214
+// when nothing was staged. Safe to call multiple times.
215
+func (gs *gitSourceHandler) releaseStagedBundle() error {
216
+	cleanup := gs.stagedBundleCleanup
217
+	gs.stagedBundleCleanup = nil
218
+	gs.stagedBundleURL = ""
219
+	if cleanup == nil {
220
+		return nil
221
+	}
222
+	return cleanup()
223
+}
224
+
225
+// downloadBundleToFile streams the bundle blob into a file named bundleName
226
+// inside bundleDir while verifying its digest against the locator. The file
227
+// is created via os.Root so a pre-existing symlink at bundleName inside
228
+// bundleDir cannot redirect the write outside of bundleDir.
229
+func (gs *gitSourceHandler) downloadBundleToFile(ctx context.Context, g session.Group, scheme, ref, storeID string, expectedDigest digest.Digest, bundleDir, bundleName string) (retErr error) {
230
+	rc, err := gs.openBundleBlob(ctx, g, scheme, ref, storeID, expectedDigest)
231
+	if err != nil {
232
+		return errors.Wrapf(err, "failed to fetch git bundle %s", ref)
233
+	}
234
+	defer rc.Close()
235
+
236
+	bundleDirRoot, err := os.OpenRoot(bundleDir)
237
+	if err != nil {
238
+		return errors.Wrapf(err, "failed to open bundle dir root %s", bundleDir)
239
+	}
240
+	defer bundleDirRoot.Close()
241
+
242
+	f, err := bundleDirRoot.Create(bundleName)
243
+	if err != nil {
244
+		return errors.Wrapf(err, "failed to create bundle file %s", bundleName)
245
+	}
246
+	defer func() {
247
+		if retErr != nil {
248
+			f.Close()
249
+			bundleDirRoot.Remove(bundleName)
250
+		}
251
+	}()
252
+
253
+	d := digest.SHA256.Digester()
254
+	if _, err := io.Copy(io.MultiWriter(f, d.Hash()), rc); err != nil {
255
+		return errors.Wrapf(err, "failed to download git bundle %s", ref)
256
+	}
257
+	if err := f.Close(); err != nil {
258
+		return errors.Wrapf(err, "failed to close bundle file %s", bundleName)
259
+	}
260
+
261
+	got := d.Digest()
262
+	if expectedDigest != "" && got != expectedDigest {
263
+		return errors.Errorf("expected checksum to match %s, got %s", expectedDigest, got)
264
+	}
265
+	return nil
266
+}
267
+
268
+// openBundleBlob resolves the blob using blobfetch. storeID is the pre-derived
269
+// OCI-layout store id (empty for registry blobs).
270
+func (gs *gitSourceHandler) openBundleBlob(ctx context.Context, g session.Group, scheme, ref, storeID string, dgst digest.Digest) (io.ReadCloser, error) {
271
+	opt := blobfetch.FetchOpt{
272
+		Scheme:         scheme,
273
+		Ref:            ref,
274
+		Digest:         dgst,
275
+		RegistryHosts:  gs.registryHosts,
276
+		SessionManager: gs.sm,
277
+	}
278
+	if scheme == srctypes.OCIBlobScheme {
279
+		opt.SessionID = gs.src.BundleOCISessionID
280
+		opt.StoreID = storeID
281
+		if gs.src.BundleOCIStoreID != "" {
282
+			opt.StoreID = gs.src.BundleOCIStoreID
283
+		}
284
+	}
285
+	rc, _, err := blobfetch.FetchBlob(ctx, g, opt)
286
+	return rc, err
287
+}
288
+
289
+// resolveBundleMetadata stages the bundle in an isolated temp bare repo and
290
+// delegates to the shared ls-remote code path using that repo's file:// URL.
291
+// Bundle mode lands refs in refs/heads/* and refs/tags/* just like a normal
292
+// origin fetch, so the resolution logic is identical once the URL swap is
293
+// done — and the resulting Metadata.Ref has the same shape as non-bundle
294
+// mode (e.g. "refs/heads/master" for symbolic ref "master").
295
+//
296
+// For a pinned-commit / empty user ref, short-circuit: the user has already
297
+// told us the commit, so there is nothing for ls-remote to add and no point
298
+// downloading the bundle just for metadata resolution. The bundle will still
299
+// be staged during the Snapshot path if the commit is not already present.
300
+func (gs *gitSourceHandler) resolveBundleMetadata(ctx context.Context, jobCtx solver.JobContext) (*Metadata, error) {
301
+	// Bundle mode requires git.checksum; if Ref is empty or a SHA, the
302
+	// commit is already pinned and no lookup is needed. Report Ref and
303
+	// Checksum as the commit SHA — same shape as the commit-SHA short
304
+	// circuit in non-bundle resolveMetadata.
305
+	if gs.src.Ref == "" || gitutil.IsCommitSHA(gs.src.Ref) {
306
+		ref := gs.src.Ref
307
+		if ref == "" {
308
+			ref = gs.src.Checksum
309
+		}
310
+		if gs.src.Checksum != "" && !strings.HasPrefix(ref, gs.src.Checksum) {
311
+			return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, ref)
312
+		}
313
+		return &Metadata{Ref: ref, Checksum: ref}, nil
314
+	}
315
+
316
+	var g session.Group
317
+	if jobCtx != nil {
318
+		g = jobCtx.Session()
319
+	}
320
+	stagedURL, err := gs.ensureStagedBundle(ctx, jobCtx, g)
321
+	if err != nil {
322
+		return nil, err
323
+	}
324
+	return gs.resolveMetadataFromURL(ctx, g, stagedURL)
325
+}
326
+
327
+// checkoutAsBundle writes a single-file git bundle at the checkout mount root
328
+// instead of a worktree. This is the checkout counterpart of bundle import and
329
+// is used when GitCheckoutBundle() is set on the identifier. Submodules are
330
+// not included in the bundle.
331
+func (gs *gitSourceHandler) checkoutAsBundle(ctx context.Context, repo *gitRepo, g session.Group) (_ cache.ImmutableRef, retErr error) {
332
+	ref := gs.src.Ref
333
+	commit := gs.src.Checksum
334
+	if commit == "" {
335
+		commit = ref
336
+	}
337
+
338
+	checkoutRef, err := gs.cache.New(ctx, nil, g, cache.CachePolicyRetain, cache.WithDescription(fmt.Sprintf("git bundle checkout for %s#%s", gs.src.Remote, ref)))
339
+	if err != nil {
340
+		return nil, errors.Wrapf(err, "failed to create new mutable for bundle checkout")
341
+	}
342
+	defer func() {
343
+		if retErr != nil && checkoutRef != nil {
344
+			checkoutRef.Release(context.WithoutCancel(ctx))
345
+		}
346
+	}()
347
+
348
+	mount, err := checkoutRef.Mount(ctx, false, g)
349
+	if err != nil {
350
+		return nil, err
351
+	}
352
+	lm := snapshot.LocalMounter(mount)
353
+	checkoutDir, err := lm.Mount()
354
+	if err != nil {
355
+		return nil, err
356
+	}
357
+	defer func() {
358
+		if retErr != nil && lm != nil {
359
+			lm.Unmount()
360
+		}
361
+	}()
362
+
363
+	// The bundle needs to carry a named ref that points at the target
364
+	// commit, so `git fetch <bundle> +refs/*:refs/*` on the consumer side
365
+	// has something to land. We preserve the user's ref name (normalized
366
+	// into refs/heads/* or refs/tags/*) so the bundle reads as if it were
367
+	// produced by the upstream remote; no fake internal namespace.
368
+	targetRef := bundleTargetRef(ref)
369
+
370
+	// Stage the bundle in an isolated temp bare repo rather than mutating
371
+	// the shared bare repo's ref namespace. We fetch the pinned commit
372
+	// from the shared repo into the temp bare repo and give it the
373
+	// target-ref name there, so `git bundle create` sees a clean,
374
+	// self-contained repo with exactly the ref we want in the bundle.
375
+	tmpDir, err := os.MkdirTemp("", "buildkit-bundle-create-")
376
+	if err != nil {
377
+		return nil, errors.Wrap(err, "failed to create temp dir for bundle creation")
378
+	}
379
+	defer os.RemoveAll(tmpDir)
380
+
381
+	tmpRepoDir := filepath.Join(tmpDir, "repo.git")
382
+	if err := os.Mkdir(tmpRepoDir, 0700); err != nil {
383
+		return nil, errors.Wrap(err, "failed to create temp bare repo dir for bundle creation")
384
+	}
385
+	tmpGit := repo.New(gitutil.WithGitDir(tmpRepoDir))
386
+	initArgs := []string{"-c", "init.defaultBranch=master", "init", "--bare"}
387
+	if gs.sha256 {
388
+		initArgs = append(initArgs, "--object-format=sha256")
389
+	}
390
+	if _, err := tmpGit.Run(ctx, initArgs...); err != nil {
391
+		return nil, errors.Wrap(err, "failed to init temp bare repo for bundle creation")
392
+	}
393
+
394
+	// Pull the pinned commit from the shared bare repo into temp, then
395
+	// point the target ref at it. The shared bare repo maps user refs to
396
+	// refs/tags/<name> via the main-path fetch refspec, so there is no
397
+	// natural-name ref to pull; update-ref alone places the tip against
398
+	// the pinned commit under the target name.
399
+	sharedURL := "file://" + repo.dir
400
+	if _, err := tmpGit.Run(ctx, "fetch", sharedURL, commit); err != nil {
401
+		return nil, errors.Wrapf(err, "failed to fetch commit %s from shared repo for bundle creation", commit)
402
+	}
403
+
404
+	if _, err := tmpGit.Run(ctx, "update-ref", targetRef, commit); err != nil {
405
+		return nil, errors.Wrapf(err, "failed to create target ref for bundle %s", commit)
406
+	}
407
+
408
+	// Write the bundle under a temp path we fully own, then move it into
409
+	// the checkout mount through os.OpenRoot so a pre-existing symlink at
410
+	// <checkoutDir>/bundle cannot redirect the final artifact outside the
411
+	// mount. git bundle create is an external process that only accepts
412
+	// path strings, so the only safe landing pattern is: write to a path
413
+	// outside the mount, then atomically transfer the bytes through an
414
+	// os.Root-scoped file handle.
415
+	stagePath := filepath.Join(tmpDir, "out.bundle")
416
+	if _, err := tmpGit.Run(ctx, "bundle", "create", stagePath, targetRef); err != nil {
417
+		return nil, errors.Wrapf(err, "failed to create git bundle for %s", commit)
418
+	}
419
+
420
+	if err := writeBundleToMount(checkoutDir, bundleFileName, stagePath); err != nil {
421
+		return nil, err
422
+	}
423
+
424
+	outPath := filepath.Join(checkoutDir, bundleFileName)
425
+	if idmap := mount.IdentityMapping(); idmap != nil {
426
+		uid, gid := idmap.RootPair()
427
+		if err := os.Lchown(outPath, uid, gid); err != nil {
428
+			return nil, errors.Wrap(err, "failed to remap git bundle ownership")
429
+		}
430
+	}
431
+
432
+	lm.Unmount()
433
+	lm = nil
434
+
435
+	snap, err := checkoutRef.Commit(ctx)
436
+	if err != nil {
437
+		return nil, err
438
+	}
439
+	checkoutRef = nil
440
+	return snap, nil
441
+}
442
+
443
+// writeBundleToMount copies the bytes at stagePath into
444
+// <checkoutDir>/<bundleName> via os.OpenRoot, so a symlink at the destination
445
+// cannot redirect the write outside checkoutDir. The staging file is expected
446
+// to be a path the caller fully controls (created under its own temp dir),
447
+// not a path inside checkoutDir.
448
+func writeBundleToMount(checkoutDir, bundleName, stagePath string) (retErr error) {
449
+	src, err := os.Open(stagePath)
450
+	if err != nil {
451
+		return errors.Wrapf(err, "failed to open staged bundle %s", stagePath)
452
+	}
453
+	defer src.Close()
454
+
455
+	checkoutRoot, err := os.OpenRoot(checkoutDir)
456
+	if err != nil {
457
+		return errors.Wrapf(err, "failed to open checkout dir root %s", checkoutDir)
458
+	}
459
+	defer checkoutRoot.Close()
460
+
461
+	// Remove any pre-existing entry at bundleName. If it's a symlink,
462
+	// os.Root.Remove deletes the link itself, not the target, so a symlink
463
+	// planted in the mount cannot redirect the subsequent create.
464
+	if err := checkoutRoot.Remove(bundleName); err != nil && !errors.Is(err, os.ErrNotExist) {
465
+		return errors.Wrapf(err, "failed to clear bundle dest %s", bundleName)
466
+	}
467
+
468
+	dst, err := checkoutRoot.Create(bundleName)
469
+	if err != nil {
470
+		return errors.Wrapf(err, "failed to create bundle file %s", bundleName)
471
+	}
472
+	defer func() {
473
+		if retErr != nil {
474
+			dst.Close()
475
+			checkoutRoot.Remove(bundleName)
476
+		}
477
+	}()
478
+
479
+	if _, err := io.Copy(dst, src); err != nil {
480
+		return errors.Wrapf(err, "failed to write bundle to %s", bundleName)
481
+	}
482
+	if err := dst.Close(); err != nil {
483
+		return errors.Wrapf(err, "failed to close bundle file %s", bundleName)
484
+	}
485
+	return nil
486
+}
... ...
@@ -1,11 +1,16 @@
1 1
 package git
2 2
 
3 3
 import (
4
+	"strings"
5
+
6
+	"github.com/containerd/containerd/v2/pkg/reference"
4 7
 	"github.com/moby/buildkit/solver/llbsolver/provenance"
5 8
 	provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
6 9
 	"github.com/moby/buildkit/source"
7 10
 	srctypes "github.com/moby/buildkit/source/types"
8 11
 	"github.com/moby/buildkit/util/gitutil"
12
+	digest "github.com/opencontainers/go-digest"
13
+	"github.com/pkg/errors"
9 14
 )
10 15
 
11 16
 type GitIdentifier struct {
... ...
@@ -20,6 +25,24 @@ type GitIdentifier struct {
20 20
 	KnownSSHHosts    string
21 21
 	SkipSubmodules   bool
22 22
 	MTime            string // "checkout" (default) or "commit"
23
+	FetchByCommit    bool
24
+
25
+	// Bundle, when non-empty, instructs the git source to fetch commits from a
26
+	// pre-built git bundle stored as a blob instead of fetching from the remote
27
+	// repository. The locator must use the "docker-image+blob://" or
28
+	// "oci-layout+blob://" scheme.
29
+	Bundle string
30
+	// BundleOCISessionID, when set, pins the OCI-layout bundle fetch to a
31
+	// specific client session. Only meaningful when Bundle uses the
32
+	// oci-layout+blob:// scheme.
33
+	BundleOCISessionID string
34
+	// BundleOCIStoreID, when set, overrides the OCI-layout store name
35
+	// derived from the bundle locator. Only meaningful when Bundle uses the
36
+	// oci-layout+blob:// scheme.
37
+	BundleOCIStoreID string
38
+	// CheckoutBundle, when true, produces a single-file git bundle at the
39
+	// checkout mount root (filename "bundle") instead of a worktree.
40
+	CheckoutBundle bool
23 41
 
24 42
 	VerifySignature *GitSignatureVerifyOptions
25 43
 }
... ...
@@ -55,14 +78,18 @@ func (GitIdentifier) Scheme() string {
55 55
 var _ source.Identifier = (*GitIdentifier)(nil)
56 56
 
57 57
 func (id *GitIdentifier) Capture(c *provenance.Capture, pin string) error {
58
-	url := id.Remote
58
+	remoteURL := id.Remote
59 59
 	if id.Ref != "" {
60
-		url += "#" + id.Ref
60
+		remoteURL += "#" + id.Ref
61 61
 	}
62
-	c.AddGit(provenancetypes.GitSource{
63
-		URL:    url,
62
+	gs := provenancetypes.GitSource{
63
+		URL:    remoteURL,
64 64
 		Commit: pin,
65
-	})
65
+	}
66
+	if id.Bundle != "" {
67
+		gs.Bundle = &provenancetypes.GitBundle{URL: id.Bundle}
68
+	}
69
+	c.AddGit(gs)
66 70
 	if id.AuthTokenSecret != "" {
67 71
 		c.AddSecret(provenancetypes.Secret{
68 72
 			ID:       id.AuthTokenSecret,
... ...
@@ -83,3 +110,73 @@ func (id *GitIdentifier) Capture(c *provenance.Capture, pin string) error {
83 83
 	}
84 84
 	return nil
85 85
 }
86
+
87
+// validateBundleAttrs enforces the bundle-mode constraints that are statically
88
+// decidable at identifier parse time.
89
+func validateBundleAttrs(id *GitIdentifier) error {
90
+	if id.Bundle != "" {
91
+		if _, _, _, _, err := parseBundleLocator(id.Bundle); err != nil {
92
+			return err
93
+		}
94
+		if id.Checksum == "" {
95
+			return errors.Errorf("git.bundle requires git.checksum to be set")
96
+		}
97
+		// Reject the pathological "Ref=sha1, Checksum=sha2" case: if both
98
+		// are set and Ref is itself a commit SHA, the two must agree.
99
+		// Without this check bundle mode silently overwrites Ref with
100
+		// Checksum in tryRemoteFetch and the mismatch never surfaces.
101
+		if id.Ref != "" && gitutil.IsCommitSHA(id.Ref) && !strings.HasPrefix(id.Ref, id.Checksum) {
102
+			return errors.Errorf("expected checksum to match %s, got %s", id.Checksum, id.Ref)
103
+		}
104
+	}
105
+	if id.CheckoutBundle {
106
+		if id.KeepGitDir {
107
+			return errors.Errorf("git.checkoutbundle is incompatible with git.keepgitdir")
108
+		}
109
+		if id.Subdir != "" {
110
+			return errors.Errorf("git.checkoutbundle is incompatible with git.subdir")
111
+		}
112
+	}
113
+	return nil
114
+}
115
+
116
+// splitBundleLocator splits a locator of the form "<scheme>://<body>" into
117
+// (scheme, body). It returns ("","") if raw does not contain a "://" separator.
118
+// This avoids net/url for schemes like "oci-layout+blob" whose body contains
119
+// a colon after '@' (the digest) and confuses url.Parse.
120
+func splitBundleLocator(raw string) (string, string) {
121
+	const sep = "://"
122
+	i := strings.Index(raw, sep)
123
+	if i < 0 {
124
+		return "", ""
125
+	}
126
+	return raw[:i], raw[i+len(sep):]
127
+}
128
+
129
+// parseBundleLocator extracts the scheme, normalized reference string, the
130
+// reference locator (i.e. the body before '@digest', used as the OCI-layout
131
+// store id), and the digest from a bundle locator of the form
132
+// "<scheme>://<ref>@sha256:<hex>". Only sha256 digests are accepted because
133
+// downloadBundleToFile hashes with sha256 and would fail to match any other
134
+// algorithm.
135
+func parseBundleLocator(raw string) (string, string, string, digest.Digest, error) {
136
+	scheme, body := splitBundleLocator(raw)
137
+	if scheme == "" {
138
+		return "", "", "", "", errors.Errorf("failed to parse git.bundle locator %q: missing scheme", raw)
139
+	}
140
+	if scheme != srctypes.DockerImageBlobScheme && scheme != srctypes.OCIBlobScheme {
141
+		return "", "", "", "", errors.Errorf("git.bundle locator scheme %q is not supported; expected %q or %q", scheme, srctypes.DockerImageBlobScheme, srctypes.OCIBlobScheme)
142
+	}
143
+	ref, err := reference.Parse(body)
144
+	if err != nil {
145
+		return "", "", "", "", errors.Wrapf(err, "failed to parse git.bundle locator %q", raw)
146
+	}
147
+	dgst := ref.Digest()
148
+	if err := dgst.Validate(); err != nil {
149
+		return "", "", "", "", errors.Wrapf(err, "failed to parse git.bundle locator %q: invalid digest", raw)
150
+	}
151
+	if dgst.Algorithm() != digest.SHA256 {
152
+		return "", "", "", "", errors.Errorf("git.bundle locator %q: digest algorithm %q not supported, expected %q", raw, dgst.Algorithm(), digest.SHA256)
153
+	}
154
+	return scheme, ref.String(), ref.Locator, dgst, nil
155
+}
... ...
@@ -16,6 +16,7 @@ import (
16 16
 	"strings"
17 17
 	"time"
18 18
 
19
+	"github.com/containerd/containerd/v2/core/remotes/docker"
19 20
 	"github.com/moby/buildkit/cache"
20 21
 	"github.com/moby/buildkit/client"
21 22
 	"github.com/moby/buildkit/identity"
... ...
@@ -24,6 +25,7 @@ import (
24 24
 	"github.com/moby/buildkit/session/sshforward"
25 25
 	"github.com/moby/buildkit/snapshot"
26 26
 	"github.com/moby/buildkit/solver"
27
+	"github.com/moby/buildkit/solver/llbsolver/compat"
27 28
 	"github.com/moby/buildkit/solver/pb"
28 29
 	"github.com/moby/buildkit/source"
29 30
 	srctypes "github.com/moby/buildkit/source/types"
... ...
@@ -44,11 +46,16 @@ var defaultBranch = regexp.MustCompile(`refs/heads/(\S+)`)
44 44
 
45 45
 type Opt struct {
46 46
 	CacheAccessor cache.Accessor
47
+	// RegistryHosts is used to fetch bundle blobs from a docker registry
48
+	// when a git source uses GitBundleURL("docker-image+blob://...").
49
+	// Optional: when unset, bundle mode requires oci-layout+blob.
50
+	RegistryHosts docker.RegistryHosts
47 51
 }
48 52
 
49 53
 type Source struct {
50
-	cache  cache.Accessor
51
-	locker *locker.Locker
54
+	cache         cache.Accessor
55
+	locker        *locker.Locker
56
+	registryHosts docker.RegistryHosts
52 57
 }
53 58
 
54 59
 type Metadata struct {
... ...
@@ -74,8 +81,9 @@ func Supported() error {
74 74
 
75 75
 func NewSource(opt Opt) (*Source, error) {
76 76
 	gs := &Source{
77
-		cache:  opt.CacheAccessor,
78
-		locker: locker.New(),
77
+		cache:         opt.CacheAccessor,
78
+		locker:        locker.New(),
79
+		registryHosts: opt.RegistryHosts,
79 80
 	}
80 81
 	return gs, nil
81 82
 }
... ...
@@ -137,11 +145,24 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo
137 137
 			id.VerifySignature.IgnoreSignedTag = v == "true"
138 138
 		case pb.AttrGitMTime:
139 139
 			id.MTime = v
140
+		case pb.AttrGitFetchByCommit:
141
+			id.FetchByCommit = v == "true"
142
+		case pb.AttrGitBundle:
143
+			id.Bundle = v
144
+		case pb.AttrGitCheckoutBundle:
145
+			id.CheckoutBundle = v == "true"
146
+		case pb.AttrOCILayoutSessionID:
147
+			id.BundleOCISessionID = v
148
+		case pb.AttrOCILayoutStoreID:
149
+			id.BundleOCIStoreID = v
140 150
 		}
141 151
 	}
142 152
 	if err := validateGitRef(id.Ref); err != nil {
143 153
 		return nil, err
144 154
 	}
155
+	if err := validateBundleAttrs(id); err != nil {
156
+		return nil, err
157
+	}
145 158
 
146 159
 	return id, nil
147 160
 }
... ...
@@ -254,6 +275,12 @@ type gitSourceHandler struct {
254 254
 	sha256      bool
255 255
 	sm          *session.Manager
256 256
 	authArgs    []string
257
+
258
+	// stagedBundleURL / stagedBundleCleanup cache the temp bundle staging
259
+	// across resolveBundleMetadata (CacheKey) and tryRemoteFetch (Snapshot)
260
+	// so the bundle blob is downloaded only once per handler.
261
+	stagedBundleURL     string
262
+	stagedBundleCleanup func() error
257 263
 }
258 264
 
259 265
 func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string {
... ...
@@ -273,6 +300,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string {
273 273
 	if gs.src.MTime != "" && gs.src.MTime != "checkout" {
274 274
 		key += "(mtime=" + gs.src.MTime + ")"
275 275
 	}
276
+	if gs.src.CheckoutBundle {
277
+		key += "(bundle)"
278
+	}
276 279
 	return key
277 280
 }
278 281
 
... ...
@@ -282,18 +312,28 @@ func (gs *Source) ResolveMetadata(ctx context.Context, id *GitIdentifier, sm *se
282 282
 		Source: gs,
283 283
 		sm:     sm,
284 284
 	}
285
+	// The handler is scoped to this call, so any staged bundle it owns
286
+	// must be released before returning. When a jobCtx is present,
287
+	// ensureStagedBundle hands ownership to the job and this is a no-op;
288
+	// without a jobCtx (e.g. direct ResolveMetadata callers) this is the
289
+	// only hook that frees the temp dir.
290
+	defer func() {
291
+		if err := gsh.releaseStagedBundle(); err != nil {
292
+			bklog.G(ctx).Warnf("failed to release staged git bundle: %v", err)
293
+		}
294
+	}()
285 295
 	md, err := gsh.resolveMetadata(ctx, jobCtx)
286 296
 	if err != nil {
287 297
 		return nil, err
288 298
 	}
289 299
 
300
+	gsh.cacheCommit = md.Checksum
301
+	gsh.sha256 = len(md.Checksum) == 64
302
+
290 303
 	if !opt.ReturnObject && id.VerifySignature == nil {
291 304
 		return md, nil
292 305
 	}
293 306
 
294
-	gsh.cacheCommit = md.Checksum
295
-	gsh.sha256 = len(md.Checksum) == 64
296
-
297 307
 	if err := gsh.addGitObjectsToMetadata(ctx, jobCtx, md); err != nil {
298 308
 		return nil, err
299 309
 	}
... ...
@@ -487,10 +527,6 @@ func (gs *gitSourceHandler) remoteKey() string {
487 487
 }
488 488
 
489 489
 func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.JobContext) (md *Metadata, retErr error) {
490
-	remote := gs.src.Remote
491
-	gs.locker.Lock(remote)
492
-	defer gs.locker.Unlock(remote)
493
-
494 490
 	if gs.src.Checksum != "" {
495 491
 		matched, err := regexp.MatchString("^[a-fA-F0-9]+$", gs.src.Checksum)
496 492
 		if err != nil || !matched {
... ...
@@ -498,6 +534,19 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
498 498
 		}
499 499
 	}
500 500
 
501
+	if gs.src.Bundle != "" {
502
+		// Bundle mode stages the bundle in a temp bare repo and runs the
503
+		// shared ls-remote code path against its file:// URL so the
504
+		// resulting Metadata has the same ref shape as the non-bundle
505
+		// path. Bundle mode does not need the per-remote lock: the temp
506
+		// bare repo is private to this call.
507
+		return gs.resolveBundleMetadata(ctx, jobCtx)
508
+	}
509
+
510
+	remote := gs.src.Remote
511
+	gs.locker.Lock(remote)
512
+	defer gs.locker.Unlock(remote)
513
+
501 514
 	if gitutil.IsCommitSHA(gs.src.Ref) {
502 515
 		if gs.src.Checksum != "" && !strings.HasPrefix(gs.src.Ref, gs.src.Checksum) {
503 516
 			return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, gs.src.Ref)
... ...
@@ -508,6 +557,29 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
508 508
 		}, nil
509 509
 	}
510 510
 
511
+	if gs.src.FetchByCommit {
512
+		if gs.src.Checksum == "" {
513
+			return nil, errors.Errorf("fetch-by-commit requires a checksum or a commit SHA ref")
514
+		}
515
+		if !gitutil.IsCommitSHA(gs.src.Checksum) {
516
+			return nil, errors.Errorf("fetch-by-commit requires a full commit SHA checksum, got %q", gs.src.Checksum)
517
+		}
518
+		// Canonicalize unqualified refs so that cache keys match those
519
+		// produced by the normal path's ls-remote-driven normalization.
520
+		// Unqualified names are treated as branches (git's preferred
521
+		// resolution); tags must be passed as "refs/tags/<name>".
522
+		if gs.src.Ref != "" && !strings.HasPrefix(gs.src.Ref, "refs/") {
523
+			gs.src.Ref = "refs/heads/" + gs.src.Ref
524
+		}
525
+		if gs.src.Ref == "" {
526
+			gs.src.Ref = gs.src.Checksum
527
+		}
528
+		return &Metadata{
529
+			Ref:      gs.src.Ref,
530
+			Checksum: gs.src.Checksum,
531
+		}, nil
532
+	}
533
+
511 534
 	var g session.Group
512 535
 	if jobCtx != nil {
513 536
 		g = jobCtx.Session()
... ...
@@ -544,6 +616,14 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
544 544
 
545 545
 	gs.getAuthToken(ctx, g)
546 546
 
547
+	return gs.resolveMetadataFromURL(ctx, g, gs.src.Remote)
548
+}
549
+
550
+// resolveMetadataFromURL runs ls-remote against the given URL and parses the
551
+// result into a Metadata. The URL may be the user's configured remote (for
552
+// non-bundle mode) or a file:// URL for a staged bundle. Auth and lock
553
+// management are the caller's responsibility.
554
+func (gs *gitSourceHandler) resolveMetadataFromURL(ctx context.Context, g session.Group, remoteURL string) (*Metadata, error) {
547 555
 	tmpGit, cleanup, err := gs.emptyGitCli(ctx, g)
548 556
 	if err != nil {
549 557
 		return nil, err
... ...
@@ -552,16 +632,16 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
552 552
 
553 553
 	ref := gs.src.Ref
554 554
 	if ref == "" {
555
-		ref, err = getDefaultBranch(ctx, tmpGit, gs.src.Remote)
555
+		ref, err = getDefaultBranch(ctx, tmpGit, remoteURL)
556 556
 		if err != nil {
557 557
 			return nil, err
558 558
 		}
559 559
 	}
560 560
 	// TODO: should we assume that remote tag is immutable? add a timer?
561 561
 
562
-	buf, err := tmpGit.Run(ctx, "ls-remote", "--", gs.src.Remote, ref, ref+"^{}")
562
+	buf, err := tmpGit.Run(ctx, "ls-remote", "--", remoteURL, ref, ref+"^{}")
563 563
 	if err != nil {
564
-		return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote))
564
+		return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remoteURL))
565 565
 	}
566 566
 	lines := strings.Split(string(buf), "\n")
567 567
 
... ...
@@ -616,7 +696,7 @@ func (gs *gitSourceHandler) resolveMetadata(ctx context.Context, jobCtx solver.J
616 616
 			return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, exp)
617 617
 		}
618 618
 	}
619
-	md = &Metadata{
619
+	md := &Metadata{
620 620
 		Ref:      usedRef,
621 621
 		Checksum: sha,
622 622
 	}
... ...
@@ -725,12 +805,12 @@ func (gs *gitSourceHandler) remoteFetch(ctx context.Context, jobCtx solver.JobCo
725 725
 		g = jobCtx.Session()
726 726
 	}
727 727
 
728
-	repo, err := gs.tryRemoteFetch(ctx, g, false)
728
+	repo, err := gs.tryRemoteFetch(ctx, jobCtx, g, false)
729 729
 	if err != nil {
730 730
 		var wce *wouldClobberExistingTagError
731 731
 		var ulre *unableToUpdateLocalRefError
732 732
 		if errors.As(err, &wce) || errors.As(err, &ulre) {
733
-			repo, err = gs.tryRemoteFetch(ctx, g, true)
733
+			repo, err = gs.tryRemoteFetch(ctx, jobCtx, g, true)
734 734
 			if err != nil {
735 735
 				return nil, err
736 736
 			}
... ...
@@ -748,6 +828,11 @@ func (gs *gitSourceHandler) remoteFetch(ctx context.Context, jobCtx solver.JobCo
748 748
 	}()
749 749
 
750 750
 	ref := gs.src.Ref
751
+	// With fetch-by-commit, the user-provided ref is not written into the
752
+	// bare repo, so resolve the checksum directly.
753
+	if gs.src.FetchByCommit && gs.src.Checksum != "" {
754
+		ref = gs.src.Checksum
755
+	}
751 756
 	git := repo.GitCLI
752 757
 	if gs.src.Checksum != "" {
753 758
 		actualHashBuf, err := repo.Run(ctx, "rev-parse", ref)
... ...
@@ -789,6 +874,14 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobConte
789 789
 		g = jobCtx.Session()
790 790
 	}
791 791
 	gs.getAuthToken(ctx, g)
792
+	compatibilityVersion := 0
793
+	if jobCtx != nil {
794
+		var err error
795
+		compatibilityVersion, err = jobCtx.CompatibilityVersion()
796
+		if err != nil {
797
+			return nil, err
798
+		}
799
+	}
792 800
 
793 801
 	snapshotKey := cacheKey + ":" + gs.src.Subdir
794 802
 	gs.locker.Lock(snapshotKey)
... ...
@@ -808,7 +901,12 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, jobCtx solver.JobConte
808 808
 	}
809 809
 	defer repo.Release()
810 810
 
811
-	ref, err := gs.checkout(ctx, repo, g)
811
+	var ref cache.ImmutableRef
812
+	if gs.src.CheckoutBundle {
813
+		ref, err = gs.checkoutAsBundle(ctx, repo, g)
814
+	} else {
815
+		ref, err = gs.checkout(ctx, repo, g, compatibilityVersion)
816
+	}
812 817
 	if err != nil {
813 818
 		return nil, err
814 819
 	}
... ...
@@ -836,7 +934,7 @@ func (g *gitRepo) Release() error {
836 836
 	return err
837 837
 }
838 838
 
839
-func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, reset bool) (_ *gitRepo, retErr error) {
839
+func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, jobCtx solver.JobContext, g session.Group, reset bool) (_ *gitRepo, retErr error) {
840 840
 	repo := &gitRepo{}
841 841
 
842 842
 	defer func() {
... ...
@@ -846,13 +944,32 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
846 846
 		}
847 847
 	}()
848 848
 
849
+	// Auth args only apply to non-bundle fetches. In bundle mode the
850
+	// payload comes from a blob fetch, so there is no http remote to
851
+	// authenticate against.
852
+	authArgs := gs.authArgs
853
+	if gs.src.Bundle != "" {
854
+		authArgs = nil
855
+	}
856
+
849 857
 	git, cleanup, err := gs.emptyGitCli(ctx, g)
850 858
 	if err != nil {
851 859
 		return nil, err
852 860
 	}
853 861
 	repo.releasers = append(repo.releasers, cleanup)
854 862
 
855
-	gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.authArgs, gs.sha256, reset, g)
863
+	// Bundle mode may need to create the shared bare repo before the main
864
+	// fetch. Stage the bundle first so stageBundle can probe the bundle's
865
+	// object format via ls-remote and set gs.sha256 for mountRemote.
866
+	stagedURL := ""
867
+	if gs.src.Bundle != "" {
868
+		stagedURL, err = gs.ensureStagedBundle(ctx, jobCtx, g)
869
+		if err != nil {
870
+			return nil, err
871
+		}
872
+	}
873
+
874
+	gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, authArgs, gs.sha256, reset, g)
856 875
 	if err != nil {
857 876
 		return nil, err
858 877
 	}
... ...
@@ -862,6 +979,40 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
862 862
 	git = git.New(gitutil.WithGitDir(gitDir))
863 863
 	repo.GitCLI = git
864 864
 
865
+	// fetchSource is the URL used in place of the "origin" remote name for
866
+	// the main fetch. In bundle mode it points at a staged bundle's file://
867
+	// URL; an empty value means fetch from the configured "origin" remote.
868
+	fetchSource := ""
869
+	// Bundle mode: stage the bundle into an isolated temp bare repo and
870
+	// fetch from it as a file:// remote. The rest of this function runs
871
+	// unchanged — git accepts a URL anywhere a remote name is expected,
872
+	// so the fetch flow treats the staged repo like a normal origin.
873
+	if gs.src.Bundle != "" {
874
+		// When the user did not supply a symbolic ref, use the pinned
875
+		// commit as the ref directly. validateBundleAttrs guarantees
876
+		// Checksum is set.
877
+		if gs.src.Ref == "" || gitutil.IsCommitSHA(gs.src.Ref) {
878
+			gs.src.Ref = gs.src.Checksum
879
+		}
880
+		// Bundle mode enters tryRemoteFetch from Snapshot. If CacheKey
881
+		// has not run yet, prime gs.cacheCommit from the pinned checksum
882
+		// to make the downstream cacheCommit validation a no-op rather
883
+		// than a spurious mismatch.
884
+		if gs.cacheCommit == "" {
885
+			gs.cacheCommit = gs.src.Checksum
886
+		}
887
+		// If the pinned commit is already present in the shared bare
888
+		// repo (from a prior origin or bundle fetch), skip staging
889
+		// entirely. The doFetch check below will no-op the fetch.
890
+		if _, err := repo.Run(ctx, "cat-file", "-e", gs.src.Checksum+"^{commit}"); err != nil {
891
+			// Reuse the staged bundle if CacheKey or the eager bundle-format
892
+			// probe above already built one. ensureStagedBundle owns the
893
+			// teardown (wired to jobCtx.Cleanup on first call), so the
894
+			// cleanup is intentionally not appended to repo.releasers.
895
+			fetchSource = stagedURL
896
+		}
897
+	}
898
+
865 899
 	ref := gs.src.Ref
866 900
 	if ref == "" {
867 901
 		ref, err = getDefaultBranch(ctx, git, gs.src.Remote)
... ...
@@ -870,10 +1021,16 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
870 870
 		}
871 871
 		gs.src.Ref = ref
872 872
 	}
873
+	// fetchRef is the identifier used to fetch from the remote. For fetch-by-commit
874
+	// mode this is the checksum (commit SHA); otherwise it is the ref itself.
875
+	fetchRef := ref
876
+	if gs.src.FetchByCommit && gs.src.Checksum != "" {
877
+		fetchRef = gs.src.Checksum
878
+	}
873 879
 	doFetch := true
874
-	if gitutil.IsCommitSHA(ref) {
880
+	if gitutil.IsCommitSHA(fetchRef) {
875 881
 		// skip fetch if commit already exists
876
-		if _, err := git.Run(ctx, "cat-file", "-e", "--", ref+"^{commit}"); err == nil {
882
+		if _, err := git.Run(ctx, "cat-file", "-e", "--", fetchRef+"^{commit}"); err == nil {
877 883
 			doFetch = false
878 884
 		}
879 885
 	}
... ...
@@ -896,8 +1053,16 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
896 896
 		// make sure no old lock files have leaked
897 897
 		gitDirRoot.RemoveAll("shallow.lock")
898 898
 
899
+		origin := "origin"
900
+		if fetchSource != "" {
901
+			origin = fetchSource
902
+		}
899 903
 		args := []string{"fetch"}
900
-		if !gitutil.IsCommitSHA(ref) { // TODO: find a branch from ls-remote?
904
+		// For fetch-by-commit, assume the server supports
905
+		// uploadpack.allowReachableSHA1InWant / allowAnySHA1InWant and
906
+		// fetch only the requested commit, without the tag/unshallow
907
+		// fallback used by the generic SHA-ref path.
908
+		if gs.src.FetchByCommit || !gitutil.IsCommitSHA(fetchRef) { // TODO: find a branch from ls-remote?
901 909
 			args = append(args, "--depth=1", "--no-tags")
902 910
 		} else {
903 911
 			args = append(args, "--tags")
... ...
@@ -905,11 +1070,11 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
905 905
 				args = append(args, "--unshallow")
906 906
 			}
907 907
 		}
908
-		args = append(args, "origin")
908
+		args = append(args, origin)
909 909
 		if gitutil.IsCommitSHA(ref) {
910 910
 			args = append(args, ref)
911 911
 		} else {
912
-			args = append(args, "--force", "--", ref+":"+targetRef)
912
+			args = append(args, "--force", "--", fetchRef+":"+targetRef)
913 913
 		}
914 914
 		if _, err := git.Run(ctx, args...); err != nil {
915 915
 			err := errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote))
... ...
@@ -928,7 +1093,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
928 928
 		}
929 929
 
930 930
 		// verify that commit matches the cache key commit
931
-		dt, err := git.Run(ctx, "rev-parse", ref)
931
+		dt, err := git.Run(ctx, "rev-parse", fetchRef)
932 932
 		if err != nil {
933 933
 			return nil, err
934 934
 		}
... ...
@@ -951,7 +1116,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
951 951
 				if _, err := gitDirRoot.Lstat("shallow"); err == nil {
952 952
 					args = append(args, "--unshallow")
953 953
 				}
954
-				args = append(args, "origin", gs.cacheCommit)
954
+				args = append(args, origin, gs.cacheCommit)
955 955
 				if _, err := git.Run(ctx, args...); err != nil {
956 956
 					return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote))
957 957
 				}
... ...
@@ -975,9 +1140,27 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group,
975 975
 	return repo, nil
976 976
 }
977 977
 
978
-func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g session.Group) (_ cache.ImmutableRef, retErr error) {
978
+func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g session.Group, compatibilityVersion int) (_ cache.ImmutableRef, retErr error) {
979 979
 	ref := gs.src.Ref
980
-	checkoutRef, err := gs.cache.New(ctx, nil, g, cache.WithRecordType(client.UsageRecordTypeGitCheckout), cache.WithDescription(fmt.Sprintf("git snapshot for %s#%s", urlutil.RedactCredentials(gs.src.Remote), ref)))
980
+	// refOrCommit is the identifier to use against the bare repo. For
981
+	// fetch-by-commit, the user-provided ref is not written into the bare
982
+	// repo, so fall back to the commit checksum which is always present.
983
+	refOrCommit := ref
984
+	if gs.src.FetchByCommit && gs.src.Checksum != "" {
985
+		refOrCommit = gs.src.Checksum
986
+	}
987
+	// In bundle mode, if the user did not supply a symbolic Ref (empty or
988
+	// a SHA), fall back to the pinned checksum so the downstream
989
+	// `git checkout` / `git fetch` has a valid tip. With a symbolic ref
990
+	// present, use it as-is: bundle import lands refs in the natural
991
+	// refs/heads/* / refs/tags/* namespace in the shared bare repo, so
992
+	// `git checkout <ref>` and `git fetch origin <ref>` resolve it
993
+	// normally and the resulting .git carries the natural ref name.
994
+	if gs.src.Bundle != "" && (ref == "" || gitutil.IsCommitSHA(ref)) {
995
+		ref = gs.src.Checksum
996
+		refOrCommit = ref
997
+	}
998
+	checkoutRef, err := gs.cache.New(ctx, nil, g, cache.WithRecordType(client.UsageRecordTypeGitCheckout), cache.WithDescription(fmt.Sprintf("git snapshot for %s#%s", urlutil.RedactCredentials(gs.src.Remote), gs.src.Ref)))
981 999
 	if err != nil {
982 1000
 		return nil, errors.Wrapf(err, "failed to create new mutable for %s", urlutil.RedactCredentials(gs.src.Remote))
983 1001
 	}
... ...
@@ -1034,7 +1217,7 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi
1034 1034
 			return nil, err
1035 1035
 		}
1036 1036
 
1037
-		gitCatFileBuf, err := git.Run(ctx, "cat-file", "-t", ref)
1037
+		gitCatFileBuf, err := git.Run(ctx, "cat-file", "-t", refOrCommit)
1038 1038
 		if err != nil {
1039 1039
 			return nil, err
1040 1040
 		}
... ...
@@ -1047,6 +1230,11 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi
1047 1047
 				targetRef = "refs/tags/" + pullref
1048 1048
 			}
1049 1049
 			pullref += ":" + targetRef
1050
+		} else if gs.src.FetchByCommit && ref != refOrCommit {
1051
+			// Fetch the commit object from the bare repo and save it under the
1052
+			// user-provided ref name in the checkout clone. The ref does not
1053
+			// need to exist in the bare repo.
1054
+			pullref = refOrCommit + ":" + ref
1050 1055
 		} else if gitutil.IsCommitSHA(ref) {
1051 1056
 			pullref = "refs/buildkit/" + identity.NewID()
1052 1057
 			_, err = git.Run(ctx, "update-ref", pullref, ref)
... ...
@@ -1084,7 +1272,7 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi
1084 1084
 			}
1085 1085
 		}
1086 1086
 		checkoutGit := git.New(gitutil.WithWorkTree(cd), gitutil.WithGitDir(gitDir))
1087
-		_, err = checkoutGit.Run(ctx, "checkout", "--no-overlay", ref, "--", ".")
1087
+		_, err = checkoutGit.Run(ctx, "checkout", "--no-overlay", refOrCommit, "--", ".")
1088 1088
 		if err != nil {
1089 1089
 			return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote))
1090 1090
 		}
... ...
@@ -1138,8 +1326,14 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi
1138 1138
 		}
1139 1139
 	}
1140 1140
 
1141
+	if compatibilityVersion == compat.CompatibilityVersion013 {
1142
+		if err := resetCompatibility014FileModes(checkoutDir); err != nil {
1143
+			return nil, errors.Wrapf(err, "failed to normalize compatibility file modes for %s", urlutil.RedactCredentials(gs.src.Remote))
1144
+		}
1145
+	}
1146
+
1141 1147
 	if gs.src.MTime == "commit" {
1142
-		commitTime, err := getCommitTime(ctx, git, ref)
1148
+		commitTime, err := getCommitTime(ctx, git, refOrCommit)
1143 1149
 		if err != nil {
1144 1150
 			return nil, errors.Wrapf(err, "failed to get commit time for %s", urlutil.RedactCredentials(gs.src.Remote))
1145 1151
 		}
... ...
@@ -1220,6 +1414,33 @@ func resetSnapshotMtimes(dir string, t time.Time) error {
1220 1220
 	return nil
1221 1221
 }
1222 1222
 
1223
+// resetCompatibility014FileModes restores the pre-v0.15 git checkout file
1224
+// mode for non-executable regular files, which were stored with group/other
1225
+// write bits set before the exec-option propagation fix. Executable files are
1226
+// left untouched: their pre-v0.15 behavior is not covered by the current
1227
+// compatibility matrix, and blindly adding write bits to 0o755 would be a
1228
+// guess.
1229
+func resetCompatibility014FileModes(dir string) error {
1230
+	return filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
1231
+		if err != nil {
1232
+			return err
1233
+		}
1234
+		if d.IsDir() || d.Type()&os.ModeSymlink != 0 {
1235
+			return nil
1236
+		}
1237
+
1238
+		info, err := d.Info()
1239
+		if err != nil {
1240
+			return err
1241
+		}
1242
+		mode := info.Mode()
1243
+		if !mode.IsRegular() || mode&0o111 != 0 {
1244
+			return nil
1245
+		}
1246
+		return os.Chmod(p, mode|0o222)
1247
+	})
1248
+}
1249
+
1223 1250
 type wouldClobberExistingTagError struct {
1224 1251
 	error
1225 1252
 }
1226 1253
new file mode 100644
... ...
@@ -0,0 +1,48 @@
0
+package appcontext
1
+
2
+import (
3
+	"context"
4
+	"os"
5
+	"os/signal"
6
+	"sync"
7
+
8
+	"github.com/moby/buildkit/util/bklog"
9
+	"github.com/pkg/errors"
10
+)
11
+
12
+var appContextCache context.Context
13
+var appContextOnce sync.Once
14
+
15
+// Context returns a static context that reacts to termination signals of the
16
+// running process. Useful in CLI tools.
17
+func Context() context.Context {
18
+	appContextOnce.Do(func() {
19
+		signals := make(chan os.Signal, 2048)
20
+		signal.Notify(signals, terminationSignals...)
21
+
22
+		const exitLimit = 3
23
+		retries := 0
24
+
25
+		ctx := context.Background()
26
+		for _, f := range inits {
27
+			ctx = f(ctx) //nolint:fatcontext
28
+		}
29
+
30
+		ctx, cancel := context.WithCancelCause(ctx)
31
+		appContextCache = ctx //nolint:fatcontext
32
+
33
+		go func() {
34
+			for {
35
+				<-signals
36
+				retries++
37
+				err := errors.Errorf("got %d SIGTERM/SIGINTs, forcing shutdown", retries)
38
+				cancel(err)
39
+				if retries >= exitLimit {
40
+					bklog.G(ctx).Error(err.Error())
41
+					os.Exit(1)
42
+				}
43
+			}
44
+		}()
45
+	})
46
+	return appContextCache
47
+}
0 48
new file mode 100644
... ...
@@ -0,0 +1,11 @@
0
+//go:build !windows
1
+
2
+package appcontext
3
+
4
+import (
5
+	"os"
6
+
7
+	"golang.org/x/sys/unix"
8
+)
9
+
10
+var terminationSignals = []os.Signal{unix.SIGTERM, unix.SIGINT}
0 11
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+package appcontext
1
+
2
+import (
3
+	"os"
4
+)
5
+
6
+var terminationSignals = []os.Signal{os.Interrupt}
0 7
new file mode 100644
... ...
@@ -0,0 +1,14 @@
0
+package appcontext
1
+
2
+import (
3
+	"context"
4
+)
5
+
6
+type Initializer func(context.Context) context.Context
7
+
8
+var inits []Initializer
9
+
10
+// Register stores a new context initializer that runs on app context creation
11
+func Register(f Initializer) {
12
+	inits = append(inits, f)
13
+}
... ...
@@ -27,6 +27,7 @@ type GitCLI struct {
27 27
 
28 28
 	sshAuthSock   string
29 29
 	sshKnownHosts string
30
+	hostGitConfig bool
30 31
 }
31 32
 
32 33
 // Option provides a variadic option for configuring the git client.
... ...
@@ -97,6 +98,15 @@ func WithSSHKnownHosts(sshKnownHosts string) Option {
97 97
 	}
98 98
 }
99 99
 
100
+// WithHostGitConfig allows git to read the host system and user git config.
101
+// This is intended for client-side local git inspection. The default remains
102
+// isolated so daemon-side callers do not leak host configuration into git.
103
+func WithHostGitConfig() Option {
104
+	return func(b *GitCLI) {
105
+		b.hostGitConfig = true
106
+	}
107
+}
108
+
100 109
 type StreamFunc func(context.Context) (io.WriteCloser, io.WriteCloser, func())
101 110
 
102 111
 // WithStreams configures a callback for getting the streams for a command. The
... ...
@@ -108,7 +118,7 @@ func WithStreams(streams StreamFunc) Option {
108 108
 	}
109 109
 }
110 110
 
111
-// New initializes a new git client
111
+// NewGitCLI initializes a new git client
112 112
 func NewGitCLI(opts ...Option) *GitCLI {
113 113
 	c := &GitCLI{}
114 114
 	for _, opt := range opts {
... ...
@@ -191,9 +201,28 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
191 191
 			"GIT_TERMINAL_PROMPT=0",
192 192
 			"GIT_SSH_COMMAND=" + getGitSSHCommand(cli.sshKnownHosts),
193 193
 			//	"GIT_TRACE=1",
194
-			"GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig.
195
-			"HOME=/dev/null",        // Disable reading from user gitconfig.
196
-			"LC_ALL=C",              // Ensure consistent output.
194
+			"LC_ALL=C", // Ensure consistent output.
195
+		}
196
+		if cli.hostGitConfig {
197
+			for _, ev := range [...]string{
198
+				"HOME",
199
+				"XDG_CONFIG_HOME",
200
+				"USERPROFILE",
201
+				"HOMEDRIVE",
202
+				"HOMEPATH",
203
+				"GIT_CONFIG_GLOBAL",
204
+				"GIT_CONFIG_SYSTEM",
205
+			} {
206
+				if v, ok := os.LookupEnv(ev); ok {
207
+					cmd.Env = append(cmd.Env, ev+"="+v)
208
+				}
209
+			}
210
+		} else {
211
+			cmd.Env = append(cmd.Env,
212
+				"GIT_CONFIG_NOSYSTEM=1",         // Disable reading from system gitconfig.
213
+				"HOME="+os.DevNull,              // Disable reading from user gitconfig.
214
+				"GIT_CONFIG_GLOBAL="+os.DevNull, // Disable reading from global gitconfig.
215
+			)
197 216
 		}
198 217
 		for _, ev := range proxyEnvVars {
199 218
 			if v, ok := os.LookupEnv(ev); ok {
... ...
@@ -244,7 +273,7 @@ func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ []byte, err error
244 244
 }
245 245
 
246 246
 func getGitSSHCommand(knownHosts string) string {
247
-	gitSSHCommand := "ssh -F /dev/null"
247
+	gitSSHCommand := "ssh -F " + os.DevNull
248 248
 	if knownHosts != "" {
249 249
 		gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts
250 250
 	} else {
... ...
@@ -1,7 +1,6 @@
1 1
 package progress
2 2
 
3 3
 import (
4
-	"maps"
5 4
 	"slices"
6 5
 	"sync"
7 6
 	"time"
... ...
@@ -81,18 +80,7 @@ func (ps *MultiWriter) Write(id string, v any) error {
81 81
 }
82 82
 
83 83
 func (ps *MultiWriter) WriteRawProgress(p *Progress) error {
84
-	meta := p.meta
85
-	if len(ps.meta) > 0 {
86
-		meta = map[string]any{}
87
-		maps.Copy(meta, p.meta)
88
-		for k, v := range ps.meta {
89
-			if _, ok := meta[k]; !ok {
90
-				meta[k] = v
91
-			}
92
-		}
93
-	}
94
-	p.meta = meta
95
-	return ps.writeRawProgress(p)
84
+	return ps.writeRawProgress(p.Decorate(ps.meta))
96 85
 }
97 86
 
98 87
 func (ps *MultiWriter) writeRawProgress(p *Progress) error {
... ...
@@ -235,18 +235,7 @@ func (pw *progressWriter) Write(id string, v any) error {
235 235
 }
236 236
 
237 237
 func (pw *progressWriter) WriteRawProgress(p *Progress) error {
238
-	meta := p.meta
239
-	if len(pw.meta) > 0 {
240
-		meta = map[string]any{}
241
-		maps.Copy(meta, p.meta)
242
-		for k, v := range pw.meta {
243
-			if _, ok := meta[k]; !ok {
244
-				meta[k] = v
245
-			}
246
-		}
247
-	}
248
-	p.meta = meta
249
-	return pw.writeRawProgress(p)
238
+	return pw.writeRawProgress(p.Decorate(pw.meta))
250 239
 }
251 240
 
252 241
 func (pw *progressWriter) writeRawProgress(p *Progress) error {
... ...
@@ -271,6 +260,21 @@ func (p *Progress) Meta(key string) (any, bool) {
271 271
 	return v, ok
272 272
 }
273 273
 
274
+// Decorate merges in missing metadata without overwriting existing keys.
275
+func (p *Progress) Decorate(meta map[string]any) *Progress {
276
+	if len(meta) == 0 {
277
+		return p
278
+	}
279
+
280
+	merged := make(map[string]any, len(p.meta)+len(meta))
281
+	maps.Copy(merged, meta)
282
+	maps.Copy(merged, p.meta)
283
+
284
+	newP := *p
285
+	newP.meta = merged
286
+	return &newP
287
+}
288
+
274 289
 type noOpWriter struct{}
275 290
 
276 291
 func (pw *noOpWriter) Write(_ string, _ any) error {
277 292
new file mode 100644
... ...
@@ -0,0 +1,110 @@
0
+package registrar
1
+
2
+import (
3
+	"context"
4
+	"sync"
5
+	"time"
6
+)
7
+
8
+type Registrar[K comparable, V any] struct {
9
+	mu     sync.Mutex
10
+	values map[K]*registrarValue[V]
11
+}
12
+
13
+func New[K comparable, V any]() *Registrar[K, V] {
14
+	return &Registrar[K, V]{
15
+		values: make(map[K]*registrarValue[V]),
16
+	}
17
+}
18
+
19
+// Register will register the value with the given id.
20
+// This value will persist until Discard is called with the same id.
21
+func (r *Registrar[K, V]) Register(id K, val V) {
22
+	reg := r.getOrCreateRegistrar(id, nil)
23
+	reg.Register(val, nil)
24
+}
25
+
26
+// Get will retrieve a registered value and will wait a small time period for that
27
+// value to appear if it hasn't been registered yet.
28
+func (r *Registrar[K, V]) Get(ctx context.Context, id K) (v V, _ error) {
29
+	onCreate := func(reg *registrarValue[V]) {
30
+		select {
31
+		case <-reg.notifyCh:
32
+			return
33
+		case <-time.After(3 * time.Second):
34
+			r.Discard(id)
35
+		}
36
+	}
37
+
38
+	reg := r.getOrCreateRegistrar(id, onCreate)
39
+
40
+	select {
41
+	case <-ctx.Done():
42
+		return v, context.Cause(ctx)
43
+	case <-reg.notifyCh:
44
+		return reg.value, reg.err
45
+	}
46
+}
47
+
48
+// Discard will remove the given value from the registrar after it has been registered
49
+// with Register.
50
+func (r *Registrar[K, V]) Discard(id K) {
51
+	r.mu.Lock()
52
+	reg, ok := r.values[id]
53
+	delete(r.values, id)
54
+	r.mu.Unlock()
55
+
56
+	if ok {
57
+		var value V
58
+		reg.Register(value, context.Canceled)
59
+	}
60
+}
61
+
62
+// getOrCreateRegistrar will create a registrar with the given id to be retrieved at a later time.
63
+// The same id will return the same registrar.
64
+//
65
+// If the registrar is newly created, the onCreate function is invoked in a separate goroutine
66
+// if it is present. If nil, this function is ignored.
67
+func (r *Registrar[K, V]) getOrCreateRegistrar(id K, onCreate func(*registrarValue[V])) *registrarValue[V] {
68
+	r.mu.Lock()
69
+	defer r.mu.Unlock()
70
+
71
+	reg, ok := r.values[id]
72
+	if !ok {
73
+		reg = &registrarValue[V]{
74
+			notifyCh: make(chan struct{}),
75
+		}
76
+		r.values[id] = reg
77
+
78
+		if onCreate != nil {
79
+			go onCreate(reg)
80
+		}
81
+	}
82
+	return reg
83
+}
84
+
85
+type registrarValue[V any] struct {
86
+	// notifyCh is the notification channel that gets closed when
87
+	// the bridge is registered.
88
+	notifyCh chan struct{}
89
+
90
+	value V
91
+	err   error
92
+	isSet bool
93
+
94
+	mu sync.Mutex
95
+}
96
+
97
+func (r *registrarValue[V]) Register(value V, err error) {
98
+	r.mu.Lock()
99
+	defer r.mu.Unlock()
100
+
101
+	if r.isSet {
102
+		return
103
+	}
104
+
105
+	r.value = value
106
+	r.err = err
107
+	r.isSet = true
108
+	close(r.notifyCh)
109
+}
... ...
@@ -47,6 +47,7 @@ func newAuthHandlerNS(sm *session.Manager) *authHandlerNS {
47 47
 }
48 48
 
49 49
 func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manager, g session.Group) *authFetcher {
50
+	hasSession := false
50 51
 	if g != nil {
51 52
 		if iter := g.SessionIterator(); iter != nil {
52 53
 			for {
... ...
@@ -54,6 +55,7 @@ func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manage
54 54
 				if id == "" {
55 55
 					break
56 56
 				}
57
+				hasSession = true
57 58
 				h, ok := a.fetchers[host+"/"+id]
58 59
 				if ok {
59 60
 					h.lastUsed = time.Now()
... ...
@@ -63,6 +65,14 @@ func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manage
63 63
 		}
64 64
 	}
65 65
 
66
+	if !hasSession {
67
+		h, ok := a.fetchers[host+"/"]
68
+		if ok {
69
+			h.lastUsed = time.Now()
70
+			return h
71
+		}
72
+	}
73
+
66 74
 	// link existing fetcher
67 75
 	for k, h := range a.fetchers {
68 76
 		parts := strings.SplitN(k, "/", 2)
... ...
@@ -78,7 +88,7 @@ func (a *authHandlerNS) get(ctx context.Context, host string, sm *session.Manage
78 78
 					return h
79 79
 				}
80 80
 			} else {
81
-				sessionID, username, password, err := sessionauth.CredentialsFunc(sm, g)(host)
81
+				sessionID, username, password, err := sessionauth.CredentialsFunc(ctx, sm, g)(host)
82 82
 				if err == nil {
83 83
 					if username == h.common.Username && password == h.common.Secret {
84 84
 						a.fetchers[host+"/"+sessionID] = h
... ...
@@ -140,8 +150,8 @@ func (a *dockerAuthorizer) Authorize(ctx context.Context, req *http.Request) err
140 140
 	return nil
141 141
 }
142 142
 
143
-func (a *dockerAuthorizer) getCredentials(host string) (sessionID, username, secret string, err error) {
144
-	return sessionauth.CredentialsFunc(a.sm, a.session)(host)
143
+func (a *dockerAuthorizer) getCredentials(ctx context.Context, host string) (sessionID, username, secret string, err error) {
144
+	return sessionauth.CredentialsFunc(ctx, a.sm, a.session)(host)
145 145
 }
146 146
 
147 147
 func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.Response) error {
... ...
@@ -185,12 +195,12 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R
185 185
 
186 186
 			var username, secret string
187 187
 			sessionID, pubKey, err := sessionauth.GetTokenAuthority(ctx, host, a.sm, a.session)
188
-			if err != nil {
188
+			if err != nil && !errors.Is(err, session.ErrNoActiveSessions) {
189 189
 				return err
190 190
 			}
191 191
 			if pubKey == nil {
192
-				sessionID, username, secret, err = a.getCredentials(host)
193
-				if err != nil {
192
+				sessionID, username, secret, err = a.getCredentials(ctx, host)
193
+				if err != nil && !errors.Is(err, session.ErrNoActiveSessions) {
194 194
 					return err
195 195
 				}
196 196
 			}
... ...
@@ -205,7 +215,7 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R
205 205
 
206 206
 			return nil
207 207
 		case auth.BasicAuth:
208
-			sessionID, username, secret, err := a.getCredentials(host)
208
+			sessionID, username, secret, err := a.getCredentials(ctx, host)
209 209
 			if err != nil {
210 210
 				return err
211 211
 			}
212 212
new file mode 100644
... ...
@@ -0,0 +1,43 @@
0
+package childprocess
1
+
2
+const (
3
+	// go.opentelemetry.io/otel/propagation doesn't export these as constants.
4
+	traceparentHeader = "traceparent"
5
+	tracestateHeader  = "tracestate"
6
+)
7
+
8
+type textMap struct {
9
+	parent string
10
+	state  string
11
+}
12
+
13
+func (tm *textMap) Get(key string) string {
14
+	switch key {
15
+	case traceparentHeader:
16
+		return tm.parent
17
+	case tracestateHeader:
18
+		return tm.state
19
+	default:
20
+		return ""
21
+	}
22
+}
23
+
24
+func (tm *textMap) Set(key string, value string) {
25
+	switch key {
26
+	case traceparentHeader:
27
+		tm.parent = value
28
+	case tracestateHeader:
29
+		tm.state = value
30
+	}
31
+}
32
+
33
+func (tm *textMap) Keys() []string {
34
+	var k []string
35
+	if tm.parent != "" {
36
+		k = append(k, traceparentHeader)
37
+	}
38
+	if tm.state != "" {
39
+		k = append(k, tracestateHeader)
40
+	}
41
+	return k
42
+}
0 43
new file mode 100644
... ...
@@ -0,0 +1,36 @@
0
+package childprocess
1
+
2
+import (
3
+	"context"
4
+	"os"
5
+
6
+	"github.com/moby/buildkit/util/appcontext"
7
+	"go.opentelemetry.io/otel/propagation"
8
+)
9
+
10
+func init() {
11
+	appcontext.Register(initContext)
12
+}
13
+
14
+func initContext(ctx context.Context) context.Context {
15
+	// open-telemetry/opentelemetry-specification#740
16
+	parent := os.Getenv("TRACEPARENT")
17
+	state := os.Getenv("TRACESTATE")
18
+
19
+	if parent != "" {
20
+		tc := propagation.TraceContext{}
21
+		return tc.Extract(ctx, &textMap{parent: parent, state: state})
22
+	}
23
+
24
+	// deprecated: removed in v0.11.0
25
+	// previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15
26
+	parent = os.Getenv("OTEL_TRACE_PARENT")
27
+	state = os.Getenv("OTEL_TRACE_STATE")
28
+
29
+	if parent == "" {
30
+		return ctx
31
+	}
32
+
33
+	tc := propagation.TraceContext{}
34
+	return tc.Extract(ctx, &textMap{parent: parent, state: state})
35
+}
0 36
new file mode 100644
... ...
@@ -0,0 +1,36 @@
0
+package childprocess
1
+
2
+import (
3
+	"context"
4
+
5
+	"go.opentelemetry.io/otel/propagation"
6
+)
7
+
8
+// Environ returns list of environment variables that need to be sent to the child process
9
+// in order for it to pick up cross-process tracing from same state.
10
+func Environ(ctx context.Context) []string {
11
+	var tm textMap
12
+	tc := propagation.TraceContext{}
13
+	tc.Inject(ctx, &tm)
14
+
15
+	var env []string
16
+
17
+	// deprecated: removed in v0.11.0
18
+	// previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15
19
+	if tm.parent != "" {
20
+		env = append(env, "OTEL_TRACE_PARENT="+tm.parent)
21
+	}
22
+	if tm.state != "" {
23
+		env = append(env, "OTEL_TRACE_STATE="+tm.state)
24
+	}
25
+
26
+	// open-telemetry/opentelemetry-specification#740
27
+	if tm.parent != "" {
28
+		env = append(env, "TRACEPARENT="+tm.parent)
29
+	}
30
+	if tm.state != "" {
31
+		env = append(env, "TRACESTATE="+tm.state)
32
+	}
33
+
34
+	return env
35
+}
0 36
deleted file mode 100644
... ...
@@ -1,77 +0,0 @@
1
-package detect
2
-
3
-import (
4
-	"context"
5
-
6
-	"go.opentelemetry.io/otel/propagation"
7
-)
8
-
9
-const (
10
-	traceparentHeader = "traceparent"
11
-	tracestateHeader  = "tracestate"
12
-)
13
-
14
-// Environ returns list of environment variables that need to be sent to the child process
15
-// in order for it to pick up cross-process tracing from same state.
16
-func Environ(ctx context.Context) []string {
17
-	var tm textMap
18
-	tc := propagation.TraceContext{}
19
-	tc.Inject(ctx, &tm)
20
-
21
-	var env []string
22
-
23
-	// deprecated: removed in v0.11.0
24
-	// previously defined in https://github.com/open-telemetry/opentelemetry-swift/blob/4ea467ed4b881d7329bf2254ca7ed7f2d9d6e1eb/Sources/OpenTelemetrySdk/Trace/Propagation/EnvironmentContextPropagator.swift#L14-L15
25
-	if tm.parent != "" {
26
-		env = append(env, "OTEL_TRACE_PARENT="+tm.parent)
27
-	}
28
-	if tm.state != "" {
29
-		env = append(env, "OTEL_TRACE_STATE="+tm.state)
30
-	}
31
-
32
-	// open-telemetry/opentelemetry-specification#740
33
-	if tm.parent != "" {
34
-		env = append(env, "TRACEPARENT="+tm.parent)
35
-	}
36
-	if tm.state != "" {
37
-		env = append(env, "TRACESTATE="+tm.state)
38
-	}
39
-
40
-	return env
41
-}
42
-
43
-type textMap struct {
44
-	parent string
45
-	state  string
46
-}
47
-
48
-func (tm *textMap) Get(key string) string {
49
-	switch key {
50
-	case traceparentHeader:
51
-		return tm.parent
52
-	case tracestateHeader:
53
-		return tm.state
54
-	default:
55
-		return ""
56
-	}
57
-}
58
-
59
-func (tm *textMap) Set(key string, value string) {
60
-	switch key {
61
-	case traceparentHeader:
62
-		tm.parent = value
63
-	case tracestateHeader:
64
-		tm.state = value
65
-	}
66
-}
67
-
68
-func (tm *textMap) Keys() []string {
69
-	var k []string
70
-	if tm.parent != "" {
71
-		k = append(k, traceparentHeader)
72
-	}
73
-	if tm.state != "" {
74
-		k = append(k, tracestateHeader)
75
-	}
76
-	return k
77
-}
... ...
@@ -80,7 +80,7 @@ func (c *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.Resourc
80 80
 		c.lock.Lock()
81 81
 		defer c.lock.Unlock()
82 82
 		if c.tracesClient == nil {
83
-			return errNoClient
83
+			return errors.New("no client")
84 84
 		}
85 85
 
86 86
 		_, err := c.tracesClient.Export(ctx, &coltracepb.ExportTraceServiceRequest{
87 87
deleted file mode 100644
... ...
@@ -1,7 +0,0 @@
1
-package otlptracegrpc
2
-
3
-import "errors"
4
-
5
-var (
6
-	errNoClient = errors.New("no client")
7
-)
... ...
@@ -159,6 +159,7 @@ func NewWorker(ctx context.Context, opt WorkerOpt) (*Worker, error) {
159 159
 	if err := git.Supported(); err == nil {
160 160
 		gs, err := git.NewSource(git.Opt{
161 161
 			CacheAccessor: cm,
162
+			RegistryHosts: opt.RegistryHosts,
162 163
 		})
163 164
 		if err != nil {
164 165
 			return nil, err
... ...
@@ -238,6 +238,8 @@ type CommonLanguageSettings struct {
238 238
 	// The destination where API teams want this client library to be published.
239 239
 	Destinations []ClientLibraryDestination `protobuf:"varint,2,rep,packed,name=destinations,proto3,enum=google.api.ClientLibraryDestination" json:"destinations,omitempty"`
240 240
 	// Configuration for which RPCs should be generated in the GAPIC client.
241
+	//
242
+	// Note: This field should not be used in most cases.
241 243
 	SelectiveGapicGeneration *SelectiveGapicGeneration `protobuf:"bytes,3,opt,name=selective_gapic_generation,json=selectiveGapicGeneration,proto3" json:"selective_gapic_generation,omitempty"`
242 244
 }
243 245
 
... ...
@@ -1249,6 +1251,8 @@ func (x *MethodSettings) GetBatching() *BatchingConfigProto {
1249 1249
 
1250 1250
 // This message is used to configure the generation of a subset of the RPCs in
1251 1251
 // a service for client libraries.
1252
+//
1253
+// Note: This feature should not be used in most cases.
1252 1254
 type SelectiveGapicGeneration struct {
1253 1255
 	state         protoimpl.MessageState
1254 1256
 	sizeCache     protoimpl.SizeCache
... ...
@@ -950,7 +950,7 @@ github.com/ishidawataru/sctp
950 950
 # github.com/jmoiron/sqlx v1.4.0
951 951
 ## explicit; go 1.10
952 952
 github.com/jmoiron/sqlx/types
953
-# github.com/klauspost/compress v1.18.5
953
+# github.com/klauspost/compress v1.18.6
954 954
 ## explicit; go 1.24
955 955
 github.com/klauspost/compress
956 956
 github.com/klauspost/compress/fse
... ...
@@ -978,7 +978,7 @@ github.com/mitchellh/hashstructure/v2
978 978
 # github.com/mitchellh/reflectwalk v1.0.2
979 979
 ## explicit
980 980
 github.com/mitchellh/reflectwalk
981
-# github.com/moby/buildkit v0.29.0
981
+# github.com/moby/buildkit v0.30.0-rc1
982 982
 ## explicit; go 1.25.5
983 983
 github.com/moby/buildkit/api/services/control
984 984
 github.com/moby/buildkit/api/types
... ...
@@ -1065,6 +1065,7 @@ github.com/moby/buildkit/solver/errdefs
1065 1065
 github.com/moby/buildkit/solver/internal/pipe
1066 1066
 github.com/moby/buildkit/solver/llbsolver
1067 1067
 github.com/moby/buildkit/solver/llbsolver/cdidevices
1068
+github.com/moby/buildkit/solver/llbsolver/compat
1068 1069
 github.com/moby/buildkit/solver/llbsolver/errdefs
1069 1070
 github.com/moby/buildkit/solver/llbsolver/file
1070 1071
 github.com/moby/buildkit/solver/llbsolver/history
... ...
@@ -1079,6 +1080,7 @@ github.com/moby/buildkit/solver/pb
1079 1079
 github.com/moby/buildkit/solver/result
1080 1080
 github.com/moby/buildkit/source
1081 1081
 github.com/moby/buildkit/source/containerblob
1082
+github.com/moby/buildkit/source/containerblob/blobfetch
1082 1083
 github.com/moby/buildkit/source/containerimage
1083 1084
 github.com/moby/buildkit/source/git
1084 1085
 github.com/moby/buildkit/source/http
... ...
@@ -1090,6 +1092,7 @@ github.com/moby/buildkit/sourcepolicy/pb
1090 1090
 github.com/moby/buildkit/sourcepolicy/policysession
1091 1091
 github.com/moby/buildkit/util/apicaps
1092 1092
 github.com/moby/buildkit/util/apicaps/pb
1093
+github.com/moby/buildkit/util/appcontext
1093 1094
 github.com/moby/buildkit/util/appdefaults
1094 1095
 github.com/moby/buildkit/util/archutil
1095 1096
 github.com/moby/buildkit/util/attestation
... ...
@@ -1132,6 +1135,7 @@ github.com/moby/buildkit/util/pull
1132 1132
 github.com/moby/buildkit/util/pull/pullprogress
1133 1133
 github.com/moby/buildkit/util/purl
1134 1134
 github.com/moby/buildkit/util/push
1135
+github.com/moby/buildkit/util/registrar
1135 1136
 github.com/moby/buildkit/util/resolvconf
1136 1137
 github.com/moby/buildkit/util/resolver
1137 1138
 github.com/moby/buildkit/util/resolver/config
... ...
@@ -1146,8 +1150,8 @@ github.com/moby/buildkit/util/suggest
1146 1146
 github.com/moby/buildkit/util/system
1147 1147
 github.com/moby/buildkit/util/throttle
1148 1148
 github.com/moby/buildkit/util/tracing
1149
+github.com/moby/buildkit/util/tracing/childprocess
1149 1150
 github.com/moby/buildkit/util/tracing/detect
1150
-github.com/moby/buildkit/util/tracing/exec
1151 1151
 github.com/moby/buildkit/util/tracing/otlptracegrpc
1152 1152
 github.com/moby/buildkit/util/tracing/transform
1153 1153
 github.com/moby/buildkit/util/urlutil
... ...
@@ -1970,7 +1974,7 @@ google.golang.org/api/transport/http
1970 1970
 # google.golang.org/genproto v0.0.0-20260401024825-9d38bb4040a9
1971 1971
 ## explicit; go 1.25.0
1972 1972
 google.golang.org/genproto/googleapis/logging/type
1973
-# google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9
1973
+# google.golang.org/genproto/googleapis/api v0.0.0-20260406210006-6f92a3bedf2d
1974 1974
 ## explicit; go 1.25.0
1975 1975
 google.golang.org/genproto/googleapis/api
1976 1976
 google.golang.org/genproto/googleapis/api/annotations