full diff: https://github.com/moby/go-archive/compare/21f3f3385ab7...v0.1.0
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
| ... | ... |
@@ -9,8 +9,8 @@ import ( |
| 9 | 9 |
"github.com/docker/docker/pkg/longpath" |
| 10 | 10 |
"github.com/docker/docker/pkg/system" |
| 11 | 11 |
"github.com/docker/docker/pkg/tarsum" |
| 12 |
- "github.com/moby/go-archive" |
|
| 13 | 12 |
"github.com/moby/go-archive/chrootarchive" |
| 13 |
+ "github.com/moby/go-archive/compression" |
|
| 14 | 14 |
"github.com/moby/sys/symlink" |
| 15 | 15 |
"github.com/pkg/errors" |
| 16 | 16 |
) |
| ... | ... |
@@ -66,7 +66,7 @@ func FromArchive(tarStream io.Reader) (builder.Source, error) {
|
| 66 | 66 |
} |
| 67 | 67 |
}() |
| 68 | 68 |
|
| 69 |
- decompressedStream, err := archive.DecompressStream(tarStream) |
|
| 69 |
+ decompressedStream, err := compression.DecompressStream(tarStream) |
|
| 70 | 70 |
if err != nil {
|
| 71 | 71 |
return nil, err |
| 72 | 72 |
} |
| ... | ... |
@@ -19,7 +19,7 @@ import ( |
| 19 | 19 |
"github.com/docker/docker/daemon/images" |
| 20 | 20 |
"github.com/docker/docker/errdefs" |
| 21 | 21 |
"github.com/docker/docker/pkg/streamformatter" |
| 22 |
- dockerarchive "github.com/moby/go-archive" |
|
| 22 |
+ "github.com/moby/go-archive/compression" |
|
| 23 | 23 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 24 | 24 |
"github.com/pkg/errors" |
| 25 | 25 |
) |
| ... | ... |
@@ -230,7 +230,7 @@ func (i *ImageService) leaseContent(ctx context.Context, store content.Store, de |
| 230 | 230 |
// complement of ExportImage. The input stream is an uncompressed tar |
| 231 | 231 |
// ball containing images and metadata. |
| 232 | 232 |
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platform *ocispec.Platform, outStream io.Writer, quiet bool) error {
|
| 233 |
- decompressed, err := dockerarchive.DecompressStream(inTar) |
|
| 233 |
+ decompressed, err := compression.DecompressStream(inTar) |
|
| 234 | 234 |
if err != nil {
|
| 235 | 235 |
return errors.Wrap(err, "failed to decompress input tar archive") |
| 236 | 236 |
} |
| ... | ... |
@@ -23,7 +23,7 @@ import ( |
| 23 | 23 |
"github.com/docker/docker/pkg/pools" |
| 24 | 24 |
"github.com/google/uuid" |
| 25 | 25 |
imagespec "github.com/moby/docker-image-spec/specs-go/v1" |
| 26 |
- "github.com/moby/go-archive" |
|
| 26 |
+ "github.com/moby/go-archive/compression" |
|
| 27 | 27 |
"github.com/opencontainers/go-digest" |
| 28 | 28 |
"github.com/opencontainers/image-spec/specs-go" |
| 29 | 29 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -168,17 +168,17 @@ func saveArchive(ctx context.Context, cs content.Store, layerReader io.Reader) ( |
| 168 | 168 |
bufRd := p.Get(layerReader) |
| 169 | 169 |
defer p.Put(bufRd) |
| 170 | 170 |
|
| 171 |
- compression, err := detectCompression(bufRd) |
|
| 171 |
+ comp, err := detectCompression(bufRd) |
|
| 172 | 172 |
if err != nil {
|
| 173 | 173 |
return "", "", "", err |
| 174 | 174 |
} |
| 175 | 175 |
|
| 176 | 176 |
var uncompressedReader io.Reader = bufRd |
| 177 |
- switch compression {
|
|
| 178 |
- case archive.Gzip, archive.Zstd: |
|
| 177 |
+ switch comp {
|
|
| 178 |
+ case compression.Gzip, compression.Zstd: |
|
| 179 | 179 |
// If the input is already a compressed layer, just save it as is. |
| 180 | 180 |
mediaType := ocispec.MediaTypeImageLayerGzip |
| 181 |
- if compression == archive.Zstd {
|
|
| 181 |
+ if comp == compression.Zstd {
|
|
| 182 | 182 |
mediaType = ocispec.MediaTypeImageLayerZstd |
| 183 | 183 |
} |
| 184 | 184 |
|
| ... | ... |
@@ -188,19 +188,17 @@ func saveArchive(ctx context.Context, cs content.Store, layerReader io.Reader) ( |
| 188 | 188 |
} |
| 189 | 189 |
|
| 190 | 190 |
return compressedDigest, uncompressedDigest, mediaType, nil |
| 191 |
- case archive.Bzip2, archive.Xz: |
|
| 192 |
- r, err := archive.DecompressStream(bufRd) |
|
| 191 |
+ case compression.Bzip2, compression.Xz: |
|
| 192 |
+ r, err := compression.DecompressStream(bufRd) |
|
| 193 | 193 |
if err != nil {
|
| 194 | 194 |
return "", "", "", errdefs.InvalidParameter(err) |
| 195 | 195 |
} |
| 196 | 196 |
defer r.Close() |
| 197 | 197 |
uncompressedReader = r |
| 198 | 198 |
fallthrough |
| 199 |
- case archive.Uncompressed: |
|
| 199 |
+ case compression.None: |
|
| 200 | 200 |
mediaType := ocispec.MediaTypeImageLayerGzip |
| 201 |
- compression := archive.Gzip |
|
| 202 |
- |
|
| 203 |
- compressedDigest, uncompressedDigest, err := compressAndWriteBlob(ctx, cs, compression, mediaType, uncompressedReader) |
|
| 201 |
+ compressedDigest, uncompressedDigest, err := compressAndWriteBlob(ctx, cs, compression.Gzip, mediaType, uncompressedReader) |
|
| 204 | 202 |
if err != nil {
|
| 205 | 203 |
return "", "", "", err |
| 206 | 204 |
} |
| ... | ... |
@@ -228,7 +226,7 @@ func writeCompressedBlob(ctx context.Context, cs content.Store, mediaType string |
| 228 | 228 |
digester := digest.Canonical.Digester() |
| 229 | 229 |
|
| 230 | 230 |
// Decompress the piped blob. |
| 231 |
- decompressedStream, err := archive.DecompressStream(pr) |
|
| 231 |
+ decompressedStream, err := compression.DecompressStream(pr) |
|
| 232 | 232 |
if err == nil {
|
| 233 | 233 |
// Feed the digester with decompressed data. |
| 234 | 234 |
_, err = io.Copy(digester.Hash(), decompressedStream) |
| ... | ... |
@@ -249,12 +247,12 @@ func writeCompressedBlob(ctx context.Context, cs content.Store, mediaType string |
| 249 | 249 |
} |
| 250 | 250 |
|
| 251 | 251 |
// compressAndWriteBlob compresses the uncompressedReader and stores it in the content store. |
| 252 |
-func compressAndWriteBlob(ctx context.Context, cs content.Store, compression archive.Compression, mediaType string, uncompressedLayerReader io.Reader) (digest.Digest, digest.Digest, error) {
|
|
| 252 |
+func compressAndWriteBlob(ctx context.Context, cs content.Store, comp compression.Compression, mediaType string, uncompressedLayerReader io.Reader) (digest.Digest, digest.Digest, error) {
|
|
| 253 | 253 |
pr, pw := io.Pipe() |
| 254 | 254 |
defer pr.Close() |
| 255 | 255 |
defer pw.Close() |
| 256 | 256 |
|
| 257 |
- compressor, err := archive.CompressStream(pw, compression) |
|
| 257 |
+ compressor, err := compression.CompressStream(pw, comp) |
|
| 258 | 258 |
if err != nil {
|
| 259 | 259 |
return "", "", errdefs.InvalidParameter(err) |
| 260 | 260 |
} |
| ... | ... |
@@ -312,7 +310,7 @@ func (i *ImageService) unpackImage(ctx context.Context, snapshotter string, img |
| 312 | 312 |
} |
| 313 | 313 |
|
| 314 | 314 |
// detectCompression detects the reader compression type. |
| 315 |
-func detectCompression(bufRd *bufio.Reader) (archive.Compression, error) {
|
|
| 315 |
+func detectCompression(bufRd *bufio.Reader) (compression.Compression, error) {
|
|
| 316 | 316 |
bs, err := bufRd.Peek(10) |
| 317 | 317 |
if err != nil && err != io.EOF {
|
| 318 | 318 |
// Note: we'll ignore any io.EOF error because there are some odd |
| ... | ... |
@@ -321,10 +319,10 @@ func detectCompression(bufRd *bufio.Reader) (archive.Compression, error) {
|
| 321 | 321 |
// cases we'll just treat it as a non-compressed stream and |
| 322 | 322 |
// that means just create an empty layer. |
| 323 | 323 |
// See Issue 18170 |
| 324 |
- return archive.Uncompressed, errdefs.Unknown(err) |
|
| 324 |
+ return compression.None, errdefs.Unknown(err) |
|
| 325 | 325 |
} |
| 326 | 326 |
|
| 327 |
- return archive.DetectCompression(bs), nil |
|
| 327 |
+ return compression.Detect(bs), nil |
|
| 328 | 328 |
} |
| 329 | 329 |
|
| 330 | 330 |
// fillUncompressedLabel sets the uncompressed digest label on the compressed blob metadata |
| ... | ... |
@@ -15,7 +15,7 @@ import ( |
| 15 | 15 |
"github.com/docker/docker/errdefs" |
| 16 | 16 |
"github.com/docker/docker/image" |
| 17 | 17 |
"github.com/docker/docker/layer" |
| 18 |
- "github.com/moby/go-archive" |
|
| 18 |
+ "github.com/moby/go-archive/compression" |
|
| 19 | 19 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 20 | 20 |
) |
| 21 | 21 |
|
| ... | ... |
@@ -40,7 +40,7 @@ func (i *ImageService) ImportImage(ctx context.Context, newRef reference.Named, |
| 40 | 40 |
return "", errdefs.InvalidParameter(err) |
| 41 | 41 |
} |
| 42 | 42 |
|
| 43 |
- inflatedLayerData, err := archive.DecompressStream(layerReader) |
|
| 43 |
+ inflatedLayerData, err := compression.DecompressStream(layerReader) |
|
| 44 | 44 |
if err != nil {
|
| 45 | 45 |
return "", err |
| 46 | 46 |
} |
| ... | ... |
@@ -13,7 +13,7 @@ import ( |
| 13 | 13 |
"github.com/docker/docker/layer" |
| 14 | 14 |
"github.com/docker/docker/pkg/ioutils" |
| 15 | 15 |
"github.com/docker/docker/pkg/progress" |
| 16 |
- "github.com/moby/go-archive" |
|
| 16 |
+ "github.com/moby/go-archive/compression" |
|
| 17 | 17 |
) |
| 18 | 18 |
|
| 19 | 19 |
const maxDownloadAttempts = 5 |
| ... | ... |
@@ -338,7 +338,7 @@ func (ldm *LayerDownloadManager) makeDownloadFunc(descriptor DownloadDescriptor, |
| 338 | 338 |
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(d.transfer.context(), downloadReader), progressOutput, size, descriptor.ID(), "Extracting") |
| 339 | 339 |
defer reader.Close() |
| 340 | 340 |
|
| 341 |
- inflatedLayerData, err := archive.DecompressStream(reader) |
|
| 341 |
+ inflatedLayerData, err := compression.DecompressStream(reader) |
|
| 342 | 342 |
if err != nil {
|
| 343 | 343 |
d.err = fmt.Errorf("could not get decompression stream: %v", err)
|
| 344 | 344 |
return |
| ... | ... |
@@ -23,8 +23,8 @@ import ( |
| 23 | 23 |
"github.com/docker/docker/pkg/progress" |
| 24 | 24 |
"github.com/docker/docker/pkg/streamformatter" |
| 25 | 25 |
"github.com/docker/docker/pkg/stringid" |
| 26 |
- "github.com/moby/go-archive" |
|
| 27 | 26 |
"github.com/moby/go-archive/chrootarchive" |
| 27 |
+ "github.com/moby/go-archive/compression" |
|
| 28 | 28 |
"github.com/moby/sys/sequential" |
| 29 | 29 |
"github.com/moby/sys/symlink" |
| 30 | 30 |
"github.com/opencontainers/go-digest" |
| ... | ... |
@@ -231,7 +231,7 @@ func (l *tarexporter) loadLayer(ctx context.Context, filename string, rootFS ima |
| 231 | 231 |
r = rawTar |
| 232 | 232 |
} |
| 233 | 233 |
|
| 234 |
- inflatedLayerData, err := archive.DecompressStream(ioutils.NewCtxReader(ctx, r)) |
|
| 234 |
+ inflatedLayerData, err := compression.DecompressStream(ioutils.NewCtxReader(ctx, r)) |
|
| 235 | 235 |
if err != nil {
|
| 236 | 236 |
return nil, err |
| 237 | 237 |
} |
| ... | ... |
@@ -26,6 +26,7 @@ import ( |
| 26 | 26 |
"github.com/docker/docker/testutil/fakestorage" |
| 27 | 27 |
"github.com/moby/buildkit/frontend/dockerfile/command" |
| 28 | 28 |
"github.com/moby/go-archive" |
| 29 |
+ "github.com/moby/go-archive/compression" |
|
| 29 | 30 |
"github.com/opencontainers/go-digest" |
| 30 | 31 |
"gotest.tools/v3/assert" |
| 31 | 32 |
is "gotest.tools/v3/assert/cmp" |
| ... | ... |
@@ -2008,7 +2009,7 @@ func (s *DockerCLIBuildSuite) TestBuildAddLocalAndRemoteFilesWithAndWithoutCache |
| 2008 | 2008 |
} |
| 2009 | 2009 |
} |
| 2010 | 2010 |
|
| 2011 |
-func testContextTar(c *testing.T, compression archive.Compression) {
|
|
| 2011 |
+func testContextTar(c *testing.T, comp compression.Compression) {
|
|
| 2012 | 2012 |
ctx := fakecontext.New(c, "", |
| 2013 | 2013 |
fakecontext.WithDockerfile(`FROM busybox |
| 2014 | 2014 |
ADD foo /foo |
| ... | ... |
@@ -2018,13 +2019,13 @@ CMD ["cat", "/foo"]`), |
| 2018 | 2018 |
}), |
| 2019 | 2019 |
) |
| 2020 | 2020 |
defer ctx.Close() |
| 2021 |
- context, err := archive.Tar(ctx.Dir, compression) |
|
| 2021 |
+ buildContext, err := archive.Tar(ctx.Dir, comp) |
|
| 2022 | 2022 |
if err != nil {
|
| 2023 | 2023 |
c.Fatalf("failed to build context tar: %v", err)
|
| 2024 | 2024 |
} |
| 2025 | 2025 |
const name = "contexttar" |
| 2026 | 2026 |
|
| 2027 |
- cli.BuildCmd(c, name, build.WithStdinContext(context)) |
|
| 2027 |
+ cli.BuildCmd(c, name, build.WithStdinContext(buildContext)) |
|
| 2028 | 2028 |
} |
| 2029 | 2029 |
|
| 2030 | 2030 |
func (s *DockerCLIBuildSuite) TestBuildContextTarGzip(c *testing.T) {
|
| ... | ... |
@@ -23,7 +23,7 @@ import ( |
| 23 | 23 |
"github.com/docker/docker/internal/testutils" |
| 24 | 24 |
"github.com/docker/docker/internal/testutils/specialimage" |
| 25 | 25 |
"github.com/docker/docker/testutil/fakecontext" |
| 26 |
- "github.com/moby/go-archive" |
|
| 26 |
+ "github.com/moby/go-archive/compression" |
|
| 27 | 27 |
"github.com/opencontainers/go-digest" |
| 28 | 28 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 29 | 29 |
"gotest.tools/v3/assert" |
| ... | ... |
@@ -360,7 +360,7 @@ RUN touch /opt/a/b/c && chown user:user /opt/a/b/c` |
| 360 | 360 |
|
| 361 | 361 |
func listTar(f io.Reader) ([]string, error) {
|
| 362 | 362 |
// If using the containerd snapshotter, the tar file may be compressed |
| 363 |
- dec, err := archive.DecompressStream(f) |
|
| 363 |
+ dec, err := compression.DecompressStream(f) |
|
| 364 | 364 |
if err != nil {
|
| 365 | 365 |
return nil, err |
| 366 | 366 |
} |
| ... | ... |
@@ -3,13 +3,13 @@ package testutils |
| 3 | 3 |
import ( |
| 4 | 4 |
"io" |
| 5 | 5 |
|
| 6 |
- "github.com/moby/go-archive" |
|
| 6 |
+ "github.com/moby/go-archive/compression" |
|
| 7 | 7 |
"github.com/opencontainers/go-digest" |
| 8 | 8 |
) |
| 9 | 9 |
|
| 10 | 10 |
// UncompressedTarDigest returns the canonical digest of the uncompressed tar stream. |
| 11 | 11 |
func UncompressedTarDigest(compressedTar io.Reader) (digest.Digest, error) {
|
| 12 |
- rd, err := archive.DecompressStream(compressedTar) |
|
| 12 |
+ rd, err := compression.DecompressStream(compressedTar) |
|
| 13 | 13 |
if err != nil {
|
| 14 | 14 |
return "", err |
| 15 | 15 |
} |
| ... | ... |
@@ -8,6 +8,8 @@ import ( |
| 8 | 8 |
|
| 9 | 9 |
"github.com/docker/docker/pkg/idtools" |
| 10 | 10 |
"github.com/moby/go-archive" |
| 11 |
+ "github.com/moby/go-archive/compression" |
|
| 12 |
+ "github.com/moby/go-archive/tarheader" |
|
| 11 | 13 |
) |
| 12 | 14 |
|
| 13 | 15 |
// ImpliedDirectoryMode represents the mode (Unix permissions) applied to directories that are implied by files in a |
| ... | ... |
@@ -19,8 +21,8 @@ const ImpliedDirectoryMode = archive.ImpliedDirectoryMode |
| 19 | 19 |
type ( |
| 20 | 20 |
// Compression is the state represents if compressed or not. |
| 21 | 21 |
// |
| 22 |
- // Deprecated: use [archive.Compression] instead. |
|
| 23 |
- Compression = archive.Compression |
|
| 22 |
+ // Deprecated: use [compression.Compression] instead. |
|
| 23 |
+ Compression = compression.Compression |
|
| 24 | 24 |
// WhiteoutFormat is the format of whiteouts unpacked |
| 25 | 25 |
// |
| 26 | 26 |
// Deprecated: use [archive.WhiteoutFormat] instead. |
| ... | ... |
@@ -32,7 +34,7 @@ type ( |
| 32 | 32 |
TarOptions struct {
|
| 33 | 33 |
IncludeFiles []string |
| 34 | 34 |
ExcludePatterns []string |
| 35 |
- Compression archive.Compression |
|
| 35 |
+ Compression compression.Compression |
|
| 36 | 36 |
NoLchown bool |
| 37 | 37 |
IDMap idtools.IdentityMapping |
| 38 | 38 |
ChownOpts *idtools.Identity |
| ... | ... |
@@ -75,11 +77,11 @@ func NewDefaultArchiver() *Archiver {
|
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 | 77 |
const ( |
| 78 |
- Uncompressed = archive.Uncompressed // Deprecated: use [archive.Uncompressed] instead. |
|
| 79 |
- Bzip2 = archive.Bzip2 // Deprecated: use [archive.Bzip2] instead. |
|
| 80 |
- Gzip = archive.Gzip // Deprecated: use [archive.Gzip] instead. |
|
| 81 |
- Xz = archive.Xz // Deprecated: use [archive.Xz] instead. |
|
| 82 |
- Zstd = archive.Zstd // Deprecated: use [archive.Zstd] instead. |
|
| 78 |
+ Uncompressed = compression.None // Deprecated: use [compression.None] instead. |
|
| 79 |
+ Bzip2 = compression.Bzip2 // Deprecated: use [compression.Bzip2] instead. |
|
| 80 |
+ Gzip = compression.Gzip // Deprecated: use [compression.Gzip] instead. |
|
| 81 |
+ Xz = compression.Xz // Deprecated: use [compression.Xz] instead. |
|
| 82 |
+ Zstd = compression.Zstd // Deprecated: use [compression.Zstd] instead. |
|
| 83 | 83 |
) |
| 84 | 84 |
|
| 85 | 85 |
const ( |
| ... | ... |
@@ -97,23 +99,23 @@ func IsArchivePath(path string) bool {
|
| 97 | 97 |
|
| 98 | 98 |
// DetectCompression detects the compression algorithm of the source. |
| 99 | 99 |
// |
| 100 |
-// Deprecated: use [archive.DetectCompression] instead. |
|
| 100 |
+// Deprecated: use [compression.Detect] instead. |
|
| 101 | 101 |
func DetectCompression(source []byte) archive.Compression {
|
| 102 |
- return archive.DetectCompression(source) |
|
| 102 |
+ return compression.Detect(source) |
|
| 103 | 103 |
} |
| 104 | 104 |
|
| 105 | 105 |
// DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. |
| 106 | 106 |
// |
| 107 |
-// Deprecated: use [archive.DecompressStream] instead. |
|
| 107 |
+// Deprecated: use [compression.DecompressStream] instead. |
|
| 108 | 108 |
func DecompressStream(arch io.Reader) (io.ReadCloser, error) {
|
| 109 |
- return archive.DecompressStream(arch) |
|
| 109 |
+ return compression.DecompressStream(arch) |
|
| 110 | 110 |
} |
| 111 | 111 |
|
| 112 | 112 |
// CompressStream compresses the dest with specified compression algorithm. |
| 113 | 113 |
// |
| 114 |
-// Deprecated: use [archive.CompressStream] instead. |
|
| 115 |
-func CompressStream(dest io.Writer, compression archive.Compression) (io.WriteCloser, error) {
|
|
| 116 |
- return archive.CompressStream(dest, compression) |
|
| 114 |
+// Deprecated: use [compression.CompressStream] instead. |
|
| 115 |
+func CompressStream(dest io.Writer, comp compression.Compression) (io.WriteCloser, error) {
|
|
| 116 |
+ return compression.CompressStream(dest, comp) |
|
| 117 | 117 |
} |
| 118 | 118 |
|
| 119 | 119 |
// TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper. |
| ... | ... |
@@ -130,9 +132,9 @@ func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]archive |
| 130 | 130 |
|
| 131 | 131 |
// FileInfoHeaderNoLookups creates a partially-populated tar.Header from fi. |
| 132 | 132 |
// |
| 133 |
-// Deprecated: use [archive.FileInfoHeaderNoLookups] instead. |
|
| 133 |
+// Deprecated: use [tarheader.FileInfoHeaderNoLookups] instead. |
|
| 134 | 134 |
func FileInfoHeaderNoLookups(fi os.FileInfo, link string) (*tar.Header, error) {
|
| 135 |
- return archive.FileInfoHeaderNoLookups(fi, link) |
|
| 135 |
+ return tarheader.FileInfoHeaderNoLookups(fi, link) |
|
| 136 | 136 |
} |
| 137 | 137 |
|
| 138 | 138 |
// FileInfoHeader creates a populated Header from fi. |
| ... | ... |
@@ -4,6 +4,7 @@ import ( |
| 4 | 4 |
"io" |
| 5 | 5 |
|
| 6 | 6 |
"github.com/moby/go-archive" |
| 7 |
+ "github.com/moby/go-archive/compression" |
|
| 7 | 8 |
) |
| 8 | 9 |
|
| 9 | 10 |
var ( |
| ... | ... |
@@ -51,7 +52,7 @@ func TarResourceRebase(sourcePath, rebaseName string) (content io.ReadCloser, _ |
| 51 | 51 |
func TarResourceRebaseOpts(sourceBase string, rebaseName string) *TarOptions {
|
| 52 | 52 |
filter := []string{sourceBase}
|
| 53 | 53 |
return &TarOptions{
|
| 54 |
- Compression: archive.Uncompressed, |
|
| 54 |
+ Compression: compression.None, |
|
| 55 | 55 |
IncludeFiles: filter, |
| 56 | 56 |
IncludeSourceDir: true, |
| 57 | 57 |
RebaseNames: map[string]string{
|
| ... | ... |
@@ -64,7 +64,7 @@ require ( |
| 64 | 64 |
github.com/mitchellh/copystructure v1.2.0 |
| 65 | 65 |
github.com/moby/buildkit v0.21.0 |
| 66 | 66 |
github.com/moby/docker-image-spec v1.3.1 |
| 67 |
- github.com/moby/go-archive v0.0.0-20250404171912-21f3f3385ab7 |
|
| 67 |
+ github.com/moby/go-archive v0.1.0 |
|
| 68 | 68 |
github.com/moby/ipvs v1.1.0 |
| 69 | 69 |
github.com/moby/locker v1.0.1 |
| 70 | 70 |
github.com/moby/patternmatcher v0.6.0 |
| ... | ... |
@@ -387,8 +387,8 @@ github.com/moby/buildkit v0.21.0 h1:+z4vVqgt0spLrOSxi4DLedRbIh2gbNVlZ5q4rsnNp60= |
| 387 | 387 |
github.com/moby/buildkit v0.21.0/go.mod h1:mBq0D44uCyz2PdX8T/qym5LBbkBO3GGv0wqgX9ABYYw= |
| 388 | 388 |
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= |
| 389 | 389 |
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= |
| 390 |
-github.com/moby/go-archive v0.0.0-20250404171912-21f3f3385ab7 h1:CWAY9uG9JhmLmnM7T64+bV0C9IraDrvxEwXq1HJ7hhk= |
|
| 391 |
-github.com/moby/go-archive v0.0.0-20250404171912-21f3f3385ab7/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= |
|
| 390 |
+github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= |
|
| 391 |
+github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= |
|
| 392 | 392 |
github.com/moby/ipvs v1.1.0 h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ= |
| 393 | 393 |
github.com/moby/ipvs v1.1.0/go.mod h1:4VJMWuf098bsUMmZEiD4Tjk/O7mOn3l1PTD3s4OoYAs= |
| 394 | 394 |
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= |
| 0 | 1 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,33 @@ |
| 0 |
+version: "2" |
|
| 1 |
+ |
|
| 2 |
+issues: |
|
| 3 |
+ # Disable maximum issues count per one linter. |
|
| 4 |
+ max-issues-per-linter: 0 |
|
| 5 |
+ # Disable maximum count of issues with the same text. |
|
| 6 |
+ max-same-issues: 0 |
|
| 7 |
+ |
|
| 8 |
+linters: |
|
| 9 |
+ enable: |
|
| 10 |
+ - errorlint |
|
| 11 |
+ - unconvert |
|
| 12 |
+ - unparam |
|
| 13 |
+ exclusions: |
|
| 14 |
+ generated: disable |
|
| 15 |
+ presets: |
|
| 16 |
+ - comments |
|
| 17 |
+ - std-error-handling |
|
| 18 |
+ settings: |
|
| 19 |
+ staticcheck: |
|
| 20 |
+ # Enable all options, with some exceptions. |
|
| 21 |
+ # For defaults, see https://golangci-lint.run/usage/linters/#staticcheck |
|
| 22 |
+ checks: |
|
| 23 |
+ - all |
|
| 24 |
+ - -QF1008 # Omit embedded fields from selector expression; https://staticcheck.dev/docs/checks/#QF1008 |
|
| 25 |
+ - -ST1003 # Poorly chosen identifier; https://staticcheck.dev/docs/checks/#ST1003 |
|
| 26 |
+ |
|
| 27 |
+formatters: |
|
| 28 |
+ enable: |
|
| 29 |
+ - gofumpt |
|
| 30 |
+ - goimports |
|
| 31 |
+ exclusions: |
|
| 32 |
+ generated: disable |
| ... | ... |
@@ -3,32 +3,24 @@ package archive |
| 3 | 3 |
|
| 4 | 4 |
import ( |
| 5 | 5 |
"archive/tar" |
| 6 |
- "bufio" |
|
| 7 |
- "bytes" |
|
| 8 |
- "compress/bzip2" |
|
| 9 |
- "compress/gzip" |
|
| 10 | 6 |
"context" |
| 11 |
- "encoding/binary" |
|
| 12 | 7 |
"errors" |
| 13 | 8 |
"fmt" |
| 14 | 9 |
"io" |
| 15 | 10 |
"os" |
| 16 |
- "os/exec" |
|
| 17 | 11 |
"path/filepath" |
| 18 | 12 |
"runtime" |
| 19 |
- "runtime/debug" |
|
| 20 |
- "strconv" |
|
| 21 | 13 |
"strings" |
| 22 |
- "sync" |
|
| 23 |
- "sync/atomic" |
|
| 24 | 14 |
"syscall" |
| 25 | 15 |
"time" |
| 26 | 16 |
|
| 27 | 17 |
"github.com/containerd/log" |
| 28 |
- "github.com/klauspost/compress/zstd" |
|
| 29 | 18 |
"github.com/moby/patternmatcher" |
| 30 | 19 |
"github.com/moby/sys/sequential" |
| 31 | 20 |
"github.com/moby/sys/user" |
| 21 |
+ |
|
| 22 |
+ "github.com/moby/go-archive/compression" |
|
| 23 |
+ "github.com/moby/go-archive/tarheader" |
|
| 32 | 24 |
) |
| 33 | 25 |
|
| 34 | 26 |
// ImpliedDirectoryMode represents the mode (Unix permissions) applied to directories that are implied by files in a |
| ... | ... |
@@ -45,7 +37,9 @@ const ImpliedDirectoryMode = 0o755 |
| 45 | 45 |
|
| 46 | 46 |
type ( |
| 47 | 47 |
// Compression is the state represents if compressed or not. |
| 48 |
- Compression int |
|
| 48 |
+ // |
|
| 49 |
+ // Deprecated: use [compression.Compression]. |
|
| 50 |
+ Compression = compression.Compression |
|
| 49 | 51 |
// WhiteoutFormat is the format of whiteouts unpacked |
| 50 | 52 |
WhiteoutFormat int |
| 51 | 53 |
|
| ... | ... |
@@ -58,7 +52,7 @@ type ( |
| 58 | 58 |
TarOptions struct {
|
| 59 | 59 |
IncludeFiles []string |
| 60 | 60 |
ExcludePatterns []string |
| 61 |
- Compression Compression |
|
| 61 |
+ Compression compression.Compression |
|
| 62 | 62 |
NoLchown bool |
| 63 | 63 |
IDMap user.IdentityMapping |
| 64 | 64 |
ChownOpts *ChownOpts |
| ... | ... |
@@ -102,11 +96,11 @@ func NewDefaultArchiver() *Archiver {
|
| 102 | 102 |
type breakoutError error |
| 103 | 103 |
|
| 104 | 104 |
const ( |
| 105 |
- Uncompressed Compression = 0 // Uncompressed represents the uncompressed. |
|
| 106 |
- Bzip2 Compression = 1 // Bzip2 is bzip2 compression algorithm. |
|
| 107 |
- Gzip Compression = 2 // Gzip is gzip compression algorithm. |
|
| 108 |
- Xz Compression = 3 // Xz is xz compression algorithm. |
|
| 109 |
- Zstd Compression = 4 // Zstd is zstd compression algorithm. |
|
| 105 |
+ Uncompressed = compression.None // Deprecated: use [compression.None]. |
|
| 106 |
+ Bzip2 = compression.Bzip2 // Deprecated: use [compression.Bzip2]. |
|
| 107 |
+ Gzip = compression.Gzip // Deprecated: use [compression.Gzip]. |
|
| 108 |
+ Xz = compression.Xz // Deprecated: use [compression.Xz]. |
|
| 109 |
+ Zstd = compression.Zstd // Deprecated: use [compression.Zstd]. |
|
| 110 | 110 |
) |
| 111 | 111 |
|
| 112 | 112 |
const ( |
| ... | ... |
@@ -122,7 +116,7 @@ func IsArchivePath(path string) bool {
|
| 122 | 122 |
return false |
| 123 | 123 |
} |
| 124 | 124 |
defer file.Close() |
| 125 |
- rdr, err := DecompressStream(file) |
|
| 125 |
+ rdr, err := compression.DecompressStream(file) |
|
| 126 | 126 |
if err != nil {
|
| 127 | 127 |
return false |
| 128 | 128 |
} |
| ... | ... |
@@ -132,242 +126,25 @@ func IsArchivePath(path string) bool {
|
| 132 | 132 |
return err == nil |
| 133 | 133 |
} |
| 134 | 134 |
|
| 135 |
-const ( |
|
| 136 |
- zstdMagicSkippableStart = 0x184D2A50 |
|
| 137 |
- zstdMagicSkippableMask = 0xFFFFFFF0 |
|
| 138 |
-) |
|
| 139 |
- |
|
| 140 |
-var ( |
|
| 141 |
- bzip2Magic = []byte{0x42, 0x5A, 0x68}
|
|
| 142 |
- gzipMagic = []byte{0x1F, 0x8B, 0x08}
|
|
| 143 |
- xzMagic = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}
|
|
| 144 |
- zstdMagic = []byte{0x28, 0xb5, 0x2f, 0xfd}
|
|
| 145 |
-) |
|
| 146 |
- |
|
| 147 |
-type matcher = func([]byte) bool |
|
| 148 |
- |
|
| 149 |
-func magicNumberMatcher(m []byte) matcher {
|
|
| 150 |
- return func(source []byte) bool {
|
|
| 151 |
- return bytes.HasPrefix(source, m) |
|
| 152 |
- } |
|
| 153 |
-} |
|
| 154 |
- |
|
| 155 |
-// zstdMatcher detects zstd compression algorithm. |
|
| 156 |
-// Zstandard compressed data is made of one or more frames. |
|
| 157 |
-// There are two frame formats defined by Zstandard: Zstandard frames and Skippable frames. |
|
| 158 |
-// See https://datatracker.ietf.org/doc/html/rfc8878#section-3 for more details. |
|
| 159 |
-func zstdMatcher() matcher {
|
|
| 160 |
- return func(source []byte) bool {
|
|
| 161 |
- if bytes.HasPrefix(source, zstdMagic) {
|
|
| 162 |
- // Zstandard frame |
|
| 163 |
- return true |
|
| 164 |
- } |
|
| 165 |
- // skippable frame |
|
| 166 |
- if len(source) < 8 {
|
|
| 167 |
- return false |
|
| 168 |
- } |
|
| 169 |
- // magic number from 0x184D2A50 to 0x184D2A5F. |
|
| 170 |
- if binary.LittleEndian.Uint32(source[:4])&zstdMagicSkippableMask == zstdMagicSkippableStart {
|
|
| 171 |
- return true |
|
| 172 |
- } |
|
| 173 |
- return false |
|
| 174 |
- } |
|
| 175 |
-} |
|
| 176 |
- |
|
| 177 | 135 |
// DetectCompression detects the compression algorithm of the source. |
| 178 |
-func DetectCompression(source []byte) Compression {
|
|
| 179 |
- compressionMap := map[Compression]matcher{
|
|
| 180 |
- Bzip2: magicNumberMatcher(bzip2Magic), |
|
| 181 |
- Gzip: magicNumberMatcher(gzipMagic), |
|
| 182 |
- Xz: magicNumberMatcher(xzMagic), |
|
| 183 |
- Zstd: zstdMatcher(), |
|
| 184 |
- } |
|
| 185 |
- for _, compression := range []Compression{Bzip2, Gzip, Xz, Zstd} {
|
|
| 186 |
- fn := compressionMap[compression] |
|
| 187 |
- if fn(source) {
|
|
| 188 |
- return compression |
|
| 189 |
- } |
|
| 190 |
- } |
|
| 191 |
- return Uncompressed |
|
| 192 |
-} |
|
| 193 |
- |
|
| 194 |
-func xzDecompress(ctx context.Context, archive io.Reader) (io.ReadCloser, error) {
|
|
| 195 |
- args := []string{"xz", "-d", "-c", "-q"}
|
|
| 196 |
- |
|
| 197 |
- return cmdStream(exec.CommandContext(ctx, args[0], args[1:]...), archive) |
|
| 198 |
-} |
|
| 199 |
- |
|
| 200 |
-func gzDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) {
|
|
| 201 |
- if noPigzEnv := os.Getenv("MOBY_DISABLE_PIGZ"); noPigzEnv != "" {
|
|
| 202 |
- noPigz, err := strconv.ParseBool(noPigzEnv) |
|
| 203 |
- if err != nil {
|
|
| 204 |
- log.G(ctx).WithError(err).Warn("invalid value in MOBY_DISABLE_PIGZ env var")
|
|
| 205 |
- } |
|
| 206 |
- if noPigz {
|
|
| 207 |
- log.G(ctx).Debugf("Use of pigz is disabled due to MOBY_DISABLE_PIGZ=%s", noPigzEnv)
|
|
| 208 |
- return gzip.NewReader(buf) |
|
| 209 |
- } |
|
| 210 |
- } |
|
| 211 |
- |
|
| 212 |
- unpigzPath, err := exec.LookPath("unpigz")
|
|
| 213 |
- if err != nil {
|
|
| 214 |
- log.G(ctx).Debugf("unpigz binary not found, falling back to go gzip library")
|
|
| 215 |
- return gzip.NewReader(buf) |
|
| 216 |
- } |
|
| 217 |
- |
|
| 218 |
- log.G(ctx).Debugf("Using %s to decompress", unpigzPath)
|
|
| 219 |
- |
|
| 220 |
- return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf) |
|
| 221 |
-} |
|
| 222 |
- |
|
| 223 |
-type readCloserWrapper struct {
|
|
| 224 |
- io.Reader |
|
| 225 |
- closer func() error |
|
| 226 |
- closed atomic.Bool |
|
| 227 |
-} |
|
| 228 |
- |
|
| 229 |
-func (r *readCloserWrapper) Close() error {
|
|
| 230 |
- if !r.closed.CompareAndSwap(false, true) {
|
|
| 231 |
- log.G(context.TODO()).Error("subsequent attempt to close readCloserWrapper")
|
|
| 232 |
- if log.GetLevel() >= log.DebugLevel {
|
|
| 233 |
- log.G(context.TODO()).Errorf("stack trace: %s", string(debug.Stack()))
|
|
| 234 |
- } |
|
| 235 |
- |
|
| 236 |
- return nil |
|
| 237 |
- } |
|
| 238 |
- if r.closer != nil {
|
|
| 239 |
- return r.closer() |
|
| 240 |
- } |
|
| 241 |
- return nil |
|
| 242 |
-} |
|
| 243 |
- |
|
| 244 |
-var bufioReader32KPool = &sync.Pool{
|
|
| 245 |
- New: func() interface{} { return bufio.NewReaderSize(nil, 32*1024) },
|
|
| 246 |
-} |
|
| 247 |
- |
|
| 248 |
-type bufferedReader struct {
|
|
| 249 |
- buf *bufio.Reader |
|
| 250 |
-} |
|
| 251 |
- |
|
| 252 |
-func newBufferedReader(r io.Reader) *bufferedReader {
|
|
| 253 |
- buf := bufioReader32KPool.Get().(*bufio.Reader) |
|
| 254 |
- buf.Reset(r) |
|
| 255 |
- return &bufferedReader{buf}
|
|
| 256 |
-} |
|
| 257 |
- |
|
| 258 |
-func (r *bufferedReader) Read(p []byte) (int, error) {
|
|
| 259 |
- if r.buf == nil {
|
|
| 260 |
- return 0, io.EOF |
|
| 261 |
- } |
|
| 262 |
- n, err := r.buf.Read(p) |
|
| 263 |
- if err == io.EOF {
|
|
| 264 |
- r.buf.Reset(nil) |
|
| 265 |
- bufioReader32KPool.Put(r.buf) |
|
| 266 |
- r.buf = nil |
|
| 267 |
- } |
|
| 268 |
- return n, err |
|
| 269 |
-} |
|
| 270 |
- |
|
| 271 |
-func (r *bufferedReader) Peek(n int) ([]byte, error) {
|
|
| 272 |
- if r.buf == nil {
|
|
| 273 |
- return nil, io.EOF |
|
| 274 |
- } |
|
| 275 |
- return r.buf.Peek(n) |
|
| 136 |
+// |
|
| 137 |
+// Deprecated: use [compression.Detect]. |
|
| 138 |
+func DetectCompression(source []byte) compression.Compression {
|
|
| 139 |
+ return compression.Detect(source) |
|
| 276 | 140 |
} |
| 277 | 141 |
|
| 278 | 142 |
// DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. |
| 143 |
+// |
|
| 144 |
+// Deprecated: use [compression.DecompressStream]. |
|
| 279 | 145 |
func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
|
| 280 |
- buf := newBufferedReader(archive) |
|
| 281 |
- bs, err := buf.Peek(10) |
|
| 282 |
- if err != nil && err != io.EOF {
|
|
| 283 |
- // Note: we'll ignore any io.EOF error because there are some odd |
|
| 284 |
- // cases where the layer.tar file will be empty (zero bytes) and |
|
| 285 |
- // that results in an io.EOF from the Peek() call. So, in those |
|
| 286 |
- // cases we'll just treat it as a non-compressed stream and |
|
| 287 |
- // that means just create an empty layer. |
|
| 288 |
- // See Issue 18170 |
|
| 289 |
- return nil, err |
|
| 290 |
- } |
|
| 291 |
- |
|
| 292 |
- compression := DetectCompression(bs) |
|
| 293 |
- switch compression {
|
|
| 294 |
- case Uncompressed: |
|
| 295 |
- return &readCloserWrapper{
|
|
| 296 |
- Reader: buf, |
|
| 297 |
- }, nil |
|
| 298 |
- case Gzip: |
|
| 299 |
- ctx, cancel := context.WithCancel(context.Background()) |
|
| 300 |
- |
|
| 301 |
- gzReader, err := gzDecompress(ctx, buf) |
|
| 302 |
- if err != nil {
|
|
| 303 |
- cancel() |
|
| 304 |
- return nil, err |
|
| 305 |
- } |
|
| 306 |
- return &readCloserWrapper{
|
|
| 307 |
- Reader: gzReader, |
|
| 308 |
- closer: func() error {
|
|
| 309 |
- cancel() |
|
| 310 |
- return gzReader.Close() |
|
| 311 |
- }, |
|
| 312 |
- }, nil |
|
| 313 |
- case Bzip2: |
|
| 314 |
- bz2Reader := bzip2.NewReader(buf) |
|
| 315 |
- return &readCloserWrapper{
|
|
| 316 |
- Reader: bz2Reader, |
|
| 317 |
- }, nil |
|
| 318 |
- case Xz: |
|
| 319 |
- ctx, cancel := context.WithCancel(context.Background()) |
|
| 320 |
- |
|
| 321 |
- xzReader, err := xzDecompress(ctx, buf) |
|
| 322 |
- if err != nil {
|
|
| 323 |
- cancel() |
|
| 324 |
- return nil, err |
|
| 325 |
- } |
|
| 326 |
- |
|
| 327 |
- return &readCloserWrapper{
|
|
| 328 |
- Reader: xzReader, |
|
| 329 |
- closer: func() error {
|
|
| 330 |
- cancel() |
|
| 331 |
- return xzReader.Close() |
|
| 332 |
- }, |
|
| 333 |
- }, nil |
|
| 334 |
- case Zstd: |
|
| 335 |
- zstdReader, err := zstd.NewReader(buf) |
|
| 336 |
- if err != nil {
|
|
| 337 |
- return nil, err |
|
| 338 |
- } |
|
| 339 |
- return &readCloserWrapper{
|
|
| 340 |
- Reader: zstdReader, |
|
| 341 |
- closer: func() error {
|
|
| 342 |
- zstdReader.Close() |
|
| 343 |
- return nil |
|
| 344 |
- }, |
|
| 345 |
- }, nil |
|
| 346 |
- default: |
|
| 347 |
- return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
|
|
| 348 |
- } |
|
| 146 |
+ return compression.DecompressStream(archive) |
|
| 349 | 147 |
} |
| 350 | 148 |
|
| 351 |
-type nopWriteCloser struct {
|
|
| 352 |
- io.Writer |
|
| 353 |
-} |
|
| 354 |
- |
|
| 355 |
-func (nopWriteCloser) Close() error { return nil }
|
|
| 356 |
- |
|
| 357 | 149 |
// CompressStream compresses the dest with specified compression algorithm. |
| 358 |
-func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) {
|
|
| 359 |
- switch compression {
|
|
| 360 |
- case Uncompressed: |
|
| 361 |
- return nopWriteCloser{dest}, nil
|
|
| 362 |
- case Gzip: |
|
| 363 |
- return gzip.NewWriter(dest), nil |
|
| 364 |
- case Bzip2, Xz: |
|
| 365 |
- // archive/bzip2 does not support writing, and there is no xz support at all |
|
| 366 |
- // However, this is not a problem as docker only currently generates gzipped tars |
|
| 367 |
- return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
|
|
| 368 |
- default: |
|
| 369 |
- return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
|
|
| 370 |
- } |
|
| 150 |
+// |
|
| 151 |
+// Deprecated: use [compression.CompressStream]. |
|
| 152 |
+func CompressStream(dest io.Writer, comp compression.Compression) (io.WriteCloser, error) {
|
|
| 153 |
+ return compression.CompressStream(dest, comp) |
|
| 371 | 154 |
} |
| 372 | 155 |
|
| 373 | 156 |
// TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to |
| ... | ... |
@@ -416,7 +193,7 @@ func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModi |
| 416 | 416 |
var originalHeader *tar.Header |
| 417 | 417 |
for {
|
| 418 | 418 |
originalHeader, err = tarReader.Next() |
| 419 |
- if err == io.EOF {
|
|
| 419 |
+ if errors.Is(err, io.EOF) {
|
|
| 420 | 420 |
break |
| 421 | 421 |
} |
| 422 | 422 |
if err != nil {
|
| ... | ... |
@@ -458,90 +235,11 @@ func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModi |
| 458 | 458 |
return pipeReader |
| 459 | 459 |
} |
| 460 | 460 |
|
| 461 |
-// Extension returns the extension of a file that uses the specified compression algorithm. |
|
| 462 |
-func (compression *Compression) Extension() string {
|
|
| 463 |
- switch *compression {
|
|
| 464 |
- case Uncompressed: |
|
| 465 |
- return "tar" |
|
| 466 |
- case Bzip2: |
|
| 467 |
- return "tar.bz2" |
|
| 468 |
- case Gzip: |
|
| 469 |
- return "tar.gz" |
|
| 470 |
- case Xz: |
|
| 471 |
- return "tar.xz" |
|
| 472 |
- case Zstd: |
|
| 473 |
- return "tar.zst" |
|
| 474 |
- } |
|
| 475 |
- return "" |
|
| 476 |
-} |
|
| 477 |
- |
|
| 478 |
-// assert that we implement [tar.FileInfoNames]. |
|
| 479 |
-// |
|
| 480 |
-// TODO(thaJeztah): disabled to allow compiling on < go1.23. un-comment once we drop support for older versions of go. |
|
| 481 |
-// var _ tar.FileInfoNames = (*nosysFileInfo)(nil) |
|
| 482 |
- |
|
| 483 |
-// nosysFileInfo hides the system-dependent info of the wrapped FileInfo to |
|
| 484 |
-// prevent tar.FileInfoHeader from introspecting it and potentially calling into |
|
| 485 |
-// glibc. |
|
| 486 |
-// |
|
| 487 |
-// It implements [tar.FileInfoNames] to further prevent [tar.FileInfoHeader] |
|
| 488 |
-// from performing any lookups on go1.23 and up. see https://go.dev/issue/50102 |
|
| 489 |
-type nosysFileInfo struct {
|
|
| 490 |
- os.FileInfo |
|
| 491 |
-} |
|
| 492 |
- |
|
| 493 |
-// Uname stubs out looking up username. It implements [tar.FileInfoNames] |
|
| 494 |
-// to prevent [tar.FileInfoHeader] from loading libraries to perform |
|
| 495 |
-// username lookups. |
|
| 496 |
-func (fi nosysFileInfo) Uname() (string, error) {
|
|
| 497 |
- return "", nil |
|
| 498 |
-} |
|
| 499 |
- |
|
| 500 |
-// Gname stubs out looking up group-name. It implements [tar.FileInfoNames] |
|
| 501 |
-// to prevent [tar.FileInfoHeader] from loading libraries to perform |
|
| 502 |
-// username lookups. |
|
| 503 |
-func (fi nosysFileInfo) Gname() (string, error) {
|
|
| 504 |
- return "", nil |
|
| 505 |
-} |
|
| 506 |
- |
|
| 507 |
-func (fi nosysFileInfo) Sys() interface{} {
|
|
| 508 |
- // A Sys value of type *tar.Header is safe as it is system-independent. |
|
| 509 |
- // The tar.FileInfoHeader function copies the fields into the returned |
|
| 510 |
- // header without performing any OS lookups. |
|
| 511 |
- if sys, ok := fi.FileInfo.Sys().(*tar.Header); ok {
|
|
| 512 |
- return sys |
|
| 513 |
- } |
|
| 514 |
- return nil |
|
| 515 |
-} |
|
| 516 |
- |
|
| 517 |
-// sysStat, if non-nil, populates hdr from system-dependent fields of fi. |
|
| 518 |
-var sysStat func(fi os.FileInfo, hdr *tar.Header) error |
|
| 519 |
- |
|
| 520 | 461 |
// FileInfoHeaderNoLookups creates a partially-populated tar.Header from fi. |
| 521 | 462 |
// |
| 522 |
-// Compared to the archive/tar.FileInfoHeader function, this function is safe to |
|
| 523 |
-// call from a chrooted process as it does not populate fields which would |
|
| 524 |
-// require operating system lookups. It behaves identically to |
|
| 525 |
-// tar.FileInfoHeader when fi is a FileInfo value returned from |
|
| 526 |
-// tar.Header.FileInfo(). |
|
| 527 |
-// |
|
| 528 |
-// When fi is a FileInfo for a native file, such as returned from os.Stat() and |
|
| 529 |
-// os.Lstat(), the returned Header value differs from one returned from |
|
| 530 |
-// tar.FileInfoHeader in the following ways. The Uname and Gname fields are not |
|
| 531 |
-// set as OS lookups would be required to populate them. The AccessTime and |
|
| 532 |
-// ChangeTime fields are not currently set (not yet implemented) although that |
|
| 533 |
-// is subject to change. Callers which require the AccessTime or ChangeTime |
|
| 534 |
-// fields to be zeroed should explicitly zero them out in the returned Header |
|
| 535 |
-// value to avoid any compatibility issues in the future. |
|
| 463 |
+// Deprecated: use [tarheader.FileInfoHeaderNoLookups]. |
|
| 536 | 464 |
func FileInfoHeaderNoLookups(fi os.FileInfo, link string) (*tar.Header, error) {
|
| 537 |
- hdr, err := tar.FileInfoHeader(nosysFileInfo{fi}, link)
|
|
| 538 |
- if err != nil {
|
|
| 539 |
- return nil, err |
|
| 540 |
- } |
|
| 541 |
- if sysStat != nil {
|
|
| 542 |
- return hdr, sysStat(fi, hdr) |
|
| 543 |
- } |
|
| 544 |
- return hdr, nil |
|
| 465 |
+ return tarheader.FileInfoHeaderNoLookups(fi, link) |
|
| 545 | 466 |
} |
| 546 | 467 |
|
| 547 | 468 |
// FileInfoHeader creates a populated Header from fi. |
| ... | ... |
@@ -552,7 +250,7 @@ func FileInfoHeaderNoLookups(fi os.FileInfo, link string) (*tar.Header, error) {
|
| 552 | 552 |
// precision, and the Uname and Gname fields are only set when fi is a FileInfo |
| 553 | 553 |
// value returned from tar.Header.FileInfo(). |
| 554 | 554 |
func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) {
|
| 555 |
- hdr, err := FileInfoHeaderNoLookups(fi, link) |
|
| 555 |
+ hdr, err := tarheader.FileInfoHeaderNoLookups(fi, link) |
|
| 556 | 556 |
if err != nil {
|
| 557 | 557 |
return nil, err |
| 558 | 558 |
} |
| ... | ... |
@@ -768,7 +466,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, o |
| 768 | 768 |
case tar.TypeDir: |
| 769 | 769 |
// Create directory unless it exists as a directory already. |
| 770 | 770 |
// In that case we just want to merge the two |
| 771 |
- if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
|
|
| 771 |
+ if fi, err := os.Lstat(path); err != nil || !fi.IsDir() {
|
|
| 772 | 772 |
if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {
|
| 773 | 773 |
return err |
| 774 | 774 |
} |
| ... | ... |
@@ -908,8 +606,8 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, o |
| 908 | 908 |
|
| 909 | 909 |
// Tar creates an archive from the directory at `path`, and returns it as a |
| 910 | 910 |
// stream of bytes. |
| 911 |
-func Tar(path string, compression Compression) (io.ReadCloser, error) {
|
|
| 912 |
- return TarWithOptions(path, &TarOptions{Compression: compression})
|
|
| 911 |
+func Tar(path string, comp compression.Compression) (io.ReadCloser, error) {
|
|
| 912 |
+ return TarWithOptions(path, &TarOptions{Compression: comp})
|
|
| 913 | 913 |
} |
| 914 | 914 |
|
| 915 | 915 |
// TarWithOptions creates an archive from the directory at `path`, only including files whose relative |
| ... | ... |
@@ -945,7 +643,7 @@ func NewTarballer(srcPath string, options *TarOptions) (*Tarballer, error) {
|
| 945 | 945 |
|
| 946 | 946 |
pipeReader, pipeWriter := io.Pipe() |
| 947 | 947 |
|
| 948 |
- compressWriter, err := CompressStream(pipeWriter, options.Compression) |
|
| 948 |
+ compressWriter, err := compression.CompressStream(pipeWriter, options.Compression) |
|
| 949 | 949 |
if err != nil {
|
| 950 | 950 |
return nil, err |
| 951 | 951 |
} |
| ... | ... |
@@ -1031,7 +729,8 @@ func (t *Tarballer) Do() {
|
| 1031 | 1031 |
) |
| 1032 | 1032 |
|
| 1033 | 1033 |
walkRoot := getWalkRoot(t.srcPath, include) |
| 1034 |
- filepath.WalkDir(walkRoot, func(filePath string, f os.DirEntry, err error) error {
|
|
| 1034 |
+ // TODO(thaJeztah): should this error be handled? |
|
| 1035 |
+ _ = filepath.WalkDir(walkRoot, func(filePath string, f os.DirEntry, err error) error {
|
|
| 1035 | 1036 |
if err != nil {
|
| 1036 | 1037 |
log.G(context.TODO()).Errorf("Tar: Can't stat file %s to tar: %s", t.srcPath, err)
|
| 1037 | 1038 |
return nil |
| ... | ... |
@@ -1135,7 +834,7 @@ func (t *Tarballer) Do() {
|
| 1135 | 1135 |
if err := ta.addTarFile(filePath, relFilePath); err != nil {
|
| 1136 | 1136 |
log.G(context.TODO()).Errorf("Can't add file %s to tar: %s", filePath, err)
|
| 1137 | 1137 |
// if pipe is broken, stop writing tar stream to it |
| 1138 |
- if err == io.ErrClosedPipe {
|
|
| 1138 |
+ if errors.Is(err, io.ErrClosedPipe) {
|
|
| 1139 | 1139 |
return err |
| 1140 | 1140 |
} |
| 1141 | 1141 |
} |
| ... | ... |
@@ -1155,7 +854,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err |
| 1155 | 1155 |
loop: |
| 1156 | 1156 |
for {
|
| 1157 | 1157 |
hdr, err := tr.Next() |
| 1158 |
- if err == io.EOF {
|
|
| 1158 |
+ if errors.Is(err, io.EOF) {
|
|
| 1159 | 1159 |
// end of tar archive |
| 1160 | 1160 |
break |
| 1161 | 1161 |
} |
| ... | ... |
@@ -1217,7 +916,7 @@ loop: |
| 1217 | 1217 |
continue |
| 1218 | 1218 |
} |
| 1219 | 1219 |
|
| 1220 |
- if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
|
|
| 1220 |
+ if !fi.IsDir() || hdr.Typeflag != tar.TypeDir {
|
|
| 1221 | 1221 |
if err := os.RemoveAll(path); err != nil {
|
| 1222 | 1222 |
return err |
| 1223 | 1223 |
} |
| ... | ... |
@@ -1309,7 +1008,7 @@ func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) e |
| 1309 | 1309 |
// Handler for teasing out the automatic decompression |
| 1310 | 1310 |
func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
|
| 1311 | 1311 |
if tarArchive == nil {
|
| 1312 |
- return fmt.Errorf("Empty archive")
|
|
| 1312 |
+ return errors.New("empty archive")
|
|
| 1313 | 1313 |
} |
| 1314 | 1314 |
dest = filepath.Clean(dest) |
| 1315 | 1315 |
if options == nil {
|
| ... | ... |
@@ -1321,7 +1020,7 @@ func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decomp |
| 1321 | 1321 |
|
| 1322 | 1322 |
r := tarArchive |
| 1323 | 1323 |
if decompress {
|
| 1324 |
- decompressedArchive, err := DecompressStream(tarArchive) |
|
| 1324 |
+ decompressedArchive, err := compression.DecompressStream(tarArchive) |
|
| 1325 | 1325 |
if err != nil {
|
| 1326 | 1326 |
return err |
| 1327 | 1327 |
} |
| ... | ... |
@@ -1335,15 +1034,14 @@ func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decomp |
| 1335 | 1335 |
// TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other. |
| 1336 | 1336 |
// If either Tar or Untar fails, TarUntar aborts and returns the error. |
| 1337 | 1337 |
func (archiver *Archiver) TarUntar(src, dst string) error {
|
| 1338 |
- archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed})
|
|
| 1338 |
+ archive, err := Tar(src, compression.None) |
|
| 1339 | 1339 |
if err != nil {
|
| 1340 | 1340 |
return err |
| 1341 | 1341 |
} |
| 1342 | 1342 |
defer archive.Close() |
| 1343 |
- options := &TarOptions{
|
|
| 1343 |
+ return archiver.Untar(archive, dst, &TarOptions{
|
|
| 1344 | 1344 |
IDMap: archiver.IDMapping, |
| 1345 |
- } |
|
| 1346 |
- return archiver.Untar(archive, dst, options) |
|
| 1345 |
+ }) |
|
| 1347 | 1346 |
} |
| 1348 | 1347 |
|
| 1349 | 1348 |
// UntarPath untar a file from path to a destination, src is the source tar file path. |
| ... | ... |
@@ -1353,10 +1051,9 @@ func (archiver *Archiver) UntarPath(src, dst string) error {
|
| 1353 | 1353 |
return err |
| 1354 | 1354 |
} |
| 1355 | 1355 |
defer archive.Close() |
| 1356 |
- options := &TarOptions{
|
|
| 1356 |
+ return archiver.Untar(archive, dst, &TarOptions{
|
|
| 1357 | 1357 |
IDMap: archiver.IDMapping, |
| 1358 |
- } |
|
| 1359 |
- return archiver.Untar(archive, dst, options) |
|
| 1358 |
+ }) |
|
| 1360 | 1359 |
} |
| 1361 | 1360 |
|
| 1362 | 1361 |
// CopyWithTar creates a tar archive of filesystem path `src`, and |
| ... | ... |
@@ -1393,7 +1090,7 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
|
| 1393 | 1393 |
} |
| 1394 | 1394 |
|
| 1395 | 1395 |
if srcSt.IsDir() {
|
| 1396 |
- return fmt.Errorf("Can't copy a directory")
|
|
| 1396 |
+ return errors.New("can't copy a directory")
|
|
| 1397 | 1397 |
} |
| 1398 | 1398 |
|
| 1399 | 1399 |
// Clean up the trailing slash. This must be done in an operating |
| ... | ... |
@@ -1421,7 +1118,7 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
|
| 1421 | 1421 |
} |
| 1422 | 1422 |
defer srcF.Close() |
| 1423 | 1423 |
|
| 1424 |
- hdr, err := FileInfoHeaderNoLookups(srcSt, "") |
|
| 1424 |
+ hdr, err := tarheader.FileInfoHeaderNoLookups(srcSt, "") |
|
| 1425 | 1425 |
if err != nil {
|
| 1426 | 1426 |
return err |
| 1427 | 1427 |
} |
| ... | ... |
@@ -1470,43 +1167,3 @@ func remapIDs(idMapping user.IdentityMapping, hdr *tar.Header) error {
|
| 1470 | 1470 |
hdr.Uid, hdr.Gid = uid, gid |
| 1471 | 1471 |
return err |
| 1472 | 1472 |
} |
| 1473 |
- |
|
| 1474 |
-// cmdStream executes a command, and returns its stdout as a stream. |
|
| 1475 |
-// If the command fails to run or doesn't complete successfully, an error |
|
| 1476 |
-// will be returned, including anything written on stderr. |
|
| 1477 |
-func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) {
|
|
| 1478 |
- cmd.Stdin = input |
|
| 1479 |
- pipeR, pipeW := io.Pipe() |
|
| 1480 |
- cmd.Stdout = pipeW |
|
| 1481 |
- var errBuf bytes.Buffer |
|
| 1482 |
- cmd.Stderr = &errBuf |
|
| 1483 |
- |
|
| 1484 |
- // Run the command and return the pipe |
|
| 1485 |
- if err := cmd.Start(); err != nil {
|
|
| 1486 |
- return nil, err |
|
| 1487 |
- } |
|
| 1488 |
- |
|
| 1489 |
- // Ensure the command has exited before we clean anything up |
|
| 1490 |
- done := make(chan struct{})
|
|
| 1491 |
- |
|
| 1492 |
- // Copy stdout to the returned pipe |
|
| 1493 |
- go func() {
|
|
| 1494 |
- if err := cmd.Wait(); err != nil {
|
|
| 1495 |
- pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String()))
|
|
| 1496 |
- } else {
|
|
| 1497 |
- pipeW.Close() |
|
| 1498 |
- } |
|
| 1499 |
- close(done) |
|
| 1500 |
- }() |
|
| 1501 |
- |
|
| 1502 |
- return &readCloserWrapper{
|
|
| 1503 |
- Reader: pipeR, |
|
| 1504 |
- closer: func() error {
|
|
| 1505 |
- // Close pipeR, and then wait for the command to complete before returning. We have to close pipeR first, as |
|
| 1506 |
- // cmd.Wait waits for any non-file stdout/stderr/stdin to close. |
|
| 1507 |
- err := pipeR.Close() |
|
| 1508 |
- <-done |
|
| 1509 |
- return err |
|
| 1510 |
- }, |
|
| 1511 |
- }, nil |
|
| 1512 |
-} |
| ... | ... |
@@ -7,17 +7,12 @@ import ( |
| 7 | 7 |
"errors" |
| 8 | 8 |
"os" |
| 9 | 9 |
"path/filepath" |
| 10 |
- "runtime" |
|
| 11 | 10 |
"strings" |
| 12 | 11 |
"syscall" |
| 13 | 12 |
|
| 14 | 13 |
"golang.org/x/sys/unix" |
| 15 | 14 |
) |
| 16 | 15 |
|
| 17 |
-func init() {
|
|
| 18 |
- sysStat = statUnix |
|
| 19 |
-} |
|
| 20 |
- |
|
| 21 | 16 |
// addLongPathPrefix adds the Windows long path prefix to the path provided if |
| 22 | 17 |
// it does not already have it. It is a no-op on platforms other than Windows. |
| 23 | 18 |
func addLongPathPrefix(srcPath string) string {
|
| ... | ... |
@@ -38,40 +33,6 @@ func chmodTarEntry(perm os.FileMode) os.FileMode {
|
| 38 | 38 |
return perm // noop for unix as golang APIs provide perm bits correctly |
| 39 | 39 |
} |
| 40 | 40 |
|
| 41 |
-// statUnix populates hdr from system-dependent fields of fi without performing |
|
| 42 |
-// any OS lookups. |
|
| 43 |
-func statUnix(fi os.FileInfo, hdr *tar.Header) error {
|
|
| 44 |
- // Devmajor and Devminor are only needed for special devices. |
|
| 45 |
- |
|
| 46 |
- // In FreeBSD, RDev for regular files is -1 (unless overridden by FS): |
|
| 47 |
- // https://cgit.freebsd.org/src/tree/sys/kern/vfs_default.c?h=stable/13#n1531 |
|
| 48 |
- // (NODEV is -1: https://cgit.freebsd.org/src/tree/sys/sys/param.h?h=stable/13#n241). |
|
| 49 |
- |
|
| 50 |
- // ZFS in particular does not override the default: |
|
| 51 |
- // https://cgit.freebsd.org/src/tree/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c?h=stable/13#n2027 |
|
| 52 |
- |
|
| 53 |
- // Since `Stat_t.Rdev` is uint64, the cast turns -1 into (2^64 - 1). |
|
| 54 |
- // Such large values cannot be encoded in a tar header. |
|
| 55 |
- if runtime.GOOS == "freebsd" && hdr.Typeflag != tar.TypeBlock && hdr.Typeflag != tar.TypeChar {
|
|
| 56 |
- return nil |
|
| 57 |
- } |
|
| 58 |
- s, ok := fi.Sys().(*syscall.Stat_t) |
|
| 59 |
- if !ok {
|
|
| 60 |
- return nil |
|
| 61 |
- } |
|
| 62 |
- |
|
| 63 |
- hdr.Uid = int(s.Uid) |
|
| 64 |
- hdr.Gid = int(s.Gid) |
|
| 65 |
- |
|
| 66 |
- if s.Mode&unix.S_IFBLK != 0 || |
|
| 67 |
- s.Mode&unix.S_IFCHR != 0 {
|
|
| 68 |
- hdr.Devmajor = int64(unix.Major(uint64(s.Rdev))) //nolint: unconvert |
|
| 69 |
- hdr.Devminor = int64(unix.Minor(uint64(s.Rdev))) //nolint: unconvert |
|
| 70 |
- } |
|
| 71 |
- |
|
| 72 |
- return nil |
|
| 73 |
-} |
|
| 74 |
- |
|
| 75 | 41 |
func getInodeFromStat(stat interface{}) (uint64, error) {
|
| 76 | 42 |
s, ok := stat.(*syscall.Stat_t) |
| 77 | 43 |
if !ok {
|
| ... | ... |
@@ -41,11 +41,6 @@ func chmodTarEntry(perm os.FileMode) os.FileMode {
|
| 41 | 41 |
return perm | 0o111 |
| 42 | 42 |
} |
| 43 | 43 |
|
| 44 |
-func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (err error) {
|
|
| 45 |
- // do nothing. no notion of Rdev, Nlink in stat on Windows |
|
| 46 |
- return |
|
| 47 |
-} |
|
| 48 |
- |
|
| 49 | 44 |
func getInodeFromStat(stat interface{}) (uint64, error) {
|
| 50 | 45 |
// do nothing. no notion of Inode in stat on Windows |
| 51 | 46 |
return 0, nil |
| ... | ... |
@@ -75,7 +75,7 @@ func sameFsTime(a, b time.Time) bool {
|
| 75 | 75 |
// Changes walks the path rw and determines changes for the files in the path, |
| 76 | 76 |
// with respect to the parent layers |
| 77 | 77 |
func Changes(layers []string, rw string) ([]Change, error) {
|
| 78 |
- return changes(layers, rw, aufsDeletedFile, aufsMetadataSkip) |
|
| 78 |
+ return collectChanges(layers, rw, aufsDeletedFile, aufsMetadataSkip) |
|
| 79 | 79 |
} |
| 80 | 80 |
|
| 81 | 81 |
func aufsMetadataSkip(path string) (skip bool, err error) {
|
| ... | ... |
@@ -103,7 +103,7 @@ type ( |
| 103 | 103 |
deleteChange func(string, string, os.FileInfo) (string, error) |
| 104 | 104 |
) |
| 105 | 105 |
|
| 106 |
-func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Change, error) {
|
|
| 106 |
+func collectChanges(layers []string, rw string, dc deleteChange, sc skipChange) ([]Change, error) {
|
|
| 107 | 107 |
var ( |
| 108 | 108 |
changes []Change |
| 109 | 109 |
changedDirs = make(map[string]struct{})
|
| ... | ... |
@@ -132,14 +132,7 @@ func (w *walker) walk(path string, i1, i2 os.FileInfo) (err error) {
|
| 132 | 132 |
ix1 := 0 |
| 133 | 133 |
ix2 := 0 |
| 134 | 134 |
|
| 135 |
- for {
|
|
| 136 |
- if ix1 >= len(names1) {
|
|
| 137 |
- break |
|
| 138 |
- } |
|
| 139 |
- if ix2 >= len(names2) {
|
|
| 140 |
- break |
|
| 141 |
- } |
|
| 142 |
- |
|
| 135 |
+ for ix1 < len(names1) && ix2 < len(names2) {
|
|
| 143 | 136 |
ni1 := names1[ix1] |
| 144 | 137 |
ni2 := names2[ix2] |
| 145 | 138 |
|
| ... | ... |
@@ -1,7 +1,7 @@ |
| 1 | 1 |
package chrootarchive |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
- "fmt" |
|
| 4 |
+ "errors" |
|
| 5 | 5 |
"io" |
| 6 | 6 |
"os" |
| 7 | 7 |
"path/filepath" |
| ... | ... |
@@ -9,6 +9,7 @@ import ( |
| 9 | 9 |
"github.com/moby/sys/user" |
| 10 | 10 |
|
| 11 | 11 |
"github.com/moby/go-archive" |
| 12 |
+ "github.com/moby/go-archive/compression" |
|
| 12 | 13 |
) |
| 13 | 14 |
|
| 14 | 15 |
// NewArchiver returns a new Archiver which uses chrootarchive.Untar |
| ... | ... |
@@ -53,7 +54,7 @@ func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOp |
| 53 | 53 |
// Handler for teasing out the automatic decompression |
| 54 | 54 |
func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool, root string) error {
|
| 55 | 55 |
if tarArchive == nil {
|
| 56 |
- return fmt.Errorf("Empty archive")
|
|
| 56 |
+ return errors.New("empty archive")
|
|
| 57 | 57 |
} |
| 58 | 58 |
if options == nil {
|
| 59 | 59 |
options = &archive.TarOptions{}
|
| ... | ... |
@@ -77,7 +78,7 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions |
| 77 | 77 |
|
| 78 | 78 |
r := io.NopCloser(tarArchive) |
| 79 | 79 |
if decompress {
|
| 80 |
- decompressedArchive, err := archive.DecompressStream(tarArchive) |
|
| 80 |
+ decompressedArchive, err := compression.DecompressStream(tarArchive) |
|
| 81 | 81 |
if err != nil {
|
| 82 | 82 |
return err |
| 83 | 83 |
} |
| ... | ... |
@@ -9,6 +9,7 @@ import ( |
| 9 | 9 |
"github.com/moby/sys/userns" |
| 10 | 10 |
|
| 11 | 11 |
"github.com/moby/go-archive" |
| 12 |
+ "github.com/moby/go-archive/compression" |
|
| 12 | 13 |
) |
| 13 | 14 |
|
| 14 | 15 |
// applyLayerHandler parses a diff in the standard layer format from `layer`, and |
| ... | ... |
@@ -16,7 +17,7 @@ import ( |
| 16 | 16 |
// contents of the layer. |
| 17 | 17 |
func applyLayerHandler(dest string, layer io.Reader, options *archive.TarOptions, decompress bool) (size int64, _ error) {
|
| 18 | 18 |
if decompress {
|
| 19 |
- decompressed, err := archive.DecompressStream(layer) |
|
| 19 |
+ decompressed, err := compression.DecompressStream(layer) |
|
| 20 | 20 |
if err != nil {
|
| 21 | 21 |
return 0, err |
| 22 | 22 |
} |
| ... | ... |
@@ -6,6 +6,7 @@ import ( |
| 6 | 6 |
"path/filepath" |
| 7 | 7 |
|
| 8 | 8 |
"github.com/moby/go-archive" |
| 9 |
+ "github.com/moby/go-archive/compression" |
|
| 9 | 10 |
) |
| 10 | 11 |
|
| 11 | 12 |
// applyLayerHandler parses a diff in the standard layer format from `layer`, and |
| ... | ... |
@@ -13,7 +14,7 @@ import ( |
| 13 | 13 |
// contents of the layer. |
| 14 | 14 |
func applyLayerHandler(dest string, layer io.Reader, options *archive.TarOptions, decompress bool) (size int64, err error) {
|
| 15 | 15 |
if decompress {
|
| 16 |
- decompressed, err := archive.DecompressStream(layer) |
|
| 16 |
+ decompressed, err := compression.DecompressStream(layer) |
|
| 17 | 17 |
if err != nil {
|
| 18 | 18 |
return 0, err |
| 19 | 19 |
} |
| ... | ... |
@@ -26,7 +27,7 @@ func applyLayerHandler(dest string, layer io.Reader, options *archive.TarOptions |
| 26 | 26 |
dest = addLongPathPrefix(filepath.Clean(dest)) |
| 27 | 27 |
s, err := archive.UnpackLayer(dest, layer, nil) |
| 28 | 28 |
if err != nil {
|
| 29 |
- return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s: %s", layer, dest, err)
|
|
| 29 |
+ return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s: %w", layer, dest, err)
|
|
| 30 | 30 |
} |
| 31 | 31 |
|
| 32 | 32 |
return s, nil |
| 33 | 33 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,263 @@ |
| 0 |
+package compression |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "bufio" |
|
| 4 |
+ "bytes" |
|
| 5 |
+ "compress/bzip2" |
|
| 6 |
+ "compress/gzip" |
|
| 7 |
+ "context" |
|
| 8 |
+ "errors" |
|
| 9 |
+ "fmt" |
|
| 10 |
+ "io" |
|
| 11 |
+ "os" |
|
| 12 |
+ "os/exec" |
|
| 13 |
+ "strconv" |
|
| 14 |
+ "sync" |
|
| 15 |
+ |
|
| 16 |
+ "github.com/containerd/log" |
|
| 17 |
+ "github.com/klauspost/compress/zstd" |
|
| 18 |
+) |
|
| 19 |
+ |
|
| 20 |
+// Compression is the state represents if compressed or not. |
|
| 21 |
+type Compression int |
|
| 22 |
+ |
|
| 23 |
+const ( |
|
| 24 |
+ None Compression = 0 // None represents the uncompressed. |
|
| 25 |
+ Bzip2 Compression = 1 // Bzip2 is bzip2 compression algorithm. |
|
| 26 |
+ Gzip Compression = 2 // Gzip is gzip compression algorithm. |
|
| 27 |
+ Xz Compression = 3 // Xz is xz compression algorithm. |
|
| 28 |
+ Zstd Compression = 4 // Zstd is zstd compression algorithm. |
|
| 29 |
+) |
|
| 30 |
+ |
|
| 31 |
+// Extension returns the extension of a file that uses the specified compression algorithm. |
|
| 32 |
+func (c *Compression) Extension() string {
|
|
| 33 |
+ switch *c {
|
|
| 34 |
+ case None: |
|
| 35 |
+ return "tar" |
|
| 36 |
+ case Bzip2: |
|
| 37 |
+ return "tar.bz2" |
|
| 38 |
+ case Gzip: |
|
| 39 |
+ return "tar.gz" |
|
| 40 |
+ case Xz: |
|
| 41 |
+ return "tar.xz" |
|
| 42 |
+ case Zstd: |
|
| 43 |
+ return "tar.zst" |
|
| 44 |
+ default: |
|
| 45 |
+ return "" |
|
| 46 |
+ } |
|
| 47 |
+} |
|
| 48 |
+ |
|
| 49 |
+type readCloserWrapper struct {
|
|
| 50 |
+ io.Reader |
|
| 51 |
+ closer func() error |
|
| 52 |
+} |
|
| 53 |
+ |
|
| 54 |
+func (r *readCloserWrapper) Close() error {
|
|
| 55 |
+ if r.closer != nil {
|
|
| 56 |
+ return r.closer() |
|
| 57 |
+ } |
|
| 58 |
+ return nil |
|
| 59 |
+} |
|
| 60 |
+ |
|
| 61 |
+type nopWriteCloser struct {
|
|
| 62 |
+ io.Writer |
|
| 63 |
+} |
|
| 64 |
+ |
|
| 65 |
+func (nopWriteCloser) Close() error { return nil }
|
|
| 66 |
+ |
|
| 67 |
+var bufioReader32KPool = &sync.Pool{
|
|
| 68 |
+ New: func() interface{} { return bufio.NewReaderSize(nil, 32*1024) },
|
|
| 69 |
+} |
|
| 70 |
+ |
|
| 71 |
+type bufferedReader struct {
|
|
| 72 |
+ buf *bufio.Reader |
|
| 73 |
+} |
|
| 74 |
+ |
|
| 75 |
+func newBufferedReader(r io.Reader) *bufferedReader {
|
|
| 76 |
+ buf := bufioReader32KPool.Get().(*bufio.Reader) |
|
| 77 |
+ buf.Reset(r) |
|
| 78 |
+ return &bufferedReader{buf}
|
|
| 79 |
+} |
|
| 80 |
+ |
|
| 81 |
+func (r *bufferedReader) Read(p []byte) (int, error) {
|
|
| 82 |
+ if r.buf == nil {
|
|
| 83 |
+ return 0, io.EOF |
|
| 84 |
+ } |
|
| 85 |
+ n, err := r.buf.Read(p) |
|
| 86 |
+ if errors.Is(err, io.EOF) {
|
|
| 87 |
+ r.buf.Reset(nil) |
|
| 88 |
+ bufioReader32KPool.Put(r.buf) |
|
| 89 |
+ r.buf = nil |
|
| 90 |
+ } |
|
| 91 |
+ return n, err |
|
| 92 |
+} |
|
| 93 |
+ |
|
| 94 |
+func (r *bufferedReader) Peek(n int) ([]byte, error) {
|
|
| 95 |
+ if r.buf == nil {
|
|
| 96 |
+ return nil, io.EOF |
|
| 97 |
+ } |
|
| 98 |
+ return r.buf.Peek(n) |
|
| 99 |
+} |
|
| 100 |
+ |
|
| 101 |
+// DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive. |
|
| 102 |
+func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
|
|
| 103 |
+ buf := newBufferedReader(archive) |
|
| 104 |
+ bs, err := buf.Peek(10) |
|
| 105 |
+ if err != nil && !errors.Is(err, io.EOF) {
|
|
| 106 |
+ // Note: we'll ignore any io.EOF error because there are some odd |
|
| 107 |
+ // cases where the layer.tar file will be empty (zero bytes) and |
|
| 108 |
+ // that results in an io.EOF from the Peek() call. So, in those |
|
| 109 |
+ // cases we'll just treat it as a non-compressed stream and |
|
| 110 |
+ // that means just create an empty layer. |
|
| 111 |
+ // See Issue 18170 |
|
| 112 |
+ return nil, err |
|
| 113 |
+ } |
|
| 114 |
+ |
|
| 115 |
+ switch compression := Detect(bs); compression {
|
|
| 116 |
+ case None: |
|
| 117 |
+ return &readCloserWrapper{
|
|
| 118 |
+ Reader: buf, |
|
| 119 |
+ }, nil |
|
| 120 |
+ case Gzip: |
|
| 121 |
+ ctx, cancel := context.WithCancel(context.Background()) |
|
| 122 |
+ gzReader, err := gzipDecompress(ctx, buf) |
|
| 123 |
+ if err != nil {
|
|
| 124 |
+ cancel() |
|
| 125 |
+ return nil, err |
|
| 126 |
+ } |
|
| 127 |
+ |
|
| 128 |
+ return &readCloserWrapper{
|
|
| 129 |
+ Reader: gzReader, |
|
| 130 |
+ closer: func() error {
|
|
| 131 |
+ cancel() |
|
| 132 |
+ return gzReader.Close() |
|
| 133 |
+ }, |
|
| 134 |
+ }, nil |
|
| 135 |
+ case Bzip2: |
|
| 136 |
+ bz2Reader := bzip2.NewReader(buf) |
|
| 137 |
+ return &readCloserWrapper{
|
|
| 138 |
+ Reader: bz2Reader, |
|
| 139 |
+ }, nil |
|
| 140 |
+ case Xz: |
|
| 141 |
+ ctx, cancel := context.WithCancel(context.Background()) |
|
| 142 |
+ |
|
| 143 |
+ xzReader, err := xzDecompress(ctx, buf) |
|
| 144 |
+ if err != nil {
|
|
| 145 |
+ cancel() |
|
| 146 |
+ return nil, err |
|
| 147 |
+ } |
|
| 148 |
+ |
|
| 149 |
+ return &readCloserWrapper{
|
|
| 150 |
+ Reader: xzReader, |
|
| 151 |
+ closer: func() error {
|
|
| 152 |
+ cancel() |
|
| 153 |
+ return xzReader.Close() |
|
| 154 |
+ }, |
|
| 155 |
+ }, nil |
|
| 156 |
+ case Zstd: |
|
| 157 |
+ zstdReader, err := zstd.NewReader(buf) |
|
| 158 |
+ if err != nil {
|
|
| 159 |
+ return nil, err |
|
| 160 |
+ } |
|
| 161 |
+ return &readCloserWrapper{
|
|
| 162 |
+ Reader: zstdReader, |
|
| 163 |
+ closer: func() error {
|
|
| 164 |
+ zstdReader.Close() |
|
| 165 |
+ return nil |
|
| 166 |
+ }, |
|
| 167 |
+ }, nil |
|
| 168 |
+ |
|
| 169 |
+ default: |
|
| 170 |
+ return nil, fmt.Errorf("unsupported compression format (%d)", compression)
|
|
| 171 |
+ } |
|
| 172 |
+} |
|
| 173 |
+ |
|
| 174 |
+// CompressStream compresses the dest with specified compression algorithm. |
|
| 175 |
+func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) {
|
|
| 176 |
+ switch compression {
|
|
| 177 |
+ case None: |
|
| 178 |
+ return nopWriteCloser{dest}, nil
|
|
| 179 |
+ case Gzip: |
|
| 180 |
+ return gzip.NewWriter(dest), nil |
|
| 181 |
+ case Bzip2: |
|
| 182 |
+ // archive/bzip2 does not support writing. |
|
| 183 |
+ return nil, errors.New("unsupported compression format: tar.bz2")
|
|
| 184 |
+ case Xz: |
|
| 185 |
+ // there is no xz support at all |
|
| 186 |
+ // However, this is not a problem as docker only currently generates gzipped tars |
|
| 187 |
+ return nil, errors.New("unsupported compression format: tar.xz")
|
|
| 188 |
+ default: |
|
| 189 |
+ return nil, fmt.Errorf("unsupported compression format (%d)", compression)
|
|
| 190 |
+ } |
|
| 191 |
+} |
|
| 192 |
+ |
|
| 193 |
+func xzDecompress(ctx context.Context, archive io.Reader) (io.ReadCloser, error) {
|
|
| 194 |
+ args := []string{"xz", "-d", "-c", "-q"}
|
|
| 195 |
+ |
|
| 196 |
+ return cmdStream(exec.CommandContext(ctx, args[0], args[1:]...), archive) |
|
| 197 |
+} |
|
| 198 |
+ |
|
| 199 |
+func gzipDecompress(ctx context.Context, buf io.Reader) (io.ReadCloser, error) {
|
|
| 200 |
+ if noPigzEnv := os.Getenv("MOBY_DISABLE_PIGZ"); noPigzEnv != "" {
|
|
| 201 |
+ noPigz, err := strconv.ParseBool(noPigzEnv) |
|
| 202 |
+ if err != nil {
|
|
| 203 |
+ log.G(ctx).WithError(err).Warn("invalid value in MOBY_DISABLE_PIGZ env var")
|
|
| 204 |
+ } |
|
| 205 |
+ if noPigz {
|
|
| 206 |
+ log.G(ctx).Debugf("Use of pigz is disabled due to MOBY_DISABLE_PIGZ=%s", noPigzEnv)
|
|
| 207 |
+ return gzip.NewReader(buf) |
|
| 208 |
+ } |
|
| 209 |
+ } |
|
| 210 |
+ |
|
| 211 |
+ unpigzPath, err := exec.LookPath("unpigz")
|
|
| 212 |
+ if err != nil {
|
|
| 213 |
+ log.G(ctx).Debugf("unpigz binary not found, falling back to go gzip library")
|
|
| 214 |
+ return gzip.NewReader(buf) |
|
| 215 |
+ } |
|
| 216 |
+ |
|
| 217 |
+ log.G(ctx).Debugf("Using %s to decompress", unpigzPath)
|
|
| 218 |
+ |
|
| 219 |
+ return cmdStream(exec.CommandContext(ctx, unpigzPath, "-d", "-c"), buf) |
|
| 220 |
+} |
|
| 221 |
+ |
|
| 222 |
+// cmdStream executes a command, and returns its stdout as a stream. |
|
| 223 |
+// If the command fails to run or doesn't complete successfully, an error |
|
| 224 |
+// will be returned, including anything written on stderr. |
|
| 225 |
+func cmdStream(cmd *exec.Cmd, in io.Reader) (io.ReadCloser, error) {
|
|
| 226 |
+ reader, writer := io.Pipe() |
|
| 227 |
+ |
|
| 228 |
+ cmd.Stdin = in |
|
| 229 |
+ cmd.Stdout = writer |
|
| 230 |
+ |
|
| 231 |
+ var errBuf bytes.Buffer |
|
| 232 |
+ cmd.Stderr = &errBuf |
|
| 233 |
+ |
|
| 234 |
+ // Run the command and return the pipe |
|
| 235 |
+ if err := cmd.Start(); err != nil {
|
|
| 236 |
+ return nil, err |
|
| 237 |
+ } |
|
| 238 |
+ |
|
| 239 |
+ // Ensure the command has exited before we clean anything up |
|
| 240 |
+ done := make(chan struct{})
|
|
| 241 |
+ |
|
| 242 |
+ // Copy stdout to the returned pipe |
|
| 243 |
+ go func() {
|
|
| 244 |
+ if err := cmd.Wait(); err != nil {
|
|
| 245 |
+ _ = writer.CloseWithError(fmt.Errorf("%w: %s", err, errBuf.String()))
|
|
| 246 |
+ } else {
|
|
| 247 |
+ _ = writer.Close() |
|
| 248 |
+ } |
|
| 249 |
+ close(done) |
|
| 250 |
+ }() |
|
| 251 |
+ |
|
| 252 |
+ return &readCloserWrapper{
|
|
| 253 |
+ Reader: reader, |
|
| 254 |
+ closer: func() error {
|
|
| 255 |
+ // Close pipeR, and then wait for the command to complete before returning. We have to close pipeR first, as |
|
| 256 |
+ // cmd.Wait waits for any non-file stdout/stderr/stdin to close. |
|
| 257 |
+ err := reader.Close() |
|
| 258 |
+ <-done |
|
| 259 |
+ return err |
|
| 260 |
+ }, |
|
| 261 |
+ }, nil |
|
| 262 |
+} |
| 0 | 263 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,65 @@ |
| 0 |
+package compression |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "bytes" |
|
| 4 |
+ "encoding/binary" |
|
| 5 |
+) |
|
| 6 |
+ |
|
| 7 |
+const ( |
|
| 8 |
+ zstdMagicSkippableStart = 0x184D2A50 |
|
| 9 |
+ zstdMagicSkippableMask = 0xFFFFFFF0 |
|
| 10 |
+) |
|
| 11 |
+ |
|
| 12 |
+var ( |
|
| 13 |
+ bzip2Magic = []byte{0x42, 0x5A, 0x68}
|
|
| 14 |
+ gzipMagic = []byte{0x1F, 0x8B, 0x08}
|
|
| 15 |
+ xzMagic = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}
|
|
| 16 |
+ zstdMagic = []byte{0x28, 0xb5, 0x2f, 0xfd}
|
|
| 17 |
+) |
|
| 18 |
+ |
|
| 19 |
+type matcher = func([]byte) bool |
|
| 20 |
+ |
|
| 21 |
+// Detect detects the compression algorithm of the source. |
|
| 22 |
+func Detect(source []byte) Compression {
|
|
| 23 |
+ compressionMap := map[Compression]matcher{
|
|
| 24 |
+ Bzip2: magicNumberMatcher(bzip2Magic), |
|
| 25 |
+ Gzip: magicNumberMatcher(gzipMagic), |
|
| 26 |
+ Xz: magicNumberMatcher(xzMagic), |
|
| 27 |
+ Zstd: zstdMatcher(), |
|
| 28 |
+ } |
|
| 29 |
+ for _, compression := range []Compression{Bzip2, Gzip, Xz, Zstd} {
|
|
| 30 |
+ fn := compressionMap[compression] |
|
| 31 |
+ if fn(source) {
|
|
| 32 |
+ return compression |
|
| 33 |
+ } |
|
| 34 |
+ } |
|
| 35 |
+ return None |
|
| 36 |
+} |
|
| 37 |
+ |
|
| 38 |
+func magicNumberMatcher(m []byte) matcher {
|
|
| 39 |
+ return func(source []byte) bool {
|
|
| 40 |
+ return bytes.HasPrefix(source, m) |
|
| 41 |
+ } |
|
| 42 |
+} |
|
| 43 |
+ |
|
| 44 |
+// zstdMatcher detects zstd compression algorithm. |
|
| 45 |
+// Zstandard compressed data is made of one or more frames. |
|
| 46 |
+// There are two frame formats defined by Zstandard: Zstandard frames and Skippable frames. |
|
| 47 |
+// See https://datatracker.ietf.org/doc/html/rfc8878#section-3 for more details. |
|
| 48 |
+func zstdMatcher() matcher {
|
|
| 49 |
+ return func(source []byte) bool {
|
|
| 50 |
+ if bytes.HasPrefix(source, zstdMagic) {
|
|
| 51 |
+ // Zstandard frame |
|
| 52 |
+ return true |
|
| 53 |
+ } |
|
| 54 |
+ // skippable frame |
|
| 55 |
+ if len(source) < 8 {
|
|
| 56 |
+ return false |
|
| 57 |
+ } |
|
| 58 |
+ // magic number from 0x184D2A50 to 0x184D2A5F. |
|
| 59 |
+ if binary.LittleEndian.Uint32(source[:4])&zstdMagicSkippableMask == zstdMagicSkippableStart {
|
|
| 60 |
+ return true |
|
| 61 |
+ } |
|
| 62 |
+ return false |
|
| 63 |
+ } |
|
| 64 |
+} |
| ... | ... |
@@ -128,7 +128,6 @@ func TarResourceRebase(sourcePath, rebaseName string) (content io.ReadCloser, _ |
| 128 | 128 |
func TarResourceRebaseOpts(sourceBase string, rebaseName string) *TarOptions {
|
| 129 | 129 |
filter := []string{sourceBase}
|
| 130 | 130 |
return &TarOptions{
|
| 131 |
- Compression: Uncompressed, |
|
| 132 | 131 |
IncludeFiles: filter, |
| 133 | 132 |
IncludeSourceDir: true, |
| 134 | 133 |
RebaseNames: map[string]string{
|
| ... | ... |
@@ -335,7 +334,7 @@ func RebaseArchiveEntries(srcContent io.Reader, oldBase, newBase string) io.Read |
| 335 | 335 |
|
| 336 | 336 |
for {
|
| 337 | 337 |
hdr, err := srcTar.Next() |
| 338 |
- if err == io.EOF {
|
|
| 338 |
+ if errors.Is(err, io.EOF) {
|
|
| 339 | 339 |
// Signals end of archive. |
| 340 | 340 |
rebasedTar.Close() |
| 341 | 341 |
w.Close() |
| ... | ... |
@@ -3,6 +3,7 @@ package archive |
| 3 | 3 |
import ( |
| 4 | 4 |
"archive/tar" |
| 5 | 5 |
"context" |
| 6 |
+ "errors" |
|
| 6 | 7 |
"fmt" |
| 7 | 8 |
"io" |
| 8 | 9 |
"os" |
| ... | ... |
@@ -11,6 +12,8 @@ import ( |
| 11 | 11 |
"strings" |
| 12 | 12 |
|
| 13 | 13 |
"github.com/containerd/log" |
| 14 |
+ |
|
| 15 |
+ "github.com/moby/go-archive/compression" |
|
| 14 | 16 |
) |
| 15 | 17 |
|
| 16 | 18 |
// UnpackLayer unpack `layer` to a `dest`. The stream `layer` can be |
| ... | ... |
@@ -35,7 +38,7 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, |
| 35 | 35 |
// Iterate through the files in the archive. |
| 36 | 36 |
for {
|
| 37 | 37 |
hdr, err := tr.Next() |
| 38 |
- if err == io.EOF {
|
|
| 38 |
+ if errors.Is(err, io.EOF) {
|
|
| 39 | 39 |
// end of tar archive |
| 40 | 40 |
break |
| 41 | 41 |
} |
| ... | ... |
@@ -149,7 +152,7 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, |
| 149 | 149 |
// the layer is also a directory. Then we want to merge them (i.e. |
| 150 | 150 |
// just apply the metadata from the layer). |
| 151 | 151 |
if fi, err := os.Lstat(path); err == nil {
|
| 152 |
- if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
|
|
| 152 |
+ if !fi.IsDir() || hdr.Typeflag != tar.TypeDir {
|
|
| 153 | 153 |
if err := os.RemoveAll(path); err != nil {
|
| 154 | 154 |
return 0, err |
| 155 | 155 |
} |
| ... | ... |
@@ -165,7 +168,7 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, |
| 165 | 165 |
linkBasename := filepath.Base(hdr.Linkname) |
| 166 | 166 |
srcHdr = aufsHardlinks[linkBasename] |
| 167 | 167 |
if srcHdr == nil {
|
| 168 |
- return 0, fmt.Errorf("Invalid aufs hardlink")
|
|
| 168 |
+ return 0, errors.New("invalid aufs hardlink")
|
|
| 169 | 169 |
} |
| 170 | 170 |
tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename)) |
| 171 | 171 |
if err != nil {
|
| ... | ... |
@@ -221,18 +224,18 @@ func ApplyUncompressedLayer(dest string, layer io.Reader, options *TarOptions) ( |
| 221 | 221 |
|
| 222 | 222 |
// IsEmpty checks if the tar archive is empty (doesn't contain any entries). |
| 223 | 223 |
func IsEmpty(rd io.Reader) (bool, error) {
|
| 224 |
- decompRd, err := DecompressStream(rd) |
|
| 224 |
+ decompRd, err := compression.DecompressStream(rd) |
|
| 225 | 225 |
if err != nil {
|
| 226 |
- return true, fmt.Errorf("failed to decompress archive: %v", err)
|
|
| 226 |
+ return true, fmt.Errorf("failed to decompress archive: %w", err)
|
|
| 227 | 227 |
} |
| 228 | 228 |
defer decompRd.Close() |
| 229 | 229 |
|
| 230 | 230 |
tarReader := tar.NewReader(decompRd) |
| 231 | 231 |
if _, err := tarReader.Next(); err != nil {
|
| 232 |
- if err == io.EOF {
|
|
| 232 |
+ if errors.Is(err, io.EOF) {
|
|
| 233 | 233 |
return true, nil |
| 234 | 234 |
} |
| 235 |
- return false, fmt.Errorf("failed to read next archive header: %v", err)
|
|
| 235 |
+ return false, fmt.Errorf("failed to read next archive header: %w", err)
|
|
| 236 | 236 |
} |
| 237 | 237 |
|
| 238 | 238 |
return false, nil |
| ... | ... |
@@ -247,7 +250,7 @@ func applyLayerHandler(dest string, layer io.Reader, options *TarOptions, decomp |
| 247 | 247 |
defer restore() |
| 248 | 248 |
|
| 249 | 249 |
if decompress {
|
| 250 |
- decompLayer, err := DecompressStream(layer) |
|
| 250 |
+ decompLayer, err := compression.DecompressStream(layer) |
|
| 251 | 251 |
if err != nil {
|
| 252 | 252 |
return 0, err |
| 253 | 253 |
} |
| ... | ... |
@@ -26,7 +26,7 @@ func SwitchRoot(path string) error {
|
| 26 | 26 |
// setup oldRoot for pivot_root |
| 27 | 27 |
pivotDir, err := os.MkdirTemp(path, ".pivot_root") |
| 28 | 28 |
if err != nil {
|
| 29 |
- return fmt.Errorf("Error setting up pivot dir: %v", err)
|
|
| 29 |
+ return fmt.Errorf("error setting up pivot dir: %w", err)
|
|
| 30 | 30 |
} |
| 31 | 31 |
|
| 32 | 32 |
var mounted bool |
| ... | ... |
@@ -45,7 +45,7 @@ func SwitchRoot(path string) error {
|
| 45 | 45 |
// pivotDir doesn't exist if pivot_root failed and chroot+chdir was successful |
| 46 | 46 |
// because we already cleaned it up on failed pivot_root |
| 47 | 47 |
if errCleanup != nil && !os.IsNotExist(errCleanup) {
|
| 48 |
- errCleanup = fmt.Errorf("Error cleaning up after pivot: %v", errCleanup)
|
|
| 48 |
+ errCleanup = fmt.Errorf("error cleaning up after pivot: %w", errCleanup)
|
|
| 49 | 49 |
if err == nil {
|
| 50 | 50 |
err = errCleanup |
| 51 | 51 |
} |
| ... | ... |
@@ -55,7 +55,7 @@ func SwitchRoot(path string) error {
|
| 55 | 55 |
if err := unix.PivotRoot(path, pivotDir); err != nil {
|
| 56 | 56 |
// If pivot fails, fall back to the normal chroot after cleaning up temp dir |
| 57 | 57 |
if err := os.Remove(pivotDir); err != nil {
|
| 58 |
- return fmt.Errorf("Error cleaning up after failed pivot: %v", err)
|
|
| 58 |
+ return fmt.Errorf("error cleaning up after failed pivot: %w", err)
|
|
| 59 | 59 |
} |
| 60 | 60 |
return realChroot(path) |
| 61 | 61 |
} |
| ... | ... |
@@ -66,17 +66,17 @@ func SwitchRoot(path string) error {
|
| 66 | 66 |
pivotDir = filepath.Join("/", filepath.Base(pivotDir))
|
| 67 | 67 |
|
| 68 | 68 |
if err := unix.Chdir("/"); err != nil {
|
| 69 |
- return fmt.Errorf("Error changing to new root: %v", err)
|
|
| 69 |
+ return fmt.Errorf("error changing to new root: %w", err)
|
|
| 70 | 70 |
} |
| 71 | 71 |
|
| 72 | 72 |
// Make the pivotDir (where the old root lives) private so it can be unmounted without propagating to the host |
| 73 | 73 |
if err := unix.Mount("", pivotDir, "", unix.MS_PRIVATE|unix.MS_REC, ""); err != nil {
|
| 74 |
- return fmt.Errorf("Error making old root private after pivot: %v", err)
|
|
| 74 |
+ return fmt.Errorf("error making old root private after pivot: %w", err)
|
|
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 | 77 |
// Now unmount the old root so it's no longer visible from the new root |
| 78 | 78 |
if err := unix.Unmount(pivotDir, unix.MNT_DETACH); err != nil {
|
| 79 |
- return fmt.Errorf("Error while unmounting old root after pivot: %v", err)
|
|
| 79 |
+ return fmt.Errorf("error while unmounting old root after pivot: %w", err)
|
|
| 80 | 80 |
} |
| 81 | 81 |
mounted = false |
| 82 | 82 |
|
| ... | ... |
@@ -85,10 +85,10 @@ func SwitchRoot(path string) error {
|
| 85 | 85 |
|
| 86 | 86 |
func realChroot(path string) error {
|
| 87 | 87 |
if err := unix.Chroot(path); err != nil {
|
| 88 |
- return fmt.Errorf("Error after fallback to chroot: %v", err)
|
|
| 88 |
+ return fmt.Errorf("error after fallback to chroot: %w", err)
|
|
| 89 | 89 |
} |
| 90 | 90 |
if err := unix.Chdir("/"); err != nil {
|
| 91 |
- return fmt.Errorf("Error changing to new root after chroot: %v", err)
|
|
| 91 |
+ return fmt.Errorf("error changing to new root after chroot: %w", err)
|
|
| 92 | 92 |
} |
| 93 | 93 |
return nil |
| 94 | 94 |
} |
| 95 | 95 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,67 @@ |
| 0 |
+package tarheader |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "archive/tar" |
|
| 4 |
+ "os" |
|
| 5 |
+) |
|
| 6 |
+ |
|
| 7 |
+// assert that we implement [tar.FileInfoNames]. |
|
| 8 |
+var _ tar.FileInfoNames = (*nosysFileInfo)(nil) |
|
| 9 |
+ |
|
| 10 |
+// nosysFileInfo hides the system-dependent info of the wrapped FileInfo to |
|
| 11 |
+// prevent tar.FileInfoHeader from introspecting it and potentially calling into |
|
| 12 |
+// glibc. |
|
| 13 |
+// |
|
| 14 |
+// It implements [tar.FileInfoNames] to further prevent [tar.FileInfoHeader] |
|
| 15 |
+// from performing any lookups on go1.23 and up. see https://go.dev/issue/50102 |
|
| 16 |
+type nosysFileInfo struct {
|
|
| 17 |
+ os.FileInfo |
|
| 18 |
+} |
|
| 19 |
+ |
|
| 20 |
+// Uname stubs out looking up username. It implements [tar.FileInfoNames] |
|
| 21 |
+// to prevent [tar.FileInfoHeader] from loading libraries to perform |
|
| 22 |
+// username lookups. |
|
| 23 |
+func (fi nosysFileInfo) Uname() (string, error) {
|
|
| 24 |
+ return "", nil |
|
| 25 |
+} |
|
| 26 |
+ |
|
| 27 |
+// Gname stubs out looking up group-name. It implements [tar.FileInfoNames] |
|
| 28 |
+// to prevent [tar.FileInfoHeader] from loading libraries to perform |
|
| 29 |
+// username lookups. |
|
| 30 |
+func (fi nosysFileInfo) Gname() (string, error) {
|
|
| 31 |
+ return "", nil |
|
| 32 |
+} |
|
| 33 |
+ |
|
| 34 |
+func (fi nosysFileInfo) Sys() interface{} {
|
|
| 35 |
+ // A Sys value of type *tar.Header is safe as it is system-independent. |
|
| 36 |
+ // The tar.FileInfoHeader function copies the fields into the returned |
|
| 37 |
+ // header without performing any OS lookups. |
|
| 38 |
+ if sys, ok := fi.FileInfo.Sys().(*tar.Header); ok {
|
|
| 39 |
+ return sys |
|
| 40 |
+ } |
|
| 41 |
+ return nil |
|
| 42 |
+} |
|
| 43 |
+ |
|
| 44 |
+// FileInfoHeaderNoLookups creates a partially-populated tar.Header from fi. |
|
| 45 |
+// |
|
| 46 |
+// Compared to the archive/tar.FileInfoHeader function, this function is safe to |
|
| 47 |
+// call from a chrooted process as it does not populate fields which would |
|
| 48 |
+// require operating system lookups. It behaves identically to |
|
| 49 |
+// tar.FileInfoHeader when fi is a FileInfo value returned from |
|
| 50 |
+// tar.Header.FileInfo(). |
|
| 51 |
+// |
|
| 52 |
+// When fi is a FileInfo for a native file, such as returned from os.Stat() and |
|
| 53 |
+// os.Lstat(), the returned Header value differs from one returned from |
|
| 54 |
+// tar.FileInfoHeader in the following ways. The Uname and Gname fields are not |
|
| 55 |
+// set as OS lookups would be required to populate them. The AccessTime and |
|
| 56 |
+// ChangeTime fields are not currently set (not yet implemented) although that |
|
| 57 |
+// is subject to change. Callers which require the AccessTime or ChangeTime |
|
| 58 |
+// fields to be zeroed should explicitly zero them out in the returned Header |
|
| 59 |
+// value to avoid any compatibility issues in the future. |
|
| 60 |
+func FileInfoHeaderNoLookups(fi os.FileInfo, link string) (*tar.Header, error) {
|
|
| 61 |
+ hdr, err := tar.FileInfoHeader(nosysFileInfo{fi}, link)
|
|
| 62 |
+ if err != nil {
|
|
| 63 |
+ return nil, err |
|
| 64 |
+ } |
|
| 65 |
+ return hdr, sysStat(fi, hdr) |
|
| 66 |
+} |
| 0 | 67 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,46 @@ |
| 0 |
+//go:build !windows |
|
| 1 |
+ |
|
| 2 |
+package tarheader |
|
| 3 |
+ |
|
| 4 |
+import ( |
|
| 5 |
+ "archive/tar" |
|
| 6 |
+ "os" |
|
| 7 |
+ "runtime" |
|
| 8 |
+ "syscall" |
|
| 9 |
+ |
|
| 10 |
+ "golang.org/x/sys/unix" |
|
| 11 |
+) |
|
| 12 |
+ |
|
| 13 |
+// sysStat populates hdr from system-dependent fields of fi without performing |
|
| 14 |
+// any OS lookups. |
|
| 15 |
+func sysStat(fi os.FileInfo, hdr *tar.Header) error {
|
|
| 16 |
+ // Devmajor and Devminor are only needed for special devices. |
|
| 17 |
+ |
|
| 18 |
+ // In FreeBSD, RDev for regular files is -1 (unless overridden by FS): |
|
| 19 |
+ // https://cgit.freebsd.org/src/tree/sys/kern/vfs_default.c?h=stable/13#n1531 |
|
| 20 |
+ // (NODEV is -1: https://cgit.freebsd.org/src/tree/sys/sys/param.h?h=stable/13#n241). |
|
| 21 |
+ |
|
| 22 |
+ // ZFS in particular does not override the default: |
|
| 23 |
+ // https://cgit.freebsd.org/src/tree/sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c?h=stable/13#n2027 |
|
| 24 |
+ |
|
| 25 |
+ // Since `Stat_t.Rdev` is uint64, the cast turns -1 into (2^64 - 1). |
|
| 26 |
+ // Such large values cannot be encoded in a tar header. |
|
| 27 |
+ if runtime.GOOS == "freebsd" && hdr.Typeflag != tar.TypeBlock && hdr.Typeflag != tar.TypeChar {
|
|
| 28 |
+ return nil |
|
| 29 |
+ } |
|
| 30 |
+ s, ok := fi.Sys().(*syscall.Stat_t) |
|
| 31 |
+ if !ok {
|
|
| 32 |
+ return nil |
|
| 33 |
+ } |
|
| 34 |
+ |
|
| 35 |
+ hdr.Uid = int(s.Uid) |
|
| 36 |
+ hdr.Gid = int(s.Gid) |
|
| 37 |
+ |
|
| 38 |
+ if s.Mode&unix.S_IFBLK != 0 || |
|
| 39 |
+ s.Mode&unix.S_IFCHR != 0 {
|
|
| 40 |
+ hdr.Devmajor = int64(unix.Major(uint64(s.Rdev))) //nolint: unconvert |
|
| 41 |
+ hdr.Devminor = int64(unix.Minor(uint64(s.Rdev))) //nolint: unconvert |
|
| 42 |
+ } |
|
| 43 |
+ |
|
| 44 |
+ return nil |
|
| 45 |
+} |
| 0 | 46 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,12 @@ |
| 0 |
+package tarheader |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "archive/tar" |
|
| 4 |
+ "os" |
|
| 5 |
+) |
|
| 6 |
+ |
|
| 7 |
+// sysStat populates hdr from system-dependent fields of fi without performing |
|
| 8 |
+// any OS lookups. It is a no-op on Windows. |
|
| 9 |
+func sysStat(os.FileInfo, *tar.Header) error {
|
|
| 10 |
+ return nil |
|
| 11 |
+} |
| ... | ... |
@@ -925,12 +925,14 @@ github.com/moby/buildkit/worker/label |
| 925 | 925 |
# github.com/moby/docker-image-spec v1.3.1 |
| 926 | 926 |
## explicit; go 1.18 |
| 927 | 927 |
github.com/moby/docker-image-spec/specs-go/v1 |
| 928 |
-# github.com/moby/go-archive v0.0.0-20250404171912-21f3f3385ab7 |
|
| 928 |
+# github.com/moby/go-archive v0.1.0 |
|
| 929 | 929 |
## explicit; go 1.23.0 |
| 930 | 930 |
github.com/moby/go-archive |
| 931 | 931 |
github.com/moby/go-archive/chrootarchive |
| 932 |
+github.com/moby/go-archive/compression |
|
| 932 | 933 |
github.com/moby/go-archive/internal/mounttree |
| 933 | 934 |
github.com/moby/go-archive/internal/unshare |
| 935 |
+github.com/moby/go-archive/tarheader |
|
| 934 | 936 |
# github.com/moby/ipvs v1.1.0 |
| 935 | 937 |
## explicit; go 1.17 |
| 936 | 938 |
github.com/moby/ipvs |