package archive

 
 import (

 	"archive/tar"

 	"bufio"

 	"bytes"

 	"compress/bzip2"

 	"compress/gzip"

 	"fmt"

 	"io"
 	"io/ioutil"

 	"os"

 	"os/exec"

 	"path/filepath"

 	"runtime"

 	"strings"

 	"syscall"

 	"github.com/docker/docker/pkg/fileutils"

 	"github.com/docker/docker/pkg/idtools"

 	"github.com/docker/docker/pkg/ioutils"

 	"github.com/docker/docker/pkg/pools"

 	"github.com/docker/docker/pkg/promise"

 	"github.com/docker/docker/pkg/system"

 	"github.com/sirupsen/logrus"

 )
 

 type (

 	// Compression is the state represents if compressed or not.

 	Compression int

 	// WhiteoutFormat is the format of whiteouts unpacked
 	WhiteoutFormat int

 	// TarOptions wraps the tar options.

 	TarOptions struct {
 		IncludeFiles     []string
 		ExcludePatterns  []string
 		Compression      Compression
 		NoLchown         bool

 		UIDMaps          []idtools.IDMap
 		GIDMaps          []idtools.IDMap

 		ChownOpts        *idtools.IDPair

 		IncludeSourceDir bool

 		// WhiteoutFormat is the expected on disk format for whiteout files.
 		// This format will be converted to the standard format on pack
 		// and from the standard format on unpack.
 		WhiteoutFormat WhiteoutFormat

 		// When unpacking, specifies whether overwriting a directory with a
 		// non-directory is allowed and vice versa.
 		NoOverwriteDirNonDir bool

 		// For each include when creating an archive, the included name will be
 		// replaced with the matching name from this map.
 		RebaseNames map[string]string

 		InUserNS    bool

 	}
 )

 // Archiver implements the Archiver interface and allows the reuse of most utility functions of
 // this package with a pluggable Untar function. Also, to facilitate the passing of specific id
 // mappings for untar, an Archiver can be created with maps which will then be passed to Untar operations.

 type Archiver struct {

 	Untar         func(io.Reader, string, *TarOptions) error
 	IDMappingsVar *idtools.IDMappings

 }

 // NewDefaultArchiver returns a new Archiver without any IDMappings
 func NewDefaultArchiver() *Archiver {

 	return &Archiver{Untar: Untar, IDMappingsVar: &idtools.IDMappings{}}

 }
 
 // breakoutError is used to differentiate errors related to breaking out
 // When testing archive breakout in the unit tests, this error is expected
 // in order for the test to pass.
 type breakoutError error

 
 const (

 	// Uncompressed represents the uncompressed.

 	Uncompressed Compression = iota

 	// Bzip2 is bzip2 compression algorithm.

 	Bzip2

 	// Gzip is gzip compression algorithm.

 	Gzip

 	// Xz is xz compression algorithm.

 	Xz

 )
 

 const (

 	// AUFSWhiteoutFormat is the default format for whiteouts

 	AUFSWhiteoutFormat WhiteoutFormat = iota
 	// OverlayWhiteoutFormat formats whiteout according to the overlay
 	// standard.
 	OverlayWhiteoutFormat
 )
 

 const (
 	modeISDIR  = 040000  // Directory
 	modeISFIFO = 010000  // FIFO
 	modeISREG  = 0100000 // Regular file
 	modeISLNK  = 0120000 // Symbolic link
 	modeISBLK  = 060000  // Block special file
 	modeISCHR  = 020000  // Character special file
 	modeISSOCK = 0140000 // Socket
 )
 

 // IsArchivePath checks if the (possibly compressed) file at the given path
 // starts with a tar file header.
 func IsArchivePath(path string) bool {
 	file, err := os.Open(path)
 	if err != nil {
 		return false
 	}
 	defer file.Close()
 	rdr, err := DecompressStream(file)
 	if err != nil {
 		return false
 	}
 	r := tar.NewReader(rdr)
 	_, err = r.Next()
 	return err == nil
 }
 

 // DetectCompression detects the compression algorithm of the source.

 func DetectCompression(source []byte) Compression {
 	for compression, m := range map[Compression][]byte{
 		Bzip2: {0x42, 0x5A, 0x68},
 		Gzip:  {0x1F, 0x8B, 0x08},
 		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
 	} {

 		if len(source) < len(m) {

 			logrus.Debug("Len too short")

 			continue
 		}

 		if bytes.Equal(m, source[:len(m)]) {

 			return compression
 		}
 	}
 	return Uncompressed
 }
 

 func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) {

 	args := []string{"xz", "-d", "-c", "-q"}
 

 	return cmdStream(exec.Command(args[0], args[1:]...), archive)

 }
 

 // DecompressStream decompresses the archive and returns a ReaderCloser with the decompressed archive.

 func DecompressStream(archive io.Reader) (io.ReadCloser, error) {

 	p := pools.BufioReader32KPool
 	buf := p.Get(archive)

 	bs, err := buf.Peek(10)

 	if err != nil && err != io.EOF {
 		// Note: we'll ignore any io.EOF error because there are some odd
 		// cases where the layer.tar file will be empty (zero bytes) and
 		// that results in an io.EOF from the Peek() call. So, in those
 		// cases we'll just treat it as a non-compressed stream and
 		// that means just create an empty layer.
 		// See Issue 18170

 		return nil, err

 	}

 
 	compression := DetectCompression(bs)

 	switch compression {
 	case Uncompressed:

 		readBufWrapper := p.NewReadCloserWrapper(buf, buf)
 		return readBufWrapper, nil

 	case Gzip:

 		gzReader, err := gzip.NewReader(buf)
 		if err != nil {
 			return nil, err
 		}
 		readBufWrapper := p.NewReadCloserWrapper(buf, gzReader)
 		return readBufWrapper, nil

 	case Bzip2:

 		bz2Reader := bzip2.NewReader(buf)
 		readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader)
 		return readBufWrapper, nil

 	case Xz:

 		xzReader, chdone, err := xzDecompress(buf)

 		if err != nil {
 			return nil, err
 		}
 		readBufWrapper := p.NewReadCloserWrapper(buf, xzReader)

 		return ioutils.NewReadCloserWrapper(readBufWrapper, func() error {
 			<-chdone
 			return readBufWrapper.Close()
 		}), nil

 	default:
 		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
 	}
 }
 

 // CompressStream compresses the dest with specified compression algorithm.

 func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, error) {

 	p := pools.BufioWriter32KPool
 	buf := p.Get(dest)

 	switch compression {
 	case Uncompressed:

 		writeBufWrapper := p.NewWriteCloserWrapper(buf, buf)
 		return writeBufWrapper, nil

 	case Gzip:

 		gzWriter := gzip.NewWriter(dest)
 		writeBufWrapper := p.NewWriteCloserWrapper(buf, gzWriter)
 		return writeBufWrapper, nil

 	case Bzip2, Xz:
 		// archive/bzip2 does not support writing, and there is no xz support at all
 		// However, this is not a problem as docker only currently generates gzipped tars
 		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())
 	default:
 		return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension())

 	}
 }
 

 // TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to

 // modify the contents or header of an entry in the archive. If the file already
 // exists in the archive the TarModifierFunc will be called with the Header and
 // a reader which will return the files content. If the file does not exist both
 // header and content will be nil.

 type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
 

 // ReplaceFileTarWrapper converts inputTarStream to a new tar stream. Files in the
 // tar stream are modified if they match any of the keys in mods.

 func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser {
 	pipeReader, pipeWriter := io.Pipe()
 
 	go func() {
 		tarReader := tar.NewReader(inputTarStream)
 		tarWriter := tar.NewWriter(pipeWriter)
 		defer inputTarStream.Close()

 		defer tarWriter.Close()
 
 		modify := func(name string, original *tar.Header, modifier TarModifierFunc, tarReader io.Reader) error {
 			header, data, err := modifier(name, original, tarReader)
 			switch {
 			case err != nil:
 				return err
 			case header == nil:
 				return nil
 			}

 			header.Name = name
 			header.Size = int64(len(data))
 			if err := tarWriter.WriteHeader(header); err != nil {
 				return err
 			}
 			if len(data) != 0 {
 				if _, err := tarWriter.Write(data); err != nil {
 					return err

 				}
 			}

 			return nil
 		}

 		var err error
 		var originalHeader *tar.Header
 		for {
 			originalHeader, err = tarReader.Next()

 			if err == io.EOF {

 				break

 			}
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				return
 			}
 

 			modifier, ok := mods[originalHeader.Name]
 			if !ok {
 				// No modifiers for this file, copy the header and data
 				if err := tarWriter.WriteHeader(originalHeader); err != nil {
 					pipeWriter.CloseWithError(err)
 					return
 				}
 				if _, err := pools.Copy(tarWriter, tarReader); err != nil {
 					pipeWriter.CloseWithError(err)
 					return
 				}
 				continue
 			}
 			delete(mods, originalHeader.Name)
 
 			if err := modify(originalHeader.Name, originalHeader, modifier, tarReader); err != nil {

 				pipeWriter.CloseWithError(err)
 				return
 			}

 		}

 		// Apply the modifiers that haven't matched any files in the archive
 		for name, modifier := range mods {
 			if err := modify(name, nil, modifier, nil); err != nil {

 				pipeWriter.CloseWithError(err)
 				return
 			}
 		}

 
 		pipeWriter.Close()
 

 	}()
 	return pipeReader
 }
 

 // Extension returns the extension of a file that uses the specified compression algorithm.

 func (compression *Compression) Extension() string {
 	switch *compression {
 	case Uncompressed:
 		return "tar"
 	case Bzip2:
 		return "tar.bz2"
 	case Gzip:
 		return "tar.gz"
 	case Xz:
 		return "tar.xz"
 	}
 	return ""
 }
 

 // FileInfoHeader creates a populated Header from fi.
 // Compared to archive pkg this function fills in more information.

 // Also, regardless of Go version, this function fills file type bits (e.g. hdr.Mode |= modeISDIR),
 // which have been deleted since Go 1.9 archive/tar.

 func FileInfoHeader(name string, fi os.FileInfo, link string) (*tar.Header, error) {

 	hdr, err := tar.FileInfoHeader(fi, link)
 	if err != nil {
 		return nil, err
 	}

 	hdr.Mode = fillGo18FileTypeBits(int64(chmodTarEntry(os.FileMode(hdr.Mode))), fi)

 	name, err = canonicalTarName(name, fi.IsDir())
 	if err != nil {
 		return nil, fmt.Errorf("tar: cannot canonicalize path: %v", err)
 	}
 	hdr.Name = name
 	if err := setHeaderForSpecialDevice(hdr, name, fi.Sys()); err != nil {
 		return nil, err
 	}

 	return hdr, nil
 }
 

 // fillGo18FileTypeBits fills type bits which have been removed on Go 1.9 archive/tar
 // https://github.com/golang/go/commit/66b5a2f
 func fillGo18FileTypeBits(mode int64, fi os.FileInfo) int64 {
 	fm := fi.Mode()
 	switch {
 	case fm.IsRegular():
 		mode |= modeISREG
 	case fi.IsDir():
 		mode |= modeISDIR
 	case fm&os.ModeSymlink != 0:
 		mode |= modeISLNK
 	case fm&os.ModeDevice != 0:
 		if fm&os.ModeCharDevice != 0 {
 			mode |= modeISCHR
 		} else {
 			mode |= modeISBLK
 		}
 	case fm&os.ModeNamedPipe != 0:
 		mode |= modeISFIFO
 	case fm&os.ModeSocket != 0:
 		mode |= modeISSOCK
 	}
 	return mode
 }
 

 // ReadSecurityXattrToTarHeader reads security.capability xattr from filesystem
 // to a tar header
 func ReadSecurityXattrToTarHeader(path string, hdr *tar.Header) error {

 	capability, _ := system.Lgetxattr(path, "security.capability")
 	if capability != nil {
 		hdr.Xattrs = make(map[string]string)
 		hdr.Xattrs["security.capability"] = string(capability)
 	}

 	return nil

 }
 

 type tarWhiteoutConverter interface {

 	ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error)

 	ConvertRead(*tar.Header, string) (bool, error)
 }
 

 type tarAppender struct {
 	TarWriter *tar.Writer
 	Buffer    *bufio.Writer
 
 	// for hardlink mapping

 	SeenFiles  map[uint64]string
 	IDMappings *idtools.IDMappings

 	ChownOpts  *idtools.IDPair

 	// For packing and unpacking whiteout files in the
 	// non standard format. The whiteout files defined
 	// by the AUFS standard are used as the tar whiteout
 	// standard.
 	WhiteoutConverter tarWhiteoutConverter

 }
 

 func newTarAppender(idMapping *idtools.IDMappings, writer io.Writer, chownOpts *idtools.IDPair) *tarAppender {

 	return &tarAppender{
 		SeenFiles:  make(map[uint64]string),
 		TarWriter:  tar.NewWriter(writer),
 		Buffer:     pools.BufioWriter32KPool.Get(nil),
 		IDMappings: idMapping,

 		ChownOpts:  chownOpts,

 	}
 }
 

 // canonicalTarName provides a platform-independent and consistent posix-style
 //path for files and directories to be archived regardless of the platform.
 func canonicalTarName(name string, isDir bool) (string, error) {

 	name, err := CanonicalTarNameForPath(name)

 	if err != nil {
 		return "", err
 	}
 
 	// suffix with '/' for directories
 	if isDir && !strings.HasSuffix(name, "/") {
 		name += "/"
 	}
 	return name, nil
 }
 

 // addTarFile adds to the tar archive a file from `path` as `name`

 func (ta *tarAppender) addTarFile(path, name string) error {

 	fi, err := os.Lstat(path)
 	if err != nil {

 		return err
 	}
 

 	var link string
 	if fi.Mode()&os.ModeSymlink != 0 {
 		var err error
 		link, err = os.Readlink(path)
 		if err != nil {
 			return err
 		}
 	}
 
 	hdr, err := FileInfoHeader(name, fi, link)

 	if err != nil {
 		return err

 	}

 	if err := ReadSecurityXattrToTarHeader(path, hdr); err != nil {
 		return err
 	}

 	// if it's not a directory and has more than 1 link,

 	// it's hard linked, so set the type flag accordingly

 	if !fi.IsDir() && hasHardlinks(fi) {

 		inode, err := getInodeFromStat(fi.Sys())
 		if err != nil {
 			return err
 		}

 		// a link should have a name that it links too
 		// and that linked name should be first in the tar archive

 		if oldpath, ok := ta.SeenFiles[inode]; ok {

 			hdr.Typeflag = tar.TypeLink
 			hdr.Linkname = oldpath
 			hdr.Size = 0 // This Must be here for the writer math to add up!
 		} else {

 			ta.SeenFiles[inode] = name

 		}
 	}
 

 	//handle re-mapping container ID mappings back to host ID mappings before

 	//writing tar headers/files. We skip whiteout files because they were written
 	//by the kernel and already have proper ownership relative to the host

 	if !strings.HasPrefix(filepath.Base(hdr.Name), WhiteoutPrefix) && !ta.IDMappings.Empty() {

 		fileIDPair, err := getFileUIDGID(fi.Sys())

 		if err != nil {
 			return err
 		}

 		hdr.Uid, hdr.Gid, err = ta.IDMappings.ToContainer(fileIDPair)

 		if err != nil {
 			return err
 		}
 	}
 

 	// explicitly override with ChownOpts
 	if ta.ChownOpts != nil {
 		hdr.Uid = ta.ChownOpts.UID
 		hdr.Gid = ta.ChownOpts.GID
 	}
 

 	if ta.WhiteoutConverter != nil {

 		wo, err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi)
 		if err != nil {

 			return err

 		}

 
 		// If a new whiteout file exists, write original hdr, then
 		// replace hdr with wo to be written after. Whiteouts should
 		// always be written after the original. Note the original
 		// hdr may have been updated to be a whiteout with returning
 		// a whiteout header
 		if wo != nil {
 			if err := ta.TarWriter.WriteHeader(hdr); err != nil {
 				return err
 			}
 			if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {
 				return fmt.Errorf("tar: cannot use whiteout for non-empty file")
 			}
 			hdr = wo
 		}

 	}
 

 	if err := ta.TarWriter.WriteHeader(hdr); err != nil {

 		return err
 	}
 

 	if hdr.Typeflag == tar.TypeReg && hdr.Size > 0 {

 		// We use system.OpenSequential to ensure we use sequential file
 		// access on Windows to avoid depleting the standby list.
 		// On Linux, this equates to a regular os.Open.
 		file, err := system.OpenSequential(path)

 		if err != nil {

 			return err

 		}

 		ta.Buffer.Reset(ta.TarWriter)

 		defer ta.Buffer.Reset(nil)

 		_, err = io.Copy(ta.Buffer, file)

 		file.Close()
 		if err != nil {

 			return err

 		}

 		err = ta.Buffer.Flush()

 		if err != nil {
 			return err
 		}

 	}
 
 	return nil
 }
 

 func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns bool) error {

 	// hdr.Mode is in linux format, which we can use for sycalls,
 	// but for os.Foo() calls we need the mode converted to os.FileMode,
 	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
 	hdrInfo := hdr.FileInfo()
 

 	switch hdr.Typeflag {
 	case tar.TypeDir:
 		// Create directory unless it exists as a directory already.
 		// In that case we just want to merge the two
 		if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {

 			if err := os.Mkdir(path, hdrInfo.Mode()); err != nil {

 				return err
 			}
 		}
 
 	case tar.TypeReg, tar.TypeRegA:

 		// Source is regular file. We use system.OpenFileSequential to use sequential
 		// file access to avoid depleting the standby list on Windows.
 		// On Linux, this equates to a regular os.OpenFile
 		file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode())

 		if err != nil {
 			return err
 		}
 		if _, err := io.Copy(file, reader); err != nil {
 			file.Close()
 			return err
 		}
 		file.Close()
 

 	case tar.TypeBlock, tar.TypeChar:
 		if inUserns { // cannot create devices in a userns
 			return nil
 		}
 		// Handle this is an OS-specific way
 		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {
 			return err
 		}
 
 	case tar.TypeFifo:

 		// Handle this is an OS-specific way
 		if err := handleTarTypeBlockCharFifo(hdr, path); err != nil {

 			return err
 		}
 
 	case tar.TypeLink:

 		targetPath := filepath.Join(extractDir, hdr.Linkname)
 		// check for hardlink breakout
 		if !strings.HasPrefix(targetPath, extractDir) {
 			return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname))
 		}
 		if err := os.Link(targetPath, path); err != nil {

 			return err
 		}
 
 	case tar.TypeSymlink:

 		// 	path 				-> hdr.Linkname = targetPath
 		// e.g. /extractDir/path/to/symlink 	-> ../2/file	= /extractDir/path/2/file
 		targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname)
 
 		// the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because
 		// that symlink would first have to be created, which would be caught earlier, at this very check:
 		if !strings.HasPrefix(targetPath, extractDir) {
 			return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname))

 		}

 		if err := os.Symlink(hdr.Linkname, path); err != nil {
 			return err
 		}
 

 	case tar.TypeXGlobalHeader:

 		logrus.Debug("PAX Global Extended Headers found and ignored")

 		return nil
 

 	default:

 		return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag)

 	}
 

 	// Lchown is not supported on Windows.
 	if Lchown && runtime.GOOS != "windows" {
 		if chownOpts == nil {

 			chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}

 		}
 		if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil {

 			return err
 		}

 	}
 

 	var errors []string

 	for key, value := range hdr.Xattrs {

 		if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil {

 			if err == syscall.ENOTSUP {
 				// We ignore errors here because not all graphdrivers support
 				// xattrs *cough* old versions of AUFS *cough*. However only
 				// ENOTSUP should be emitted in that case, otherwise we still
 				// bail.
 				errors = append(errors, err.Error())
 				continue
 			}
 			return err

 		}

 
 	}
 
 	if len(errors) > 0 {
 		logrus.WithFields(logrus.Fields{
 			"errors": errors,
 		}).Warn("ignored xattrs in archive: underlying filesystem doesn't support them")

 	}
 

 	// There is no LChmod, so ignore mode for symlink. Also, this
 	// must happen after chown, as that can modify the file mode

 	if err := handleLChmod(hdr, path, hdrInfo); err != nil {
 		return err

 	}
 

 	aTime := hdr.AccessTime
 	if aTime.Before(hdr.ModTime) {
 		// Last access time should never be before last modified time.
 		aTime = hdr.ModTime
 	}
 

 	// system.Chtimes doesn't support a NOFOLLOW flag atm

 	if hdr.Typeflag == tar.TypeLink {
 		if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) {

 			if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {

 				return err
 			}
 		}
 	} else if hdr.Typeflag != tar.TypeSymlink {

 		if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil {

 			return err
 		}
 	} else {

 		ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)}

 		if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform {

 			return err
 		}
 	}
 	return nil
 }
 

 // Tar creates an archive from the directory at `path`, and returns it as a
 // stream of bytes.

 func Tar(path string, compression Compression) (io.ReadCloser, error) {

 	return TarWithOptions(path, &TarOptions{Compression: compression})

 }
 

 // TarWithOptions creates an archive from the directory at `path`, only including files whose relative

 // paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.

 func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {

 	// Fix the source path to work with long path names. This is a no-op
 	// on platforms other than Windows.
 	srcPath = fixVolumePathPrefix(srcPath)
 

 	pm, err := fileutils.NewPatternMatcher(options.ExcludePatterns)

 	if err != nil {
 		return nil, err
 	}
 

 	pipeReader, pipeWriter := io.Pipe()

 	compressWriter, err := CompressStream(pipeWriter, options.Compression)
 	if err != nil {
 		return nil, err

 	}

 	go func() {

 		ta := newTarAppender(
 			idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps),
 			compressWriter,

 			options.ChownOpts,

 		)
 		ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat)

 
 		defer func() {
 			// Make sure to check the error on Close.
 			if err := ta.TarWriter.Close(); err != nil {

 				logrus.Errorf("Can't close tar writer: %s", err)

 			}
 			if err := compressWriter.Close(); err != nil {

 				logrus.Errorf("Can't close compress writer: %s", err)

 			}
 			if err := pipeWriter.Close(); err != nil {

 				logrus.Errorf("Can't close pipe writer: %s", err)

 			}
 		}()
 

 		// this buffer is needed for the duration of this piped stream
 		defer pools.BufioWriter32KPool.Put(ta.Buffer)
 

 		// In general we log errors here but ignore them because
 		// during e.g. a diff operation the container can continue
 		// mutating the filesystem and we can see transient errors
 		// from this
 

 		stat, err := os.Lstat(srcPath)
 		if err != nil {
 			return
 		}
 
 		if !stat.IsDir() {
 			// We can't later join a non-dir with any includes because the
 			// 'walk' will error if "file/." is stat-ed and "file" is not a
 			// directory. So, we must split the source path and use the
 			// basename as the include.
 			if len(options.IncludeFiles) > 0 {
 				logrus.Warn("Tar: Can't archive a file with includes")
 			}
 
 			dir, base := SplitPathDirEntry(srcPath)
 			srcPath = dir
 			options.IncludeFiles = []string{base}
 		}
 
 		if len(options.IncludeFiles) == 0 {

 			options.IncludeFiles = []string{"."}

 		}
 

 		seen := make(map[string]bool)
 
 		for _, include := range options.IncludeFiles {

 			rebaseName := options.RebaseNames[include]
 

 			walkRoot := getWalkRoot(srcPath, include)

 			filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error {

 				if err != nil {

 					logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err)

 					return nil
 				}
 
 				relFilePath, err := filepath.Rel(srcPath, filePath)

 				if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) {

 					// Error getting relative path OR we are looking

 					// at the source directory path. Skip in both situations.

 					return nil
 				}
 

 				if options.IncludeSourceDir && include == "." && relFilePath != "." {
 					relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator))
 				}
 

 				skip := false
 
 				// If "include" is an exact match for the current file
 				// then even if there's an "excludePatterns" pattern that
 				// matches it, don't skip it. IOW, assume an explicit 'include'
 				// is asking for that file no matter what - which is true
 				// for some files, like .dockerignore and Dockerfile (sometimes)
 				if include != relFilePath {

 					skip, err = pm.Matches(relFilePath)

 					if err != nil {

 						logrus.Errorf("Error matching %s: %v", relFilePath, err)

 						return err
 					}

 				}

 				if skip {

 					// If we want to skip this file and its a directory
 					// then we should first check to see if there's an

 					// excludes pattern (e.g. !dir/file) that starts with this

 					// dir. If so then we can't skip this dir.
 
 					// Its not a dir then so we can just return/skip.
 					if !f.IsDir() {
 						return nil
 					}
 
 					// No exceptions (!...) in patterns so just skip dir

 					if !pm.Exclusions() {

 						return filepath.SkipDir

 					}

 
 					dirSlash := relFilePath + string(filepath.Separator)
 

 					for _, pat := range pm.Patterns() {
 						if !pat.Exclusion() {

 							continue
 						}

 						if strings.HasPrefix(pat.String()+string(filepath.Separator), dirSlash) {

 							// found a match - so can't skip this dir
 							return nil
 						}
 					}
 
 					// No matching exclusion dir so just skip dir
 					return filepath.SkipDir

 				}
 

 				if seen[relFilePath] {
 					return nil
 				}
 				seen[relFilePath] = true
 

 				// Rename the base resource.
 				if rebaseName != "" {
 					var replacement string
 					if rebaseName != string(filepath.Separator) {
 						// Special case the root directory to replace with an
 						// empty string instead so that we don't end up with
 						// double slashes in the paths.
 						replacement = rebaseName
 					}
 
 					relFilePath = strings.Replace(relFilePath, include, replacement, 1)

 				}
 

 				if err := ta.addTarFile(filePath, relFilePath); err != nil {

 					logrus.Errorf("Can't add file %s to tar: %s", filePath, err)

 					// if pipe is broken, stop writing tar stream to it

 					if err == io.ErrClosedPipe {
 						return err
 					}

 				}
 				return nil
 			})
 		}
 	}()
 
 	return pipeReader, nil

 }
 

 // Unpack unpacks the decompressedArchive to dest with options.

 func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error {

 	tr := tar.NewReader(decompressedArchive)

 	trBuf := pools.BufioReader32KPool.Get(nil)
 	defer pools.BufioReader32KPool.Put(trBuf)

 	var dirs []*tar.Header

 	idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)

 	rootIDs := idMappings.RootPair()

 	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat)

 
 	// Iterate through the files in the archive.

 loop:

 	for {
 		hdr, err := tr.Next()
 		if err == io.EOF {
 			// end of tar archive
 			break
 		}

 		if err != nil {

 			return err
 		}

 		// Normalize name, for safety and for a simple is-root check

 		// This keeps "../" as-is, but normalizes "/../" to "/". Or Windows:
 		// This keeps "..\" as-is, but normalizes "\..\" to "\".

 		hdr.Name = filepath.Clean(hdr.Name)
 

 		for _, exclude := range options.ExcludePatterns {

 			if strings.HasPrefix(hdr.Name, exclude) {
 				continue loop
 			}
 		}
 

 		// After calling filepath.Clean(hdr.Name) above, hdr.Name will now be in
 		// the filepath format for the OS on which the daemon is running. Hence
 		// the check for a slash-suffix MUST be done in an OS-agnostic way.
 		if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) {

 			// Not the root directory, ensure that the parent directory exists
 			parent := filepath.Dir(hdr.Name)
 			parentPath := filepath.Join(dest, parent)
 			if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {

 				err = idtools.MkdirAllAndChownNew(parentPath, 0777, rootIDs)

 				if err != nil {
 					return err
 				}
 			}
 		}
 

 		path := filepath.Join(dest, hdr.Name)

 		rel, err := filepath.Rel(dest, path)
 		if err != nil {
 			return err
 		}

 		if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {

 			return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))

 		}

 
 		// If path exits we almost always just want to remove and replace it
 		// The only exception is when it is a directory *and* the file from
 		// the layer is also a directory. Then we want to merge them (i.e.
 		// just apply the metadata from the layer).
 		if fi, err := os.Lstat(path); err == nil {

 			if options.NoOverwriteDirNonDir && fi.IsDir() && hdr.Typeflag != tar.TypeDir {
 				// If NoOverwriteDirNonDir is true then we cannot replace
 				// an existing directory with a non-directory from the archive.
 				return fmt.Errorf("cannot overwrite directory %q with non-directory %q", path, dest)
 			}
 
 			if options.NoOverwriteDirNonDir && !fi.IsDir() && hdr.Typeflag == tar.TypeDir {
 				// If NoOverwriteDirNonDir is true then we cannot replace
 				// an existing non-directory with a directory from the archive.
 				return fmt.Errorf("cannot overwrite non-directory %q with directory %q", path, dest)
 			}
 

 			if fi.IsDir() && hdr.Name == "." {
 				continue
 			}

 			if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
 				if err := os.RemoveAll(path); err != nil {
 					return err
 				}
 			}
 		}

 		trBuf.Reset(tr)

 		if err := remapIDs(idMappings, hdr); err != nil {
 			return err

 		}
 

 		if whiteoutConverter != nil {
 			writeFile, err := whiteoutConverter.ConvertRead(hdr, path)
 			if err != nil {
 				return err

 			}

 			if !writeFile {

 				continue
 			}
 		}
 

 		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts, options.InUserNS); err != nil {

 			return err
 		}

 		// Directory mtimes must be handled at the end to avoid further
 		// file creation in them to modify the directory mtime
 		if hdr.Typeflag == tar.TypeDir {
 			dirs = append(dirs, hdr)

 		}
 	}
 

 	for _, hdr := range dirs {
 		path := filepath.Join(dest, hdr.Name)

 
 		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {

 			return err
 		}

 	}
 	return nil
 }
 

 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
 // and unpacks it into the directory at `dest`.
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.

 func Untar(tarArchive io.Reader, dest string, options *TarOptions) error {
 	return untarHandler(tarArchive, dest, options, true)
 }
 

 // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive,

 // and unpacks it into the directory at `dest`.
 // The archive must be an uncompressed stream.
 func UntarUncompressed(tarArchive io.Reader, dest string, options *TarOptions) error {
 	return untarHandler(tarArchive, dest, options, false)
 }
 
 // Handler for teasing out the automatic decompression
 func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decompress bool) error {
 	if tarArchive == nil {

 		return fmt.Errorf("Empty archive")
 	}
 	dest = filepath.Clean(dest)
 	if options == nil {
 		options = &TarOptions{}
 	}

 	if options.ExcludePatterns == nil {
 		options.ExcludePatterns = []string{}

 	}

 	r := tarArchive

 	if decompress {
 		decompressedArchive, err := DecompressStream(tarArchive)
 		if err != nil {
 			return err
 		}
 		defer decompressedArchive.Close()
 		r = decompressedArchive

 	}

 
 	return Unpack(r, dest, options)

 }
 

 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
 // If either Tar or Untar fails, TarUntar aborts and returns the error.

 func (archiver *Archiver) TarUntar(src, dst string) error {

 	logrus.Debugf("TarUntar(%s %s)", src, dst)

 	archive, err := TarWithOptions(src, &TarOptions{Compression: Uncompressed})

 	if err != nil {
 		return err
 	}

 	defer archive.Close()

 	options := &TarOptions{

 		UIDMaps: archiver.IDMappingsVar.UIDs(),
 		GIDMaps: archiver.IDMappingsVar.GIDs(),

 	}
 	return archiver.Untar(archive, dst, options)

 }
 

 // UntarPath untar a file from path to a destination, src is the source tar file path.

 func (archiver *Archiver) UntarPath(src, dst string) error {

 	archive, err := os.Open(src)
 	if err != nil {

 		return err

 	}
 	defer archive.Close()

 	options := &TarOptions{

 		UIDMaps: archiver.IDMappingsVar.UIDs(),
 		GIDMaps: archiver.IDMappingsVar.GIDs(),

 	}

 	return archiver.Untar(archive, dst, options)

 }
 

 // CopyWithTar creates a tar archive of filesystem path `src`, and
 // unpacks it at filesystem path `dst`.
 // The archive is streamed directly with fixed buffering and no
 // intermediary disk IO.

 func (archiver *Archiver) CopyWithTar(src, dst string) error {

 	srcSt, err := os.Stat(src)

 	if err != nil {
 		return err
 	}

 	if !srcSt.IsDir() {

 		return archiver.CopyFileWithTar(src, dst)

 	}

 	// if this Archiver is set up with ID mapping we need to create

 	// the new destination directory with the remapped root UID/GID pair
 	// as owner

 	rootIDs := archiver.IDMappingsVar.RootPair()

 	// Create dst, copy src's content into it

 	logrus.Debugf("Creating dest directory: %s", dst)

 	if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil {

 		return err
 	}

 	logrus.Debugf("Calling TarUntar(%s, %s)", src, dst)

 	return archiver.TarUntar(src, dst)

 }
 

 // CopyFileWithTar emulates the behavior of the 'cp' command-line
 // for a single file. It copies a regular file from path `src` to
 // path `dst`, and preserves all its metadata.

 func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {

 	logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst)

 	srcSt, err := os.Stat(src)

 	if err != nil {

 		return err

 	}

 	if srcSt.IsDir() {

 		return fmt.Errorf("Can't copy a directory")

 	}

 
 	// Clean up the trailing slash. This must be done in an operating
 	// system specific manner.

 	if dst[len(dst)-1] == os.PathSeparator {
 		dst = filepath.Join(dst, filepath.Base(src))

 	}

 	// Create the holding directory if necessary

 	if err := system.MkdirAll(filepath.Dir(dst), 0700, ""); err != nil {

 		return err
 	}

 
 	r, w := io.Pipe()

 	errC := promise.Go(func() error {

 		defer w.Close()
 
 		srcF, err := os.Open(src)
 		if err != nil {
 			return err
 		}
 		defer srcF.Close()
 
 		hdr, err := tar.FileInfoHeader(srcSt, "")
 		if err != nil {
 			return err
 		}
 		hdr.Name = filepath.Base(dst)

 		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
 

 		if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {

 			return err
 		}
 

 		tw := tar.NewWriter(w)
 		defer tw.Close()

 		if err := tw.WriteHeader(hdr); err != nil {
 			return err
 		}
 		if _, err := io.Copy(tw, srcF); err != nil {
 			return err
 		}
 		return nil
 	})
 	defer func() {

 		if er := <-errC; err == nil && er != nil {

 			err = er
 		}
 	}()

 	err = archiver.Untar(r, filepath.Dir(dst), nil)
 	if err != nil {
 		r.CloseWithError(err)
 	}
 	return err

 }
 

 // IDMappings returns the IDMappings of the archiver.
 func (archiver *Archiver) IDMappings() *idtools.IDMappings {
 	return archiver.IDMappingsVar
 }
 

 func remapIDs(idMappings *idtools.IDMappings, hdr *tar.Header) error {
 	ids, err := idMappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid})
 	hdr.Uid, hdr.Gid = ids.UID, ids.GID
 	return err
 }
 

 // cmdStream executes a command, and returns its stdout as a stream.

 // If the command fails to run or doesn't complete successfully, an error
 // will be returned, including anything written on stderr.

 func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) {
 	chdone := make(chan struct{})
 	cmd.Stdin = input

 	pipeR, pipeW := io.Pipe()

 	cmd.Stdout = pipeW
 	var errBuf bytes.Buffer
 	cmd.Stderr = &errBuf
 
 	// Run the command and return the pipe
 	if err := cmd.Start(); err != nil {
 		return nil, nil, err
 	}
 

 	// Copy stdout to the returned pipe

 	go func() {

 		if err := cmd.Wait(); err != nil {

 			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String()))

 		} else {
 			pipeW.Close()
 		}

 		close(chdone)

 	}()

 
 	return pipeR, chdone, nil

 }

 
 // NewTempArchive reads the content of src into a temporary file, and returns the contents
 // of that file as an archive. The archive can only be read once - as soon as reading completes,
 // the file will be deleted.

 func NewTempArchive(src io.Reader, dir string) (*TempArchive, error) {

 	f, err := ioutil.TempFile(dir, "")
 	if err != nil {
 		return nil, err
 	}
 	if _, err := io.Copy(f, src); err != nil {
 		return nil, err
 	}
 	if _, err := f.Seek(0, 0); err != nil {
 		return nil, err
 	}
 	st, err := f.Stat()
 	if err != nil {
 		return nil, err
 	}
 	size := st.Size()

 	return &TempArchive{File: f, Size: size}, nil

 }
 

 // TempArchive is a temporary archive. The archive can only be read once - as soon as reading completes,
 // the file will be deleted.

 type TempArchive struct {
 	*os.File

 	Size   int64 // Pre-computed from Stat().Size() as a convenience
 	read   int64
 	closed bool
 }
 
 // Close closes the underlying file if it's still open, or does a no-op
 // to allow callers to try to close the TempArchive multiple times safely.
 func (archive *TempArchive) Close() error {
 	if archive.closed {
 		return nil
 	}
 
 	archive.closed = true
 
 	return archive.File.Close()

 }
 
 func (archive *TempArchive) Read(data []byte) (int, error) {
 	n, err := archive.File.Read(data)

 	archive.read += int64(n)
 	if err != nil || archive.read == archive.Size {

 		archive.Close()

 		os.Remove(archive.File.Name())
 	}
 	return n, err
 }