Browse code

Use continuity fs package for volume copy

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Brian Goff authored on 2018/02/13 05:27:34
Showing 40 changed files
... ...
@@ -6,13 +6,12 @@ import (
6 6
 	"io/ioutil"
7 7
 	"os"
8 8
 
9
+	"github.com/containerd/continuity/fs"
9 10
 	"github.com/docker/docker/api/types"
10 11
 	containertypes "github.com/docker/docker/api/types/container"
11 12
 	mounttypes "github.com/docker/docker/api/types/mount"
12
-	"github.com/docker/docker/pkg/chrootarchive"
13 13
 	"github.com/docker/docker/pkg/mount"
14 14
 	"github.com/docker/docker/pkg/stringid"
15
-	"github.com/docker/docker/pkg/system"
16 15
 	"github.com/docker/docker/volume"
17 16
 	"github.com/opencontainers/selinux/go-selinux/label"
18 17
 	"github.com/pkg/errors"
... ...
@@ -398,53 +397,15 @@ func (container *Container) DetachAndUnmount(volumeEventLog func(name, action st
398 398
 // copyExistingContents copies from the source to the destination and
399 399
 // ensures the ownership is appropriately set.
400 400
 func copyExistingContents(source, destination string) error {
401
-	volList, err := ioutil.ReadDir(source)
401
+	dstList, err := ioutil.ReadDir(destination)
402 402
 	if err != nil {
403 403
 		return err
404 404
 	}
405
-	if len(volList) > 0 {
406
-		srcList, err := ioutil.ReadDir(destination)
407
-		if err != nil {
408
-			return err
409
-		}
410
-		if len(srcList) == 0 {
411
-			// If the source volume is empty, copies files from the root into the volume
412
-			if err := chrootarchive.NewArchiver(nil).CopyWithTar(source, destination); err != nil {
413
-				return err
414
-			}
415
-		}
416
-	}
417
-	return copyOwnership(source, destination)
418
-}
419
-
420
-// copyOwnership copies the permissions and uid:gid of the source file
421
-// to the destination file
422
-func copyOwnership(source, destination string) error {
423
-	stat, err := system.Stat(source)
424
-	if err != nil {
425
-		return err
426
-	}
427
-
428
-	destStat, err := system.Stat(destination)
429
-	if err != nil {
430
-		return err
431
-	}
432
-
433
-	// In some cases, even though UID/GID match and it would effectively be a no-op,
434
-	// this can return a permission denied error... for example if this is an NFS
435
-	// mount.
436
-	// Since it's not really an error that we can't chown to the same UID/GID, don't
437
-	// even bother trying in such cases.
438
-	if stat.UID() != destStat.UID() || stat.GID() != destStat.GID() {
439
-		if err := os.Chown(destination, int(stat.UID()), int(stat.GID())); err != nil {
440
-			return err
441
-		}
442
-	}
443
-
444
-	if stat.Mode() != destStat.Mode() {
445
-		return os.Chmod(destination, os.FileMode(stat.Mode()))
405
+	if len(dstList) != 0 {
406
+		// destination is not empty, do not copy
407
+		return nil
446 408
 	}
447
-	return nil
409
+	return fs.CopyDir(destination, source)
448 410
 }
449 411
 
450 412
 // TmpfsMounts returns the list of tmpfs mounts
... ...
@@ -106,7 +106,7 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
106 106
 # containerd
107 107
 github.com/containerd/containerd 3fa104f843ec92328912e042b767d26825f202aa
108 108
 github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6
109
-github.com/containerd/continuity 35d55c5e8dd23b32037d56cf97174aff3efdfa83
109
+github.com/containerd/continuity 992a5f112bd2211d0983a1cc8562d2882848f3a3
110 110
 github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f
111 111
 github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e
112 112
 github.com/containerd/go-runc ed1cbe1fc31f5fb2359d3a54b6330d1a097858b7
113 113
deleted file mode 100644
... ...
@@ -1,15 +0,0 @@
1
-package devices
2
-
3
-// from /usr/include/sys/types.h
4
-
5
-func getmajor(dev int32) uint64 {
6
-	return (uint64(dev) >> 24) & 0xff
7
-}
8
-
9
-func getminor(dev int32) uint64 {
10
-	return uint64(dev) & 0xffffff
11
-}
12
-
13
-func makedev(major int, minor int) int {
14
-	return ((major << 24) | minor)
15
-}
16 1
deleted file mode 100644
... ...
@@ -1,23 +0,0 @@
1
-// +build solaris,!cgo
2
-
3
-//
4
-// Implementing the functions below requires cgo support.  Non-cgo stubs
5
-// versions are defined below to enable cross-compilation of source code
6
-// that depends on these functions, but the resultant cross-compiled
7
-// binaries cannot actually be used.  If the stub function(s) below are
8
-// actually invoked they will cause the calling process to exit.
9
-//
10
-
11
-package devices
12
-
13
-func getmajor(dev uint64) uint64 {
14
-	panic("getmajor() support requires cgo.")
15
-}
16
-
17
-func getminor(dev uint64) uint64 {
18
-	panic("getminor() support requires cgo.")
19
-}
20
-
21
-func makedev(major int, minor int) int {
22
-	panic("makedev() support requires cgo.")
23
-}
24 1
deleted file mode 100644
... ...
@@ -1,15 +0,0 @@
1
-package devices
2
-
3
-// from /usr/include/sys/types.h
4
-
5
-func getmajor(dev uint32) uint64 {
6
-	return (uint64(dev) >> 24) & 0xff
7
-}
8
-
9
-func getminor(dev uint32) uint64 {
10
-	return uint64(dev) & 0xffffff
11
-}
12
-
13
-func makedev(major int, minor int) int {
14
-	return ((major << 24) | minor)
15
-}
16 1
deleted file mode 100644
... ...
@@ -1,15 +0,0 @@
1
-package devices
2
-
3
-// from /usr/include/linux/kdev_t.h
4
-
5
-func getmajor(dev uint64) uint64 {
6
-	return dev >> 8
7
-}
8
-
9
-func getminor(dev uint64) uint64 {
10
-	return dev & 0xff
11
-}
12
-
13
-func makedev(major int, minor int) int {
14
-	return ((major << 8) | minor)
15
-}
16 1
deleted file mode 100644
... ...
@@ -1,18 +0,0 @@
1
-// +build cgo
2
-
3
-package devices
4
-
5
-//#include <sys/mkdev.h>
6
-import "C"
7
-
8
-func getmajor(dev uint64) uint64 {
9
-	return uint64(C.major(C.dev_t(dev)))
10
-}
11
-
12
-func getminor(dev uint64) uint64 {
13
-	return uint64(C.minor(C.dev_t(dev)))
14
-}
15
-
16
-func makedev(major int, minor int) int {
17
-	return int(C.makedev(C.major_t(major), C.minor_t(minor)))
18
-}
... ...
@@ -6,6 +6,8 @@ import (
6 6
 	"fmt"
7 7
 	"os"
8 8
 	"syscall"
9
+
10
+	"golang.org/x/sys/unix"
9 11
 )
10 12
 
11 13
 func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) {
... ...
@@ -14,42 +16,43 @@ func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) {
14 14
 		return 0, 0, fmt.Errorf("cannot extract device from os.FileInfo")
15 15
 	}
16 16
 
17
-	return getmajor(sys.Rdev), getminor(sys.Rdev), nil
17
+	dev := uint64(sys.Rdev)
18
+	return uint64(unix.Major(dev)), uint64(unix.Minor(dev)), nil
18 19
 }
19 20
 
20 21
 // mknod provides a shortcut for syscall.Mknod
21 22
 func Mknod(p string, mode os.FileMode, maj, min int) error {
22 23
 	var (
23 24
 		m   = syscallMode(mode.Perm())
24
-		dev int
25
+		dev uint64
25 26
 	)
26 27
 
27 28
 	if mode&os.ModeDevice != 0 {
28
-		dev = makedev(maj, min)
29
+		dev = unix.Mkdev(uint32(maj), uint32(min))
29 30
 
30 31
 		if mode&os.ModeCharDevice != 0 {
31
-			m |= syscall.S_IFCHR
32
+			m |= unix.S_IFCHR
32 33
 		} else {
33
-			m |= syscall.S_IFBLK
34
+			m |= unix.S_IFBLK
34 35
 		}
35 36
 	} else if mode&os.ModeNamedPipe != 0 {
36
-		m |= syscall.S_IFIFO
37
+		m |= unix.S_IFIFO
37 38
 	}
38 39
 
39
-	return syscall.Mknod(p, m, dev)
40
+	return unix.Mknod(p, m, int(dev))
40 41
 }
41 42
 
42 43
 // syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
43 44
 func syscallMode(i os.FileMode) (o uint32) {
44 45
 	o |= uint32(i.Perm())
45 46
 	if i&os.ModeSetuid != 0 {
46
-		o |= syscall.S_ISUID
47
+		o |= unix.S_ISUID
47 48
 	}
48 49
 	if i&os.ModeSetgid != 0 {
49
-		o |= syscall.S_ISGID
50
+		o |= unix.S_ISGID
50 51
 	}
51 52
 	if i&os.ModeSticky != 0 {
52
-		o |= syscall.S_ISVTX
53
+		o |= unix.S_ISVTX
53 54
 	}
54 55
 	return
55 56
 }
56 57
new file mode 100644
... ...
@@ -0,0 +1,119 @@
0
+package fs
1
+
2
+import (
3
+	"io/ioutil"
4
+	"os"
5
+	"path/filepath"
6
+	"sync"
7
+
8
+	"github.com/pkg/errors"
9
+)
10
+
11
+var bufferPool = &sync.Pool{
12
+	New: func() interface{} {
13
+		buffer := make([]byte, 32*1024)
14
+		return &buffer
15
+	},
16
+}
17
+
18
+// CopyDir copies the directory from src to dst.
19
+// Most efficient copy of files is attempted.
20
+func CopyDir(dst, src string) error {
21
+	inodes := map[uint64]string{}
22
+	return copyDirectory(dst, src, inodes)
23
+}
24
+
25
+func copyDirectory(dst, src string, inodes map[uint64]string) error {
26
+	stat, err := os.Stat(src)
27
+	if err != nil {
28
+		return errors.Wrapf(err, "failed to stat %s", src)
29
+	}
30
+	if !stat.IsDir() {
31
+		return errors.Errorf("source is not directory")
32
+	}
33
+
34
+	if st, err := os.Stat(dst); err != nil {
35
+		if err := os.Mkdir(dst, stat.Mode()); err != nil {
36
+			return errors.Wrapf(err, "failed to mkdir %s", dst)
37
+		}
38
+	} else if !st.IsDir() {
39
+		return errors.Errorf("cannot copy to non-directory: %s", dst)
40
+	} else {
41
+		if err := os.Chmod(dst, stat.Mode()); err != nil {
42
+			return errors.Wrapf(err, "failed to chmod on %s", dst)
43
+		}
44
+	}
45
+
46
+	fis, err := ioutil.ReadDir(src)
47
+	if err != nil {
48
+		return errors.Wrapf(err, "failed to read %s", src)
49
+	}
50
+
51
+	if err := copyFileInfo(stat, dst); err != nil {
52
+		return errors.Wrapf(err, "failed to copy file info for %s", dst)
53
+	}
54
+
55
+	for _, fi := range fis {
56
+		source := filepath.Join(src, fi.Name())
57
+		target := filepath.Join(dst, fi.Name())
58
+
59
+		switch {
60
+		case fi.IsDir():
61
+			if err := copyDirectory(target, source, inodes); err != nil {
62
+				return err
63
+			}
64
+			continue
65
+		case (fi.Mode() & os.ModeType) == 0:
66
+			link, err := getLinkSource(target, fi, inodes)
67
+			if err != nil {
68
+				return errors.Wrap(err, "failed to get hardlink")
69
+			}
70
+			if link != "" {
71
+				if err := os.Link(link, target); err != nil {
72
+					return errors.Wrap(err, "failed to create hard link")
73
+				}
74
+			} else if err := copyFile(source, target); err != nil {
75
+				return errors.Wrap(err, "failed to copy files")
76
+			}
77
+		case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink:
78
+			link, err := os.Readlink(source)
79
+			if err != nil {
80
+				return errors.Wrapf(err, "failed to read link: %s", source)
81
+			}
82
+			if err := os.Symlink(link, target); err != nil {
83
+				return errors.Wrapf(err, "failed to create symlink: %s", target)
84
+			}
85
+		case (fi.Mode() & os.ModeDevice) == os.ModeDevice:
86
+			if err := copyDevice(target, fi); err != nil {
87
+				return errors.Wrapf(err, "failed to create device")
88
+			}
89
+		default:
90
+			// TODO: Support pipes and sockets
91
+			return errors.Wrapf(err, "unsupported mode %s", fi.Mode())
92
+		}
93
+		if err := copyFileInfo(fi, target); err != nil {
94
+			return errors.Wrap(err, "failed to copy file info")
95
+		}
96
+
97
+		if err := copyXAttrs(target, source); err != nil {
98
+			return errors.Wrap(err, "failed to copy xattrs")
99
+		}
100
+	}
101
+
102
+	return nil
103
+}
104
+
105
+func copyFile(source, target string) error {
106
+	src, err := os.Open(source)
107
+	if err != nil {
108
+		return errors.Wrapf(err, "failed to open source %s", source)
109
+	}
110
+	defer src.Close()
111
+	tgt, err := os.Create(target)
112
+	if err != nil {
113
+		return errors.Wrapf(err, "failed to open target %s", target)
114
+	}
115
+	defer tgt.Close()
116
+
117
+	return copyFileContent(tgt, src)
118
+}
0 119
new file mode 100644
... ...
@@ -0,0 +1,95 @@
0
+package fs
1
+
2
+import (
3
+	"io"
4
+	"os"
5
+	"syscall"
6
+
7
+	"github.com/containerd/continuity/sysx"
8
+	"github.com/pkg/errors"
9
+	"golang.org/x/sys/unix"
10
+)
11
+
12
+func copyFileInfo(fi os.FileInfo, name string) error {
13
+	st := fi.Sys().(*syscall.Stat_t)
14
+	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
15
+		if os.IsPermission(err) {
16
+			// Normally if uid/gid are the same this would be a no-op, but some
17
+			// filesystems may still return EPERM... for instance NFS does this.
18
+			// In such a case, this is not an error.
19
+			if dstStat, err2 := os.Lstat(name); err2 == nil {
20
+				st2 := dstStat.Sys().(*syscall.Stat_t)
21
+				if st.Uid == st2.Uid && st.Gid == st2.Gid {
22
+					err = nil
23
+				}
24
+			}
25
+		}
26
+		if err != nil {
27
+			return errors.Wrapf(err, "failed to chown %s", name)
28
+		}
29
+	}
30
+
31
+	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
32
+		if err := os.Chmod(name, fi.Mode()); err != nil {
33
+			return errors.Wrapf(err, "failed to chmod %s", name)
34
+		}
35
+	}
36
+
37
+	timespec := []unix.Timespec{unix.Timespec(StatAtime(st)), unix.Timespec(StatMtime(st))}
38
+	if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil {
39
+		return errors.Wrapf(err, "failed to utime %s", name)
40
+	}
41
+
42
+	return nil
43
+}
44
+
45
+func copyFileContent(dst, src *os.File) error {
46
+	st, err := src.Stat()
47
+	if err != nil {
48
+		return errors.Wrap(err, "unable to stat source")
49
+	}
50
+
51
+	n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, int(st.Size()), 0)
52
+	if err != nil {
53
+		if err != unix.ENOSYS && err != unix.EXDEV {
54
+			return errors.Wrap(err, "copy file range failed")
55
+		}
56
+
57
+		buf := bufferPool.Get().(*[]byte)
58
+		_, err = io.CopyBuffer(dst, src, *buf)
59
+		bufferPool.Put(buf)
60
+		return err
61
+	}
62
+
63
+	if int64(n) != st.Size() {
64
+		return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size())
65
+	}
66
+
67
+	return nil
68
+}
69
+
70
+func copyXAttrs(dst, src string) error {
71
+	xattrKeys, err := sysx.LListxattr(src)
72
+	if err != nil {
73
+		return errors.Wrapf(err, "failed to list xattrs on %s", src)
74
+	}
75
+	for _, xattr := range xattrKeys {
76
+		data, err := sysx.LGetxattr(src, xattr)
77
+		if err != nil {
78
+			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
79
+		}
80
+		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
81
+			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
82
+		}
83
+	}
84
+
85
+	return nil
86
+}
87
+
88
+func copyDevice(dst string, fi os.FileInfo) error {
89
+	st, ok := fi.Sys().(*syscall.Stat_t)
90
+	if !ok {
91
+		return errors.New("unsupported stat type")
92
+	}
93
+	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
94
+}
0 95
new file mode 100644
... ...
@@ -0,0 +1,80 @@
0
+// +build solaris darwin freebsd
1
+
2
+package fs
3
+
4
+import (
5
+	"io"
6
+	"os"
7
+	"syscall"
8
+
9
+	"github.com/containerd/continuity/sysx"
10
+	"github.com/pkg/errors"
11
+	"golang.org/x/sys/unix"
12
+)
13
+
14
+func copyFileInfo(fi os.FileInfo, name string) error {
15
+	st := fi.Sys().(*syscall.Stat_t)
16
+	if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil {
17
+		if os.IsPermission(err) {
18
+			// Normally if uid/gid are the same this would be a no-op, but some
19
+			// filesystems may still return EPERM... for instance NFS does this.
20
+			// In such a case, this is not an error.
21
+			if dstStat, err2 := os.Lstat(name); err2 == nil {
22
+				st2 := dstStat.Sys().(*syscall.Stat_t)
23
+				if st.Uid == st2.Uid && st.Gid == st2.Gid {
24
+					err = nil
25
+				}
26
+			}
27
+		}
28
+		if err != nil {
29
+			return errors.Wrapf(err, "failed to chown %s", name)
30
+		}
31
+	}
32
+
33
+	if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink {
34
+		if err := os.Chmod(name, fi.Mode()); err != nil {
35
+			return errors.Wrapf(err, "failed to chmod %s", name)
36
+		}
37
+	}
38
+
39
+	timespec := []syscall.Timespec{StatAtime(st), StatMtime(st)}
40
+	if err := syscall.UtimesNano(name, timespec); err != nil {
41
+		return errors.Wrapf(err, "failed to utime %s", name)
42
+	}
43
+
44
+	return nil
45
+}
46
+
47
+func copyFileContent(dst, src *os.File) error {
48
+	buf := bufferPool.Get().(*[]byte)
49
+	_, err := io.CopyBuffer(dst, src, *buf)
50
+	bufferPool.Put(buf)
51
+
52
+	return err
53
+}
54
+
55
+func copyXAttrs(dst, src string) error {
56
+	xattrKeys, err := sysx.LListxattr(src)
57
+	if err != nil {
58
+		return errors.Wrapf(err, "failed to list xattrs on %s", src)
59
+	}
60
+	for _, xattr := range xattrKeys {
61
+		data, err := sysx.LGetxattr(src, xattr)
62
+		if err != nil {
63
+			return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src)
64
+		}
65
+		if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil {
66
+			return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst)
67
+		}
68
+	}
69
+
70
+	return nil
71
+}
72
+
73
+func copyDevice(dst string, fi os.FileInfo) error {
74
+	st, ok := fi.Sys().(*syscall.Stat_t)
75
+	if !ok {
76
+		return errors.New("unsupported stat type")
77
+	}
78
+	return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev))
79
+}
0 80
new file mode 100644
... ...
@@ -0,0 +1,33 @@
0
+package fs
1
+
2
+import (
3
+	"io"
4
+	"os"
5
+
6
+	"github.com/pkg/errors"
7
+)
8
+
9
+func copyFileInfo(fi os.FileInfo, name string) error {
10
+	if err := os.Chmod(name, fi.Mode()); err != nil {
11
+		return errors.Wrapf(err, "failed to chmod %s", name)
12
+	}
13
+
14
+	// TODO: copy windows specific metadata
15
+
16
+	return nil
17
+}
18
+
19
+func copyFileContent(dst, src *os.File) error {
20
+	buf := bufferPool.Get().(*[]byte)
21
+	_, err := io.CopyBuffer(dst, src, *buf)
22
+	bufferPool.Put(buf)
23
+	return err
24
+}
25
+
26
+func copyXAttrs(dst, src string) error {
27
+	return nil
28
+}
29
+
30
+func copyDevice(dst string, fi os.FileInfo) error {
31
+	return errors.New("device copy not supported")
32
+}
0 33
new file mode 100644
... ...
@@ -0,0 +1,310 @@
0
+package fs
1
+
2
+import (
3
+	"context"
4
+	"os"
5
+	"path/filepath"
6
+	"strings"
7
+
8
+	"golang.org/x/sync/errgroup"
9
+
10
+	"github.com/sirupsen/logrus"
11
+)
12
+
13
+// ChangeKind is the type of modification that
14
+// a change is making.
15
+type ChangeKind int
16
+
17
+const (
18
+	// ChangeKindUnmodified represents an unmodified
19
+	// file
20
+	ChangeKindUnmodified = iota
21
+
22
+	// ChangeKindAdd represents an addition of
23
+	// a file
24
+	ChangeKindAdd
25
+
26
+	// ChangeKindModify represents a change to
27
+	// an existing file
28
+	ChangeKindModify
29
+
30
+	// ChangeKindDelete represents a delete of
31
+	// a file
32
+	ChangeKindDelete
33
+)
34
+
35
+func (k ChangeKind) String() string {
36
+	switch k {
37
+	case ChangeKindUnmodified:
38
+		return "unmodified"
39
+	case ChangeKindAdd:
40
+		return "add"
41
+	case ChangeKindModify:
42
+		return "modify"
43
+	case ChangeKindDelete:
44
+		return "delete"
45
+	default:
46
+		return ""
47
+	}
48
+}
49
+
50
+// Change represents single change between a diff and its parent.
51
+type Change struct {
52
+	Kind ChangeKind
53
+	Path string
54
+}
55
+
56
+// ChangeFunc is the type of function called for each change
57
+// computed during a directory changes calculation.
58
+type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error
59
+
60
+// Changes computes changes between two directories calling the
61
+// given change function for each computed change. The first
62
+// directory is intended to the base directory and second
63
+// directory the changed directory.
64
+//
65
+// The change callback is called by the order of path names and
66
+// should be appliable in that order.
67
+//  Due to this apply ordering, the following is true
68
+//  - Removed directory trees only create a single change for the root
69
+//    directory removed. Remaining changes are implied.
70
+//  - A directory which is modified to become a file will not have
71
+//    delete entries for sub-path items, their removal is implied
72
+//    by the removal of the parent directory.
73
+//
74
+// Opaque directories will not be treated specially and each file
75
+// removed from the base directory will show up as a removal.
76
+//
77
+// File content comparisons will be done on files which have timestamps
78
+// which may have been truncated. If either of the files being compared
79
+// has a zero value nanosecond value, each byte will be compared for
80
+// differences. If 2 files have the same seconds value but different
81
+// nanosecond values where one of those values is zero, the files will
82
+// be considered unchanged if the content is the same. This behavior
83
+// is to account for timestamp truncation during archiving.
84
+func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error {
85
+	if a == "" {
86
+		logrus.Debugf("Using single walk diff for %s", b)
87
+		return addDirChanges(ctx, changeFn, b)
88
+	} else if diffOptions := detectDirDiff(b, a); diffOptions != nil {
89
+		logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, a)
90
+		return diffDirChanges(ctx, changeFn, a, diffOptions)
91
+	}
92
+
93
+	logrus.Debugf("Using double walk diff for %s from %s", b, a)
94
+	return doubleWalkDiff(ctx, changeFn, a, b)
95
+}
96
+
97
+func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error {
98
+	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
99
+		if err != nil {
100
+			return err
101
+		}
102
+
103
+		// Rebase path
104
+		path, err = filepath.Rel(root, path)
105
+		if err != nil {
106
+			return err
107
+		}
108
+
109
+		path = filepath.Join(string(os.PathSeparator), path)
110
+
111
+		// Skip root
112
+		if path == string(os.PathSeparator) {
113
+			return nil
114
+		}
115
+
116
+		return changeFn(ChangeKindAdd, path, f, nil)
117
+	})
118
+}
119
+
120
+// diffDirOptions is used when the diff can be directly calculated from
121
+// a diff directory to its base, without walking both trees.
122
+type diffDirOptions struct {
123
+	diffDir      string
124
+	skipChange   func(string) (bool, error)
125
+	deleteChange func(string, string, os.FileInfo) (string, error)
126
+}
127
+
128
+// diffDirChanges walks the diff directory and compares changes against the base.
129
+func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *diffDirOptions) error {
130
+	changedDirs := make(map[string]struct{})
131
+	return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error {
132
+		if err != nil {
133
+			return err
134
+		}
135
+
136
+		// Rebase path
137
+		path, err = filepath.Rel(o.diffDir, path)
138
+		if err != nil {
139
+			return err
140
+		}
141
+
142
+		path = filepath.Join(string(os.PathSeparator), path)
143
+
144
+		// Skip root
145
+		if path == string(os.PathSeparator) {
146
+			return nil
147
+		}
148
+
149
+		// TODO: handle opaqueness, start new double walker at this
150
+		// location to get deletes, and skip tree in single walker
151
+
152
+		if o.skipChange != nil {
153
+			if skip, err := o.skipChange(path); skip {
154
+				return err
155
+			}
156
+		}
157
+
158
+		var kind ChangeKind
159
+
160
+		deletedFile, err := o.deleteChange(o.diffDir, path, f)
161
+		if err != nil {
162
+			return err
163
+		}
164
+
165
+		// Find out what kind of modification happened
166
+		if deletedFile != "" {
167
+			path = deletedFile
168
+			kind = ChangeKindDelete
169
+			f = nil
170
+		} else {
171
+			// Otherwise, the file was added
172
+			kind = ChangeKindAdd
173
+
174
+			// ...Unless it already existed in a base, in which case, it's a modification
175
+			stat, err := os.Stat(filepath.Join(base, path))
176
+			if err != nil && !os.IsNotExist(err) {
177
+				return err
178
+			}
179
+			if err == nil {
180
+				// The file existed in the base, so that's a modification
181
+
182
+				// However, if it's a directory, maybe it wasn't actually modified.
183
+				// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
184
+				if stat.IsDir() && f.IsDir() {
185
+					if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
186
+						// Both directories are the same, don't record the change
187
+						return nil
188
+					}
189
+				}
190
+				kind = ChangeKindModify
191
+			}
192
+		}
193
+
194
+		// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
195
+		// This block is here to ensure the change is recorded even if the
196
+		// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
197
+		// Check https://github.com/docker/docker/pull/13590 for details.
198
+		if f.IsDir() {
199
+			changedDirs[path] = struct{}{}
200
+		}
201
+		if kind == ChangeKindAdd || kind == ChangeKindDelete {
202
+			parent := filepath.Dir(path)
203
+			if _, ok := changedDirs[parent]; !ok && parent != "/" {
204
+				pi, err := os.Stat(filepath.Join(o.diffDir, parent))
205
+				if err := changeFn(ChangeKindModify, parent, pi, err); err != nil {
206
+					return err
207
+				}
208
+				changedDirs[parent] = struct{}{}
209
+			}
210
+		}
211
+
212
+		return changeFn(kind, path, f, nil)
213
+	})
214
+}
215
+
216
+// doubleWalkDiff walks both directories to create a diff
217
+func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err error) {
218
+	g, ctx := errgroup.WithContext(ctx)
219
+
220
+	var (
221
+		c1 = make(chan *currentPath)
222
+		c2 = make(chan *currentPath)
223
+
224
+		f1, f2 *currentPath
225
+		rmdir  string
226
+	)
227
+	g.Go(func() error {
228
+		defer close(c1)
229
+		return pathWalk(ctx, a, c1)
230
+	})
231
+	g.Go(func() error {
232
+		defer close(c2)
233
+		return pathWalk(ctx, b, c2)
234
+	})
235
+	g.Go(func() error {
236
+		for c1 != nil || c2 != nil {
237
+			if f1 == nil && c1 != nil {
238
+				f1, err = nextPath(ctx, c1)
239
+				if err != nil {
240
+					return err
241
+				}
242
+				if f1 == nil {
243
+					c1 = nil
244
+				}
245
+			}
246
+
247
+			if f2 == nil && c2 != nil {
248
+				f2, err = nextPath(ctx, c2)
249
+				if err != nil {
250
+					return err
251
+				}
252
+				if f2 == nil {
253
+					c2 = nil
254
+				}
255
+			}
256
+			if f1 == nil && f2 == nil {
257
+				continue
258
+			}
259
+
260
+			var f os.FileInfo
261
+			k, p := pathChange(f1, f2)
262
+			switch k {
263
+			case ChangeKindAdd:
264
+				if rmdir != "" {
265
+					rmdir = ""
266
+				}
267
+				f = f2.f
268
+				f2 = nil
269
+			case ChangeKindDelete:
270
+				// Check if this file is already removed by being
271
+				// under of a removed directory
272
+				if rmdir != "" && strings.HasPrefix(f1.path, rmdir) {
273
+					f1 = nil
274
+					continue
275
+				} else if f1.f.IsDir() {
276
+					rmdir = f1.path + string(os.PathSeparator)
277
+				} else if rmdir != "" {
278
+					rmdir = ""
279
+				}
280
+				f1 = nil
281
+			case ChangeKindModify:
282
+				same, err := sameFile(f1, f2)
283
+				if err != nil {
284
+					return err
285
+				}
286
+				if f1.f.IsDir() && !f2.f.IsDir() {
287
+					rmdir = f1.path + string(os.PathSeparator)
288
+				} else if rmdir != "" {
289
+					rmdir = ""
290
+				}
291
+				f = f2.f
292
+				f1 = nil
293
+				f2 = nil
294
+				if same {
295
+					if !isLinked(f) {
296
+						continue
297
+					}
298
+					k = ChangeKindUnmodified
299
+				}
300
+			}
301
+			if err := changeFn(k, p, f, nil); err != nil {
302
+				return err
303
+			}
304
+		}
305
+		return nil
306
+	})
307
+
308
+	return g.Wait()
309
+}
0 310
new file mode 100644
... ...
@@ -0,0 +1,58 @@
0
+// +build !windows
1
+
2
+package fs
3
+
4
+import (
5
+	"bytes"
6
+	"os"
7
+	"syscall"
8
+
9
+	"github.com/containerd/continuity/sysx"
10
+	"github.com/pkg/errors"
11
+)
12
+
13
+// detectDirDiff returns diff dir options if a directory could
14
+// be found in the mount info for upper which is the direct
15
+// diff with the provided lower directory
16
+func detectDirDiff(upper, lower string) *diffDirOptions {
17
+	// TODO: get mount options for upper
18
+	// TODO: detect AUFS
19
+	// TODO: detect overlay
20
+	return nil
21
+}
22
+
23
+// compareSysStat returns whether the stats are equivalent,
24
+// whether the files are considered the same file, and
25
+// an error
26
+func compareSysStat(s1, s2 interface{}) (bool, error) {
27
+	ls1, ok := s1.(*syscall.Stat_t)
28
+	if !ok {
29
+		return false, nil
30
+	}
31
+	ls2, ok := s2.(*syscall.Stat_t)
32
+	if !ok {
33
+		return false, nil
34
+	}
35
+
36
+	return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil
37
+}
38
+
39
+func compareCapabilities(p1, p2 string) (bool, error) {
40
+	c1, err := sysx.LGetxattr(p1, "security.capability")
41
+	if err != nil && err != sysx.ENODATA {
42
+		return false, errors.Wrapf(err, "failed to get xattr for %s", p1)
43
+	}
44
+	c2, err := sysx.LGetxattr(p2, "security.capability")
45
+	if err != nil && err != sysx.ENODATA {
46
+		return false, errors.Wrapf(err, "failed to get xattr for %s", p2)
47
+	}
48
+	return bytes.Equal(c1, c2), nil
49
+}
50
+
51
+func isLinked(f os.FileInfo) bool {
52
+	s, ok := f.Sys().(*syscall.Stat_t)
53
+	if !ok {
54
+		return false
55
+	}
56
+	return !f.IsDir() && s.Nlink > 1
57
+}
0 58
new file mode 100644
... ...
@@ -0,0 +1,32 @@
0
+package fs
1
+
2
+import (
3
+	"os"
4
+
5
+	"golang.org/x/sys/windows"
6
+)
7
+
8
+func detectDirDiff(upper, lower string) *diffDirOptions {
9
+	return nil
10
+}
11
+
12
+func compareSysStat(s1, s2 interface{}) (bool, error) {
13
+	f1, ok := s1.(windows.Win32FileAttributeData)
14
+	if !ok {
15
+		return false, nil
16
+	}
17
+	f2, ok := s2.(windows.Win32FileAttributeData)
18
+	if !ok {
19
+		return false, nil
20
+	}
21
+	return f1.FileAttributes == f2.FileAttributes, nil
22
+}
23
+
24
+func compareCapabilities(p1, p2 string) (bool, error) {
25
+	// TODO: Use windows equivalent
26
+	return true, nil
27
+}
28
+
29
+func isLinked(os.FileInfo) bool {
30
+	return false
31
+}
0 32
new file mode 100644
... ...
@@ -0,0 +1,87 @@
0
+// +build linux
1
+
2
+package fs
3
+
4
+import (
5
+	"fmt"
6
+	"io/ioutil"
7
+	"os"
8
+	"syscall"
9
+	"unsafe"
10
+)
11
+
12
+func locateDummyIfEmpty(path string) (string, error) {
13
+	children, err := ioutil.ReadDir(path)
14
+	if err != nil {
15
+		return "", err
16
+	}
17
+	if len(children) != 0 {
18
+		return "", nil
19
+	}
20
+	dummyFile, err := ioutil.TempFile(path, "fsutils-dummy")
21
+	if err != nil {
22
+		return "", err
23
+	}
24
+	name := dummyFile.Name()
25
+	err = dummyFile.Close()
26
+	return name, err
27
+}
28
+
29
+// SupportsDType returns whether the filesystem mounted on path supports d_type
30
+func SupportsDType(path string) (bool, error) {
31
+	// locate dummy so that we have at least one dirent
32
+	dummy, err := locateDummyIfEmpty(path)
33
+	if err != nil {
34
+		return false, err
35
+	}
36
+	if dummy != "" {
37
+		defer os.Remove(dummy)
38
+	}
39
+
40
+	visited := 0
41
+	supportsDType := true
42
+	fn := func(ent *syscall.Dirent) bool {
43
+		visited++
44
+		if ent.Type == syscall.DT_UNKNOWN {
45
+			supportsDType = false
46
+			// stop iteration
47
+			return true
48
+		}
49
+		// continue iteration
50
+		return false
51
+	}
52
+	if err = iterateReadDir(path, fn); err != nil {
53
+		return false, err
54
+	}
55
+	if visited == 0 {
56
+		return false, fmt.Errorf("did not hit any dirent during iteration %s", path)
57
+	}
58
+	return supportsDType, nil
59
+}
60
+
61
+func iterateReadDir(path string, fn func(*syscall.Dirent) bool) error {
62
+	d, err := os.Open(path)
63
+	if err != nil {
64
+		return err
65
+	}
66
+	defer d.Close()
67
+	fd := int(d.Fd())
68
+	buf := make([]byte, 4096)
69
+	for {
70
+		nbytes, err := syscall.ReadDirent(fd, buf)
71
+		if err != nil {
72
+			return err
73
+		}
74
+		if nbytes == 0 {
75
+			break
76
+		}
77
+		for off := 0; off < nbytes; {
78
+			ent := (*syscall.Dirent)(unsafe.Pointer(&buf[off]))
79
+			if stop := fn(ent); stop {
80
+				return nil
81
+			}
82
+			off += int(ent.Reclen)
83
+		}
84
+	}
85
+	return nil
86
+}
0 87
new file mode 100644
... ...
@@ -0,0 +1,22 @@
0
+package fs
1
+
2
+import "context"
3
+
4
+// Usage of disk information
5
+type Usage struct {
6
+	Inodes int64
7
+	Size   int64
8
+}
9
+
10
+// DiskUsage counts the number of inodes and disk usage for the resources under
11
+// path.
12
+func DiskUsage(roots ...string) (Usage, error) {
13
+	return diskUsage(roots...)
14
+}
15
+
16
+// DiffUsage counts the numbers of inodes and disk usage in the
17
+// diff between the 2 directories. The first path is intended
18
+// as the base directory and the second as the changed directory.
19
+func DiffUsage(ctx context.Context, a, b string) (Usage, error) {
20
+	return diffUsage(ctx, a, b)
21
+}
0 22
new file mode 100644
... ...
@@ -0,0 +1,88 @@
0
+// +build !windows
1
+
2
+package fs
3
+
4
+import (
5
+	"context"
6
+	"os"
7
+	"path/filepath"
8
+	"syscall"
9
+)
10
+
11
+type inode struct {
12
+	// TODO(stevvooe): Can probably reduce memory usage by not tracking
13
+	// device, but we can leave this right for now.
14
+	dev, ino uint64
15
+}
16
+
17
+func newInode(stat *syscall.Stat_t) inode {
18
+	return inode{
19
+		// Dev is uint32 on darwin/bsd, uint64 on linux/solaris
20
+		dev: uint64(stat.Dev), // nolint: unconvert
21
+		// Ino is uint32 on bsd, uint64 on darwin/linux/solaris
22
+		ino: uint64(stat.Ino), // nolint: unconvert
23
+	}
24
+}
25
+
26
+func diskUsage(roots ...string) (Usage, error) {
27
+
28
+	var (
29
+		size   int64
30
+		inodes = map[inode]struct{}{} // expensive!
31
+	)
32
+
33
+	for _, root := range roots {
34
+		if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
35
+			if err != nil {
36
+				return err
37
+			}
38
+
39
+			inoKey := newInode(fi.Sys().(*syscall.Stat_t))
40
+			if _, ok := inodes[inoKey]; !ok {
41
+				inodes[inoKey] = struct{}{}
42
+				size += fi.Size()
43
+			}
44
+
45
+			return nil
46
+		}); err != nil {
47
+			return Usage{}, err
48
+		}
49
+	}
50
+
51
+	return Usage{
52
+		Inodes: int64(len(inodes)),
53
+		Size:   size,
54
+	}, nil
55
+}
56
+
57
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
58
+	var (
59
+		size   int64
60
+		inodes = map[inode]struct{}{} // expensive!
61
+	)
62
+
63
+	if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
64
+		if err != nil {
65
+			return err
66
+		}
67
+
68
+		if kind == ChangeKindAdd || kind == ChangeKindModify {
69
+			inoKey := newInode(fi.Sys().(*syscall.Stat_t))
70
+			if _, ok := inodes[inoKey]; !ok {
71
+				inodes[inoKey] = struct{}{}
72
+				size += fi.Size()
73
+			}
74
+
75
+			return nil
76
+
77
+		}
78
+		return nil
79
+	}); err != nil {
80
+		return Usage{}, err
81
+	}
82
+
83
+	return Usage{
84
+		Inodes: int64(len(inodes)),
85
+		Size:   size,
86
+	}, nil
87
+}
0 88
new file mode 100644
... ...
@@ -0,0 +1,60 @@
0
+// +build windows
1
+
2
+package fs
3
+
4
+import (
5
+	"context"
6
+	"os"
7
+	"path/filepath"
8
+)
9
+
10
+func diskUsage(roots ...string) (Usage, error) {
11
+	var (
12
+		size int64
13
+	)
14
+
15
+	// TODO(stevvooe): Support inodes (or equivalent) for windows.
16
+
17
+	for _, root := range roots {
18
+		if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
19
+			if err != nil {
20
+				return err
21
+			}
22
+
23
+			size += fi.Size()
24
+			return nil
25
+		}); err != nil {
26
+			return Usage{}, err
27
+		}
28
+	}
29
+
30
+	return Usage{
31
+		Size: size,
32
+	}, nil
33
+}
34
+
35
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
36
+	var (
37
+		size int64
38
+	)
39
+
40
+	if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
41
+		if err != nil {
42
+			return err
43
+		}
44
+
45
+		if kind == ChangeKindAdd || kind == ChangeKindModify {
46
+			size += fi.Size()
47
+
48
+			return nil
49
+
50
+		}
51
+		return nil
52
+	}); err != nil {
53
+		return Usage{}, err
54
+	}
55
+
56
+	return Usage{
57
+		Size: size,
58
+	}, nil
59
+}
0 60
new file mode 100644
... ...
@@ -0,0 +1,27 @@
0
+package fs
1
+
2
+import "os"
3
+
4
+// GetLinkInfo returns an identifier representing the node a hardlink is pointing
5
+// to. If the file is not hard linked then 0 will be returned.
6
+func GetLinkInfo(fi os.FileInfo) (uint64, bool) {
7
+	return getLinkInfo(fi)
8
+}
9
+
10
+// getLinkSource returns a path for the given name and
11
+// file info to its link source in the provided inode
12
+// map. If the given file name is not in the map and
13
+// has other links, it is added to the inode map
14
+// to be a source for other link locations.
15
+func getLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) {
16
+	inode, isHardlink := getLinkInfo(fi)
17
+	if !isHardlink {
18
+		return "", nil
19
+	}
20
+
21
+	path, ok := inodes[inode]
22
+	if !ok {
23
+		inodes[inode] = name
24
+	}
25
+	return path, nil
26
+}
0 27
new file mode 100644
... ...
@@ -0,0 +1,18 @@
0
+// +build !windows
1
+
2
+package fs
3
+
4
+import (
5
+	"os"
6
+	"syscall"
7
+)
8
+
9
+func getLinkInfo(fi os.FileInfo) (uint64, bool) {
10
+	s, ok := fi.Sys().(*syscall.Stat_t)
11
+	if !ok {
12
+		return 0, false
13
+	}
14
+
15
+	// Ino is uint32 on bsd, uint64 on darwin/linux/solaris
16
+	return uint64(s.Ino), !fi.IsDir() && s.Nlink > 1 // nolint: unconvert
17
+}
0 18
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+package fs
1
+
2
+import "os"
3
+
4
+func getLinkInfo(fi os.FileInfo) (uint64, bool) {
5
+	return 0, false
6
+}
0 7
new file mode 100644
... ...
@@ -0,0 +1,276 @@
0
+package fs
1
+
2
+import (
3
+	"bytes"
4
+	"context"
5
+	"io"
6
+	"os"
7
+	"path/filepath"
8
+	"strings"
9
+
10
+	"github.com/pkg/errors"
11
+)
12
+
13
+var (
14
+	errTooManyLinks = errors.New("too many links")
15
+)
16
+
17
+type currentPath struct {
18
+	path     string
19
+	f        os.FileInfo
20
+	fullPath string
21
+}
22
+
23
+func pathChange(lower, upper *currentPath) (ChangeKind, string) {
24
+	if lower == nil {
25
+		if upper == nil {
26
+			panic("cannot compare nil paths")
27
+		}
28
+		return ChangeKindAdd, upper.path
29
+	}
30
+	if upper == nil {
31
+		return ChangeKindDelete, lower.path
32
+	}
33
+	// TODO: compare by directory
34
+
35
+	switch i := strings.Compare(lower.path, upper.path); {
36
+	case i < 0:
37
+		// File in lower that is not in upper
38
+		return ChangeKindDelete, lower.path
39
+	case i > 0:
40
+		// File in upper that is not in lower
41
+		return ChangeKindAdd, upper.path
42
+	default:
43
+		return ChangeKindModify, upper.path
44
+	}
45
+}
46
+
47
+func sameFile(f1, f2 *currentPath) (bool, error) {
48
+	if os.SameFile(f1.f, f2.f) {
49
+		return true, nil
50
+	}
51
+
52
+	equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys())
53
+	if err != nil || !equalStat {
54
+		return equalStat, err
55
+	}
56
+
57
+	if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq {
58
+		return eq, err
59
+	}
60
+
61
+	// If not a directory also check size, modtime, and content
62
+	if !f1.f.IsDir() {
63
+		if f1.f.Size() != f2.f.Size() {
64
+			return false, nil
65
+		}
66
+		t1 := f1.f.ModTime()
67
+		t2 := f2.f.ModTime()
68
+
69
+		if t1.Unix() != t2.Unix() {
70
+			return false, nil
71
+		}
72
+
73
+		// If the timestamp may have been truncated in both of the
74
+		// files, check content of file to determine difference
75
+		if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 {
76
+			var eq bool
77
+			if (f1.f.Mode() & os.ModeSymlink) == os.ModeSymlink {
78
+				eq, err = compareSymlinkTarget(f1.fullPath, f2.fullPath)
79
+			} else if f1.f.Size() > 0 {
80
+				eq, err = compareFileContent(f1.fullPath, f2.fullPath)
81
+			}
82
+			if err != nil || !eq {
83
+				return eq, err
84
+			}
85
+		} else if t1.Nanosecond() != t2.Nanosecond() {
86
+			return false, nil
87
+		}
88
+	}
89
+
90
+	return true, nil
91
+}
92
+
93
+func compareSymlinkTarget(p1, p2 string) (bool, error) {
94
+	t1, err := os.Readlink(p1)
95
+	if err != nil {
96
+		return false, err
97
+	}
98
+	t2, err := os.Readlink(p2)
99
+	if err != nil {
100
+		return false, err
101
+	}
102
+	return t1 == t2, nil
103
+}
104
+
105
+const compareChuckSize = 32 * 1024
106
+
107
+// compareFileContent compares the content of 2 same sized files
108
+// by comparing each byte.
109
+func compareFileContent(p1, p2 string) (bool, error) {
110
+	f1, err := os.Open(p1)
111
+	if err != nil {
112
+		return false, err
113
+	}
114
+	defer f1.Close()
115
+	f2, err := os.Open(p2)
116
+	if err != nil {
117
+		return false, err
118
+	}
119
+	defer f2.Close()
120
+
121
+	b1 := make([]byte, compareChuckSize)
122
+	b2 := make([]byte, compareChuckSize)
123
+	for {
124
+		n1, err1 := f1.Read(b1)
125
+		if err1 != nil && err1 != io.EOF {
126
+			return false, err1
127
+		}
128
+		n2, err2 := f2.Read(b2)
129
+		if err2 != nil && err2 != io.EOF {
130
+			return false, err2
131
+		}
132
+		if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) {
133
+			return false, nil
134
+		}
135
+		if err1 == io.EOF && err2 == io.EOF {
136
+			return true, nil
137
+		}
138
+	}
139
+}
140
+
141
+func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error {
142
+	return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
143
+		if err != nil {
144
+			return err
145
+		}
146
+
147
+		// Rebase path
148
+		path, err = filepath.Rel(root, path)
149
+		if err != nil {
150
+			return err
151
+		}
152
+
153
+		path = filepath.Join(string(os.PathSeparator), path)
154
+
155
+		// Skip root
156
+		if path == string(os.PathSeparator) {
157
+			return nil
158
+		}
159
+
160
+		p := &currentPath{
161
+			path:     path,
162
+			f:        f,
163
+			fullPath: filepath.Join(root, path),
164
+		}
165
+
166
+		select {
167
+		case <-ctx.Done():
168
+			return ctx.Err()
169
+		case pathC <- p:
170
+			return nil
171
+		}
172
+	})
173
+}
174
+
175
+func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) {
176
+	select {
177
+	case <-ctx.Done():
178
+		return nil, ctx.Err()
179
+	case p := <-pathC:
180
+		return p, nil
181
+	}
182
+}
183
+
184
+// RootPath joins a path with a root, evaluating and bounding any
185
+// symlink to the root directory.
186
+func RootPath(root, path string) (string, error) {
187
+	if path == "" {
188
+		return root, nil
189
+	}
190
+	var linksWalked int // to protect against cycles
191
+	for {
192
+		i := linksWalked
193
+		newpath, err := walkLinks(root, path, &linksWalked)
194
+		if err != nil {
195
+			return "", err
196
+		}
197
+		path = newpath
198
+		if i == linksWalked {
199
+			newpath = filepath.Join("/", newpath)
200
+			if path == newpath {
201
+				return filepath.Join(root, newpath), nil
202
+			}
203
+			path = newpath
204
+		}
205
+	}
206
+}
207
+
208
+func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, err error) {
209
+	if *linksWalked > 255 {
210
+		return "", false, errTooManyLinks
211
+	}
212
+
213
+	path = filepath.Join("/", path)
214
+	if path == "/" {
215
+		return path, false, nil
216
+	}
217
+	realPath := filepath.Join(root, path)
218
+
219
+	fi, err := os.Lstat(realPath)
220
+	if err != nil {
221
+		// If path does not yet exist, treat as non-symlink
222
+		if os.IsNotExist(err) {
223
+			return path, false, nil
224
+		}
225
+		return "", false, err
226
+	}
227
+	if fi.Mode()&os.ModeSymlink == 0 {
228
+		return path, false, nil
229
+	}
230
+	newpath, err = os.Readlink(realPath)
231
+	if err != nil {
232
+		return "", false, err
233
+	}
234
+	if filepath.IsAbs(newpath) && strings.HasPrefix(newpath, root) {
235
+		newpath = newpath[:len(root)]
236
+		if !strings.HasPrefix(newpath, "/") {
237
+			newpath = "/" + newpath
238
+		}
239
+	}
240
+	*linksWalked++
241
+	return newpath, true, nil
242
+}
243
+
244
+func walkLinks(root, path string, linksWalked *int) (string, error) {
245
+	switch dir, file := filepath.Split(path); {
246
+	case dir == "":
247
+		newpath, _, err := walkLink(root, file, linksWalked)
248
+		return newpath, err
249
+	case file == "":
250
+		if os.IsPathSeparator(dir[len(dir)-1]) {
251
+			if dir == "/" {
252
+				return dir, nil
253
+			}
254
+			return walkLinks(root, dir[:len(dir)-1], linksWalked)
255
+		}
256
+		newpath, _, err := walkLink(root, dir, linksWalked)
257
+		return newpath, err
258
+	default:
259
+		newdir, err := walkLinks(root, dir, linksWalked)
260
+		if err != nil {
261
+			return "", err
262
+		}
263
+		newpath, islink, err := walkLink(root, filepath.Join(newdir, file), linksWalked)
264
+		if err != nil {
265
+			return "", err
266
+		}
267
+		if !islink {
268
+			return newpath, nil
269
+		}
270
+		if filepath.IsAbs(newpath) {
271
+			return newpath, nil
272
+		}
273
+		return filepath.Join(newdir, newpath), nil
274
+	}
275
+}
0 276
new file mode 100644
... ...
@@ -0,0 +1,28 @@
0
+// +build darwin freebsd
1
+
2
+package fs
3
+
4
+import (
5
+	"syscall"
6
+	"time"
7
+)
8
+
9
+// StatAtime returns the access time from a stat struct
10
+func StatAtime(st *syscall.Stat_t) syscall.Timespec {
11
+	return st.Atimespec
12
+}
13
+
14
+// StatCtime returns the created time from a stat struct
15
+func StatCtime(st *syscall.Stat_t) syscall.Timespec {
16
+	return st.Ctimespec
17
+}
18
+
19
+// StatMtime returns the modified time from a stat struct
20
+func StatMtime(st *syscall.Stat_t) syscall.Timespec {
21
+	return st.Mtimespec
22
+}
23
+
24
+// StatATimeAsTime returns the access time as a time.Time
25
+func StatATimeAsTime(st *syscall.Stat_t) time.Time {
26
+	return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) // nolint: unconvert
27
+}
0 28
new file mode 100644
... ...
@@ -0,0 +1,26 @@
0
+package fs
1
+
2
+import (
3
+	"syscall"
4
+	"time"
5
+)
6
+
7
+// StatAtime returns the Atim
8
+func StatAtime(st *syscall.Stat_t) syscall.Timespec {
9
+	return st.Atim
10
+}
11
+
12
+// StatCtime returns the Ctim
13
+func StatCtime(st *syscall.Stat_t) syscall.Timespec {
14
+	return st.Ctim
15
+}
16
+
17
+// StatMtime returns the Mtim
18
+func StatMtime(st *syscall.Stat_t) syscall.Timespec {
19
+	return st.Mtim
20
+}
21
+
22
+// StatATimeAsTime returns st.Atim as a time.Time
23
+func StatATimeAsTime(st *syscall.Stat_t) time.Time {
24
+	return time.Unix(st.Atim.Sec, st.Atim.Nsec)
25
+}
0 26
new file mode 100644
... ...
@@ -0,0 +1,13 @@
0
+package fs
1
+
2
+import "time"
3
+
4
+// Gnu tar and the go tar writer don't have sub-second mtime
5
+// precision, which is problematic when we apply changes via tar
6
+// files, we handle this by comparing for exact times, *or* same
7
+// second count and either a or b having exactly 0 nanoseconds
8
+func sameFsTime(a, b time.Time) bool {
9
+	return a == b ||
10
+		(a.Unix() == b.Unix() &&
11
+			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
12
+}
0 13
deleted file mode 100644
... ...
@@ -1,11 +0,0 @@
1
-package sysx
2
-
3
-// These functions will be generated by generate.sh
4
-//    $ GOOS=linux GOARCH=386 ./generate.sh copy
5
-//    $ GOOS=linux GOARCH=amd64 ./generate.sh copy
6
-//    $ GOOS=linux GOARCH=arm ./generate.sh copy
7
-//    $ GOOS=linux GOARCH=arm64 ./generate.sh copy
8
-//    $ GOOS=linux GOARCH=ppc64le ./generate.sh copy
9
-//    $ GOOS=linux GOARCH=s390x ./generate.sh copy
10
-
11
-//sys CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error)
12 1
deleted file mode 100644
... ...
@@ -1,20 +0,0 @@
1
-// mksyscall.pl -l32 copy_linux.go
2
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
3
-
4
-package sysx
5
-
6
-import (
7
-	"syscall"
8
-	"unsafe"
9
-)
10
-
11
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
12
-
13
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
14
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
15
-	n = int(r0)
16
-	if e1 != 0 {
17
-		err = errnoErr(e1)
18
-	}
19
-	return
20
-}
21 1
deleted file mode 100644
... ...
@@ -1,20 +0,0 @@
1
-// mksyscall.pl copy_linux.go
2
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
3
-
4
-package sysx
5
-
6
-import (
7
-	"syscall"
8
-	"unsafe"
9
-)
10
-
11
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
12
-
13
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
14
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
15
-	n = int(r0)
16
-	if e1 != 0 {
17
-		err = errnoErr(e1)
18
-	}
19
-	return
20
-}
21 1
deleted file mode 100644
... ...
@@ -1,20 +0,0 @@
1
-// mksyscall.pl -l32 copy_linux.go
2
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
3
-
4
-package sysx
5
-
6
-import (
7
-	"syscall"
8
-	"unsafe"
9
-)
10
-
11
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
12
-
13
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
14
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
15
-	n = int(r0)
16
-	if e1 != 0 {
17
-		err = errnoErr(e1)
18
-	}
19
-	return
20
-}
21 1
deleted file mode 100644
... ...
@@ -1,20 +0,0 @@
1
-// mksyscall.pl copy_linux.go
2
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
3
-
4
-package sysx
5
-
6
-import (
7
-	"syscall"
8
-	"unsafe"
9
-)
10
-
11
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
12
-
13
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
14
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
15
-	n = int(r0)
16
-	if e1 != 0 {
17
-		err = errnoErr(e1)
18
-	}
19
-	return
20
-}
21 1
deleted file mode 100644
... ...
@@ -1,20 +0,0 @@
1
-// mksyscall.pl copy_linux.go
2
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
3
-
4
-package sysx
5
-
6
-import (
7
-	"syscall"
8
-	"unsafe"
9
-)
10
-
11
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
12
-
13
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
14
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
15
-	n = int(r0)
16
-	if e1 != 0 {
17
-		err = errnoErr(e1)
18
-	}
19
-	return
20
-}
21 1
deleted file mode 100644
... ...
@@ -1,20 +0,0 @@
1
-// mksyscall.pl copy_linux.go
2
-// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
3
-
4
-package sysx
5
-
6
-import (
7
-	"syscall"
8
-	"unsafe"
9
-)
10
-
11
-// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
12
-
13
-func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) {
14
-	r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags))
15
-	n = int(r0)
16
-	if e1 != 0 {
17
-		err = errnoErr(e1)
18
-	}
19
-	return
20
-}
21 1
deleted file mode 100644
... ...
@@ -1,7 +0,0 @@
1
-package sysx
2
-
3
-const (
4
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
5
-	// Number defined in /usr/include/asm/unistd_32.h
6
-	SYS_COPY_FILE_RANGE = 377
7
-)
8 1
deleted file mode 100644
... ...
@@ -1,7 +0,0 @@
1
-package sysx
2
-
3
-const (
4
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
5
-	// Number defined in /usr/include/asm/unistd_64.h
6
-	SYS_COPY_FILE_RANGE = 326
7
-)
8 1
deleted file mode 100644
... ...
@@ -1,7 +0,0 @@
1
-package sysx
2
-
3
-const (
4
-	// SYS_COPY_FILE_RANGE defined in Kernel 4.5+
5
-	// Number defined in /usr/include/arm-linux-gnueabihf/asm/unistd.h
6
-	SYS_COPY_FILE_RANGE = 391
7
-)
8 1
deleted file mode 100644
... ...
@@ -1,7 +0,0 @@
1
-package sysx
2
-
3
-const (
4
-	// SYS_COPY_FILE_RANGE defined in Kernel 4.5+
5
-	// Number defined in /usr/include/asm-generic/unistd.h
6
-	SYS_COPY_FILE_RANGE = 285
7
-)
8 1
deleted file mode 100644
... ...
@@ -1,7 +0,0 @@
1
-package sysx
2
-
3
-const (
4
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
5
-	// Number defined in /usr/include/asm/unistd_64.h
6
-	SYS_COPY_FILE_RANGE = 379
7
-)
8 1
deleted file mode 100644
... ...
@@ -1,7 +0,0 @@
1
-package sysx
2
-
3
-const (
4
-	// SYS_COPYFILERANGE defined in Kernel 4.5+
5
-	// Number defined in /usr/include/asm/unistd_64.h
6
-	SYS_COPY_FILE_RANGE = 375
7
-)
8 1
new file mode 100644
... ...
@@ -0,0 +1,13 @@
0
+bazil.org/fuse 371fbbdaa8987b715bdd21d6adc4c9b20155f748
1
+github.com/dustin/go-humanize bb3d318650d48840a39aa21a027c6630e198e626
2
+github.com/golang/protobuf 1e59b77b52bf8e4b449a57e6f79f21226d571845
3
+github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
4
+github.com/opencontainers/go-digest 279bed98673dd5bef374d3b6e4b09e2af76183bf
5
+github.com/pkg/errors f15c970de5b76fac0b59abb32d62c17cc7bed265
6
+github.com/sirupsen/logrus 89742aefa4b206dcf400792f3bd35b542998eb3b
7
+github.com/spf13/cobra 2da4a54c5ceefcee7ca5dd0eea1e18a3b6366489
8
+github.com/spf13/pflag 4c012f6dcd9546820e378d0bdda4d8fc772cdfea
9
+golang.org/x/crypto 9f005a07e0d31d45e6656d241bb5c0f2efd4bc94
10
+golang.org/x/net a337091b0525af65de94df2eb7e98bd9962dcbe2
11
+golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
12
+golang.org/x/sys 665f6529cca930e27b831a0d1dafffbe1c172924