Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)
| ... | ... |
@@ -33,6 +33,8 @@ type ( |
| 33 | 33 |
Reader io.Reader |
| 34 | 34 |
// Compression is the state represents if compressed or not. |
| 35 | 35 |
Compression int |
| 36 |
+ // WhiteoutFormat is the format of whiteouts unpacked |
|
| 37 |
+ WhiteoutFormat int |
|
| 36 | 38 |
// TarChownOptions wraps the chown options UID and GID. |
| 37 | 39 |
TarChownOptions struct {
|
| 38 | 40 |
UID, GID int |
| ... | ... |
@@ -47,9 +49,10 @@ type ( |
| 47 | 47 |
GIDMaps []idtools.IDMap |
| 48 | 48 |
ChownOpts *TarChownOptions |
| 49 | 49 |
IncludeSourceDir bool |
| 50 |
- // When unpacking convert whiteouts and opaque dirs from aufs format to overlayfs format |
|
| 51 |
- // When packing convert whiteouts and opaque dirs from overlayfs format to aufs format |
|
| 52 |
- OverlayFormat bool |
|
| 50 |
+ // WhiteoutFormat is the expected on disk format for whiteout files. |
|
| 51 |
+ // This format will be converted to the standard format on pack |
|
| 52 |
+ // and from the standard format on unpack. |
|
| 53 |
+ WhiteoutFormat WhiteoutFormat |
|
| 53 | 54 |
// When unpacking, specifies whether overwriting a directory with a |
| 54 | 55 |
// non-directory is allowed and vice versa. |
| 55 | 56 |
NoOverwriteDirNonDir bool |
| ... | ... |
@@ -96,6 +99,14 @@ const ( |
| 96 | 96 |
Xz |
| 97 | 97 |
) |
| 98 | 98 |
|
| 99 |
+const ( |
|
| 100 |
+ // AUFSWhiteoutFormat is the default format for whitesouts |
|
| 101 |
+ AUFSWhiteoutFormat WhiteoutFormat = iota |
|
| 102 |
+ // OverlayWhiteoutFormat formats whiteout according to the overlay |
|
| 103 |
+ // standard. |
|
| 104 |
+ OverlayWhiteoutFormat |
|
| 105 |
+) |
|
| 106 |
+ |
|
| 99 | 107 |
// IsArchive checks for the magic bytes of a tar or any supported compression |
| 100 | 108 |
// algorithm. |
| 101 | 109 |
func IsArchive(header []byte) bool {
|
| ... | ... |
@@ -231,6 +242,11 @@ func (compression *Compression) Extension() string {
|
| 231 | 231 |
return "" |
| 232 | 232 |
} |
| 233 | 233 |
|
| 234 |
+type tarWhiteoutConverter interface {
|
|
| 235 |
+ ConvertWrite(*tar.Header, string, os.FileInfo) error |
|
| 236 |
+ ConvertRead(*tar.Header, string) (bool, error) |
|
| 237 |
+} |
|
| 238 |
+ |
|
| 234 | 239 |
type tarAppender struct {
|
| 235 | 240 |
TarWriter *tar.Writer |
| 236 | 241 |
Buffer *bufio.Writer |
| ... | ... |
@@ -240,10 +256,11 @@ type tarAppender struct {
|
| 240 | 240 |
UIDMaps []idtools.IDMap |
| 241 | 241 |
GIDMaps []idtools.IDMap |
| 242 | 242 |
|
| 243 |
- // `overlayFormat` controls whether to interpret character devices with numbers 0,0 |
|
| 244 |
- // and directories with the attribute `trusted.overlay.opaque` using their overlayfs |
|
| 245 |
- // meanings and remap them to AUFS format |
|
| 246 |
- OverlayFormat bool |
|
| 243 |
+ // For packing and unpacking whiteout files in the |
|
| 244 |
+ // non standard format. The whiteout files defined |
|
| 245 |
+ // by the AUFS standard are used as the tar whiteout |
|
| 246 |
+ // standard. |
|
| 247 |
+ WhiteoutConverter tarWhiteoutConverter |
|
| 247 | 248 |
} |
| 248 | 249 |
|
| 249 | 250 |
// canonicalTarName provides a platform-independent and consistent posix-style |
| ... | ... |
@@ -332,13 +349,9 @@ func (ta *tarAppender) addTarFile(path, name string) error {
|
| 332 | 332 |
hdr.Gid = xGID |
| 333 | 333 |
} |
| 334 | 334 |
|
| 335 |
- if ta.OverlayFormat {
|
|
| 336 |
- // convert whiteouts to AUFS format |
|
| 337 |
- if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
|
|
| 338 |
- // we just rename the file and make it normal |
|
| 339 |
- hdr.Name = WhiteoutPrefix + hdr.Name |
|
| 340 |
- hdr.Mode = 0600 |
|
| 341 |
- hdr.Typeflag = tar.TypeReg |
|
| 335 |
+ if ta.WhiteoutConverter != nil {
|
|
| 336 |
+ if err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi); err != nil {
|
|
| 337 |
+ return err |
|
| 342 | 338 |
} |
| 343 | 339 |
} |
| 344 | 340 |
|
| ... | ... |
@@ -365,30 +378,6 @@ func (ta *tarAppender) addTarFile(path, name string) error {
|
| 365 | 365 |
} |
| 366 | 366 |
} |
| 367 | 367 |
|
| 368 |
- if ta.OverlayFormat {
|
|
| 369 |
- // convert opaque dirs to AUFS format by writing an empty file with the prefix |
|
| 370 |
- opaque, _ := system.Lgetxattr(path, "trusted.overlay.opaque") |
|
| 371 |
- if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
|
|
| 372 |
- // create a header for the whiteout file |
|
| 373 |
- // it should inherit some properties from the parent, but be a regular file |
|
| 374 |
- whHdr := &tar.Header{
|
|
| 375 |
- Typeflag: tar.TypeReg, |
|
| 376 |
- Mode: hdr.Mode & int64(os.ModePerm), |
|
| 377 |
- Name: filepath.Join(name, WhiteoutOpaqueDir), |
|
| 378 |
- Size: 0, |
|
| 379 |
- Uid: hdr.Uid, |
|
| 380 |
- Uname: hdr.Uname, |
|
| 381 |
- Gid: hdr.Gid, |
|
| 382 |
- Gname: hdr.Gname, |
|
| 383 |
- AccessTime: hdr.AccessTime, |
|
| 384 |
- ChangeTime: hdr.ChangeTime, |
|
| 385 |
- } |
|
| 386 |
- if err := ta.TarWriter.WriteHeader(whHdr); err != nil {
|
|
| 387 |
- return err |
|
| 388 |
- } |
|
| 389 |
- } |
|
| 390 |
- } |
|
| 391 |
- |
|
| 392 | 368 |
return nil |
| 393 | 369 |
} |
| 394 | 370 |
|
| ... | ... |
@@ -544,12 +533,12 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) |
| 544 | 544 |
|
| 545 | 545 |
go func() {
|
| 546 | 546 |
ta := &tarAppender{
|
| 547 |
- TarWriter: tar.NewWriter(compressWriter), |
|
| 548 |
- Buffer: pools.BufioWriter32KPool.Get(nil), |
|
| 549 |
- SeenFiles: make(map[uint64]string), |
|
| 550 |
- UIDMaps: options.UIDMaps, |
|
| 551 |
- GIDMaps: options.GIDMaps, |
|
| 552 |
- OverlayFormat: options.OverlayFormat, |
|
| 547 |
+ TarWriter: tar.NewWriter(compressWriter), |
|
| 548 |
+ Buffer: pools.BufioWriter32KPool.Get(nil), |
|
| 549 |
+ SeenFiles: make(map[uint64]string), |
|
| 550 |
+ UIDMaps: options.UIDMaps, |
|
| 551 |
+ GIDMaps: options.GIDMaps, |
|
| 552 |
+ WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat), |
|
| 553 | 553 |
} |
| 554 | 554 |
|
| 555 | 555 |
defer func() {
|
| ... | ... |
@@ -711,6 +700,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err |
| 711 | 711 |
if err != nil {
|
| 712 | 712 |
return err |
| 713 | 713 |
} |
| 714 |
+ whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat) |
|
| 714 | 715 |
|
| 715 | 716 |
// Iterate through the files in the archive. |
| 716 | 717 |
loop: |
| ... | ... |
@@ -810,33 +800,12 @@ loop: |
| 810 | 810 |
hdr.Gid = xGID |
| 811 | 811 |
} |
| 812 | 812 |
|
| 813 |
- base := filepath.Base(path) |
|
| 814 |
- dir := filepath.Dir(path) |
|
| 815 |
- |
|
| 816 |
- if options.OverlayFormat {
|
|
| 817 |
- // if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay |
|
| 818 |
- if base == WhiteoutOpaqueDir {
|
|
| 819 |
- if err := syscall.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0); err != nil {
|
|
| 820 |
- return err |
|
| 821 |
- } |
|
| 822 |
- |
|
| 823 |
- // don't write the file itself |
|
| 824 |
- continue |
|
| 813 |
+ if whiteoutConverter != nil {
|
|
| 814 |
+ writeFile, err := whiteoutConverter.ConvertRead(hdr, path) |
|
| 815 |
+ if err != nil {
|
|
| 816 |
+ return err |
|
| 825 | 817 |
} |
| 826 |
- |
|
| 827 |
- // if a file was deleted and we are using overlay, we need to create a character device |
|
| 828 |
- if strings.HasPrefix(base, WhiteoutPrefix) {
|
|
| 829 |
- originalBase := base[len(WhiteoutPrefix):] |
|
| 830 |
- originalPath := filepath.Join(dir, originalBase) |
|
| 831 |
- |
|
| 832 |
- if err := syscall.Mknod(originalPath, syscall.S_IFCHR, 0); err != nil {
|
|
| 833 |
- return err |
|
| 834 |
- } |
|
| 835 |
- if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
|
|
| 836 |
- return err |
|
| 837 |
- } |
|
| 838 |
- |
|
| 839 |
- // don't write the file itself |
|
| 818 |
+ if !writeFile {
|
|
| 840 | 819 |
continue |
| 841 | 820 |
} |
| 842 | 821 |
} |
| 843 | 822 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,89 @@ |
| 0 |
+package archive |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "archive/tar" |
|
| 4 |
+ "os" |
|
| 5 |
+ "path/filepath" |
|
| 6 |
+ "strings" |
|
| 7 |
+ "syscall" |
|
| 8 |
+ |
|
| 9 |
+ "github.com/docker/docker/pkg/system" |
|
| 10 |
+) |
|
| 11 |
+ |
|
| 12 |
+func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter {
|
|
| 13 |
+ if format == OverlayWhiteoutFormat {
|
|
| 14 |
+ return overlayWhiteoutConverter{}
|
|
| 15 |
+ } |
|
| 16 |
+ return nil |
|
| 17 |
+} |
|
| 18 |
+ |
|
| 19 |
+type overlayWhiteoutConverter struct{}
|
|
| 20 |
+ |
|
| 21 |
+func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) error {
|
|
| 22 |
+ // convert whiteouts to AUFS format |
|
| 23 |
+ if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
|
|
| 24 |
+ // we just rename the file and make it normal |
|
| 25 |
+ hdr.Name = WhiteoutPrefix + hdr.Name |
|
| 26 |
+ hdr.Mode = 0600 |
|
| 27 |
+ hdr.Typeflag = tar.TypeReg |
|
| 28 |
+ } |
|
| 29 |
+ |
|
| 30 |
+ if fi.Mode()&os.ModeDir != 0 {
|
|
| 31 |
+ // convert opaque dirs to AUFS format by writing an empty file with the prefix |
|
| 32 |
+ opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque") |
|
| 33 |
+ if err != nil {
|
|
| 34 |
+ return err |
|
| 35 |
+ } |
|
| 36 |
+ if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
|
|
| 37 |
+ // create a header for the whiteout file |
|
| 38 |
+ // it should inherit some properties from the parent, but be a regular file |
|
| 39 |
+ *hdr = tar.Header{
|
|
| 40 |
+ Typeflag: tar.TypeReg, |
|
| 41 |
+ Mode: hdr.Mode & int64(os.ModePerm), |
|
| 42 |
+ Name: filepath.Join(hdr.Name, WhiteoutOpaqueDir), |
|
| 43 |
+ Size: 0, |
|
| 44 |
+ Uid: hdr.Uid, |
|
| 45 |
+ Uname: hdr.Uname, |
|
| 46 |
+ Gid: hdr.Gid, |
|
| 47 |
+ Gname: hdr.Gname, |
|
| 48 |
+ AccessTime: hdr.AccessTime, |
|
| 49 |
+ ChangeTime: hdr.ChangeTime, |
|
| 50 |
+ } |
|
| 51 |
+ } |
|
| 52 |
+ } |
|
| 53 |
+ |
|
| 54 |
+ return nil |
|
| 55 |
+} |
|
| 56 |
+ |
|
| 57 |
+func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) {
|
|
| 58 |
+ base := filepath.Base(path) |
|
| 59 |
+ dir := filepath.Dir(path) |
|
| 60 |
+ |
|
| 61 |
+ // if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay |
|
| 62 |
+ if base == WhiteoutOpaqueDir {
|
|
| 63 |
+ if err := syscall.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0); err != nil {
|
|
| 64 |
+ return false, err |
|
| 65 |
+ } |
|
| 66 |
+ |
|
| 67 |
+ // don't write the file itself |
|
| 68 |
+ return false, nil |
|
| 69 |
+ } |
|
| 70 |
+ |
|
| 71 |
+ // if a file was deleted and we are using overlay, we need to create a character device |
|
| 72 |
+ if strings.HasPrefix(base, WhiteoutPrefix) {
|
|
| 73 |
+ originalBase := base[len(WhiteoutPrefix):] |
|
| 74 |
+ originalPath := filepath.Join(dir, originalBase) |
|
| 75 |
+ |
|
| 76 |
+ if err := syscall.Mknod(originalPath, syscall.S_IFCHR, 0); err != nil {
|
|
| 77 |
+ return false, err |
|
| 78 |
+ } |
|
| 79 |
+ if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
|
|
| 80 |
+ return false, err |
|
| 81 |
+ } |
|
| 82 |
+ |
|
| 83 |
+ // don't write the file itself |
|
| 84 |
+ return false, nil |
|
| 85 |
+ } |
|
| 86 |
+ |
|
| 87 |
+ return true, nil |
|
| 88 |
+} |
| ... | ... |
@@ -81,6 +81,33 @@ func sameFsTimeSpec(a, b syscall.Timespec) bool {
|
| 81 | 81 |
// Changes walks the path rw and determines changes for the files in the path, |
| 82 | 82 |
// with respect to the parent layers |
| 83 | 83 |
func Changes(layers []string, rw string) ([]Change, error) {
|
| 84 |
+ return changes(layers, rw, aufsDeletedFile, aufsMetadataSkip) |
|
| 85 |
+} |
|
| 86 |
+ |
|
| 87 |
+func aufsMetadataSkip(path string) (skip bool, err error) {
|
|
| 88 |
+ skip, err = filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path) |
|
| 89 |
+ if err != nil {
|
|
| 90 |
+ skip = true |
|
| 91 |
+ } |
|
| 92 |
+ return |
|
| 93 |
+} |
|
| 94 |
+ |
|
| 95 |
+func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) {
|
|
| 96 |
+ f := filepath.Base(path) |
|
| 97 |
+ |
|
| 98 |
+ // If there is a whiteout, then the file was removed |
|
| 99 |
+ if strings.HasPrefix(f, WhiteoutPrefix) {
|
|
| 100 |
+ originalFile := f[len(WhiteoutPrefix):] |
|
| 101 |
+ return filepath.Join(filepath.Dir(path), originalFile), nil |
|
| 102 |
+ } |
|
| 103 |
+ |
|
| 104 |
+ return "", nil |
|
| 105 |
+} |
|
| 106 |
+ |
|
| 107 |
+type skipChange func(string) (bool, error) |
|
| 108 |
+type deleteChange func(string, string, os.FileInfo) (string, error) |
|
| 109 |
+ |
|
| 110 |
+func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Change, error) {
|
|
| 84 | 111 |
var ( |
| 85 | 112 |
changes []Change |
| 86 | 113 |
changedDirs = make(map[string]struct{})
|
| ... | ... |
@@ -105,21 +132,24 @@ func Changes(layers []string, rw string) ([]Change, error) {
|
| 105 | 105 |
return nil |
| 106 | 106 |
} |
| 107 | 107 |
|
| 108 |
- // Skip AUFS metadata |
|
| 109 |
- if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched {
|
|
| 110 |
- return err |
|
| 108 |
+ if sc != nil {
|
|
| 109 |
+ if skip, err := sc(path); skip {
|
|
| 110 |
+ return err |
|
| 111 |
+ } |
|
| 111 | 112 |
} |
| 112 | 113 |
|
| 113 | 114 |
change := Change{
|
| 114 | 115 |
Path: path, |
| 115 | 116 |
} |
| 116 | 117 |
|
| 118 |
+ deletedFile, err := dc(rw, path, f) |
|
| 119 |
+ if err != nil {
|
|
| 120 |
+ return err |
|
| 121 |
+ } |
|
| 122 |
+ |
|
| 117 | 123 |
// Find out what kind of modification happened |
| 118 |
- file := filepath.Base(path) |
|
| 119 |
- // If there is a whiteout, then the file was removed |
|
| 120 |
- if strings.HasPrefix(file, WhiteoutPrefix) {
|
|
| 121 |
- originalFile := file[len(WhiteoutPrefix):] |
|
| 122 |
- change.Path = filepath.Join(filepath.Dir(path), originalFile) |
|
| 124 |
+ if deletedFile != "" {
|
|
| 125 |
+ change.Path = deletedFile |
|
| 123 | 126 |
change.Kind = ChangeDelete |
| 124 | 127 |
} else {
|
| 125 | 128 |
// Otherwise, the file was added |
| ... | ... |
@@ -283,3 +283,30 @@ func clen(n []byte) int {
|
| 283 | 283 |
} |
| 284 | 284 |
return len(n) |
| 285 | 285 |
} |
| 286 |
+ |
|
| 287 |
+// OverlayChanges walks the path rw and determines changes for the files in the path, |
|
| 288 |
+// with respect to the parent layers |
|
| 289 |
+func OverlayChanges(layers []string, rw string) ([]Change, error) {
|
|
| 290 |
+ return changes(layers, rw, overlayDeletedFile, nil) |
|
| 291 |
+} |
|
| 292 |
+ |
|
| 293 |
+func overlayDeletedFile(root, path string, fi os.FileInfo) (string, error) {
|
|
| 294 |
+ if fi.Mode()&os.ModeCharDevice != 0 {
|
|
| 295 |
+ s := fi.Sys().(*syscall.Stat_t) |
|
| 296 |
+ if major(uint64(s.Rdev)) == 0 && minor(uint64(s.Rdev)) == 0 {
|
|
| 297 |
+ return path, nil |
|
| 298 |
+ } |
|
| 299 |
+ } |
|
| 300 |
+ if fi.Mode()&os.ModeDir != 0 {
|
|
| 301 |
+ opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque") |
|
| 302 |
+ if err != nil {
|
|
| 303 |
+ return "", err |
|
| 304 |
+ } |
|
| 305 |
+ if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
|
|
| 306 |
+ return path, nil |
|
| 307 |
+ } |
|
| 308 |
+ } |
|
| 309 |
+ |
|
| 310 |
+ return "", nil |
|
| 311 |
+ |
|
| 312 |
+} |