Browse code

Update archive package to support overlay whiteouts

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)

Derek McGowan authored on 2016/04/07 07:07:29
Showing 5 changed files
... ...
@@ -33,6 +33,8 @@ type (
33 33
 	Reader io.Reader
34 34
 	// Compression is the state represents if compressed or not.
35 35
 	Compression int
36
+	// WhiteoutFormat is the format of whiteouts unpacked
37
+	WhiteoutFormat int
36 38
 	// TarChownOptions wraps the chown options UID and GID.
37 39
 	TarChownOptions struct {
38 40
 		UID, GID int
... ...
@@ -47,9 +49,10 @@ type (
47 47
 		GIDMaps          []idtools.IDMap
48 48
 		ChownOpts        *TarChownOptions
49 49
 		IncludeSourceDir bool
50
-		// When unpacking convert whiteouts and opaque dirs from aufs format to overlayfs format
51
-		// When packing convert whiteouts and opaque dirs from overlayfs format to aufs format
52
-		OverlayFormat bool
50
+		// WhiteoutFormat is the expected on disk format for whiteout files.
51
+		// This format will be converted to the standard format on pack
52
+		// and from the standard format on unpack.
53
+		WhiteoutFormat WhiteoutFormat
53 54
 		// When unpacking, specifies whether overwriting a directory with a
54 55
 		// non-directory is allowed and vice versa.
55 56
 		NoOverwriteDirNonDir bool
... ...
@@ -96,6 +99,14 @@ const (
96 96
 	Xz
97 97
 )
98 98
 
99
+const (
100
+	// AUFSWhiteoutFormat is the default format for whitesouts
101
+	AUFSWhiteoutFormat WhiteoutFormat = iota
102
+	// OverlayWhiteoutFormat formats whiteout according to the overlay
103
+	// standard.
104
+	OverlayWhiteoutFormat
105
+)
106
+
99 107
 // IsArchive checks for the magic bytes of a tar or any supported compression
100 108
 // algorithm.
101 109
 func IsArchive(header []byte) bool {
... ...
@@ -231,6 +242,11 @@ func (compression *Compression) Extension() string {
231 231
 	return ""
232 232
 }
233 233
 
234
+type tarWhiteoutConverter interface {
235
+	ConvertWrite(*tar.Header, string, os.FileInfo) error
236
+	ConvertRead(*tar.Header, string) (bool, error)
237
+}
238
+
234 239
 type tarAppender struct {
235 240
 	TarWriter *tar.Writer
236 241
 	Buffer    *bufio.Writer
... ...
@@ -240,10 +256,11 @@ type tarAppender struct {
240 240
 	UIDMaps   []idtools.IDMap
241 241
 	GIDMaps   []idtools.IDMap
242 242
 
243
-	// `overlayFormat` controls whether to interpret character devices with numbers 0,0
244
-	// and directories with the attribute `trusted.overlay.opaque` using their overlayfs
245
-	// meanings and remap them to AUFS format
246
-	OverlayFormat bool
243
+	// For packing and unpacking whiteout files in the
244
+	// non standard format. The whiteout files defined
245
+	// by the AUFS standard are used as the tar whiteout
246
+	// standard.
247
+	WhiteoutConverter tarWhiteoutConverter
247 248
 }
248 249
 
249 250
 // canonicalTarName provides a platform-independent and consistent posix-style
... ...
@@ -332,13 +349,9 @@ func (ta *tarAppender) addTarFile(path, name string) error {
332 332
 		hdr.Gid = xGID
333 333
 	}
334 334
 
335
-	if ta.OverlayFormat {
336
-		// convert whiteouts to AUFS format
337
-		if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
338
-			// we just rename the file and make it normal
339
-			hdr.Name = WhiteoutPrefix + hdr.Name
340
-			hdr.Mode = 0600
341
-			hdr.Typeflag = tar.TypeReg
335
+	if ta.WhiteoutConverter != nil {
336
+		if err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi); err != nil {
337
+			return err
342 338
 		}
343 339
 	}
344 340
 
... ...
@@ -365,30 +378,6 @@ func (ta *tarAppender) addTarFile(path, name string) error {
365 365
 		}
366 366
 	}
367 367
 
368
-	if ta.OverlayFormat {
369
-		// convert opaque dirs to AUFS format by writing an empty file with the prefix
370
-		opaque, _ := system.Lgetxattr(path, "trusted.overlay.opaque")
371
-		if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
372
-			// create a header for the whiteout file
373
-			// it should inherit some properties from the parent, but be a regular file
374
-			whHdr := &tar.Header{
375
-				Typeflag:   tar.TypeReg,
376
-				Mode:       hdr.Mode & int64(os.ModePerm),
377
-				Name:       filepath.Join(name, WhiteoutOpaqueDir),
378
-				Size:       0,
379
-				Uid:        hdr.Uid,
380
-				Uname:      hdr.Uname,
381
-				Gid:        hdr.Gid,
382
-				Gname:      hdr.Gname,
383
-				AccessTime: hdr.AccessTime,
384
-				ChangeTime: hdr.ChangeTime,
385
-			}
386
-			if err := ta.TarWriter.WriteHeader(whHdr); err != nil {
387
-				return err
388
-			}
389
-		}
390
-	}
391
-
392 368
 	return nil
393 369
 }
394 370
 
... ...
@@ -544,12 +533,12 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
544 544
 
545 545
 	go func() {
546 546
 		ta := &tarAppender{
547
-			TarWriter:     tar.NewWriter(compressWriter),
548
-			Buffer:        pools.BufioWriter32KPool.Get(nil),
549
-			SeenFiles:     make(map[uint64]string),
550
-			UIDMaps:       options.UIDMaps,
551
-			GIDMaps:       options.GIDMaps,
552
-			OverlayFormat: options.OverlayFormat,
547
+			TarWriter:         tar.NewWriter(compressWriter),
548
+			Buffer:            pools.BufioWriter32KPool.Get(nil),
549
+			SeenFiles:         make(map[uint64]string),
550
+			UIDMaps:           options.UIDMaps,
551
+			GIDMaps:           options.GIDMaps,
552
+			WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat),
553 553
 		}
554 554
 
555 555
 		defer func() {
... ...
@@ -711,6 +700,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err
711 711
 	if err != nil {
712 712
 		return err
713 713
 	}
714
+	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat)
714 715
 
715 716
 	// Iterate through the files in the archive.
716 717
 loop:
... ...
@@ -810,33 +800,12 @@ loop:
810 810
 			hdr.Gid = xGID
811 811
 		}
812 812
 
813
-		base := filepath.Base(path)
814
-		dir := filepath.Dir(path)
815
-
816
-		if options.OverlayFormat {
817
-			// if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay
818
-			if base == WhiteoutOpaqueDir {
819
-				if err := syscall.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0); err != nil {
820
-					return err
821
-				}
822
-
823
-				// don't write the file itself
824
-				continue
813
+		if whiteoutConverter != nil {
814
+			writeFile, err := whiteoutConverter.ConvertRead(hdr, path)
815
+			if err != nil {
816
+				return err
825 817
 			}
826
-
827
-			// if a file was deleted and we are using overlay, we need to create a character device
828
-			if strings.HasPrefix(base, WhiteoutPrefix) {
829
-				originalBase := base[len(WhiteoutPrefix):]
830
-				originalPath := filepath.Join(dir, originalBase)
831
-
832
-				if err := syscall.Mknod(originalPath, syscall.S_IFCHR, 0); err != nil {
833
-					return err
834
-				}
835
-				if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
836
-					return err
837
-				}
838
-
839
-				// don't write the file itself
818
+			if !writeFile {
840 819
 				continue
841 820
 			}
842 821
 		}
843 822
new file mode 100644
... ...
@@ -0,0 +1,89 @@
0
+package archive
1
+
2
+import (
3
+	"archive/tar"
4
+	"os"
5
+	"path/filepath"
6
+	"strings"
7
+	"syscall"
8
+
9
+	"github.com/docker/docker/pkg/system"
10
+)
11
+
12
+func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter {
13
+	if format == OverlayWhiteoutFormat {
14
+		return overlayWhiteoutConverter{}
15
+	}
16
+	return nil
17
+}
18
+
19
+type overlayWhiteoutConverter struct{}
20
+
21
+func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) error {
22
+	// convert whiteouts to AUFS format
23
+	if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
24
+		// we just rename the file and make it normal
25
+		hdr.Name = WhiteoutPrefix + hdr.Name
26
+		hdr.Mode = 0600
27
+		hdr.Typeflag = tar.TypeReg
28
+	}
29
+
30
+	if fi.Mode()&os.ModeDir != 0 {
31
+		// convert opaque dirs to AUFS format by writing an empty file with the prefix
32
+		opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque")
33
+		if err != nil {
34
+			return err
35
+		}
36
+		if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
37
+			// create a header for the whiteout file
38
+			// it should inherit some properties from the parent, but be a regular file
39
+			*hdr = tar.Header{
40
+				Typeflag:   tar.TypeReg,
41
+				Mode:       hdr.Mode & int64(os.ModePerm),
42
+				Name:       filepath.Join(hdr.Name, WhiteoutOpaqueDir),
43
+				Size:       0,
44
+				Uid:        hdr.Uid,
45
+				Uname:      hdr.Uname,
46
+				Gid:        hdr.Gid,
47
+				Gname:      hdr.Gname,
48
+				AccessTime: hdr.AccessTime,
49
+				ChangeTime: hdr.ChangeTime,
50
+			}
51
+		}
52
+	}
53
+
54
+	return nil
55
+}
56
+
57
+func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) {
58
+	base := filepath.Base(path)
59
+	dir := filepath.Dir(path)
60
+
61
+	// if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay
62
+	if base == WhiteoutOpaqueDir {
63
+		if err := syscall.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0); err != nil {
64
+			return false, err
65
+		}
66
+
67
+		// don't write the file itself
68
+		return false, nil
69
+	}
70
+
71
+	// if a file was deleted and we are using overlay, we need to create a character device
72
+	if strings.HasPrefix(base, WhiteoutPrefix) {
73
+		originalBase := base[len(WhiteoutPrefix):]
74
+		originalPath := filepath.Join(dir, originalBase)
75
+
76
+		if err := syscall.Mknod(originalPath, syscall.S_IFCHR, 0); err != nil {
77
+			return false, err
78
+		}
79
+		if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
80
+			return false, err
81
+		}
82
+
83
+		// don't write the file itself
84
+		return false, nil
85
+	}
86
+
87
+	return true, nil
88
+}
0 89
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+// +build !linux
1
+
2
+package archive
3
+
4
+func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter {
5
+	return nil
6
+}
... ...
@@ -81,6 +81,33 @@ func sameFsTimeSpec(a, b syscall.Timespec) bool {
81 81
 // Changes walks the path rw and determines changes for the files in the path,
82 82
 // with respect to the parent layers
83 83
 func Changes(layers []string, rw string) ([]Change, error) {
84
+	return changes(layers, rw, aufsDeletedFile, aufsMetadataSkip)
85
+}
86
+
87
+func aufsMetadataSkip(path string) (skip bool, err error) {
88
+	skip, err = filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path)
89
+	if err != nil {
90
+		skip = true
91
+	}
92
+	return
93
+}
94
+
95
+func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) {
96
+	f := filepath.Base(path)
97
+
98
+	// If there is a whiteout, then the file was removed
99
+	if strings.HasPrefix(f, WhiteoutPrefix) {
100
+		originalFile := f[len(WhiteoutPrefix):]
101
+		return filepath.Join(filepath.Dir(path), originalFile), nil
102
+	}
103
+
104
+	return "", nil
105
+}
106
+
107
+type skipChange func(string) (bool, error)
108
+type deleteChange func(string, string, os.FileInfo) (string, error)
109
+
110
+func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Change, error) {
84 111
 	var (
85 112
 		changes     []Change
86 113
 		changedDirs = make(map[string]struct{})
... ...
@@ -105,21 +132,24 @@ func Changes(layers []string, rw string) ([]Change, error) {
105 105
 			return nil
106 106
 		}
107 107
 
108
-		// Skip AUFS metadata
109
-		if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched {
110
-			return err
108
+		if sc != nil {
109
+			if skip, err := sc(path); skip {
110
+				return err
111
+			}
111 112
 		}
112 113
 
113 114
 		change := Change{
114 115
 			Path: path,
115 116
 		}
116 117
 
118
+		deletedFile, err := dc(rw, path, f)
119
+		if err != nil {
120
+			return err
121
+		}
122
+
117 123
 		// Find out what kind of modification happened
118
-		file := filepath.Base(path)
119
-		// If there is a whiteout, then the file was removed
120
-		if strings.HasPrefix(file, WhiteoutPrefix) {
121
-			originalFile := file[len(WhiteoutPrefix):]
122
-			change.Path = filepath.Join(filepath.Dir(path), originalFile)
124
+		if deletedFile != "" {
125
+			change.Path = deletedFile
123 126
 			change.Kind = ChangeDelete
124 127
 		} else {
125 128
 			// Otherwise, the file was added
... ...
@@ -283,3 +283,30 @@ func clen(n []byte) int {
283 283
 	}
284 284
 	return len(n)
285 285
 }
286
+
287
+// OverlayChanges walks the path rw and determines changes for the files in the path,
288
+// with respect to the parent layers
289
+func OverlayChanges(layers []string, rw string) ([]Change, error) {
290
+	return changes(layers, rw, overlayDeletedFile, nil)
291
+}
292
+
293
+func overlayDeletedFile(root, path string, fi os.FileInfo) (string, error) {
294
+	if fi.Mode()&os.ModeCharDevice != 0 {
295
+		s := fi.Sys().(*syscall.Stat_t)
296
+		if major(uint64(s.Rdev)) == 0 && minor(uint64(s.Rdev)) == 0 {
297
+			return path, nil
298
+		}
299
+	}
300
+	if fi.Mode()&os.ModeDir != 0 {
301
+		opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque")
302
+		if err != nil {
303
+			return "", err
304
+		}
305
+		if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
306
+			return path, nil
307
+		}
308
+	}
309
+
310
+	return "", nil
311
+
312
+}