Browse code

archive: Implement ApplyLayer directly

Rather than calling out to tar we use the golang tar parser
to directly extract the tar files. This has two major advantages:

1) We're able to replace an existing directory with a file in the
new layer. This currently breaks with the external tar, since
it refuses to recursively remove the destination directory in
this case, and there are no options to make it do that.

2) We avoid extracting the whiteout files just to later remove them.

Alexander Larsson authored on 2013/12/13 23:43:50
Showing 3 changed files
... ...
@@ -1,6 +1,9 @@
1 1
 package archive
2 2
 
3 3
 import (
4
+	"archive/tar"
5
+	"github.com/dotcloud/docker/utils"
6
+	"io"
4 7
 	"os"
5 8
 	"path/filepath"
6 9
 	"strings"
... ...
@@ -8,87 +11,159 @@ import (
8 8
 	"time"
9 9
 )
10 10
 
11
+// Linux device nodes are a bit weird due to backwards compat with 16 bit device nodes
12
+// The lower 8 bit is the lower 8 bit in the minor, the following 12 bits are the major,
13
+// and then there is the top 12 bits of then minor
14
+func mkdev(major int64, minor int64) uint32 {
15
+	return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff))
16
+}
17
+func timeToTimespec(time time.Time) (ts syscall.Timespec) {
18
+	if time.IsZero() {
19
+		// Return UTIME_OMIT special value
20
+		ts.Sec = 0
21
+		ts.Nsec = ((1 << 30) - 2)
22
+		return
23
+	}
24
+	return syscall.NsecToTimespec(time.UnixNano())
25
+}
26
+
11 27
 // ApplyLayer parses a diff in the standard layer format from `layer`, and
12 28
 // applies it to the directory `dest`.
13 29
 func ApplyLayer(dest string, layer Archive) error {
14
-	// Poor man's diff applyer in 2 steps:
30
+	// We need to be able to set any perms
31
+	oldmask := syscall.Umask(0)
32
+	defer syscall.Umask(oldmask)
15 33
 
16
-	// Step 1: untar everything in place
17
-	if err := Untar(layer, dest, nil); err != nil {
18
-		return err
19
-	}
34
+	tr := tar.NewReader(layer)
20 35
 
21
-	modifiedDirs := make(map[string]*syscall.Stat_t)
22
-	addDir := func(file string) {
23
-		d := filepath.Dir(file)
24
-		if _, exists := modifiedDirs[d]; !exists {
25
-			if s, err := os.Lstat(d); err == nil {
26
-				if sys := s.Sys(); sys != nil {
27
-					if stat, ok := sys.(*syscall.Stat_t); ok {
28
-						modifiedDirs[d] = stat
29
-					}
30
-				}
31
-			}
32
-		}
33
-	}
36
+	var dirs []*tar.Header
34 37
 
35
-	// Step 2: walk for whiteouts and apply them, removing them in the process
36
-	err := filepath.Walk(dest, func(fullPath string, f os.FileInfo, err error) error {
38
+	// Iterate through the files in the archive.
39
+	for {
40
+		hdr, err := tr.Next()
41
+		if err == io.EOF {
42
+			// end of tar archive
43
+			break
44
+		}
37 45
 		if err != nil {
38
-			if os.IsNotExist(err) {
39
-				// This happens in the case of whiteouts in parent dir removing a directory
40
-				// We just ignore it
41
-				return filepath.SkipDir
42
-			}
43 46
 			return err
44 47
 		}
45 48
 
46
-		// Rebase path
47
-		path, err := filepath.Rel(dest, fullPath)
48
-		if err != nil {
49
-			return err
49
+		// Skip AUFS metadata dirs
50
+		if strings.HasPrefix(hdr.Name, ".wh..wh.") {
51
+			continue
50 52
 		}
51
-		path = filepath.Join("/", path)
52 53
 
53
-		// Skip AUFS metadata
54
-		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil {
55
-			return err
56
-		} else if matched {
57
-			addDir(fullPath)
58
-			if err := os.RemoveAll(fullPath); err != nil {
54
+		path := filepath.Join(dest, hdr.Name)
55
+		base := filepath.Base(path)
56
+		if strings.HasPrefix(base, ".wh.") {
57
+			originalBase := base[len(".wh."):]
58
+			originalPath := filepath.Join(filepath.Dir(path), originalBase)
59
+			if err := os.RemoveAll(originalPath); err != nil {
59 60
 				return err
60 61
 			}
61
-		}
62
+		} else {
63
+			// If path exits we almost always just want to remove and replace it
64
+			// The only exception is when it is a directory *and* the file from
65
+			// the layer is also a directory. Then we want to merge them (i.e.
66
+			// just apply the metadata from the layer).
67
+			hasDir := false
68
+			if fi, err := os.Lstat(path); err == nil {
69
+				if fi.IsDir() && hdr.Typeflag == tar.TypeDir {
70
+					hasDir = true
71
+				} else {
72
+					if err := os.RemoveAll(path); err != nil {
73
+						return err
74
+					}
75
+				}
76
+			}
62 77
 
63
-		filename := filepath.Base(path)
64
-		if strings.HasPrefix(filename, ".wh.") {
65
-			rmTargetName := filename[len(".wh."):]
66
-			rmTargetPath := filepath.Join(filepath.Dir(fullPath), rmTargetName)
78
+			switch hdr.Typeflag {
79
+			case tar.TypeDir:
80
+				if !hasDir {
81
+					err = os.Mkdir(path, os.FileMode(hdr.Mode))
82
+					if err != nil {
83
+						return err
84
+					}
85
+				}
86
+				dirs = append(dirs, hdr)
67 87
 
68
-			// Remove the file targeted by the whiteout
69
-			addDir(rmTargetPath)
70
-			if err := os.RemoveAll(rmTargetPath); err != nil {
71
-				return err
88
+			case tar.TypeReg, tar.TypeRegA:
89
+				// Source is regular file
90
+				file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode))
91
+				if err != nil {
92
+					return err
93
+				}
94
+				if _, err := io.Copy(file, tr); err != nil {
95
+					file.Close()
96
+					return err
97
+				}
98
+				file.Close()
99
+
100
+			case tar.TypeBlock, tar.TypeChar, tar.TypeFifo:
101
+				mode := uint32(hdr.Mode & 07777)
102
+				switch hdr.Typeflag {
103
+				case tar.TypeBlock:
104
+					mode |= syscall.S_IFBLK
105
+				case tar.TypeChar:
106
+					mode |= syscall.S_IFCHR
107
+				case tar.TypeFifo:
108
+					mode |= syscall.S_IFIFO
109
+				}
110
+
111
+				if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil {
112
+					return err
113
+				}
114
+
115
+			case tar.TypeLink:
116
+				if err := os.Link(filepath.Join(dest, hdr.Linkname), path); err != nil {
117
+					return err
118
+				}
119
+
120
+			case tar.TypeSymlink:
121
+				if err := os.Symlink(hdr.Linkname, path); err != nil {
122
+					return err
123
+				}
124
+
125
+			default:
126
+				utils.Debugf("unhandled type %d\n", hdr.Typeflag)
72 127
 			}
73
-			// Remove the whiteout itself
74
-			addDir(fullPath)
75
-			if err := os.RemoveAll(fullPath); err != nil {
128
+
129
+			if err = syscall.Lchown(path, hdr.Uid, hdr.Gid); err != nil {
76 130
 				return err
77 131
 			}
132
+
133
+			// There is no LChmod, so ignore mode for symlink.  Also, this
134
+			// must happen after chown, as that can modify the file mode
135
+			if hdr.Typeflag != tar.TypeSymlink {
136
+				err = syscall.Chmod(path, uint32(hdr.Mode&07777))
137
+				if err != nil {
138
+					return err
139
+				}
140
+			}
141
+
142
+			// Directories must be handled at the end to avoid further
143
+			// file creation in them to modify the mtime
144
+			if hdr.Typeflag != tar.TypeDir {
145
+				ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
146
+				// syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and
147
+				if hdr.Typeflag != tar.TypeSymlink {
148
+					if err := syscall.UtimesNano(path, ts); err != nil {
149
+						return err
150
+					}
151
+				} else {
152
+					if err := LUtimesNano(path, ts); err != nil {
153
+						return err
154
+					}
155
+				}
156
+			}
78 157
 		}
79
-		return nil
80
-	})
81
-	if err != nil {
82
-		return err
83 158
 	}
84 159
 
85
-	for k, v := range modifiedDirs {
86
-		lastAccess := getLastAccess(v)
87
-		lastModification := getLastModification(v)
88
-		aTime := time.Unix(lastAccess.Unix())
89
-		mTime := time.Unix(lastModification.Unix())
90
-
91
-		if err := os.Chtimes(k, aTime, mTime); err != nil {
160
+	for _, hdr := range dirs {
161
+		path := filepath.Join(dest, hdr.Name)
162
+		ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
163
+		if err := syscall.UtimesNano(path, ts); err != nil {
92 164
 			return err
93 165
 		}
94 166
 	}
... ...
@@ -9,3 +9,7 @@ func getLastAccess(stat *syscall.Stat_t) syscall.Timespec {
9 9
 func getLastModification(stat *syscall.Stat_t) syscall.Timespec {
10 10
 	return stat.Mtimespec
11 11
 }
12
+
13
+func LUtimesNano(path string, ts []syscall.Timespec) error {
14
+	return nil
15
+}
... ...
@@ -1,6 +1,9 @@
1 1
 package archive
2 2
 
3
-import "syscall"
3
+import (
4
+	"syscall"
5
+	"unsafe"
6
+)
4 7
 
5 8
 func getLastAccess(stat *syscall.Stat_t) syscall.Timespec {
6 9
 	return stat.Atim
... ...
@@ -9,3 +12,21 @@ func getLastAccess(stat *syscall.Stat_t) syscall.Timespec {
9 9
 func getLastModification(stat *syscall.Stat_t) syscall.Timespec {
10 10
 	return stat.Mtim
11 11
 }
12
+
13
+func LUtimesNano(path string, ts []syscall.Timespec) error {
14
+	// These are not currently availible in syscall
15
+	AT_FDCWD := -100
16
+	AT_SYMLINK_NOFOLLOW := 0x100
17
+
18
+	var _path *byte
19
+	_path, err := syscall.BytePtrFromString(path)
20
+	if err != nil {
21
+		return err
22
+	}
23
+
24
+	if _, _, err := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AT_FDCWD), uintptr(unsafe.Pointer(_path)), uintptr(unsafe.Pointer(&ts[0])), uintptr(AT_SYMLINK_NOFOLLOW), 0, 0); err != 0 && err != syscall.ENOSYS {
25
+		return err
26
+	}
27
+
28
+	return nil
29
+}