1. docker
  2. Commit 8a2f9a249c28c0a80bac4b41eb318dfe68f63f0d
Browse code

Merge pull request #22126 from dmcgowan/overlay-native-diff

Overlay multiple lower directory support

Michael Crosby authored on 2016/06/14 05:15:39
Showing 20 changed files
daemon/graphdriver/graphtest/graphbench_unix.go
History View file @ 8a2f9a2
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,264 @@
0 
+// +build linux freebsd
1 
+
2 
+package graphtest
3 
+
4 
+import (
5 
+	"bytes"
6 
+	"io"
7 
+	"io/ioutil"
8 
+	"path/filepath"
9 
+	"testing"
10 
+
11 
+	"github.com/docker/docker/pkg/stringid"
12 
+)
13 
+
14 
+// DriverBenchExists benchmarks calls to exist
15 
+func DriverBenchExists(b *testing.B, drivername string, driveroptions ...string) {
16 
+	driver := GetDriver(b, drivername, driveroptions...)
17 
+	defer PutDriver(b)
18 
+
19 
+	base := stringid.GenerateRandomID()
20 
+
21 
+	if err := driver.Create(base, "", "", nil); err != nil {
22 
+		b.Fatal(err)
23 
+	}
24 
+
25 
+	b.ResetTimer()
26 
+	for i := 0; i < b.N; i++ {
27 
+		if !driver.Exists(base) {
28 
+			b.Fatal("Newly created image doesn't exist")
29 
+		}
30 
+	}
31 
+}
32 
+
33 
+// DriverBenchGetEmpty benchmarks calls to get on an empty layer
34 
+func DriverBenchGetEmpty(b *testing.B, drivername string, driveroptions ...string) {
35 
+	driver := GetDriver(b, drivername, driveroptions...)
36 
+	defer PutDriver(b)
37 
+
38 
+	base := stringid.GenerateRandomID()
39 
+
40 
+	if err := driver.Create(base, "", "", nil); err != nil {
41 
+		b.Fatal(err)
42 
+	}
43 
+
44 
+	b.ResetTimer()
45 
+	for i := 0; i < b.N; i++ {
46 
+		_, err := driver.Get(base, "")
47 
+		b.StopTimer()
48 
+		if err != nil {
49 
+			b.Fatalf("Error getting mount: %s", err)
50 
+		}
51 
+		if err := driver.Put(base); err != nil {
52 
+			b.Fatalf("Error putting mount: %s", err)
53 
+		}
54 
+		b.StartTimer()
55 
+	}
56 
+}
57 
+
58 
+// DriverBenchDiffBase benchmarks calls to diff on a root layer
59 
+func DriverBenchDiffBase(b *testing.B, drivername string, driveroptions ...string) {
60 
+	driver := GetDriver(b, drivername, driveroptions...)
61 
+	defer PutDriver(b)
62 
+
63 
+	base := stringid.GenerateRandomID()
64 
+
65 
+	if err := driver.Create(base, "", "", nil); err != nil {
66 
+		b.Fatal(err)
67 
+	}
68 
+
69 
+	if err := addFiles(driver, base, 3); err != nil {
70 
+		b.Fatal(err)
71 
+	}
72 
+
73 
+	b.ResetTimer()
74 
+	for i := 0; i < b.N; i++ {
75 
+		arch, err := driver.Diff(base, "")
76 
+		if err != nil {
77 
+			b.Fatal(err)
78 
+		}
79 
+		_, err = io.Copy(ioutil.Discard, arch)
80 
+		if err != nil {
81 
+			b.Fatalf("Error copying archive: %s", err)
82 
+		}
83 
+		arch.Close()
84 
+	}
85 
+}
86 
+
87 
+// DriverBenchDiffN benchmarks calls to diff on two layers with
88 
+// a provided number of files on the lower and upper layers.
89 
+func DriverBenchDiffN(b *testing.B, bottom, top int, drivername string, driveroptions ...string) {
90 
+	driver := GetDriver(b, drivername, driveroptions...)
91 
+	defer PutDriver(b)
92 
+	base := stringid.GenerateRandomID()
93 
+	upper := stringid.GenerateRandomID()
94 
+
95 
+	if err := driver.Create(base, "", "", nil); err != nil {
96 
+		b.Fatal(err)
97 
+	}
98 
+
99 
+	if err := addManyFiles(driver, base, bottom, 3); err != nil {
100 
+		b.Fatal(err)
101 
+	}
102 
+
103 
+	if err := driver.Create(upper, base, "", nil); err != nil {
104 
+		b.Fatal(err)
105 
+	}
106 
+
107 
+	if err := addManyFiles(driver, upper, top, 6); err != nil {
108 
+		b.Fatal(err)
109 
+	}
110 
+	b.ResetTimer()
111 
+	for i := 0; i < b.N; i++ {
112 
+		arch, err := driver.Diff(upper, "")
113 
+		if err != nil {
114 
+			b.Fatal(err)
115 
+		}
116 
+		_, err = io.Copy(ioutil.Discard, arch)
117 
+		if err != nil {
118 
+			b.Fatalf("Error copying archive: %s", err)
119 
+		}
120 
+		arch.Close()
121 
+	}
122 
+}
123 
+
124 
+// DriverBenchDiffApplyN benchmarks calls to diff and apply together
125 
+func DriverBenchDiffApplyN(b *testing.B, fileCount int, drivername string, driveroptions ...string) {
126 
+	driver := GetDriver(b, drivername, driveroptions...)
127 
+	defer PutDriver(b)
128 
+	base := stringid.GenerateRandomID()
129 
+	upper := stringid.GenerateRandomID()
130 
+
131 
+	if err := driver.Create(base, "", "", nil); err != nil {
132 
+		b.Fatal(err)
133 
+	}
134 
+
135 
+	if err := addManyFiles(driver, base, fileCount, 3); err != nil {
136 
+		b.Fatal(err)
137 
+	}
138 
+
139 
+	if err := driver.Create(upper, base, "", nil); err != nil {
140 
+		b.Fatal(err)
141 
+	}
142 
+
143 
+	if err := addManyFiles(driver, upper, fileCount, 6); err != nil {
144 
+		b.Fatal(err)
145 
+	}
146 
+	diffSize, err := driver.DiffSize(upper, "")
147 
+	if err != nil {
148 
+		b.Fatal(err)
149 
+	}
150 
+	b.ResetTimer()
151 
+	b.StopTimer()
152 
+	for i := 0; i < b.N; i++ {
153 
+		diff := stringid.GenerateRandomID()
154 
+		if err := driver.Create(diff, base, "", nil); err != nil {
155 
+			b.Fatal(err)
156 
+		}
157 
+
158 
+		if err := checkManyFiles(driver, diff, fileCount, 3); err != nil {
159 
+			b.Fatal(err)
160 
+		}
161 
+
162 
+		b.StartTimer()
163 
+
164 
+		arch, err := driver.Diff(upper, "")
165 
+		if err != nil {
166 
+			b.Fatal(err)
167 
+		}
168 
+
169 
+		applyDiffSize, err := driver.ApplyDiff(diff, "", arch)
170 
+		if err != nil {
171 
+			b.Fatal(err)
172 
+		}
173 
+
174 
+		b.StopTimer()
175 
+		arch.Close()
176 
+
177 
+		if applyDiffSize != diffSize {
178 
+			// TODO: enforce this
179 
+			//b.Fatalf("Apply diff size different, got %d, expected %s", applyDiffSize, diffSize)
180 
+		}
181 
+		if err := checkManyFiles(driver, diff, fileCount, 6); err != nil {
182 
+			b.Fatal(err)
183 
+		}
184 
+	}
185 
+}
186 
+
187 
+// DriverBenchDeepLayerDiff benchmarks calls to diff on top of a given number of layers.
188 
+func DriverBenchDeepLayerDiff(b *testing.B, layerCount int, drivername string, driveroptions ...string) {
189 
+	driver := GetDriver(b, drivername, driveroptions...)
190 
+	defer PutDriver(b)
191 
+
192 
+	base := stringid.GenerateRandomID()
193 
+
194 
+	if err := driver.Create(base, "", "", nil); err != nil {
195 
+		b.Fatal(err)
196 
+	}
197 
+
198 
+	if err := addFiles(driver, base, 50); err != nil {
199 
+		b.Fatal(err)
200 
+	}
201 
+
202 
+	topLayer, err := addManyLayers(driver, base, layerCount)
203 
+	if err != nil {
204 
+		b.Fatal(err)
205 
+	}
206 
+
207 
+	b.ResetTimer()
208 
+	for i := 0; i < b.N; i++ {
209 
+		arch, err := driver.Diff(topLayer, "")
210 
+		if err != nil {
211 
+			b.Fatal(err)
212 
+		}
213 
+		_, err = io.Copy(ioutil.Discard, arch)
214 
+		if err != nil {
215 
+			b.Fatalf("Error copying archive: %s", err)
216 
+		}
217 
+		arch.Close()
218 
+	}
219 
+}
220 
+
221 
+// DriverBenchDeepLayerRead benchmarks calls to read a file under a given number of layers.
222 
+func DriverBenchDeepLayerRead(b *testing.B, layerCount int, drivername string, driveroptions ...string) {
223 
+	driver := GetDriver(b, drivername, driveroptions...)
224 
+	defer PutDriver(b)
225 
+
226 
+	base := stringid.GenerateRandomID()
227 
+
228 
+	if err := driver.Create(base, "", "", nil); err != nil {
229 
+		b.Fatal(err)
230 
+	}
231 
+
232 
+	content := []byte("test content")
233 
+	if err := addFile(driver, base, "testfile.txt", content); err != nil {
234 
+		b.Fatal(err)
235 
+	}
236 
+
237 
+	topLayer, err := addManyLayers(driver, base, layerCount)
238 
+	if err != nil {
239 
+		b.Fatal(err)
240 
+	}
241 
+
242 
+	root, err := driver.Get(topLayer, "")
243 
+	if err != nil {
244 
+		b.Fatal(err)
245 
+	}
246 
+	defer driver.Put(topLayer)
247 
+
248 
+	b.ResetTimer()
249 
+	for i := 0; i < b.N; i++ {
250 
+
251 
+		// Read content
252 
+		c, err := ioutil.ReadFile(filepath.Join(root, "testfile.txt"))
253 
+		if err != nil {
254 
+			b.Fatal(err)
255 
+		}
256 
+
257 
+		b.StopTimer()
258 
+		if bytes.Compare(c, content) != 0 {
259 
+			b.Fatalf("Wrong content in file %v, expected %v", c, content)
260 
+		}
261 
+		b.StartTimer()
262 
+	}
263 
+}
daemon/graphdriver/graphtest/graphtest_unix.go
History View file @ 8a2f9a2
... ... 
@@ -3,7 +3,7 @@
3 3 
 package graphtest
4 4
5 5 
 import (
6 
-	"fmt"
6 
+	"bytes"
7 7 
 	"io/ioutil"
8 8 
 	"math/rand"
9 9 
 	"os"
... ... 
@@ -14,6 +14,7 @@ import (
14 14 
 	"unsafe"
15 15
16 16 
 	"github.com/docker/docker/daemon/graphdriver"
17 
+	"github.com/docker/docker/pkg/stringid"
17 18 
 	"github.com/docker/go-units"
18 19 
 )
19 20
... ... 
@@ -30,47 +31,7 @@ type Driver struct {
30 30 
 	refCount int
31 31 
 }
32 32
33 
-// InitLoopbacks ensures that the loopback devices are properly created within
34 
-// the system running the device mapper tests.
35 
-func InitLoopbacks() error {
36 
-	statT, err := getBaseLoopStats()
37 
-	if err != nil {
38 
-		return err
39 
-	}
40 
-	// create at least 8 loopback files, ya, that is a good number
41 
-	for i := 0; i < 8; i++ {
42 
-		loopPath := fmt.Sprintf("/dev/loop%d", i)
43 
-		// only create new loopback files if they don't exist
44 
-		if _, err := os.Stat(loopPath); err != nil {
45 
-			if mkerr := syscall.Mknod(loopPath,
46 
-				uint32(statT.Mode|syscall.S_IFBLK), int((7<<8)|(i&0xff)|((i&0xfff00)<<12))); mkerr != nil {
47 
-				return mkerr
48 
-			}
49 
-			os.Chown(loopPath, int(statT.Uid), int(statT.Gid))
50 
-		}
51 
-	}
52 
-	return nil
53 
-}
54 
-
55 
-// getBaseLoopStats inspects /dev/loop0 to collect uid,gid, and mode for the
56 
-// loop0 device on the system.  If it does not exist we assume 0,0,0660 for the
57 
-// stat data
58 
-func getBaseLoopStats() (*syscall.Stat_t, error) {
59 
-	loop0, err := os.Stat("/dev/loop0")
60 
-	if err != nil {
61 
-		if os.IsNotExist(err) {
62 
-			return &syscall.Stat_t{
63 
-				Uid:  0,
64 
-				Gid:  0,
65 
-				Mode: 0660,
66 
-			}, nil
67 
-		}
68 
-		return nil, err
69 
-	}
70 
-	return loop0.Sys().(*syscall.Stat_t), nil
71 
-}
72 
-
73 
-func newDriver(t *testing.T, name string) *Driver {
33 
+func newDriver(t testing.TB, name string, options []string) *Driver {
74 34 
 	root, err := ioutil.TempDir("", "docker-graphtest-")
75 35 
 	if err != nil {
76 36 
 		t.Fatal(err)
... ... 
@@ -80,7 +41,7 @@ func newDriver(t *testing.T, name string) *Driver {
80 80 
 		t.Fatal(err)
81 81 
 	}
82 82
83 
-	d, err := graphdriver.GetDriver(name, root, nil, nil, nil)
83 
+	d, err := graphdriver.GetDriver(name, root, options, nil, nil)
84 84 
 	if err != nil {
85 85 
 		t.Logf("graphdriver: %v\n", err)
86 86 
 		if err == graphdriver.ErrNotSupported || err == graphdriver.ErrPrerequisites || err == graphdriver.ErrIncompatibleFS {
... ... 
@@ -91,7 +52,7 @@ func newDriver(t *testing.T, name string) *Driver {
91 91 
 	return &Driver{d, root, 1}
92 92 
 }
93 93
94 
-func cleanup(t *testing.T, d *Driver) {
94 
+func cleanup(t testing.TB, d *Driver) {
95 95 
 	if err := drv.Cleanup(); err != nil {
96 96 
 		t.Fatal(err)
97 97 
 	}
... ... 
@@ -99,9 +60,9 @@ func cleanup(t *testing.T, d *Driver) {
99 99 
 }
100 100
101 101 
 // GetDriver create a new driver with given name or return an existing driver with the name updating the reference count.
102 
-func GetDriver(t *testing.T, name string) graphdriver.Driver {
102 
+func GetDriver(t testing.TB, name string, options ...string) graphdriver.Driver {
103 103 
 	if drv == nil {
104 
-		drv = newDriver(t, name)
104 
+		drv = newDriver(t, name, options)
105 105 
 	} else {
106 106 
 		drv.refCount++
107 107 
 	}
... ... 
@@ -109,7 +70,7 @@ func GetDriver(t *testing.T, name string) graphdriver.Driver {
109 109 
 }
110 110
111 111 
 // PutDriver removes the driver if it is no longer used and updates the reference count.
112 
-func PutDriver(t *testing.T) {
112 
+func PutDriver(t testing.TB) {
113 113 
 	if drv == nil {
114 114 
 		t.Skip("No driver to put!")
115 115 
 	}
... ... 
@@ -120,190 +81,210 @@ func PutDriver(t *testing.T) {
120 120 
 	}
121 121 
 }
122 122
123 
-func verifyFile(t *testing.T, path string, mode os.FileMode, uid, gid uint32) {
124 
-	fi, err := os.Stat(path)
125 
-	if err != nil {
123 
+// DriverTestCreateEmpty creates a new image and verifies it is empty and the right metadata
124 
+func DriverTestCreateEmpty(t testing.TB, drivername string, driverOptions ...string) {
125 
+	driver := GetDriver(t, drivername, driverOptions...)
126 
+	defer PutDriver(t)
127 
+
128 
+	if err := driver.Create("empty", "", "", nil); err != nil {
126 129 
 		t.Fatal(err)
127 130 
 	}
128 131
129 
-	if fi.Mode()&os.ModeType != mode&os.ModeType {
130 
-		t.Fatalf("Expected %s type 0x%x, got 0x%x", path, mode&os.ModeType, fi.Mode()&os.ModeType)
131 
-	}
132 
+	defer func() {
133 
+		if err := driver.Remove("empty"); err != nil {
134 
+			t.Fatal(err)
135 
+		}
136 
+	}()
132 137
133 
-	if fi.Mode()&os.ModePerm != mode&os.ModePerm {
134 
-		t.Fatalf("Expected %s mode %o, got %o", path, mode&os.ModePerm, fi.Mode()&os.ModePerm)
138 
+	if !driver.Exists("empty") {
139 
+		t.Fatal("Newly created image doesn't exist")
135 140 
 	}
136 141
137 
-	if fi.Mode()&os.ModeSticky != mode&os.ModeSticky {
138 
-		t.Fatalf("Expected %s sticky 0x%x, got 0x%x", path, mode&os.ModeSticky, fi.Mode()&os.ModeSticky)
142 
+	dir, err := driver.Get("empty", "")
143 
+	if err != nil {
144 
+		t.Fatal(err)
139 145 
 	}
140 146
141 
-	if fi.Mode()&os.ModeSetuid != mode&os.ModeSetuid {
142 
-		t.Fatalf("Expected %s setuid 0x%x, got 0x%x", path, mode&os.ModeSetuid, fi.Mode()&os.ModeSetuid)
143 
-	}
147 
+	verifyFile(t, dir, 0755|os.ModeDir, 0, 0)
144 148
145 
-	if fi.Mode()&os.ModeSetgid != mode&os.ModeSetgid {
146 
-		t.Fatalf("Expected %s setgid 0x%x, got 0x%x", path, mode&os.ModeSetgid, fi.Mode()&os.ModeSetgid)
149 
+	// Verify that the directory is empty
150 
+	fis, err := readDir(dir)
151 
+	if err != nil {
152 
+		t.Fatal(err)
147 153 
 	}
148 154
149 
-	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
150 
-		if stat.Uid != uid {
151 
-			t.Fatalf("%s no owned by uid %d", path, uid)
152 
-		}
153 
-		if stat.Gid != gid {
154 
-			t.Fatalf("%s not owned by gid %d", path, gid)
155 
-		}
155 
+	if len(fis) != 0 {
156 
+		t.Fatal("New directory not empty")
156 157 
 	}
157 158
159 
+	driver.Put("empty")
158 160 
 }
159 161
160 
-// readDir reads a directory just like ioutil.ReadDir()
161 
-// then hides specific files (currently "lost+found")
162 
-// so the tests don't "see" it
163 
-func readDir(dir string) ([]os.FileInfo, error) {
164 
-	a, err := ioutil.ReadDir(dir)
165 
-	if err != nil {
166 
-		return nil, err
167 
-	}
162 
+// DriverTestCreateBase create a base driver and verify.
163 
+func DriverTestCreateBase(t testing.TB, drivername string, driverOptions ...string) {
164 
+	driver := GetDriver(t, drivername, driverOptions...)
165 
+	defer PutDriver(t)
168 166
169 
-	b := a[:0]
170 
-	for _, x := range a {
171 
-		if x.Name() != "lost+found" { // ext4 always have this dir
172 
-			b = append(b, x)
167 
+	createBase(t, driver, "Base")
168 
+	defer func() {
169 
+		if err := driver.Remove("Base"); err != nil {
170 
+			t.Fatal(err)
173 171 
 		}
174 
-	}
175 
-
176 
-	return b, nil
172 
+	}()
173 
+	verifyBase(t, driver, "Base")
177 174 
 }
178 175
179 
-// DriverTestCreateEmpty creates a new image and verifies it is empty and the right metadata
180 
-func DriverTestCreateEmpty(t *testing.T, drivername string) {
181 
-	driver := GetDriver(t, drivername)
176 
+// DriverTestCreateSnap Create a driver and snap and verify.
177 
+func DriverTestCreateSnap(t testing.TB, drivername string, driverOptions ...string) {
178 
+	driver := GetDriver(t, drivername, driverOptions...)
182 179 
 	defer PutDriver(t)
183 180
184 
-	if err := driver.Create("empty", "", "", nil); err != nil {
181 
+	createBase(t, driver, "Base")
182 
+
183 
+	defer func() {
184 
+		if err := driver.Remove("Base"); err != nil {
185 
+			t.Fatal(err)
186 
+		}
187 
+	}()
188 
+
189 
+	if err := driver.Create("Snap", "Base", "", nil); err != nil {
185 190 
 		t.Fatal(err)
186 191 
 	}
187 192
188 193 
 	defer func() {
189 
-		if err := driver.Remove("empty"); err != nil {
194 
+		if err := driver.Remove("Snap"); err != nil {
190 195 
 			t.Fatal(err)
191 196 
 		}
192 197 
 	}()
193 198
194 
-	if !driver.Exists("empty") {
195 
-		t.Fatal("Newly created image doesn't exist")
196 
-	}
199 
+	verifyBase(t, driver, "Snap")
200 
+}
197 201
198 
-	dir, err := driver.Get("empty", "")
199 
-	if err != nil {
202 
+// DriverTestDeepLayerRead reads a file from a lower layer under a given number of layers
203 
+func DriverTestDeepLayerRead(t testing.TB, layerCount int, drivername string, driverOptions ...string) {
204 
+	driver := GetDriver(t, drivername, driverOptions...)
205 
+	defer PutDriver(t)
206 
+
207 
+	base := stringid.GenerateRandomID()
208 
+
209 
+	if err := driver.Create(base, "", "", nil); err != nil {
200 210 
 		t.Fatal(err)
201 211 
 	}
202 212
203 
-	verifyFile(t, dir, 0755|os.ModeDir, 0, 0)
213 
+	content := []byte("test content")
214 
+	if err := addFile(driver, base, "testfile.txt", content); err != nil {
215 
+		t.Fatal(err)
216 
+	}
204 217
205 
-	// Verify that the directory is empty
206 
-	fis, err := readDir(dir)
218 
+	topLayer, err := addManyLayers(driver, base, layerCount)
207 219 
 	if err != nil {
208 220 
 		t.Fatal(err)
209 221 
 	}
210 222
211 
-	if len(fis) != 0 {
212 
-		t.Fatal("New directory not empty")
223 
+	err = checkManyLayers(driver, topLayer, layerCount)
224 
+	if err != nil {
225 
+		t.Fatal(err)
213 226 
 	}
214 227
215 
-	driver.Put("empty")
228 
+	if err := checkFile(driver, topLayer, "testfile.txt", content); err != nil {
229 
+		t.Fatal(err)
230 
+	}
216 231 
 }
217 232
218 
-func createBase(t *testing.T, driver graphdriver.Driver, name string) {
219 
-	// We need to be able to set any perms
220 
-	oldmask := syscall.Umask(0)
221 
-	defer syscall.Umask(oldmask)
233 
+// DriverTestDiffApply tests diffing and applying produces the same layer
234 
+func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverOptions ...string) {
235 
+	driver := GetDriver(t, drivername, driverOptions...)
236 
+	defer PutDriver(t)
237 
+	base := stringid.GenerateRandomID()
238 
+	upper := stringid.GenerateRandomID()
222 239
223 
-	if err := driver.CreateReadWrite(name, "", "", nil); err != nil {
240 
+	if err := driver.Create(base, "", "", nil); err != nil {
224 241 
 		t.Fatal(err)
225 242 
 	}
226 243
227 
-	dir, err := driver.Get(name, "")
228 
-	if err != nil {
244 
+	if err := addManyFiles(driver, base, fileCount, 3); err != nil {
245 
+		t.Fatal(err)
246 
+	}
247 
+
248 
+	if err := driver.Create(upper, base, "", nil); err != nil {
229 249 
 		t.Fatal(err)
230 250 
 	}
231 
-	defer driver.Put(name)
232 251
233 
-	subdir := path.Join(dir, "a subdir")
234 
-	if err := os.Mkdir(subdir, 0705|os.ModeSticky); err != nil {
252 
+	if err := addManyFiles(driver, upper, fileCount, 6); err != nil {
235 253 
 		t.Fatal(err)
236 254 
 	}
237 
-	if err := os.Chown(subdir, 1, 2); err != nil {
255 
+	diffSize, err := driver.DiffSize(upper, "")
256 
+	if err != nil {
238 257 
 		t.Fatal(err)
239 258 
 	}
240 259
241 
-	file := path.Join(dir, "a file")
242 
-	if err := ioutil.WriteFile(file, []byte("Some data"), 0222|os.ModeSetuid); err != nil {
260 
+	diff := stringid.GenerateRandomID()
261 
+	if err := driver.Create(diff, base, "", nil); err != nil {
243 262 
 		t.Fatal(err)
244 263 
 	}
245 
-}
246 264
247 
-func verifyBase(t *testing.T, driver graphdriver.Driver, name string) {
248 
-	dir, err := driver.Get(name, "")
249 
-	if err != nil {
265 
+	if err := checkManyFiles(driver, diff, fileCount, 3); err != nil {
250 266 
 		t.Fatal(err)
251 267 
 	}
252 
-	defer driver.Put(name)
253 268
254 
-	subdir := path.Join(dir, "a subdir")
255 
-	verifyFile(t, subdir, 0705|os.ModeDir|os.ModeSticky, 1, 2)
269 
+	arch, err := driver.Diff(upper, base)
270 
+	if err != nil {
271 
+		t.Fatal(err)
272 
+	}
256 273
257 
-	file := path.Join(dir, "a file")
258 
-	verifyFile(t, file, 0222|os.ModeSetuid, 0, 0)
274 
+	buf := bytes.NewBuffer(nil)
275 
+	if _, err := buf.ReadFrom(arch); err != nil {
276 
+		t.Fatal(err)
277 
+	}
278 
+	if err := arch.Close(); err != nil {
279 
+		t.Fatal(err)
280 
+	}
259 281
260 
-	fis, err := readDir(dir)
282 
+	applyDiffSize, err := driver.ApplyDiff(diff, base, bytes.NewReader(buf.Bytes()))
261 283 
 	if err != nil {
262 284 
 		t.Fatal(err)
263 285 
 	}
264 286
265 
-	if len(fis) != 2 {
266 
-		t.Fatal("Unexpected files in base image")
287 
+	if applyDiffSize != diffSize {
288 
+		t.Fatalf("Apply diff size different, got %d, expected %d", applyDiffSize, diffSize)
289 
+	}
290 
+	if err := checkManyFiles(driver, diff, fileCount, 6); err != nil {
291 
+		t.Fatal(err)
267 292 
 	}
268 293 
 }
269 294
270 
-// DriverTestCreateBase create a base driver and verify.
271 
-func DriverTestCreateBase(t *testing.T, drivername string) {
272 
-	driver := GetDriver(t, drivername)
295 
+// DriverTestChanges tests computed changes on a layer matches changes made
296 
+func DriverTestChanges(t testing.TB, drivername string, driverOptions ...string) {
297 
+	driver := GetDriver(t, drivername, driverOptions...)
273 298 
 	defer PutDriver(t)
299 
+	base := stringid.GenerateRandomID()
300 
+	upper := stringid.GenerateRandomID()
274 301
275 
-	createBase(t, driver, "Base")
276 
-	defer func() {
277 
-		if err := driver.Remove("Base"); err != nil {
278 
-			t.Fatal(err)
279 
-		}
280 
-	}()
281 
-	verifyBase(t, driver, "Base")
282 
-}
302 
+	if err := driver.Create(base, "", "", nil); err != nil {
303 
+		t.Fatal(err)
304 
+	}
283 305
284 
-// DriverTestCreateSnap Create a driver and snap and verify.
285 
-func DriverTestCreateSnap(t *testing.T, drivername string) {
286 
-	driver := GetDriver(t, drivername)
287 
-	defer PutDriver(t)
306 
+	if err := addManyFiles(driver, base, 20, 3); err != nil {
307 
+		t.Fatal(err)
308 
+	}
288 309
289 
-	createBase(t, driver, "Base")
310 
+	if err := driver.Create(upper, base, "", nil); err != nil {
311 
+		t.Fatal(err)
312 
+	}
290 313
291 
-	defer func() {
292 
-		if err := driver.Remove("Base"); err != nil {
293 
-			t.Fatal(err)
294 
-		}
295 
-	}()
314 
+	expectedChanges, err := changeManyFiles(driver, upper, 20, 6)
315 
+	if err != nil {
316 
+		t.Fatal(err)
317 
+	}
296 318
297 
-	if err := driver.Create("Snap", "Base", "", nil); err != nil {
319 
+	changes, err := driver.Changes(upper, base)
320 
+	if err != nil {
298 321 
 		t.Fatal(err)
299 322 
 	}
300 
-	defer func() {
301 
-		if err := driver.Remove("Snap"); err != nil {
302 
-			t.Fatal(err)
303 
-		}
304 
-	}()
305 323
306 
-	verifyBase(t, driver, "Snap")
324 
+	if err = checkChanges(expectedChanges, changes); err != nil {
325 
+		t.Fatal(err)
326 
+	}
307 327 
 }
308 328
309 329 
 func writeRandomFile(path string, size uint64) error {
daemon/graphdriver/graphtest/testutil.go
History View file @ 8a2f9a2
310 330 
new file mode 100644
... ... 
@@ -0,0 +1,301 @@
0 
+package graphtest
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"fmt"
5 
+	"io/ioutil"
6 
+	"math/rand"
7 
+	"os"
8 
+	"path"
9 
+	"sort"
10 
+
11 
+	"github.com/docker/docker/daemon/graphdriver"
12 
+	"github.com/docker/docker/pkg/archive"
13 
+	"github.com/docker/docker/pkg/stringid"
14 
+)
15 
+
16 
+func randomContent(size int, seed int64) []byte {
17 
+	s := rand.NewSource(seed)
18 
+	content := make([]byte, size)
19 
+
20 
+	for i := 0; i < len(content); i += 7 {
21 
+		val := s.Int63()
22 
+		for j := 0; i+j < len(content) && j < 7; j++ {
23 
+			content[i+j] = byte(val)
24 
+			val >>= 8
25 
+		}
26 
+	}
27 
+
28 
+	return content
29 
+}
30 
+
31 
+func addFiles(drv graphdriver.Driver, layer string, seed int64) error {
32 
+	root, err := drv.Get(layer, "")
33 
+	if err != nil {
34 
+		return err
35 
+	}
36 
+	defer drv.Put(layer)
37 
+
38 
+	if err := ioutil.WriteFile(path.Join(root, "file-a"), randomContent(64, seed), 0755); err != nil {
39 
+		return err
40 
+	}
41 
+	if err := os.MkdirAll(path.Join(root, "dir-b"), 0755); err != nil {
42 
+		return err
43 
+	}
44 
+	if err := ioutil.WriteFile(path.Join(root, "dir-b", "file-b"), randomContent(128, seed+1), 0755); err != nil {
45 
+		return err
46 
+	}
47 
+
48 
+	return ioutil.WriteFile(path.Join(root, "file-c"), randomContent(128*128, seed+2), 0755)
49 
+}
50 
+
51 
+func checkFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
52 
+	root, err := drv.Get(layer, "")
53 
+	if err != nil {
54 
+		return err
55 
+	}
56 
+	defer drv.Put(layer)
57 
+
58 
+	fileContent, err := ioutil.ReadFile(path.Join(root, filename))
59 
+	if err != nil {
60 
+		return err
61 
+	}
62 
+
63 
+	if bytes.Compare(fileContent, content) != 0 {
64 
+		return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
65 
+	}
66 
+
67 
+	return nil
68 
+}
69 
+
70 
+func addFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
71 
+	root, err := drv.Get(layer, "")
72 
+	if err != nil {
73 
+		return err
74 
+	}
75 
+	defer drv.Put(layer)
76 
+
77 
+	return ioutil.WriteFile(path.Join(root, filename), content, 0755)
78 
+}
79 
+
80 
+func addManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
81 
+	root, err := drv.Get(layer, "")
82 
+	if err != nil {
83 
+		return err
84 
+	}
85 
+	defer drv.Put(layer)
86 
+
87 
+	for i := 0; i < count; i += 100 {
88 
+		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
89 
+		if err := os.MkdirAll(dir, 0755); err != nil {
90 
+			return err
91 
+		}
92 
+		for j := 0; i+j < count && j < 100; j++ {
93 
+			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
94 
+			if err := ioutil.WriteFile(file, randomContent(64, seed+int64(i+j)), 0755); err != nil {
95 
+				return err
96 
+			}
97 
+		}
98 
+	}
99 
+
100 
+	return nil
101 
+}
102 
+
103 
+func changeManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) ([]archive.Change, error) {
104 
+	root, err := drv.Get(layer, "")
105 
+	if err != nil {
106 
+		return nil, err
107 
+	}
108 
+	defer drv.Put(layer)
109 
+
110 
+	changes := []archive.Change{}
111 
+	for i := 0; i < count; i += 100 {
112 
+		archiveRoot := fmt.Sprintf("/directory-%d", i)
113 
+		if err := os.MkdirAll(path.Join(root, archiveRoot), 0755); err != nil {
114 
+			return nil, err
115 
+		}
116 
+		for j := 0; i+j < count && j < 100; j++ {
117 
+			if j == 0 {
118 
+				changes = append(changes, archive.Change{
119 
+					Path: archiveRoot,
120 
+					Kind: archive.ChangeModify,
121 
+				})
122 
+			}
123 
+			var change archive.Change
124 
+			switch j % 3 {
125 
+			// Update file
126 
+			case 0:
127 
+				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
128 
+				change.Kind = archive.ChangeModify
129 
+				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
130 
+					return nil, err
131 
+				}
132 
+			// Add file
133 
+			case 1:
134 
+				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d-%d", seed, i+j))
135 
+				change.Kind = archive.ChangeAdd
136 
+				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
137 
+					return nil, err
138 
+				}
139 
+			// Remove file
140 
+			case 2:
141 
+				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
142 
+				change.Kind = archive.ChangeDelete
143 
+				if err := os.Remove(path.Join(root, change.Path)); err != nil {
144 
+					return nil, err
145 
+				}
146 
+			}
147 
+			changes = append(changes, change)
148 
+		}
149 
+	}
150 
+
151 
+	return changes, nil
152 
+}
153 
+
154 
+func checkManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
155 
+	root, err := drv.Get(layer, "")
156 
+	if err != nil {
157 
+		return err
158 
+	}
159 
+	defer drv.Put(layer)
160 
+
161 
+	for i := 0; i < count; i += 100 {
162 
+		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
163 
+		for j := 0; i+j < count && j < 100; j++ {
164 
+			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
165 
+			fileContent, err := ioutil.ReadFile(file)
166 
+			if err != nil {
167 
+				return err
168 
+			}
169 
+
170 
+			content := randomContent(64, seed+int64(i+j))
171 
+
172 
+			if bytes.Compare(fileContent, content) != 0 {
173 
+				return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
174 
+			}
175 
+		}
176 
+	}
177 
+
178 
+	return nil
179 
+}
180 
+
181 
+type changeList []archive.Change
182 
+
183 
+func (c changeList) Less(i, j int) bool {
184 
+	if c[i].Path == c[j].Path {
185 
+		return c[i].Kind < c[j].Kind
186 
+	}
187 
+	return c[i].Path < c[j].Path
188 
+}
189 
+func (c changeList) Len() int      { return len(c) }
190 
+func (c changeList) Swap(i, j int) { c[j], c[i] = c[i], c[j] }
191 
+
192 
+func checkChanges(expected, actual []archive.Change) error {
193 
+	if len(expected) != len(actual) {
194 
+		return fmt.Errorf("unexpected number of changes, expected %d, got %d", len(expected), len(actual))
195 
+	}
196 
+	sort.Sort(changeList(expected))
197 
+	sort.Sort(changeList(actual))
198 
+
199 
+	for i := range expected {
200 
+		if expected[i] != actual[i] {
201 
+			return fmt.Errorf("unexpected change, expecting %v, got %v", expected[i], actual[i])
202 
+		}
203 
+	}
204 
+
205 
+	return nil
206 
+}
207 
+
208 
+func addLayerFiles(drv graphdriver.Driver, layer, parent string, i int) error {
209 
+	root, err := drv.Get(layer, "")
210 
+	if err != nil {
211 
+		return err
212 
+	}
213 
+	defer drv.Put(layer)
214 
+
215 
+	if err := ioutil.WriteFile(path.Join(root, "top-id"), []byte(layer), 0755); err != nil {
216 
+		return err
217 
+	}
218 
+	layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
219 
+	if err := os.MkdirAll(layerDir, 0755); err != nil {
220 
+		return err
221 
+	}
222 
+	if err := ioutil.WriteFile(path.Join(layerDir, "layer-id"), []byte(layer), 0755); err != nil {
223 
+		return err
224 
+	}
225 
+	if err := ioutil.WriteFile(path.Join(layerDir, "parent-id"), []byte(parent), 0755); err != nil {
226 
+		return err
227 
+	}
228 
+
229 
+	return nil
230 
+}
231 
+
232 
+func addManyLayers(drv graphdriver.Driver, baseLayer string, count int) (string, error) {
233 
+	lastLayer := baseLayer
234 
+	for i := 1; i <= count; i++ {
235 
+		nextLayer := stringid.GenerateRandomID()
236 
+		if err := drv.Create(nextLayer, lastLayer, "", nil); err != nil {
237 
+			return "", err
238 
+		}
239 
+		if err := addLayerFiles(drv, nextLayer, lastLayer, i); err != nil {
240 
+			return "", err
241 
+		}
242 
+
243 
+		lastLayer = nextLayer
244 
+
245 
+	}
246 
+	return lastLayer, nil
247 
+}
248 
+
249 
+func checkManyLayers(drv graphdriver.Driver, layer string, count int) error {
250 
+	root, err := drv.Get(layer, "")
251 
+	if err != nil {
252 
+		return err
253 
+	}
254 
+	defer drv.Put(layer)
255 
+
256 
+	layerIDBytes, err := ioutil.ReadFile(path.Join(root, "top-id"))
257 
+	if err != nil {
258 
+		return err
259 
+	}
260 
+
261 
+	if bytes.Compare(layerIDBytes, []byte(layer)) != 0 {
262 
+		return fmt.Errorf("mismatched file content %v, expecting %v", layerIDBytes, []byte(layer))
263 
+	}
264 
+
265 
+	for i := count; i > 0; i-- {
266 
+		layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
267 
+
268 
+		thisLayerIDBytes, err := ioutil.ReadFile(path.Join(layerDir, "layer-id"))
269 
+		if err != nil {
270 
+			return err
271 
+		}
272 
+		if bytes.Compare(thisLayerIDBytes, layerIDBytes) != 0 {
273 
+			return fmt.Errorf("mismatched file content %v, expecting %v", thisLayerIDBytes, layerIDBytes)
274 
+		}
275 
+		layerIDBytes, err = ioutil.ReadFile(path.Join(layerDir, "parent-id"))
276 
+		if err != nil {
277 
+			return err
278 
+		}
279 
+	}
280 
+	return nil
281 
+}
282 
+
283 
+// readDir reads a directory just like ioutil.ReadDir()
284 
+// then hides specific files (currently "lost+found")
285 
+// so the tests don't "see" it
286 
+func readDir(dir string) ([]os.FileInfo, error) {
287 
+	a, err := ioutil.ReadDir(dir)
288 
+	if err != nil {
289 
+		return nil, err
290 
+	}
291 
+
292 
+	b := a[:0]
293 
+	for _, x := range a {
294 
+		if x.Name() != "lost+found" { // ext4 always have this dir
295 
+			b = append(b, x)
296 
+		}
297 
+	}
298 
+
299 
+	return b, nil
300 
+}
daemon/graphdriver/graphtest/testutil_unix.go
History View file @ 8a2f9a2
0 301 
new file mode 100644
... ... 
@@ -0,0 +1,143 @@
0 
+// +build linux freebsd
1 
+
2 
+package graphtest
3 
+
4 
+import (
5 
+	"fmt"
6 
+	"io/ioutil"
7 
+	"os"
8 
+	"path"
9 
+	"syscall"
10 
+	"testing"
11 
+
12 
+	"github.com/docker/docker/daemon/graphdriver"
13 
+)
14 
+
15 
+// InitLoopbacks ensures that the loopback devices are properly created within
16 
+// the system running the device mapper tests.
17 
+func InitLoopbacks() error {
18 
+	statT, err := getBaseLoopStats()
19 
+	if err != nil {
20 
+		return err
21 
+	}
22 
+	// create at least 8 loopback files, ya, that is a good number
23 
+	for i := 0; i < 8; i++ {
24 
+		loopPath := fmt.Sprintf("/dev/loop%d", i)
25 
+		// only create new loopback files if they don't exist
26 
+		if _, err := os.Stat(loopPath); err != nil {
27 
+			if mkerr := syscall.Mknod(loopPath,
28 
+				uint32(statT.Mode|syscall.S_IFBLK), int((7<<8)|(i&0xff)|((i&0xfff00)<<12))); mkerr != nil {
29 
+				return mkerr
30 
+			}
31 
+			os.Chown(loopPath, int(statT.Uid), int(statT.Gid))
32 
+		}
33 
+	}
34 
+	return nil
35 
+}
36 
+
37 
+// getBaseLoopStats inspects /dev/loop0 to collect uid,gid, and mode for the
38 
+// loop0 device on the system.  If it does not exist we assume 0,0,0660 for the
39 
+// stat data
40 
+func getBaseLoopStats() (*syscall.Stat_t, error) {
41 
+	loop0, err := os.Stat("/dev/loop0")
42 
+	if err != nil {
43 
+		if os.IsNotExist(err) {
44 
+			return &syscall.Stat_t{
45 
+				Uid:  0,
46 
+				Gid:  0,
47 
+				Mode: 0660,
48 
+			}, nil
49 
+		}
50 
+		return nil, err
51 
+	}
52 
+	return loop0.Sys().(*syscall.Stat_t), nil
53 
+}
54 
+
55 
+func verifyFile(t testing.TB, path string, mode os.FileMode, uid, gid uint32) {
56 
+	fi, err := os.Stat(path)
57 
+	if err != nil {
58 
+		t.Fatal(err)
59 
+	}
60 
+
61 
+	if fi.Mode()&os.ModeType != mode&os.ModeType {
62 
+		t.Fatalf("Expected %s type 0x%x, got 0x%x", path, mode&os.ModeType, fi.Mode()&os.ModeType)
63 
+	}
64 
+
65 
+	if fi.Mode()&os.ModePerm != mode&os.ModePerm {
66 
+		t.Fatalf("Expected %s mode %o, got %o", path, mode&os.ModePerm, fi.Mode()&os.ModePerm)
67 
+	}
68 
+
69 
+	if fi.Mode()&os.ModeSticky != mode&os.ModeSticky {
70 
+		t.Fatalf("Expected %s sticky 0x%x, got 0x%x", path, mode&os.ModeSticky, fi.Mode()&os.ModeSticky)
71 
+	}
72 
+
73 
+	if fi.Mode()&os.ModeSetuid != mode&os.ModeSetuid {
74 
+		t.Fatalf("Expected %s setuid 0x%x, got 0x%x", path, mode&os.ModeSetuid, fi.Mode()&os.ModeSetuid)
75 
+	}
76 
+
77 
+	if fi.Mode()&os.ModeSetgid != mode&os.ModeSetgid {
78 
+		t.Fatalf("Expected %s setgid 0x%x, got 0x%x", path, mode&os.ModeSetgid, fi.Mode()&os.ModeSetgid)
79 
+	}
80 
+
81 
+	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
82 
+		if stat.Uid != uid {
83 
+			t.Fatalf("%s no owned by uid %d", path, uid)
84 
+		}
85 
+		if stat.Gid != gid {
86 
+			t.Fatalf("%s not owned by gid %d", path, gid)
87 
+		}
88 
+	}
89 
+}
90 
+
91 
+func createBase(t testing.TB, driver graphdriver.Driver, name string) {
92 
+	// We need to be able to set any perms
93 
+	oldmask := syscall.Umask(0)
94 
+	defer syscall.Umask(oldmask)
95 
+
96 
+	if err := driver.CreateReadWrite(name, "", "", nil); err != nil {
97 
+		t.Fatal(err)
98 
+	}
99 
+
100 
+	dir, err := driver.Get(name, "")
101 
+	if err != nil {
102 
+		t.Fatal(err)
103 
+	}
104 
+	defer driver.Put(name)
105 
+
106 
+	subdir := path.Join(dir, "a subdir")
107 
+	if err := os.Mkdir(subdir, 0705|os.ModeSticky); err != nil {
108 
+		t.Fatal(err)
109 
+	}
110 
+	if err := os.Chown(subdir, 1, 2); err != nil {
111 
+		t.Fatal(err)
112 
+	}
113 
+
114 
+	file := path.Join(dir, "a file")
115 
+	if err := ioutil.WriteFile(file, []byte("Some data"), 0222|os.ModeSetuid); err != nil {
116 
+		t.Fatal(err)
117 
+	}
118 
+}
119 
+
120 
+func verifyBase(t testing.TB, driver graphdriver.Driver, name string) {
121 
+	dir, err := driver.Get(name, "")
122 
+	if err != nil {
123 
+		t.Fatal(err)
124 
+	}
125 
+	defer driver.Put(name)
126 
+
127 
+	subdir := path.Join(dir, "a subdir")
128 
+	verifyFile(t, subdir, 0705|os.ModeDir|os.ModeSticky, 1, 2)
129 
+
130 
+	file := path.Join(dir, "a file")
131 
+	verifyFile(t, file, 0222|os.ModeSetuid, 0, 0)
132 
+
133 
+	fis, err := readDir(dir)
134 
+	if err != nil {
135 
+		t.Fatal(err)
136 
+	}
137 
+
138 
+	if len(fis) != 2 {
139 
+		t.Fatal("Unexpected files in base image")
140 
+	}
141 
+
142 
+}
daemon/graphdriver/overlay/overlay.go
History View file @ 8a2f9a2
... ... 
@@ -15,7 +15,6 @@ import (
15 15
16 16 
 	"github.com/docker/docker/daemon/graphdriver"
17 17 
 	"github.com/docker/docker/pkg/archive"
18 
-	"github.com/docker/docker/pkg/chrootarchive"
19 18 
 	"github.com/docker/docker/pkg/idtools"
20 19
21 20 
 	"github.com/docker/docker/pkg/mount"
... ... 
@@ -426,7 +425,7 @@ func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size
426 426 
 	}
427 427
428 428 
 	options := &archive.TarOptions{UIDMaps: d.uidMaps, GIDMaps: d.gidMaps}
429 
-	if size, err = chrootarchive.ApplyUncompressedLayer(tmpRootDir, diff, options); err != nil {
429 
+	if size, err = graphdriver.ApplyUncompressedLayer(tmpRootDir, diff, options); err != nil {
430 430 
 		return 0, err
431 431 
 	}
432 432
daemon/graphdriver/overlay/overlay_test.go
History View file @ 8a2f9a2
... ... 
@@ -5,9 +5,17 @@ package overlay
5 5 
 import (
6 6 
 	"testing"
7 7
8 
+	"github.com/docker/docker/daemon/graphdriver"
8 9 
 	"github.com/docker/docker/daemon/graphdriver/graphtest"
10 
+	"github.com/docker/docker/pkg/archive"
9 11 
 )
10 12
13 
+func init() {
14 
+	// Do not sure chroot to speed run time and allow archive
15 
+	// errors or hangs to be debugged directly from the test process.
16 
+	graphdriver.ApplyUncompressedLayer = archive.ApplyUncompressedLayer
17 
+}
18 
+
11 19 
 // This avoids creating a new driver for each test if all tests are run
12 20 
 // Make sure to put new tests between TestOverlaySetup and TestOverlayTeardown
13 21 
 func TestOverlaySetup(t *testing.T) {
... ... 
@@ -26,6 +34,56 @@ func TestOverlayCreateSnap(t *testing.T) {
26 26 
 	graphtest.DriverTestCreateSnap(t, "overlay")
27 27 
 }
28 28
29 
+func TestOverlay50LayerRead(t *testing.T) {
30 
+	graphtest.DriverTestDeepLayerRead(t, 50, "overlay")
31 
+}
32 
+
33 
+func TestOverlayDiffApply10Files(t *testing.T) {
34 
+	graphtest.DriverTestDiffApply(t, 10, "overlay")
35 
+}
36 
+
37 
+func TestOverlayChanges(t *testing.T) {
38 
+	graphtest.DriverTestChanges(t, "overlay")
39 
+}
40 
+
29 41 
 func TestOverlayTeardown(t *testing.T) {
30 42 
 	graphtest.PutDriver(t)
31 43 
 }
44 
+
45 
+// Benchmarks should always setup new driver
46 
+
47 
+func BenchmarkExists(b *testing.B) {
48 
+	graphtest.DriverBenchExists(b, "overlay")
49 
+}
50 
+
51 
+func BenchmarkGetEmpty(b *testing.B) {
52 
+	graphtest.DriverBenchGetEmpty(b, "overlay")
53 
+}
54 
+
55 
+func BenchmarkDiffBase(b *testing.B) {
56 
+	graphtest.DriverBenchDiffBase(b, "overlay")
57 
+}
58 
+
59 
+func BenchmarkDiffSmallUpper(b *testing.B) {
60 
+	graphtest.DriverBenchDiffN(b, 10, 10, "overlay")
61 
+}
62 
+
63 
+func BenchmarkDiff10KFileUpper(b *testing.B) {
64 
+	graphtest.DriverBenchDiffN(b, 10, 10000, "overlay")
65 
+}
66 
+
67 
+func BenchmarkDiff10KFilesBottom(b *testing.B) {
68 
+	graphtest.DriverBenchDiffN(b, 10000, 10, "overlay")
69 
+}
70 
+
71 
+func BenchmarkDiffApply100(b *testing.B) {
72 
+	graphtest.DriverBenchDiffApplyN(b, 100, "overlay")
73 
+}
74 
+
75 
+func BenchmarkDiff20Layers(b *testing.B) {
76 
+	graphtest.DriverBenchDeepLayerDiff(b, 20, "overlay")
77 
+}
78 
+
79 
+func BenchmarkRead20Layers(b *testing.B) {
80 
+	graphtest.DriverBenchDeepLayerRead(b, 20, "overlay")
81 
+}
daemon/graphdriver/overlay2/mount.go
History View file @ 8a2f9a2
32 82 
new file mode 100644
... ... 
@@ -0,0 +1,91 @@
0 
+// +build linux
1 
+
2 
+package overlay2
3 
+
4 
+import (
5 
+	"bytes"
6 
+	"encoding/json"
7 
+	"flag"
8 
+	"fmt"
9 
+	"os"
10 
+	"runtime"
11 
+	"syscall"
12 
+
13 
+	"github.com/docker/docker/pkg/reexec"
14 
+)
15 
+
16 
+func init() {
17 
+	reexec.Register("docker-mountfrom", mountFromMain)
18 
+}
19 
+
20 
+func fatal(err error) {
21 
+	fmt.Fprint(os.Stderr, err)
22 
+	os.Exit(1)
23 
+}
24 
+
25 
+type mountOptions struct {
26 
+	Device string
27 
+	Target string
28 
+	Type   string
29 
+	Label  string
30 
+	Flag   uint32
31 
+}
32 
+
33 
+func mountFrom(dir, device, target, mType, label string) error {
34 
+
35 
+	r, w, err := os.Pipe()
36 
+	if err != nil {
37 
+		return fmt.Errorf("mountfrom pipe failure: %v", err)
38 
+	}
39 
+
40 
+	options := &mountOptions{
41 
+		Device: device,
42 
+		Target: target,
43 
+		Type:   mType,
44 
+		Flag:   0,
45 
+		Label:  label,
46 
+	}
47 
+
48 
+	cmd := reexec.Command("docker-mountfrom", dir)
49 
+	cmd.Stdin = r
50 
+
51 
+	output := bytes.NewBuffer(nil)
52 
+	cmd.Stdout = output
53 
+	cmd.Stderr = output
54 
+
55 
+	if err := cmd.Start(); err != nil {
56 
+		return fmt.Errorf("mountfrom error on re-exec cmd: %v", err)
57 
+	}
58 
+	//write the options to the pipe for the untar exec to read
59 
+	if err := json.NewEncoder(w).Encode(options); err != nil {
60 
+		return fmt.Errorf("mountfrom json encode to pipe failed: %v", err)
61 
+	}
62 
+	w.Close()
63 
+
64 
+	if err := cmd.Wait(); err != nil {
65 
+		return fmt.Errorf("mountfrom re-exec error: %v: output: %s", err, output)
66 
+	}
67 
+	return nil
68 
+}
69 
+
70 
+// mountfromMain is the entry-point for docker-mountfrom on re-exec.
71 
+func mountFromMain() {
72 
+	runtime.LockOSThread()
73 
+	flag.Parse()
74 
+
75 
+	var options *mountOptions
76 
+
77 
+	if err := json.NewDecoder(os.Stdin).Decode(&options); err != nil {
78 
+		fatal(err)
79 
+	}
80 
+
81 
+	if err := os.Chdir(flag.Arg(0)); err != nil {
82 
+		fatal(err)
83 
+	}
84 
+
85 
+	if err := syscall.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil {
86 
+		fatal(err)
87 
+	}
88 
+
89 
+	os.Exit(0)
90 
+}
daemon/graphdriver/overlay2/overlay.go
History View file @ 8a2f9a2
0 91 
new file mode 100644
... ... 
@@ -0,0 +1,476 @@
0 
+// +build linux
1 
+
2 
+package overlay2
3 
+
4 
+import (
5 
+	"bufio"
6 
+	"errors"
7 
+	"fmt"
8 
+	"io/ioutil"
9 
+	"os"
10 
+	"os/exec"
11 
+	"path"
12 
+	"strings"
13 
+	"syscall"
14 
+
15 
+	"github.com/Sirupsen/logrus"
16 
+
17 
+	"github.com/docker/docker/daemon/graphdriver"
18 
+	"github.com/docker/docker/pkg/archive"
19 
+	"github.com/docker/docker/pkg/chrootarchive"
20 
+	"github.com/docker/docker/pkg/directory"
21 
+	"github.com/docker/docker/pkg/idtools"
22 
+	"github.com/docker/docker/pkg/mount"
23 
+	"github.com/docker/docker/pkg/parsers/kernel"
24 
+
25 
+	"github.com/opencontainers/runc/libcontainer/label"
26 
+)
27 
+
28 
+var (
29 
+	// untar defines the untar method
30 
+	untar = chrootarchive.UntarUncompressed
31 
+)
32 
+
33 
+// This backend uses the overlay union filesystem for containers
34 
+// with diff directories for each layer.
35 
+
36 
+// This version of the overlay driver requires at least kernel
37 
+// 4.0.0 in order to support mounting multiple diff directories.
38 
+
39 
+// Each container/image has at least a "diff" directory and "link" file.
40 
+// If there is also a "lower" file when there are diff layers
41 
+// below  as well as "merged" and "work" directories. The "diff" directory
42 
+// has the upper layer of the overlay and is used to capture any
43 
+// changes to the layer. The "lower" file contains all the lower layer
44 
+// mounts separated by ":" and ordered from uppermost to lowermost
45 
+// layers. The overlay itself is mounted in the "merged" directory,
46 
+// and the "work" dir is needed for overlay to work.
47 
+
48 
+// The "link" file for each layer contains a unique string for the layer.
49 
+// Under the "l" directory at the root there will be a symbolic link
50 
+// with that unique string pointing the "diff" directory for the layer.
51 
+// The symbolic links are used to reference lower layers in the "lower"
52 
+// file and on mount. The links are used to shorten the total length
53 
+// of a layer reference without requiring changes to the layer identifier
54 
+// or root directory. Mounts are always done relative to root and
55 
+// referencing the symbolic links in order to ensure the number of
56 
+// lower directories can fit in a single page for making the mount
57 
+// syscall. A hard upper limit of 128 lower layers is enforced to ensure
58 
+// that mounts do not fail due to length.
59 
+
60 
+const (
61 
+	driverName = "overlay2"
62 
+	linkDir    = "l"
63 
+	lowerFile  = "lower"
64 
+	maxDepth   = 128
65 
+
66 
+	// idLength represents the number of random characters
67 
+	// which can be used to create the unique link identifer
68 
+	// for every layer. If this value is too long then the
69 
+	// page size limit for the mount command may be exceeded.
70 
+	// The idLength should be selected such that following equation
71 
+	// is true (512 is a buffer for label metadata).
72 
+	// ((idLength + len(linkDir) + 1) * maxDepth) <= (pageSize - 512)
73 
+	idLength = 26
74 
+)
75 
+
76 
+// Driver contains information about the home directory and the list of active mounts that are created using this driver.
77 
+type Driver struct {
78 
+	home    string
79 
+	uidMaps []idtools.IDMap
80 
+	gidMaps []idtools.IDMap
81 
+	ctr     *graphdriver.RefCounter
82 
+}
83 
+
84 
+var backingFs = "<unknown>"
85 
+
86 
+func init() {
87 
+	graphdriver.Register(driverName, Init)
88 
+}
89 
+
90 
+// Init returns the a native diff driver for overlay filesystem.
91 
+// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
92 
+// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
93 
+func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
94 
+
95 
+	if err := supportsOverlay(); err != nil {
96 
+		return nil, graphdriver.ErrNotSupported
97 
+	}
98 
+
99 
+	// require kernel 4.0.0 to ensure multiple lower dirs are supported
100 
+	v, err := kernel.GetKernelVersion()
101 
+	if err != nil {
102 
+		return nil, err
103 
+	}
104 
+	if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 0, Minor: 0}) < 0 {
105 
+		return nil, graphdriver.ErrNotSupported
106 
+	}
107 
+
108 
+	fsMagic, err := graphdriver.GetFSMagic(home)
109 
+	if err != nil {
110 
+		return nil, err
111 
+	}
112 
+	if fsName, ok := graphdriver.FsNames[fsMagic]; ok {
113 
+		backingFs = fsName
114 
+	}
115 
+
116 
+	// check if they are running over btrfs, aufs, zfs or overlay
117 
+	switch fsMagic {
118 
+	case graphdriver.FsMagicBtrfs:
119 
+		logrus.Error("'overlay' is not supported over btrfs.")
120 
+		return nil, graphdriver.ErrIncompatibleFS
121 
+	case graphdriver.FsMagicAufs:
122 
+		logrus.Error("'overlay' is not supported over aufs.")
123 
+		return nil, graphdriver.ErrIncompatibleFS
124 
+	case graphdriver.FsMagicZfs:
125 
+		logrus.Error("'overlay' is not supported over zfs.")
126 
+		return nil, graphdriver.ErrIncompatibleFS
127 
+	case graphdriver.FsMagicOverlay:
128 
+		logrus.Error("'overlay' is not supported over overlay.")
129 
+		return nil, graphdriver.ErrIncompatibleFS
130 
+	}
131 
+
132 
+	rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
133 
+	if err != nil {
134 
+		return nil, err
135 
+	}
136 
+	// Create the driver home dir
137 
+	if err := idtools.MkdirAllAs(path.Join(home, linkDir), 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
138 
+		return nil, err
139 
+	}
140 
+
141 
+	if err := mount.MakePrivate(home); err != nil {
142 
+		return nil, err
143 
+	}
144 
+
145 
+	d := &Driver{
146 
+		home:    home,
147 
+		uidMaps: uidMaps,
148 
+		gidMaps: gidMaps,
149 
+		ctr:     graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)),
150 
+	}
151 
+
152 
+	return d, nil
153 
+}
154 
+
155 
+func supportsOverlay() error {
156 
+	// We can try to modprobe overlay first before looking at
157 
+	// proc/filesystems for when overlay is supported
158 
+	exec.Command("modprobe", "overlay").Run()
159 
+
160 
+	f, err := os.Open("/proc/filesystems")
161 
+	if err != nil {
162 
+		return err
163 
+	}
164 
+	defer f.Close()
165 
+
166 
+	s := bufio.NewScanner(f)
167 
+	for s.Scan() {
168 
+		if s.Text() == "nodev\toverlay" {
169 
+			return nil
170 
+		}
171 
+	}
172 
+	logrus.Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
173 
+	return graphdriver.ErrNotSupported
174 
+}
175 
+
176 
+func (d *Driver) String() string {
177 
+	return driverName
178 
+}
179 
+
180 
+// Status returns current driver information in a two dimensional string array.
181 
+// Output contains "Backing Filesystem" used in this implementation.
182 
+func (d *Driver) Status() [][2]string {
183 
+	return [][2]string{
184 
+		{"Backing Filesystem", backingFs},
185 
+	}
186 
+}
187 
+
188 
+// GetMetadata returns meta data about the overlay driver such as
189 
+// LowerDir, UpperDir, WorkDir and MergeDir used to store data.
190 
+func (d *Driver) GetMetadata(id string) (map[string]string, error) {
191 
+	dir := d.dir(id)
192 
+	if _, err := os.Stat(dir); err != nil {
193 
+		return nil, err
194 
+	}
195 
+
196 
+	metadata := map[string]string{
197 
+		"WorkDir":   path.Join(dir, "work"),
198 
+		"MergedDir": path.Join(dir, "merged"),
199 
+		"UpperDir":  path.Join(dir, "diff"),
200 
+	}
201 
+
202 
+	lowerDirs, err := d.getLowerDirs(id)
203 
+	if err != nil {
204 
+		return nil, err
205 
+	}
206 
+	if len(lowerDirs) > 0 {
207 
+		metadata["LowerDir"] = strings.Join(lowerDirs, ":")
208 
+	}
209 
+
210 
+	return metadata, nil
211 
+}
212 
+
213 
+// Cleanup any state created by overlay which should be cleaned when daemon
214 
+// is being shutdown. For now, we just have to unmount the bind mounted
215 
+// we had created.
216 
+func (d *Driver) Cleanup() error {
217 
+	return mount.Unmount(d.home)
218 
+}
219 
+
220 
+// CreateReadWrite creates a layer that is writable for use as a container
221 
+// file system.
222 
+func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error {
223 
+	return d.Create(id, parent, mountLabel, storageOpt)
224 
+}
225 
+
226 
+// Create is used to create the upper, lower, and merge directories required for overlay fs for a given id.
227 
+// The parent filesystem is used to configure these directories for the overlay.
228 
+func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) (retErr error) {
229 
+
230 
+	if len(storageOpt) != 0 {
231 
+		return fmt.Errorf("--storage-opt is not supported for overlay")
232 
+	}
233 
+
234 
+	dir := d.dir(id)
235 
+
236 
+	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
237 
+	if err != nil {
238 
+		return err
239 
+	}
240 
+	if err := idtools.MkdirAllAs(path.Dir(dir), 0700, rootUID, rootGID); err != nil {
241 
+		return err
242 
+	}
243 
+	if err := idtools.MkdirAs(dir, 0700, rootUID, rootGID); err != nil {
244 
+		return err
245 
+	}
246 
+
247 
+	defer func() {
248 
+		// Clean up on failure
249 
+		if retErr != nil {
250 
+			os.RemoveAll(dir)
251 
+		}
252 
+	}()
253 
+
254 
+	if err := idtools.MkdirAs(path.Join(dir, "diff"), 0755, rootUID, rootGID); err != nil {
255 
+		return err
256 
+	}
257 
+
258 
+	lid := generateID(idLength)
259 
+	if err := os.Symlink(path.Join("..", id, "diff"), path.Join(d.home, linkDir, lid)); err != nil {
260 
+		return err
261 
+	}
262 
+
263 
+	// Write link id to link file
264 
+	if err := ioutil.WriteFile(path.Join(dir, "link"), []byte(lid), 0644); err != nil {
265 
+		return err
266 
+	}
267 
+
268 
+	// if no parent directory, done
269 
+	if parent == "" {
270 
+		return nil
271 
+	}
272 
+
273 
+	if err := idtools.MkdirAs(path.Join(dir, "work"), 0700, rootUID, rootGID); err != nil {
274 
+		return err
275 
+	}
276 
+	if err := idtools.MkdirAs(path.Join(dir, "merged"), 0700, rootUID, rootGID); err != nil {
277 
+		return err
278 
+	}
279 
+
280 
+	lower, err := d.getLower(parent)
281 
+	if err != nil {
282 
+		return err
283 
+	}
284 
+	if lower != "" {
285 
+		if err := ioutil.WriteFile(path.Join(dir, lowerFile), []byte(lower), 0666); err != nil {
286 
+			return err
287 
+		}
288 
+	}
289 
+
290 
+	return nil
291 
+}
292 
+
293 
+func (d *Driver) getLower(parent string) (string, error) {
294 
+	parentDir := d.dir(parent)
295 
+
296 
+	// Ensure parent exists
297 
+	if _, err := os.Lstat(parentDir); err != nil {
298 
+		return "", err
299 
+	}
300 
+
301 
+	// Read Parent link fileA
302 
+	parentLink, err := ioutil.ReadFile(path.Join(parentDir, "link"))
303 
+	if err != nil {
304 
+		return "", err
305 
+	}
306 
+	lowers := []string{path.Join(linkDir, string(parentLink))}
307 
+
308 
+	parentLower, err := ioutil.ReadFile(path.Join(parentDir, lowerFile))
309 
+	if err == nil {
310 
+		parentLowers := strings.Split(string(parentLower), ":")
311 
+		lowers = append(lowers, parentLowers...)
312 
+	}
313 
+	if len(lowers) > maxDepth {
314 
+		return "", errors.New("max depth exceeded")
315 
+	}
316 
+	return strings.Join(lowers, ":"), nil
317 
+}
318 
+
319 
+func (d *Driver) dir(id string) string {
320 
+	return path.Join(d.home, id)
321 
+}
322 
+
323 
+func (d *Driver) getLowerDirs(id string) ([]string, error) {
324 
+	var lowersArray []string
325 
+	lowers, err := ioutil.ReadFile(path.Join(d.dir(id), lowerFile))
326 
+	if err == nil {
327 
+		for _, s := range strings.Split(string(lowers), ":") {
328 
+			lp, err := os.Readlink(path.Join(d.home, s))
329 
+			if err != nil {
330 
+				return nil, err
331 
+			}
332 
+			lowersArray = append(lowersArray, path.Clean(path.Join(d.home, "link", lp)))
333 
+		}
334 
+	} else if !os.IsNotExist(err) {
335 
+		return nil, err
336 
+	}
337 
+	return lowersArray, nil
338 
+}
339 
+
340 
+// Remove cleans the directories that are created for this id.
341 
+func (d *Driver) Remove(id string) error {
342 
+	if err := os.RemoveAll(d.dir(id)); err != nil && !os.IsNotExist(err) {
343 
+		return err
344 
+	}
345 
+	return nil
346 
+}
347 
+
348 
+// Get creates and mounts the required file system for the given id and returns the mount path.
349 
+func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
350 
+	dir := d.dir(id)
351 
+	if _, err := os.Stat(dir); err != nil {
352 
+		return "", err
353 
+	}
354 
+
355 
+	diffDir := path.Join(dir, "diff")
356 
+	lowers, err := ioutil.ReadFile(path.Join(dir, lowerFile))
357 
+	if err != nil {
358 
+		// If no lower, just return diff directory
359 
+		if os.IsNotExist(err) {
360 
+			return diffDir, nil
361 
+		}
362 
+		return "", err
363 
+	}
364 
+
365 
+	mergedDir := path.Join(dir, "merged")
366 
+	if count := d.ctr.Increment(mergedDir); count > 1 {
367 
+		return mergedDir, nil
368 
+	}
369 
+	defer func() {
370 
+		if err != nil {
371 
+			if c := d.ctr.Decrement(mergedDir); c <= 0 {
372 
+				syscall.Unmount(mergedDir, 0)
373 
+			}
374 
+		}
375 
+	}()
376 
+
377 
+	workDir := path.Join(dir, "work")
378 
+	opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", string(lowers), path.Join(id, "diff"), path.Join(id, "work"))
379 
+	mountLabel = label.FormatMountLabel(opts, mountLabel)
380 
+	if len(mountLabel) > syscall.Getpagesize() {
381 
+		return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountLabel))
382 
+	}
383 
+
384 
+	if err := mountFrom(d.home, "overlay", path.Join(id, "merged"), "overlay", mountLabel); err != nil {
385 
+		return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
386 
+	}
387 
+
388 
+	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
389 
+	// user namespace requires this to move a directory from lower to upper.
390 
+	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
391 
+	if err != nil {
392 
+		return "", err
393 
+	}
394 
+
395 
+	if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil {
396 
+		return "", err
397 
+	}
398 
+
399 
+	return mergedDir, nil
400 
+}
401 
+
402 
+// Put unmounts the mount path created for the give id.
403 
+func (d *Driver) Put(id string) error {
404 
+	mountpoint := path.Join(d.dir(id), "merged")
405 
+	if count := d.ctr.Decrement(mountpoint); count > 0 {
406 
+		return nil
407 
+	}
408 
+	if err := syscall.Unmount(mountpoint, 0); err != nil {
409 
+		logrus.Debugf("Failed to unmount %s overlay: %v", id, err)
410 
+	}
411 
+	return nil
412 
+}
413 
+
414 
+// Exists checks to see if the id is already mounted.
415 
+func (d *Driver) Exists(id string) bool {
416 
+	_, err := os.Stat(d.dir(id))
417 
+	return err == nil
418 
+}
419 
+
420 
+// ApplyDiff applies the new layer into a root
421 
+func (d *Driver) ApplyDiff(id string, parent string, diff archive.Reader) (size int64, err error) {
422 
+	applyDir := d.getDiffPath(id)
423 
+
424 
+	logrus.Debugf("Applying tar in %s", applyDir)
425 
+	// Overlay doesn't need the parent id to apply the diff
426 
+	if err := untar(diff, applyDir, &archive.TarOptions{
427 
+		UIDMaps:        d.uidMaps,
428 
+		GIDMaps:        d.gidMaps,
429 
+		WhiteoutFormat: archive.OverlayWhiteoutFormat,
430 
+	}); err != nil {
431 
+		return 0, err
432 
+	}
433 
+
434 
+	return d.DiffSize(id, parent)
435 
+}
436 
+
437 
+func (d *Driver) getDiffPath(id string) string {
438 
+	dir := d.dir(id)
439 
+
440 
+	return path.Join(dir, "diff")
441 
+}
442 
+
443 
+// DiffSize calculates the changes between the specified id
444 
+// and its parent and returns the size in bytes of the changes
445 
+// relative to its base filesystem directory.
446 
+func (d *Driver) DiffSize(id, parent string) (size int64, err error) {
447 
+	return directory.Size(d.getDiffPath(id))
448 
+}
449 
+
450 
+// Diff produces an archive of the changes between the specified
451 
+// layer and its parent layer which may be "".
452 
+func (d *Driver) Diff(id, parent string) (archive.Archive, error) {
453 
+	diffPath := d.getDiffPath(id)
454 
+	logrus.Debugf("Tar with options on %s", diffPath)
455 
+	return archive.TarWithOptions(diffPath, &archive.TarOptions{
456 
+		Compression:    archive.Uncompressed,
457 
+		UIDMaps:        d.uidMaps,
458 
+		GIDMaps:        d.gidMaps,
459 
+		WhiteoutFormat: archive.OverlayWhiteoutFormat,
460 
+	})
461 
+}
462 
+
463 
+// Changes produces a list of changes between the specified layer
464 
+// and its parent layer. If parent is "", then all changes will be ADD changes.
465 
+func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
466 
+	// Overlay doesn't have snapshots, so we need to get changes from all parent
467 
+	// layers.
468 
+	diffPath := d.getDiffPath(id)
469 
+	layers, err := d.getLowerDirs(id)
470 
+	if err != nil {
471 
+		return nil, err
472 
+	}
473 
+
474 
+	return archive.OverlayChanges(layers, diffPath)
475 
+}
daemon/graphdriver/overlay2/overlay_test.go
History View file @ 8a2f9a2
0 476 
new file mode 100644
... ... 
@@ -0,0 +1,106 @@
0 
+// +build linux
1 
+
2 
+package overlay2
3 
+
4 
+import (
5 
+	"os"
6 
+	"syscall"
7 
+	"testing"
8 
+
9 
+	"github.com/docker/docker/daemon/graphdriver"
10 
+	"github.com/docker/docker/daemon/graphdriver/graphtest"
11 
+	"github.com/docker/docker/pkg/archive"
12 
+	"github.com/docker/docker/pkg/reexec"
13 
+)
14 
+
15 
+func init() {
16 
+	// Do not sure chroot to speed run time and allow archive
17 
+	// errors or hangs to be debugged directly from the test process.
18 
+	untar = archive.UntarUncompressed
19 
+	graphdriver.ApplyUncompressedLayer = archive.ApplyUncompressedLayer
20 
+
21 
+	reexec.Init()
22 
+}
23 
+
24 
+func cdMountFrom(dir, device, target, mType, label string) error {
25 
+	wd, err := os.Getwd()
26 
+	if err != nil {
27 
+		return err
28 
+	}
29 
+	os.Chdir(dir)
30 
+	defer os.Chdir(wd)
31 
+
32 
+	return syscall.Mount(device, target, mType, 0, label)
33 
+}
34 
+
35 
+// This avoids creating a new driver for each test if all tests are run
36 
+// Make sure to put new tests between TestOverlaySetup and TestOverlayTeardown
37 
+func TestOverlaySetup(t *testing.T) {
38 
+	graphtest.GetDriver(t, driverName)
39 
+}
40 
+
41 
+func TestOverlayCreateEmpty(t *testing.T) {
42 
+	graphtest.DriverTestCreateEmpty(t, driverName)
43 
+}
44 
+
45 
+func TestOverlayCreateBase(t *testing.T) {
46 
+	graphtest.DriverTestCreateBase(t, driverName)
47 
+}
48 
+
49 
+func TestOverlayCreateSnap(t *testing.T) {
50 
+	graphtest.DriverTestCreateSnap(t, driverName)
51 
+}
52 
+
53 
+func TestOverlay128LayerRead(t *testing.T) {
54 
+	graphtest.DriverTestDeepLayerRead(t, 128, driverName)
55 
+}
56 
+
57 
+func TestOverlayDiffApply10Files(t *testing.T) {
58 
+	graphtest.DriverTestDiffApply(t, 10, driverName)
59 
+}
60 
+
61 
+func TestOverlayChanges(t *testing.T) {
62 
+	graphtest.DriverTestChanges(t, driverName)
63 
+}
64 
+
65 
+func TestOverlayTeardown(t *testing.T) {
66 
+	graphtest.PutDriver(t)
67 
+}
68 
+
69 
+// Benchmarks should always setup new driver
70 
+
71 
+func BenchmarkExists(b *testing.B) {
72 
+	graphtest.DriverBenchExists(b, driverName)
73 
+}
74 
+
75 
+func BenchmarkGetEmpty(b *testing.B) {
76 
+	graphtest.DriverBenchGetEmpty(b, driverName)
77 
+}
78 
+
79 
+func BenchmarkDiffBase(b *testing.B) {
80 
+	graphtest.DriverBenchDiffBase(b, driverName)
81 
+}
82 
+
83 
+func BenchmarkDiffSmallUpper(b *testing.B) {
84 
+	graphtest.DriverBenchDiffN(b, 10, 10, driverName)
85 
+}
86 
+
87 
+func BenchmarkDiff10KFileUpper(b *testing.B) {
88 
+	graphtest.DriverBenchDiffN(b, 10, 10000, driverName)
89 
+}
90 
+
91 
+func BenchmarkDiff10KFilesBottom(b *testing.B) {
92 
+	graphtest.DriverBenchDiffN(b, 10000, 10, driverName)
93 
+}
94 
+
95 
+func BenchmarkDiffApply100(b *testing.B) {
96 
+	graphtest.DriverBenchDiffApplyN(b, 100, driverName)
97 
+}
98 
+
99 
+func BenchmarkDiff20Layers(b *testing.B) {
100 
+	graphtest.DriverBenchDeepLayerDiff(b, 20, driverName)
101 
+}
102 
+
103 
+func BenchmarkRead20Layers(b *testing.B) {
104 
+	graphtest.DriverBenchDeepLayerRead(b, 20, driverName)
105 
+}
daemon/graphdriver/overlay2/overlay_unsupported.go
History View file @ 8a2f9a2
0 106 
new file mode 100644
... ... 
@@ -0,0 +1,3 @@
0 
+// +build !linux
1 
+
2 
+package overlay2
daemon/graphdriver/overlay2/randomid.go
History View file @ 8a2f9a2
0 3 
new file mode 100644
... ... 
@@ -0,0 +1,80 @@
0 
+// +build linux
1 
+
2 
+package overlay2
3 
+
4 
+import (
5 
+	"crypto/rand"
6 
+	"encoding/base32"
7 
+	"fmt"
8 
+	"io"
9 
+	"os"
10 
+	"syscall"
11 
+	"time"
12 
+
13 
+	"github.com/Sirupsen/logrus"
14 
+)
15 
+
16 
+// generateID creates a new random string identifier with the given length
17 
+func generateID(l int) string {
18 
+	const (
19 
+		// ensures we backoff for less than 450ms total. Use the following to
20 
+		// select new value, in units of 10ms:
21 
+		// 	n*(n+1)/2 = d -> n^2 + n - 2d -> n = (sqrt(8d + 1) - 1)/2
22 
+		maxretries = 9
23 
+		backoff    = time.Millisecond * 10
24 
+	)
25 
+
26 
+	var (
27 
+		totalBackoff time.Duration
28 
+		count        int
29 
+		retries      int
30 
+		size         = (l*5 + 7) / 8
31 
+		u            = make([]byte, size)
32 
+	)
33 
+	// TODO: Include time component, counter component, random component
34 
+
35 
+	for {
36 
+		// This should never block but the read may fail. Because of this,
37 
+		// we just try to read the random number generator until we get
38 
+		// something. This is a very rare condition but may happen.
39 
+		b := time.Duration(retries) * backoff
40 
+		time.Sleep(b)
41 
+		totalBackoff += b
42 
+
43 
+		n, err := io.ReadFull(rand.Reader, u[count:])
44 
+		if err != nil {
45 
+			if retryOnError(err) && retries < maxretries {
46 
+				count += n
47 
+				retries++
48 
+				logrus.Errorf("error generating version 4 uuid, retrying: %v", err)
49 
+				continue
50 
+			}
51 
+
52 
+			// Any other errors represent a system problem. What did someone
53 
+			// do to /dev/urandom?
54 
+			panic(fmt.Errorf("error reading random number generator, retried for %v: %v", totalBackoff.String(), err))
55 
+		}
56 
+
57 
+		break
58 
+	}
59 
+
60 
+	s := base32.StdEncoding.EncodeToString(u)
61 
+
62 
+	return s[:l]
63 
+}
64 
+
65 
+// retryOnError tries to detect whether or not retrying would be fruitful.
66 
+func retryOnError(err error) bool {
67 
+	switch err := err.(type) {
68 
+	case *os.PathError:
69 
+		return retryOnError(err.Err) // unpack the target error
70 
+	case syscall.Errno:
71 
+		if err == syscall.EPERM {
72 
+			// EPERM represents an entropy pool exhaustion, a condition under
73 
+			// which we backoff and retry.
74 
+			return true
75 
+		}
76 
+	}
77 
+
78 
+	return false
79 
+}
daemon/graphdriver/register/register_overlay.go
History View file @ 8a2f9a2
... ... 
@@ -5,4 +5,5 @@ package register
5 5 
 import (
6 6 
 	// register the overlay graphdriver
7 7 
 	_ "github.com/docker/docker/daemon/graphdriver/overlay"
8 
+	_ "github.com/docker/docker/daemon/graphdriver/overlay2"
8 9 
 )
docs/reference/commandline/dockerd.md
History View file @ 8a2f9a2
... ... 
@@ -204,7 +204,7 @@ TCP and a Unix socket
204 204 
 ### Daemon storage-driver option
205 205
206 206 
 The Docker daemon has support for several different image layer storage
207 
-drivers: `aufs`, `devicemapper`, `btrfs`, `zfs` and `overlay`.
207 
+drivers: `aufs`, `devicemapper`, `btrfs`, `zfs`, `overlay` and `overlay2`.
208 208
209 209 
 The `aufs` driver is the oldest, but is based on a Linux kernel patch-set that
210 210 
 is unlikely to be merged into the main kernel. These are also known to cause
... ... 
@@ -242,9 +242,14 @@ Linux kernel as of [3.18.0](https://lkml.org/lkml/2014/10/26/137). Call
242 242 
 > inode consumption (especially as the number of images grows), as well as
243 243 
 > being incompatible with the use of RPMs.
244 244
245 
+The `overlay2` uses the same fast union filesystem but takes advantage of
246 
+[additional features](https://lkml.org/lkml/2015/2/11/106) added in Linux
247 
+kernel 4.0 to avoid excessive inode consumption. Call `dockerd -s overlay2`
248 
+to use it.
249 
+
245 250 
 > **Note:**
246 
-> It is currently unsupported on `btrfs` or any Copy on Write filesystem
247 
-> and should only be used over `ext4` partitions.
251 
+> Both `overlay` and `overlay2` are currently unsupported on `btrfs` or any
252 
+> Copy on Write filesystem and should only be used over `ext4` partitions.
248 253
249 254 
 ### Storage driver options
250 255
docs/userguide/storagedriver/selectadriver.md
History View file @ 8a2f9a2
... ... 
@@ -34,14 +34,14 @@ and all containers created by that daemon instance use the same storage driver.
34 34 
  The table below shows the supported storage driver technologies and their
35 35 
 driver names:
36 36
37 
-|Technology    |Storage driver name  |
38 
-|--------------|---------------------|
39 
-|OverlayFS     |`overlay`            |
40 
-|AUFS          |`aufs`               |
41 
-|Btrfs         |`btrfs`              |
42 
-|Device Mapper |`devicemapper`       |
43 
-|VFS           |`vfs`                |
44 
-|ZFS           |`zfs`                |
37 
+|Technology    |Storage driver name    |
38 
+|--------------|-----------------------|
39 
+|OverlayFS     |`overlay` or `overlay2`|
40 
+|AUFS          |`aufs`                 |
41 
+|Btrfs         |`btrfs`                |
42 
+|Device Mapper |`devicemapper`         |
43 
+|VFS           |`vfs`                  |
44 
+|ZFS           |`zfs`                  |
45 45
46 46 
 To find out which storage driver is set on the daemon, you use the
47 47 
 `docker info` command:
... ... 
@@ -71,14 +71,15 @@ For example, the `btrfs` storage driver on a Btrfs backing filesystem. The
71 71 
 following table lists each storage driver and whether it must match the host's
72 72 
 backing file system:
73 73
74 
-|Storage driver |Commonly used on |Disabled on                              |
75 
-|---------------|-----------------|-----------------------------------------|
76 
-|`overlay`      |`ext4` `xfs`     |`btrfs` `aufs` `overlay` `zfs` `eCryptfs`|
77 
-|`aufs`         |`ext4` `xfs`     |`btrfs` `aufs` `eCryptfs`                |
78 
-|`btrfs`        |`btrfs` _only_   |   N/A                                   |
79 
-|`devicemapper` |`direct-lvm`     |   N/A                                   |
80 
-|`vfs`          |debugging only   |   N/A                                   |
81 
-|`zfs`          |`zfs` _only_     |   N/A                                   |
74 
+|Storage driver |Commonly used on |Disabled on                                         |
75 
+|---------------|-----------------|----------------------------------------------------|
76 
+|`overlay`      |`ext4` `xfs`     |`btrfs` `aufs` `overlay` `overlay2` `zfs` `eCryptfs`|
77 
+|`overlay2`     |`ext4` `xfs`     |`btrfs` `aufs` `overlay` `overlay2` `zfs` `eCryptfs`|
78 
+|`aufs`         |`ext4` `xfs`     |`btrfs` `aufs` `eCryptfs`                           |
79 
+|`btrfs`        |`btrfs` _only_   |   N/A                                              |
80 
+|`devicemapper` |`direct-lvm`     |   N/A                                              |
81 
+|`vfs`          |debugging only   |   N/A                                              |
82 
+|`zfs`          |`zfs` _only_     |   N/A                                              |
82 83
83 84
84 85 
 > **Note**
... ... 
@@ -198,6 +199,24 @@ the guidance offered by the table below along with the points mentioned above.
198 198
199 199 
 ![](images/driver-pros-cons.png)
200 200
201 
+### Overlay vs Overlay2
202 
+
203 
+OverlayFS has 2 storage drivers which both make use of the same OverlayFS
204 
+technology but with different implementations and incompatible on disk
205 
+storage. Since the storage is incompatible, switching between the two
206 
+will require re-creating all image content. The `overlay` driver is the
207 
+original implementation and the only option in Docker 1.11 and before.
208 
+The `overlay` driver has known limitations with inode exhaustion and
209 
+commit performance. The `overlay2` driver addresses this limitation, but
210 
+is only compatible with Linux kernel 4.0 and later. For users on a pre-4.0
211 
+kernel or with an existing `overlay` graph, it is recommended to stay
212 
+on `overlay`. For users with at least a 4.0 kernel and no existing or required
213 
+`overlay` graph data, then `overlay2` may be used.
214 
+
215 
+> **Note**
216 
+> `overlay2` graph data will not interfere with `overlay` graph data. However
217 
+> when switching to `overlay2`, the user is responsible for removing
218 
+> `overlay` graph data to avoid storage duplication.
201 219
202 220 
 ## Related information
203 221
man/dockerd.8.md
History View file @ 8a2f9a2
... ... 
@@ -226,7 +226,7 @@ output otherwise.
226 226 
   Force the Docker runtime to use a specific storage driver.
227 227
228 228 
 **--selinux-enabled**=*true*|*false*
229 
-  Enable selinux support. Default is false. SELinux does not presently support the overlay storage driver.
229 
+  Enable selinux support. Default is false. SELinux does not presently support either of the overlay storage drivers.
230 230
231 231 
 **--storage-opt**=[]
232 232 
   Set storage driver options. See STORAGE DRIVER OPTIONS.
pkg/archive/archive.go
History View file @ 8a2f9a2
... ... 
@@ -33,6 +33,8 @@ type (
33 33 
 	Reader io.Reader
34 34 
 	// Compression is the state represents if compressed or not.
35 35 
 	Compression int
36 
+	// WhiteoutFormat is the format of whiteouts unpacked
37 
+	WhiteoutFormat int
36 38 
 	// TarChownOptions wraps the chown options UID and GID.
37 39 
 	TarChownOptions struct {
38 40 
 		UID, GID int
... ... 
@@ -47,6 +49,10 @@ type (
47 47 
 		GIDMaps          []idtools.IDMap
48 48 
 		ChownOpts        *TarChownOptions
49 49 
 		IncludeSourceDir bool
50 
+		// WhiteoutFormat is the expected on disk format for whiteout files.
51 
+		// This format will be converted to the standard format on pack
52 
+		// and from the standard format on unpack.
53 
+		WhiteoutFormat WhiteoutFormat
50 54 
 		// When unpacking, specifies whether overwriting a directory with a
51 55 
 		// non-directory is allowed and vice versa.
52 56 
 		NoOverwriteDirNonDir bool
... ... 
@@ -93,6 +99,14 @@ const (
93 93 
 	Xz
94 94 
 )
95 95
96 
+const (
97 
+	// AUFSWhiteoutFormat is the default format for whitesouts
98 
+	AUFSWhiteoutFormat WhiteoutFormat = iota
99 
+	// OverlayWhiteoutFormat formats whiteout according to the overlay
100 
+	// standard.
101 
+	OverlayWhiteoutFormat
102 
+)
103 
+
96 104 
 // IsArchive checks for the magic bytes of a tar or any supported compression
97 105 
 // algorithm.
98 106 
 func IsArchive(header []byte) bool {
... ... 
@@ -228,6 +242,11 @@ func (compression *Compression) Extension() string {
228 228 
 	return ""
229 229 
 }
230 230
231 
+type tarWhiteoutConverter interface {
232 
+	ConvertWrite(*tar.Header, string, os.FileInfo) error
233 
+	ConvertRead(*tar.Header, string) (bool, error)
234 
+}
235 
+
231 236 
 type tarAppender struct {
232 237 
 	TarWriter *tar.Writer
233 238 
 	Buffer    *bufio.Writer
... ... 
@@ -236,6 +255,12 @@ type tarAppender struct {
236 236 
 	SeenFiles map[uint64]string
237 237 
 	UIDMaps   []idtools.IDMap
238 238 
 	GIDMaps   []idtools.IDMap
239 
+
240 
+	// For packing and unpacking whiteout files in the
241 
+	// non standard format. The whiteout files defined
242 
+	// by the AUFS standard are used as the tar whiteout
243 
+	// standard.
244 
+	WhiteoutConverter tarWhiteoutConverter
239 245 
 }
240 246
241 247 
 // canonicalTarName provides a platform-independent and consistent posix-style
... ... 
@@ -253,6 +278,7 @@ func canonicalTarName(name string, isDir bool) (string, error) {
253 253 
 	return name, nil
254 254 
 }
255 255
256 
+// addTarFile adds to the tar archive a file from `path` as `name`
256 257 
 func (ta *tarAppender) addTarFile(path, name string) error {
257 258 
 	fi, err := os.Lstat(path)
258 259 
 	if err != nil {
... ... 
@@ -323,6 +349,12 @@ func (ta *tarAppender) addTarFile(path, name string) error {
323 323 
 		hdr.Gid = xGID
324 324 
 	}
325 325
326 
+	if ta.WhiteoutConverter != nil {
327 
+		if err := ta.WhiteoutConverter.ConvertWrite(hdr, path, fi); err != nil {
328 
+			return err
329 
+		}
330 
+	}
331 
+
326 332 
 	if err := ta.TarWriter.WriteHeader(hdr); err != nil {
327 333 
 		return err
328 334 
 	}
... ... 
@@ -508,11 +540,12 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
508 508
509 509 
 	go func() {
510 510 
 		ta := &tarAppender{
511 
-			TarWriter: tar.NewWriter(compressWriter),
512 
-			Buffer:    pools.BufioWriter32KPool.Get(nil),
513 
-			SeenFiles: make(map[uint64]string),
514 
-			UIDMaps:   options.UIDMaps,
515 
-			GIDMaps:   options.GIDMaps,
511 
+			TarWriter:         tar.NewWriter(compressWriter),
512 
+			Buffer:            pools.BufioWriter32KPool.Get(nil),
513 
+			SeenFiles:         make(map[uint64]string),
514 
+			UIDMaps:           options.UIDMaps,
515 
+			GIDMaps:           options.GIDMaps,
516 
+			WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat),
516 517 
 		}
517 518
518 519 
 		defer func() {
... ... 
@@ -674,6 +707,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err
674 674 
 	if err != nil {
675 675 
 		return err
676 676 
 	}
677 
+	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat)
677 678
678 679 
 	// Iterate through the files in the archive.
679 680 
 loop:
... ... 
@@ -773,6 +807,16 @@ loop:
773 773 
 			hdr.Gid = xGID
774 774 
 		}
775 775
776 
+		if whiteoutConverter != nil {
777 
+			writeFile, err := whiteoutConverter.ConvertRead(hdr, path)
778 
+			if err != nil {
779 
+				return err
780 
+			}
781 
+			if !writeFile {
782 
+				continue
783 
+			}
784 
+		}
785 
+
776 786 
 		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, options.ChownOpts); err != nil {
777 787 
 			return err
778 788 
 		}
pkg/archive/archive_linux.go
History View file @ 8a2f9a2
779 789 
new file mode 100644
... ... 
@@ -0,0 +1,89 @@
0 
+package archive
1 
+
2 
+import (
3 
+	"archive/tar"
4 
+	"os"
5 
+	"path/filepath"
6 
+	"strings"
7 
+	"syscall"
8 
+
9 
+	"github.com/docker/docker/pkg/system"
10 
+)
11 
+
12 
+func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter {
13 
+	if format == OverlayWhiteoutFormat {
14 
+		return overlayWhiteoutConverter{}
15 
+	}
16 
+	return nil
17 
+}
18 
+
19 
+type overlayWhiteoutConverter struct{}
20 
+
21 
+func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) error {
22 
+	// convert whiteouts to AUFS format
23 
+	if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
24 
+		// we just rename the file and make it normal
25 
+		hdr.Name = WhiteoutPrefix + hdr.Name
26 
+		hdr.Mode = 0600
27 
+		hdr.Typeflag = tar.TypeReg
28 
+	}
29 
+
30 
+	if fi.Mode()&os.ModeDir != 0 {
31 
+		// convert opaque dirs to AUFS format by writing an empty file with the prefix
32 
+		opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque")
33 
+		if err != nil {
34 
+			return err
35 
+		}
36 
+		if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
37 
+			// create a header for the whiteout file
38 
+			// it should inherit some properties from the parent, but be a regular file
39 
+			*hdr = tar.Header{
40 
+				Typeflag:   tar.TypeReg,
41 
+				Mode:       hdr.Mode & int64(os.ModePerm),
42 
+				Name:       filepath.Join(hdr.Name, WhiteoutOpaqueDir),
43 
+				Size:       0,
44 
+				Uid:        hdr.Uid,
45 
+				Uname:      hdr.Uname,
46 
+				Gid:        hdr.Gid,
47 
+				Gname:      hdr.Gname,
48 
+				AccessTime: hdr.AccessTime,
49 
+				ChangeTime: hdr.ChangeTime,
50 
+			}
51 
+		}
52 
+	}
53 
+
54 
+	return nil
55 
+}
56 
+
57 
+func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) {
58 
+	base := filepath.Base(path)
59 
+	dir := filepath.Dir(path)
60 
+
61 
+	// if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay
62 
+	if base == WhiteoutOpaqueDir {
63 
+		if err := syscall.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0); err != nil {
64 
+			return false, err
65 
+		}
66 
+
67 
+		// don't write the file itself
68 
+		return false, nil
69 
+	}
70 
+
71 
+	// if a file was deleted and we are using overlay, we need to create a character device
72 
+	if strings.HasPrefix(base, WhiteoutPrefix) {
73 
+		originalBase := base[len(WhiteoutPrefix):]
74 
+		originalPath := filepath.Join(dir, originalBase)
75 
+
76 
+		if err := syscall.Mknod(originalPath, syscall.S_IFCHR, 0); err != nil {
77 
+			return false, err
78 
+		}
79 
+		if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil {
80 
+			return false, err
81 
+		}
82 
+
83 
+		// don't write the file itself
84 
+		return false, nil
85 
+	}
86 
+
87 
+	return true, nil
88 
+}
pkg/archive/archive_other.go
History View file @ 8a2f9a2
0 89 
new file mode 100644
... ... 
@@ -0,0 +1,7 @@
0 
+// +build !linux
1 
+
2 
+package archive
3 
+
4 
+func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter {
5 
+	return nil
6 
+}
pkg/archive/changes.go
History View file @ 8a2f9a2
... ... 
@@ -81,6 +81,33 @@ func sameFsTimeSpec(a, b syscall.Timespec) bool {
81 81 
 // Changes walks the path rw and determines changes for the files in the path,
82 82 
 // with respect to the parent layers
83 83 
 func Changes(layers []string, rw string) ([]Change, error) {
84 
+	return changes(layers, rw, aufsDeletedFile, aufsMetadataSkip)
85 
+}
86 
+
87 
+func aufsMetadataSkip(path string) (skip bool, err error) {
88 
+	skip, err = filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path)
89 
+	if err != nil {
90 
+		skip = true
91 
+	}
92 
+	return
93 
+}
94 
+
95 
+func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) {
96 
+	f := filepath.Base(path)
97 
+
98 
+	// If there is a whiteout, then the file was removed
99 
+	if strings.HasPrefix(f, WhiteoutPrefix) {
100 
+		originalFile := f[len(WhiteoutPrefix):]
101 
+		return filepath.Join(filepath.Dir(path), originalFile), nil
102 
+	}
103 
+
104 
+	return "", nil
105 
+}
106 
+
107 
+type skipChange func(string) (bool, error)
108 
+type deleteChange func(string, string, os.FileInfo) (string, error)
109 
+
110 
+func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Change, error) {
84 111 
 	var (
85 112 
 		changes     []Change
86 113 
 		changedDirs = make(map[string]struct{})
... ... 
@@ -105,21 +132,24 @@ func Changes(layers []string, rw string) ([]Change, error) {
105 105 
 			return nil
106 106 
 		}
107 107
108 
-		// Skip AUFS metadata
109 
-		if matched, err := filepath.Match(string(os.PathSeparator)+WhiteoutMetaPrefix+"*", path); err != nil || matched {
110 
-			return err
108 
+		if sc != nil {
109 
+			if skip, err := sc(path); skip {
110 
+				return err
111 
+			}
111 112 
 		}
112 113
113 114 
 		change := Change{
114 115 
 			Path: path,
115 116 
 		}
116 117
118 
+		deletedFile, err := dc(rw, path, f)
119 
+		if err != nil {
120 
+			return err
121 
+		}
122 
+
117 123 
 		// Find out what kind of modification happened
118 
-		file := filepath.Base(path)
119 
-		// If there is a whiteout, then the file was removed
120 
-		if strings.HasPrefix(file, WhiteoutPrefix) {
121 
-			originalFile := file[len(WhiteoutPrefix):]
122 
-			change.Path = filepath.Join(filepath.Dir(path), originalFile)
124 
+		if deletedFile != "" {
125 
+			change.Path = deletedFile
123 126 
 			change.Kind = ChangeDelete
124 127 
 		} else {
125 128 
 			// Otherwise, the file was added
pkg/archive/changes_linux.go
History View file @ 8a2f9a2
... ... 
@@ -283,3 +283,30 @@ func clen(n []byte) int {
283 283 
 	}
284 284 
 	return len(n)
285 285 
 }
286 
+
287 
+// OverlayChanges walks the path rw and determines changes for the files in the path,
288 
+// with respect to the parent layers
289 
+func OverlayChanges(layers []string, rw string) ([]Change, error) {
290 
+	return changes(layers, rw, overlayDeletedFile, nil)
291 
+}
292 
+
293 
+func overlayDeletedFile(root, path string, fi os.FileInfo) (string, error) {
294 
+	if fi.Mode()&os.ModeCharDevice != 0 {
295 
+		s := fi.Sys().(*syscall.Stat_t)
296 
+		if major(uint64(s.Rdev)) == 0 && minor(uint64(s.Rdev)) == 0 {
297 
+			return path, nil
298 
+		}
299 
+	}
300 
+	if fi.Mode()&os.ModeDir != 0 {
301 
+		opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque")
302 
+		if err != nil {
303 
+			return "", err
304 
+		}
305 
+		if opaque != nil && len(opaque) == 1 && opaque[0] == 'y' {
306 
+			return path, nil
307 
+		}
308 
+	}
309 
+
310 
+	return "", nil
311 
+
312 
+}