1. docker
  2. Commit 8b0441d42cca4f89cf871c1b64172cc06f9c98e6
Browse code

Expand graphtest package

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)

Derek McGowan authored on 2016/04/15 10:14:42
Showing 4 changed files
daemon/graphdriver/graphtest/graphbench_unix.go
History View file @ 8b0441d
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,264 @@
0 
+// +build linux freebsd
1 
+
2 
+package graphtest
3 
+
4 
+import (
5 
+	"bytes"
6 
+	"io"
7 
+	"io/ioutil"
8 
+	"path/filepath"
9 
+	"testing"
10 
+
11 
+	"github.com/docker/docker/pkg/stringid"
12 
+)
13 
+
14 
+// DriverBenchExists benchmarks calls to exist
15 
+func DriverBenchExists(b *testing.B, drivername string, driveroptions ...string) {
16 
+	driver := GetDriver(b, drivername, driveroptions...)
17 
+	defer PutDriver(b)
18 
+
19 
+	base := stringid.GenerateRandomID()
20 
+
21 
+	if err := driver.Create(base, "", "", nil); err != nil {
22 
+		b.Fatal(err)
23 
+	}
24 
+
25 
+	b.ResetTimer()
26 
+	for i := 0; i < b.N; i++ {
27 
+		if !driver.Exists(base) {
28 
+			b.Fatal("Newly created image doesn't exist")
29 
+		}
30 
+	}
31 
+}
32 
+
33 
+// DriverBenchGetEmpty benchmarks calls to get on an empty layer
34 
+func DriverBenchGetEmpty(b *testing.B, drivername string, driveroptions ...string) {
35 
+	driver := GetDriver(b, drivername, driveroptions...)
36 
+	defer PutDriver(b)
37 
+
38 
+	base := stringid.GenerateRandomID()
39 
+
40 
+	if err := driver.Create(base, "", "", nil); err != nil {
41 
+		b.Fatal(err)
42 
+	}
43 
+
44 
+	b.ResetTimer()
45 
+	for i := 0; i < b.N; i++ {
46 
+		_, err := driver.Get(base, "")
47 
+		b.StopTimer()
48 
+		if err != nil {
49 
+			b.Fatalf("Error getting mount: %s", err)
50 
+		}
51 
+		if err := driver.Put(base); err != nil {
52 
+			b.Fatalf("Error putting mount: %s", err)
53 
+		}
54 
+		b.StartTimer()
55 
+	}
56 
+}
57 
+
58 
+// DriverBenchDiffBase benchmarks calls to diff on a root layer
59 
+func DriverBenchDiffBase(b *testing.B, drivername string, driveroptions ...string) {
60 
+	driver := GetDriver(b, drivername, driveroptions...)
61 
+	defer PutDriver(b)
62 
+
63 
+	base := stringid.GenerateRandomID()
64 
+
65 
+	if err := driver.Create(base, "", "", nil); err != nil {
66 
+		b.Fatal(err)
67 
+	}
68 
+
69 
+	if err := addFiles(driver, base, 3); err != nil {
70 
+		b.Fatal(err)
71 
+	}
72 
+
73 
+	b.ResetTimer()
74 
+	for i := 0; i < b.N; i++ {
75 
+		arch, err := driver.Diff(base, "")
76 
+		if err != nil {
77 
+			b.Fatal(err)
78 
+		}
79 
+		_, err = io.Copy(ioutil.Discard, arch)
80 
+		if err != nil {
81 
+			b.Fatalf("Error copying archive: %s", err)
82 
+		}
83 
+		arch.Close()
84 
+	}
85 
+}
86 
+
87 
+// DriverBenchDiffN benchmarks calls to diff on two layers with
88 
+// a provided number of files on the lower and upper layers.
89 
+func DriverBenchDiffN(b *testing.B, bottom, top int, drivername string, driveroptions ...string) {
90 
+	driver := GetDriver(b, drivername, driveroptions...)
91 
+	defer PutDriver(b)
92 
+	base := stringid.GenerateRandomID()
93 
+	upper := stringid.GenerateRandomID()
94 
+
95 
+	if err := driver.Create(base, "", "", nil); err != nil {
96 
+		b.Fatal(err)
97 
+	}
98 
+
99 
+	if err := addManyFiles(driver, base, bottom, 3); err != nil {
100 
+		b.Fatal(err)
101 
+	}
102 
+
103 
+	if err := driver.Create(upper, base, "", nil); err != nil {
104 
+		b.Fatal(err)
105 
+	}
106 
+
107 
+	if err := addManyFiles(driver, upper, top, 6); err != nil {
108 
+		b.Fatal(err)
109 
+	}
110 
+	b.ResetTimer()
111 
+	for i := 0; i < b.N; i++ {
112 
+		arch, err := driver.Diff(upper, "")
113 
+		if err != nil {
114 
+			b.Fatal(err)
115 
+		}
116 
+		_, err = io.Copy(ioutil.Discard, arch)
117 
+		if err != nil {
118 
+			b.Fatalf("Error copying archive: %s", err)
119 
+		}
120 
+		arch.Close()
121 
+	}
122 
+}
123 
+
124 
+// DriverBenchDiffApplyN benchmarks calls to diff and apply together
125 
+func DriverBenchDiffApplyN(b *testing.B, fileCount int, drivername string, driveroptions ...string) {
126 
+	driver := GetDriver(b, drivername, driveroptions...)
127 
+	defer PutDriver(b)
128 
+	base := stringid.GenerateRandomID()
129 
+	upper := stringid.GenerateRandomID()
130 
+
131 
+	if err := driver.Create(base, "", "", nil); err != nil {
132 
+		b.Fatal(err)
133 
+	}
134 
+
135 
+	if err := addManyFiles(driver, base, fileCount, 3); err != nil {
136 
+		b.Fatal(err)
137 
+	}
138 
+
139 
+	if err := driver.Create(upper, base, "", nil); err != nil {
140 
+		b.Fatal(err)
141 
+	}
142 
+
143 
+	if err := addManyFiles(driver, upper, fileCount, 6); err != nil {
144 
+		b.Fatal(err)
145 
+	}
146 
+	diffSize, err := driver.DiffSize(upper, "")
147 
+	if err != nil {
148 
+		b.Fatal(err)
149 
+	}
150 
+	b.ResetTimer()
151 
+	b.StopTimer()
152 
+	for i := 0; i < b.N; i++ {
153 
+		diff := stringid.GenerateRandomID()
154 
+		if err := driver.Create(diff, base, "", nil); err != nil {
155 
+			b.Fatal(err)
156 
+		}
157 
+
158 
+		if err := checkManyFiles(driver, diff, fileCount, 3); err != nil {
159 
+			b.Fatal(err)
160 
+		}
161 
+
162 
+		b.StartTimer()
163 
+
164 
+		arch, err := driver.Diff(upper, "")
165 
+		if err != nil {
166 
+			b.Fatal(err)
167 
+		}
168 
+
169 
+		applyDiffSize, err := driver.ApplyDiff(diff, "", arch)
170 
+		if err != nil {
171 
+			b.Fatal(err)
172 
+		}
173 
+
174 
+		b.StopTimer()
175 
+		arch.Close()
176 
+
177 
+		if applyDiffSize != diffSize {
178 
+			// TODO: enforce this
179 
+			//b.Fatalf("Apply diff size different, got %d, expected %s", applyDiffSize, diffSize)
180 
+		}
181 
+		if err := checkManyFiles(driver, diff, fileCount, 6); err != nil {
182 
+			b.Fatal(err)
183 
+		}
184 
+	}
185 
+}
186 
+
187 
+// DriverBenchDeepLayerDiff benchmarks calls to diff on top of a given number of layers.
188 
+func DriverBenchDeepLayerDiff(b *testing.B, layerCount int, drivername string, driveroptions ...string) {
189 
+	driver := GetDriver(b, drivername, driveroptions...)
190 
+	defer PutDriver(b)
191 
+
192 
+	base := stringid.GenerateRandomID()
193 
+
194 
+	if err := driver.Create(base, "", "", nil); err != nil {
195 
+		b.Fatal(err)
196 
+	}
197 
+
198 
+	if err := addFiles(driver, base, 50); err != nil {
199 
+		b.Fatal(err)
200 
+	}
201 
+
202 
+	topLayer, err := addManyLayers(driver, base, layerCount)
203 
+	if err != nil {
204 
+		b.Fatal(err)
205 
+	}
206 
+
207 
+	b.ResetTimer()
208 
+	for i := 0; i < b.N; i++ {
209 
+		arch, err := driver.Diff(topLayer, "")
210 
+		if err != nil {
211 
+			b.Fatal(err)
212 
+		}
213 
+		_, err = io.Copy(ioutil.Discard, arch)
214 
+		if err != nil {
215 
+			b.Fatalf("Error copying archive: %s", err)
216 
+		}
217 
+		arch.Close()
218 
+	}
219 
+}
220 
+
221 
+// DriverBenchDeepLayerRead benchmarks calls to read a file under a given number of layers.
222 
+func DriverBenchDeepLayerRead(b *testing.B, layerCount int, drivername string, driveroptions ...string) {
223 
+	driver := GetDriver(b, drivername, driveroptions...)
224 
+	defer PutDriver(b)
225 
+
226 
+	base := stringid.GenerateRandomID()
227 
+
228 
+	if err := driver.Create(base, "", "", nil); err != nil {
229 
+		b.Fatal(err)
230 
+	}
231 
+
232 
+	content := []byte("test content")
233 
+	if err := addFile(driver, base, "testfile.txt", content); err != nil {
234 
+		b.Fatal(err)
235 
+	}
236 
+
237 
+	topLayer, err := addManyLayers(driver, base, layerCount)
238 
+	if err != nil {
239 
+		b.Fatal(err)
240 
+	}
241 
+
242 
+	root, err := driver.Get(topLayer, "")
243 
+	if err != nil {
244 
+		b.Fatal(err)
245 
+	}
246 
+	defer driver.Put(topLayer)
247 
+
248 
+	b.ResetTimer()
249 
+	for i := 0; i < b.N; i++ {
250 
+
251 
+		// Read content
252 
+		c, err := ioutil.ReadFile(filepath.Join(root, "testfile.txt"))
253 
+		if err != nil {
254 
+			b.Fatal(err)
255 
+		}
256 
+
257 
+		b.StopTimer()
258 
+		if bytes.Compare(c, content) != 0 {
259 
+			b.Fatalf("Wrong content in file %v, expected %v", c, content)
260 
+		}
261 
+		b.StartTimer()
262 
+	}
263 
+}
daemon/graphdriver/graphtest/graphtest_unix.go
History View file @ 8b0441d
... ... 
@@ -3,7 +3,7 @@
3 3 
 package graphtest
4 4
5 5 
 import (
6 
-	"fmt"
6 
+	"bytes"
7 7 
 	"io/ioutil"
8 8 
 	"math/rand"
9 9 
 	"os"
... ... 
@@ -14,6 +14,7 @@ import (
14 14 
 	"unsafe"
15 15
16 16 
 	"github.com/docker/docker/daemon/graphdriver"
17 
+	"github.com/docker/docker/pkg/stringid"
17 18 
 	"github.com/docker/go-units"
18 19 
 )
19 20
... ... 
@@ -30,47 +31,7 @@ type Driver struct {
30 30 
 	refCount int
31 31 
 }
32 32
33 
-// InitLoopbacks ensures that the loopback devices are properly created within
34 
-// the system running the device mapper tests.
35 
-func InitLoopbacks() error {
36 
-	statT, err := getBaseLoopStats()
37 
-	if err != nil {
38 
-		return err
39 
-	}
40 
-	// create at least 8 loopback files, ya, that is a good number
41 
-	for i := 0; i < 8; i++ {
42 
-		loopPath := fmt.Sprintf("/dev/loop%d", i)
43 
-		// only create new loopback files if they don't exist
44 
-		if _, err := os.Stat(loopPath); err != nil {
45 
-			if mkerr := syscall.Mknod(loopPath,
46 
-				uint32(statT.Mode|syscall.S_IFBLK), int((7<<8)|(i&0xff)|((i&0xfff00)<<12))); mkerr != nil {
47 
-				return mkerr
48 
-			}
49 
-			os.Chown(loopPath, int(statT.Uid), int(statT.Gid))
50 
-		}
51 
-	}
52 
-	return nil
53 
-}
54 
-
55 
-// getBaseLoopStats inspects /dev/loop0 to collect uid,gid, and mode for the
56 
-// loop0 device on the system.  If it does not exist we assume 0,0,0660 for the
57 
-// stat data
58 
-func getBaseLoopStats() (*syscall.Stat_t, error) {
59 
-	loop0, err := os.Stat("/dev/loop0")
60 
-	if err != nil {
61 
-		if os.IsNotExist(err) {
62 
-			return &syscall.Stat_t{
63 
-				Uid:  0,
64 
-				Gid:  0,
65 
-				Mode: 0660,
66 
-			}, nil
67 
-		}
68 
-		return nil, err
69 
-	}
70 
-	return loop0.Sys().(*syscall.Stat_t), nil
71 
-}
72 
-
73 
-func newDriver(t *testing.T, name string) *Driver {
33 
+func newDriver(t testing.TB, name string, options []string) *Driver {
74 34 
 	root, err := ioutil.TempDir("", "docker-graphtest-")
75 35 
 	if err != nil {
76 36 
 		t.Fatal(err)
... ... 
@@ -80,7 +41,7 @@ func newDriver(t *testing.T, name string) *Driver {
80 80 
 		t.Fatal(err)
81 81 
 	}
82 82
83 
-	d, err := graphdriver.GetDriver(name, root, nil, nil, nil)
83 
+	d, err := graphdriver.GetDriver(name, root, options, nil, nil)
84 84 
 	if err != nil {
85 85 
 		t.Logf("graphdriver: %v\n", err)
86 86 
 		if err == graphdriver.ErrNotSupported || err == graphdriver.ErrPrerequisites || err == graphdriver.ErrIncompatibleFS {
... ... 
@@ -91,7 +52,7 @@ func newDriver(t *testing.T, name string) *Driver {
91 91 
 	return &Driver{d, root, 1}
92 92 
 }
93 93
94 
-func cleanup(t *testing.T, d *Driver) {
94 
+func cleanup(t testing.TB, d *Driver) {
95 95 
 	if err := drv.Cleanup(); err != nil {
96 96 
 		t.Fatal(err)
97 97 
 	}
... ... 
@@ -99,9 +60,9 @@ func cleanup(t *testing.T, d *Driver) {
99 99 
 }
100 100
101 101 
 // GetDriver create a new driver with given name or return an existing driver with the name updating the reference count.
102 
-func GetDriver(t *testing.T, name string) graphdriver.Driver {
102 
+func GetDriver(t testing.TB, name string, options ...string) graphdriver.Driver {
103 103 
 	if drv == nil {
104 
-		drv = newDriver(t, name)
104 
+		drv = newDriver(t, name, options)
105 105 
 	} else {
106 106 
 		drv.refCount++
107 107 
 	}
... ... 
@@ -109,7 +70,7 @@ func GetDriver(t *testing.T, name string) graphdriver.Driver {
109 109 
 }
110 110
111 111 
 // PutDriver removes the driver if it is no longer used and updates the reference count.
112 
-func PutDriver(t *testing.T) {
112 
+func PutDriver(t testing.TB) {
113 113 
 	if drv == nil {
114 114 
 		t.Skip("No driver to put!")
115 115 
 	}
... ... 
@@ -120,190 +81,210 @@ func PutDriver(t *testing.T) {
120 120 
 	}
121 121 
 }
122 122
123 
-func verifyFile(t *testing.T, path string, mode os.FileMode, uid, gid uint32) {
124 
-	fi, err := os.Stat(path)
125 
-	if err != nil {
123 
+// DriverTestCreateEmpty creates a new image and verifies it is empty and the right metadata
124 
+func DriverTestCreateEmpty(t testing.TB, drivername string, driverOptions ...string) {
125 
+	driver := GetDriver(t, drivername, driverOptions...)
126 
+	defer PutDriver(t)
127 
+
128 
+	if err := driver.Create("empty", "", "", nil); err != nil {
126 129 
 		t.Fatal(err)
127 130 
 	}
128 131
129 
-	if fi.Mode()&os.ModeType != mode&os.ModeType {
130 
-		t.Fatalf("Expected %s type 0x%x, got 0x%x", path, mode&os.ModeType, fi.Mode()&os.ModeType)
131 
-	}
132 
+	defer func() {
133 
+		if err := driver.Remove("empty"); err != nil {
134 
+			t.Fatal(err)
135 
+		}
136 
+	}()
132 137
133 
-	if fi.Mode()&os.ModePerm != mode&os.ModePerm {
134 
-		t.Fatalf("Expected %s mode %o, got %o", path, mode&os.ModePerm, fi.Mode()&os.ModePerm)
138 
+	if !driver.Exists("empty") {
139 
+		t.Fatal("Newly created image doesn't exist")
135 140 
 	}
136 141
137 
-	if fi.Mode()&os.ModeSticky != mode&os.ModeSticky {
138 
-		t.Fatalf("Expected %s sticky 0x%x, got 0x%x", path, mode&os.ModeSticky, fi.Mode()&os.ModeSticky)
142 
+	dir, err := driver.Get("empty", "")
143 
+	if err != nil {
144 
+		t.Fatal(err)
139 145 
 	}
140 146
141 
-	if fi.Mode()&os.ModeSetuid != mode&os.ModeSetuid {
142 
-		t.Fatalf("Expected %s setuid 0x%x, got 0x%x", path, mode&os.ModeSetuid, fi.Mode()&os.ModeSetuid)
143 
-	}
147 
+	verifyFile(t, dir, 0755|os.ModeDir, 0, 0)
144 148
145 
-	if fi.Mode()&os.ModeSetgid != mode&os.ModeSetgid {
146 
-		t.Fatalf("Expected %s setgid 0x%x, got 0x%x", path, mode&os.ModeSetgid, fi.Mode()&os.ModeSetgid)
149 
+	// Verify that the directory is empty
150 
+	fis, err := readDir(dir)
151 
+	if err != nil {
152 
+		t.Fatal(err)
147 153 
 	}
148 154
149 
-	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
150 
-		if stat.Uid != uid {
151 
-			t.Fatalf("%s no owned by uid %d", path, uid)
152 
-		}
153 
-		if stat.Gid != gid {
154 
-			t.Fatalf("%s not owned by gid %d", path, gid)
155 
-		}
155 
+	if len(fis) != 0 {
156 
+		t.Fatal("New directory not empty")
156 157 
 	}
157 158
159 
+	driver.Put("empty")
158 160 
 }
159 161
160 
-// readDir reads a directory just like ioutil.ReadDir()
161 
-// then hides specific files (currently "lost+found")
162 
-// so the tests don't "see" it
163 
-func readDir(dir string) ([]os.FileInfo, error) {
164 
-	a, err := ioutil.ReadDir(dir)
165 
-	if err != nil {
166 
-		return nil, err
167 
-	}
162 
+// DriverTestCreateBase create a base driver and verify.
163 
+func DriverTestCreateBase(t testing.TB, drivername string, driverOptions ...string) {
164 
+	driver := GetDriver(t, drivername, driverOptions...)
165 
+	defer PutDriver(t)
168 166
169 
-	b := a[:0]
170 
-	for _, x := range a {
171 
-		if x.Name() != "lost+found" { // ext4 always have this dir
172 
-			b = append(b, x)
167 
+	createBase(t, driver, "Base")
168 
+	defer func() {
169 
+		if err := driver.Remove("Base"); err != nil {
170 
+			t.Fatal(err)
173 171 
 		}
174 
-	}
175 
-
176 
-	return b, nil
172 
+	}()
173 
+	verifyBase(t, driver, "Base")
177 174 
 }
178 175
179 
-// DriverTestCreateEmpty creates a new image and verifies it is empty and the right metadata
180 
-func DriverTestCreateEmpty(t *testing.T, drivername string) {
181 
-	driver := GetDriver(t, drivername)
176 
+// DriverTestCreateSnap Create a driver and snap and verify.
177 
+func DriverTestCreateSnap(t testing.TB, drivername string, driverOptions ...string) {
178 
+	driver := GetDriver(t, drivername, driverOptions...)
182 179 
 	defer PutDriver(t)
183 180
184 
-	if err := driver.Create("empty", "", "", nil); err != nil {
181 
+	createBase(t, driver, "Base")
182 
+
183 
+	defer func() {
184 
+		if err := driver.Remove("Base"); err != nil {
185 
+			t.Fatal(err)
186 
+		}
187 
+	}()
188 
+
189 
+	if err := driver.Create("Snap", "Base", "", nil); err != nil {
185 190 
 		t.Fatal(err)
186 191 
 	}
187 192
188 193 
 	defer func() {
189 
-		if err := driver.Remove("empty"); err != nil {
194 
+		if err := driver.Remove("Snap"); err != nil {
190 195 
 			t.Fatal(err)
191 196 
 		}
192 197 
 	}()
193 198
194 
-	if !driver.Exists("empty") {
195 
-		t.Fatal("Newly created image doesn't exist")
196 
-	}
199 
+	verifyBase(t, driver, "Snap")
200 
+}
197 201
198 
-	dir, err := driver.Get("empty", "")
199 
-	if err != nil {
202 
+// DriverTestDeepLayerRead reads a file from a lower layer under a given number of layers
203 
+func DriverTestDeepLayerRead(t testing.TB, layerCount int, drivername string, driverOptions ...string) {
204 
+	driver := GetDriver(t, drivername, driverOptions...)
205 
+	defer PutDriver(t)
206 
+
207 
+	base := stringid.GenerateRandomID()
208 
+
209 
+	if err := driver.Create(base, "", "", nil); err != nil {
200 210 
 		t.Fatal(err)
201 211 
 	}
202 212
203 
-	verifyFile(t, dir, 0755|os.ModeDir, 0, 0)
213 
+	content := []byte("test content")
214 
+	if err := addFile(driver, base, "testfile.txt", content); err != nil {
215 
+		t.Fatal(err)
216 
+	}
204 217
205 
-	// Verify that the directory is empty
206 
-	fis, err := readDir(dir)
218 
+	topLayer, err := addManyLayers(driver, base, layerCount)
207 219 
 	if err != nil {
208 220 
 		t.Fatal(err)
209 221 
 	}
210 222
211 
-	if len(fis) != 0 {
212 
-		t.Fatal("New directory not empty")
223 
+	err = checkManyLayers(driver, topLayer, layerCount)
224 
+	if err != nil {
225 
+		t.Fatal(err)
213 226 
 	}
214 227
215 
-	driver.Put("empty")
228 
+	if err := checkFile(driver, topLayer, "testfile.txt", content); err != nil {
229 
+		t.Fatal(err)
230 
+	}
216 231 
 }
217 232
218 
-func createBase(t *testing.T, driver graphdriver.Driver, name string) {
219 
-	// We need to be able to set any perms
220 
-	oldmask := syscall.Umask(0)
221 
-	defer syscall.Umask(oldmask)
233 
+// DriverTestDiffApply tests diffing and applying produces the same layer
234 
+func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverOptions ...string) {
235 
+	driver := GetDriver(t, drivername, driverOptions...)
236 
+	defer PutDriver(t)
237 
+	base := stringid.GenerateRandomID()
238 
+	upper := stringid.GenerateRandomID()
222 239
223 
-	if err := driver.CreateReadWrite(name, "", "", nil); err != nil {
240 
+	if err := driver.Create(base, "", "", nil); err != nil {
224 241 
 		t.Fatal(err)
225 242 
 	}
226 243
227 
-	dir, err := driver.Get(name, "")
228 
-	if err != nil {
244 
+	if err := addManyFiles(driver, base, fileCount, 3); err != nil {
245 
+		t.Fatal(err)
246 
+	}
247 
+
248 
+	if err := driver.Create(upper, base, "", nil); err != nil {
229 249 
 		t.Fatal(err)
230 250 
 	}
231 
-	defer driver.Put(name)
232 251
233 
-	subdir := path.Join(dir, "a subdir")
234 
-	if err := os.Mkdir(subdir, 0705|os.ModeSticky); err != nil {
252 
+	if err := addManyFiles(driver, upper, fileCount, 6); err != nil {
235 253 
 		t.Fatal(err)
236 254 
 	}
237 
-	if err := os.Chown(subdir, 1, 2); err != nil {
255 
+	diffSize, err := driver.DiffSize(upper, "")
256 
+	if err != nil {
238 257 
 		t.Fatal(err)
239 258 
 	}
240 259
241 
-	file := path.Join(dir, "a file")
242 
-	if err := ioutil.WriteFile(file, []byte("Some data"), 0222|os.ModeSetuid); err != nil {
260 
+	diff := stringid.GenerateRandomID()
261 
+	if err := driver.Create(diff, base, "", nil); err != nil {
243 262 
 		t.Fatal(err)
244 263 
 	}
245 
-}
246 264
247 
-func verifyBase(t *testing.T, driver graphdriver.Driver, name string) {
248 
-	dir, err := driver.Get(name, "")
249 
-	if err != nil {
265 
+	if err := checkManyFiles(driver, diff, fileCount, 3); err != nil {
250 266 
 		t.Fatal(err)
251 267 
 	}
252 
-	defer driver.Put(name)
253 268
254 
-	subdir := path.Join(dir, "a subdir")
255 
-	verifyFile(t, subdir, 0705|os.ModeDir|os.ModeSticky, 1, 2)
269 
+	arch, err := driver.Diff(upper, base)
270 
+	if err != nil {
271 
+		t.Fatal(err)
272 
+	}
256 273
257 
-	file := path.Join(dir, "a file")
258 
-	verifyFile(t, file, 0222|os.ModeSetuid, 0, 0)
274 
+	buf := bytes.NewBuffer(nil)
275 
+	if _, err := buf.ReadFrom(arch); err != nil {
276 
+		t.Fatal(err)
277 
+	}
278 
+	if err := arch.Close(); err != nil {
279 
+		t.Fatal(err)
280 
+	}
259 281
260 
-	fis, err := readDir(dir)
282 
+	applyDiffSize, err := driver.ApplyDiff(diff, base, bytes.NewReader(buf.Bytes()))
261 283 
 	if err != nil {
262 284 
 		t.Fatal(err)
263 285 
 	}
264 286
265 
-	if len(fis) != 2 {
266 
-		t.Fatal("Unexpected files in base image")
287 
+	if applyDiffSize != diffSize {
288 
+		t.Fatalf("Apply diff size different, got %d, expected %d", applyDiffSize, diffSize)
289 
+	}
290 
+	if err := checkManyFiles(driver, diff, fileCount, 6); err != nil {
291 
+		t.Fatal(err)
267 292 
 	}
268 293 
 }
269 294
270 
-// DriverTestCreateBase create a base driver and verify.
271 
-func DriverTestCreateBase(t *testing.T, drivername string) {
272 
-	driver := GetDriver(t, drivername)
295 
+// DriverTestChanges tests computed changes on a layer matches changes made
296 
+func DriverTestChanges(t testing.TB, drivername string, driverOptions ...string) {
297 
+	driver := GetDriver(t, drivername, driverOptions...)
273 298 
 	defer PutDriver(t)
299 
+	base := stringid.GenerateRandomID()
300 
+	upper := stringid.GenerateRandomID()
274 301
275 
-	createBase(t, driver, "Base")
276 
-	defer func() {
277 
-		if err := driver.Remove("Base"); err != nil {
278 
-			t.Fatal(err)
279 
-		}
280 
-	}()
281 
-	verifyBase(t, driver, "Base")
282 
-}
302 
+	if err := driver.Create(base, "", "", nil); err != nil {
303 
+		t.Fatal(err)
304 
+	}
283 305
284 
-// DriverTestCreateSnap Create a driver and snap and verify.
285 
-func DriverTestCreateSnap(t *testing.T, drivername string) {
286 
-	driver := GetDriver(t, drivername)
287 
-	defer PutDriver(t)
306 
+	if err := addManyFiles(driver, base, 20, 3); err != nil {
307 
+		t.Fatal(err)
308 
+	}
288 309
289 
-	createBase(t, driver, "Base")
310 
+	if err := driver.Create(upper, base, "", nil); err != nil {
311 
+		t.Fatal(err)
312 
+	}
290 313
291 
-	defer func() {
292 
-		if err := driver.Remove("Base"); err != nil {
293 
-			t.Fatal(err)
294 
-		}
295 
-	}()
314 
+	expectedChanges, err := changeManyFiles(driver, upper, 20, 6)
315 
+	if err != nil {
316 
+		t.Fatal(err)
317 
+	}
296 318
297 
-	if err := driver.Create("Snap", "Base", "", nil); err != nil {
319 
+	changes, err := driver.Changes(upper, base)
320 
+	if err != nil {
298 321 
 		t.Fatal(err)
299 322 
 	}
300 
-	defer func() {
301 
-		if err := driver.Remove("Snap"); err != nil {
302 
-			t.Fatal(err)
303 
-		}
304 
-	}()
305 323
306 
-	verifyBase(t, driver, "Snap")
324 
+	if err = checkChanges(expectedChanges, changes); err != nil {
325 
+		t.Fatal(err)
326 
+	}
307 327 
 }
308 328
309 329 
 func writeRandomFile(path string, size uint64) error {
daemon/graphdriver/graphtest/testutil.go
History View file @ 8b0441d
310 330 
new file mode 100644
... ... 
@@ -0,0 +1,301 @@
0 
+package graphtest
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"fmt"
5 
+	"io/ioutil"
6 
+	"math/rand"
7 
+	"os"
8 
+	"path"
9 
+	"sort"
10 
+
11 
+	"github.com/docker/docker/daemon/graphdriver"
12 
+	"github.com/docker/docker/pkg/archive"
13 
+	"github.com/docker/docker/pkg/stringid"
14 
+)
15 
+
16 
+func randomContent(size int, seed int64) []byte {
17 
+	s := rand.NewSource(seed)
18 
+	content := make([]byte, size)
19 
+
20 
+	for i := 0; i < len(content); i += 7 {
21 
+		val := s.Int63()
22 
+		for j := 0; i+j < len(content) && j < 7; j++ {
23 
+			content[i+j] = byte(val)
24 
+			val >>= 8
25 
+		}
26 
+	}
27 
+
28 
+	return content
29 
+}
30 
+
31 
+func addFiles(drv graphdriver.Driver, layer string, seed int64) error {
32 
+	root, err := drv.Get(layer, "")
33 
+	if err != nil {
34 
+		return err
35 
+	}
36 
+	defer drv.Put(layer)
37 
+
38 
+	if err := ioutil.WriteFile(path.Join(root, "file-a"), randomContent(64, seed), 0755); err != nil {
39 
+		return err
40 
+	}
41 
+	if err := os.MkdirAll(path.Join(root, "dir-b"), 0755); err != nil {
42 
+		return err
43 
+	}
44 
+	if err := ioutil.WriteFile(path.Join(root, "dir-b", "file-b"), randomContent(128, seed+1), 0755); err != nil {
45 
+		return err
46 
+	}
47 
+
48 
+	return ioutil.WriteFile(path.Join(root, "file-c"), randomContent(128*128, seed+2), 0755)
49 
+}
50 
+
51 
+func checkFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
52 
+	root, err := drv.Get(layer, "")
53 
+	if err != nil {
54 
+		return err
55 
+	}
56 
+	defer drv.Put(layer)
57 
+
58 
+	fileContent, err := ioutil.ReadFile(path.Join(root, filename))
59 
+	if err != nil {
60 
+		return err
61 
+	}
62 
+
63 
+	if bytes.Compare(fileContent, content) != 0 {
64 
+		return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
65 
+	}
66 
+
67 
+	return nil
68 
+}
69 
+
70 
+func addFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
71 
+	root, err := drv.Get(layer, "")
72 
+	if err != nil {
73 
+		return err
74 
+	}
75 
+	defer drv.Put(layer)
76 
+
77 
+	return ioutil.WriteFile(path.Join(root, filename), content, 0755)
78 
+}
79 
+
80 
+func addManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
81 
+	root, err := drv.Get(layer, "")
82 
+	if err != nil {
83 
+		return err
84 
+	}
85 
+	defer drv.Put(layer)
86 
+
87 
+	for i := 0; i < count; i += 100 {
88 
+		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
89 
+		if err := os.MkdirAll(dir, 0755); err != nil {
90 
+			return err
91 
+		}
92 
+		for j := 0; i+j < count && j < 100; j++ {
93 
+			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
94 
+			if err := ioutil.WriteFile(file, randomContent(64, seed+int64(i+j)), 0755); err != nil {
95 
+				return err
96 
+			}
97 
+		}
98 
+	}
99 
+
100 
+	return nil
101 
+}
102 
+
103 
+func changeManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) ([]archive.Change, error) {
104 
+	root, err := drv.Get(layer, "")
105 
+	if err != nil {
106 
+		return nil, err
107 
+	}
108 
+	defer drv.Put(layer)
109 
+
110 
+	changes := []archive.Change{}
111 
+	for i := 0; i < count; i += 100 {
112 
+		archiveRoot := fmt.Sprintf("/directory-%d", i)
113 
+		if err := os.MkdirAll(path.Join(root, archiveRoot), 0755); err != nil {
114 
+			return nil, err
115 
+		}
116 
+		for j := 0; i+j < count && j < 100; j++ {
117 
+			if j == 0 {
118 
+				changes = append(changes, archive.Change{
119 
+					Path: archiveRoot,
120 
+					Kind: archive.ChangeModify,
121 
+				})
122 
+			}
123 
+			var change archive.Change
124 
+			switch j % 3 {
125 
+			// Update file
126 
+			case 0:
127 
+				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
128 
+				change.Kind = archive.ChangeModify
129 
+				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
130 
+					return nil, err
131 
+				}
132 
+			// Add file
133 
+			case 1:
134 
+				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d-%d", seed, i+j))
135 
+				change.Kind = archive.ChangeAdd
136 
+				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
137 
+					return nil, err
138 
+				}
139 
+			// Remove file
140 
+			case 2:
141 
+				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
142 
+				change.Kind = archive.ChangeDelete
143 
+				if err := os.Remove(path.Join(root, change.Path)); err != nil {
144 
+					return nil, err
145 
+				}
146 
+			}
147 
+			changes = append(changes, change)
148 
+		}
149 
+	}
150 
+
151 
+	return changes, nil
152 
+}
153 
+
154 
+func checkManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) error {
155 
+	root, err := drv.Get(layer, "")
156 
+	if err != nil {
157 
+		return err
158 
+	}
159 
+	defer drv.Put(layer)
160 
+
161 
+	for i := 0; i < count; i += 100 {
162 
+		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
163 
+		for j := 0; i+j < count && j < 100; j++ {
164 
+			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
165 
+			fileContent, err := ioutil.ReadFile(file)
166 
+			if err != nil {
167 
+				return err
168 
+			}
169 
+
170 
+			content := randomContent(64, seed+int64(i+j))
171 
+
172 
+			if bytes.Compare(fileContent, content) != 0 {
173 
+				return fmt.Errorf("mismatched file content %v, expecting %v", fileContent, content)
174 
+			}
175 
+		}
176 
+	}
177 
+
178 
+	return nil
179 
+}
180 
+
181 
+type changeList []archive.Change
182 
+
183 
+func (c changeList) Less(i, j int) bool {
184 
+	if c[i].Path == c[j].Path {
185 
+		return c[i].Kind < c[j].Kind
186 
+	}
187 
+	return c[i].Path < c[j].Path
188 
+}
189 
+func (c changeList) Len() int      { return len(c) }
190 
+func (c changeList) Swap(i, j int) { c[j], c[i] = c[i], c[j] }
191 
+
192 
+func checkChanges(expected, actual []archive.Change) error {
193 
+	if len(expected) != len(actual) {
194 
+		return fmt.Errorf("unexpected number of changes, expected %d, got %d", len(expected), len(actual))
195 
+	}
196 
+	sort.Sort(changeList(expected))
197 
+	sort.Sort(changeList(actual))
198 
+
199 
+	for i := range expected {
200 
+		if expected[i] != actual[i] {
201 
+			return fmt.Errorf("unexpected change, expecting %v, got %v", expected[i], actual[i])
202 
+		}
203 
+	}
204 
+
205 
+	return nil
206 
+}
207 
+
208 
+func addLayerFiles(drv graphdriver.Driver, layer, parent string, i int) error {
209 
+	root, err := drv.Get(layer, "")
210 
+	if err != nil {
211 
+		return err
212 
+	}
213 
+	defer drv.Put(layer)
214 
+
215 
+	if err := ioutil.WriteFile(path.Join(root, "top-id"), []byte(layer), 0755); err != nil {
216 
+		return err
217 
+	}
218 
+	layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
219 
+	if err := os.MkdirAll(layerDir, 0755); err != nil {
220 
+		return err
221 
+	}
222 
+	if err := ioutil.WriteFile(path.Join(layerDir, "layer-id"), []byte(layer), 0755); err != nil {
223 
+		return err
224 
+	}
225 
+	if err := ioutil.WriteFile(path.Join(layerDir, "parent-id"), []byte(parent), 0755); err != nil {
226 
+		return err
227 
+	}
228 
+
229 
+	return nil
230 
+}
231 
+
232 
+func addManyLayers(drv graphdriver.Driver, baseLayer string, count int) (string, error) {
233 
+	lastLayer := baseLayer
234 
+	for i := 1; i <= count; i++ {
235 
+		nextLayer := stringid.GenerateRandomID()
236 
+		if err := drv.Create(nextLayer, lastLayer, "", nil); err != nil {
237 
+			return "", err
238 
+		}
239 
+		if err := addLayerFiles(drv, nextLayer, lastLayer, i); err != nil {
240 
+			return "", err
241 
+		}
242 
+
243 
+		lastLayer = nextLayer
244 
+
245 
+	}
246 
+	return lastLayer, nil
247 
+}
248 
+
249 
+func checkManyLayers(drv graphdriver.Driver, layer string, count int) error {
250 
+	root, err := drv.Get(layer, "")
251 
+	if err != nil {
252 
+		return err
253 
+	}
254 
+	defer drv.Put(layer)
255 
+
256 
+	layerIDBytes, err := ioutil.ReadFile(path.Join(root, "top-id"))
257 
+	if err != nil {
258 
+		return err
259 
+	}
260 
+
261 
+	if bytes.Compare(layerIDBytes, []byte(layer)) != 0 {
262 
+		return fmt.Errorf("mismatched file content %v, expecting %v", layerIDBytes, []byte(layer))
263 
+	}
264 
+
265 
+	for i := count; i > 0; i-- {
266 
+		layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
267 
+
268 
+		thisLayerIDBytes, err := ioutil.ReadFile(path.Join(layerDir, "layer-id"))
269 
+		if err != nil {
270 
+			return err
271 
+		}
272 
+		if bytes.Compare(thisLayerIDBytes, layerIDBytes) != 0 {
273 
+			return fmt.Errorf("mismatched file content %v, expecting %v", thisLayerIDBytes, layerIDBytes)
274 
+		}
275 
+		layerIDBytes, err = ioutil.ReadFile(path.Join(layerDir, "parent-id"))
276 
+		if err != nil {
277 
+			return err
278 
+		}
279 
+	}
280 
+	return nil
281 
+}
282 
+
283 
+// readDir reads a directory just like ioutil.ReadDir()
284 
+// then hides specific files (currently "lost+found")
285 
+// so the tests don't "see" it
286 
+func readDir(dir string) ([]os.FileInfo, error) {
287 
+	a, err := ioutil.ReadDir(dir)
288 
+	if err != nil {
289 
+		return nil, err
290 
+	}
291 
+
292 
+	b := a[:0]
293 
+	for _, x := range a {
294 
+		if x.Name() != "lost+found" { // ext4 always have this dir
295 
+			b = append(b, x)
296 
+		}
297 
+	}
298 
+
299 
+	return b, nil
300 
+}
daemon/graphdriver/graphtest/testutil_unix.go
History View file @ 8b0441d
0 301 
new file mode 100644
... ... 
@@ -0,0 +1,143 @@
0 
+// +build linux freebsd
1 
+
2 
+package graphtest
3 
+
4 
+import (
5 
+	"fmt"
6 
+	"io/ioutil"
7 
+	"os"
8 
+	"path"
9 
+	"syscall"
10 
+	"testing"
11 
+
12 
+	"github.com/docker/docker/daemon/graphdriver"
13 
+)
14 
+
15 
+// InitLoopbacks ensures that the loopback devices are properly created within
16 
+// the system running the device mapper tests.
17 
+func InitLoopbacks() error {
18 
+	statT, err := getBaseLoopStats()
19 
+	if err != nil {
20 
+		return err
21 
+	}
22 
+	// create at least 8 loopback files, ya, that is a good number
23 
+	for i := 0; i < 8; i++ {
24 
+		loopPath := fmt.Sprintf("/dev/loop%d", i)
25 
+		// only create new loopback files if they don't exist
26 
+		if _, err := os.Stat(loopPath); err != nil {
27 
+			if mkerr := syscall.Mknod(loopPath,
28 
+				uint32(statT.Mode|syscall.S_IFBLK), int((7<<8)|(i&0xff)|((i&0xfff00)<<12))); mkerr != nil {
29 
+				return mkerr
30 
+			}
31 
+			os.Chown(loopPath, int(statT.Uid), int(statT.Gid))
32 
+		}
33 
+	}
34 
+	return nil
35 
+}
36 
+
37 
+// getBaseLoopStats inspects /dev/loop0 to collect uid,gid, and mode for the
38 
+// loop0 device on the system.  If it does not exist we assume 0,0,0660 for the
39 
+// stat data
40 
+func getBaseLoopStats() (*syscall.Stat_t, error) {
41 
+	loop0, err := os.Stat("/dev/loop0")
42 
+	if err != nil {
43 
+		if os.IsNotExist(err) {
44 
+			return &syscall.Stat_t{
45 
+				Uid:  0,
46 
+				Gid:  0,
47 
+				Mode: 0660,
48 
+			}, nil
49 
+		}
50 
+		return nil, err
51 
+	}
52 
+	return loop0.Sys().(*syscall.Stat_t), nil
53 
+}
54 
+
55 
+func verifyFile(t testing.TB, path string, mode os.FileMode, uid, gid uint32) {
56 
+	fi, err := os.Stat(path)
57 
+	if err != nil {
58 
+		t.Fatal(err)
59 
+	}
60 
+
61 
+	if fi.Mode()&os.ModeType != mode&os.ModeType {
62 
+		t.Fatalf("Expected %s type 0x%x, got 0x%x", path, mode&os.ModeType, fi.Mode()&os.ModeType)
63 
+	}
64 
+
65 
+	if fi.Mode()&os.ModePerm != mode&os.ModePerm {
66 
+		t.Fatalf("Expected %s mode %o, got %o", path, mode&os.ModePerm, fi.Mode()&os.ModePerm)
67 
+	}
68 
+
69 
+	if fi.Mode()&os.ModeSticky != mode&os.ModeSticky {
70 
+		t.Fatalf("Expected %s sticky 0x%x, got 0x%x", path, mode&os.ModeSticky, fi.Mode()&os.ModeSticky)
71 
+	}
72 
+
73 
+	if fi.Mode()&os.ModeSetuid != mode&os.ModeSetuid {
74 
+		t.Fatalf("Expected %s setuid 0x%x, got 0x%x", path, mode&os.ModeSetuid, fi.Mode()&os.ModeSetuid)
75 
+	}
76 
+
77 
+	if fi.Mode()&os.ModeSetgid != mode&os.ModeSetgid {
78 
+		t.Fatalf("Expected %s setgid 0x%x, got 0x%x", path, mode&os.ModeSetgid, fi.Mode()&os.ModeSetgid)
79 
+	}
80 
+
81 
+	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
82 
+		if stat.Uid != uid {
83 
+			t.Fatalf("%s no owned by uid %d", path, uid)
84 
+		}
85 
+		if stat.Gid != gid {
86 
+			t.Fatalf("%s not owned by gid %d", path, gid)
87 
+		}
88 
+	}
89 
+}
90 
+
91 
+func createBase(t testing.TB, driver graphdriver.Driver, name string) {
92 
+	// We need to be able to set any perms
93 
+	oldmask := syscall.Umask(0)
94 
+	defer syscall.Umask(oldmask)
95 
+
96 
+	if err := driver.CreateReadWrite(name, "", "", nil); err != nil {
97 
+		t.Fatal(err)
98 
+	}
99 
+
100 
+	dir, err := driver.Get(name, "")
101 
+	if err != nil {
102 
+		t.Fatal(err)
103 
+	}
104 
+	defer driver.Put(name)
105 
+
106 
+	subdir := path.Join(dir, "a subdir")
107 
+	if err := os.Mkdir(subdir, 0705|os.ModeSticky); err != nil {
108 
+		t.Fatal(err)
109 
+	}
110 
+	if err := os.Chown(subdir, 1, 2); err != nil {
111 
+		t.Fatal(err)
112 
+	}
113 
+
114 
+	file := path.Join(dir, "a file")
115 
+	if err := ioutil.WriteFile(file, []byte("Some data"), 0222|os.ModeSetuid); err != nil {
116 
+		t.Fatal(err)
117 
+	}
118 
+}
119 
+
120 
+func verifyBase(t testing.TB, driver graphdriver.Driver, name string) {
121 
+	dir, err := driver.Get(name, "")
122 
+	if err != nil {
123 
+		t.Fatal(err)
124 
+	}
125 
+	defer driver.Put(name)
126 
+
127 
+	subdir := path.Join(dir, "a subdir")
128 
+	verifyFile(t, subdir, 0705|os.ModeDir|os.ModeSticky, 1, 2)
129 
+
130 
+	file := path.Join(dir, "a file")
131 
+	verifyFile(t, file, 0222|os.ModeSetuid, 0, 0)
132 
+
133 
+	fis, err := readDir(dir)
134 
+	if err != nil {
135 
+		t.Fatal(err)
136 
+	}
137 
+
138 
+	if len(fis) != 2 {
139 
+		t.Fatal("Unexpected files in base image")
140 
+	}
141 
+
142 
+}