Browse code

archive: preserve hardlinks in Tar and Untar

* integration test for preserving hardlinks

Signed-off-by: Vincent Batts <vbatts@redhat.com>
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>

Vincent Batts authored on 2014/09/16 03:45:53
Showing 4 changed files
... ...
@@ -101,6 +101,58 @@ func TestCommitNewFile(t *testing.T) {
101 101
 	logDone("commit - commit file and read")
102 102
 }
103 103
 
104
+func TestCommitHardlink(t *testing.T) {
105
+	cmd := exec.Command(dockerBinary, "run", "-t", "--name", "hardlinks", "busybox", "sh", "-c", "touch file1 && ln file1 file2 && ls -di file1 file2")
106
+	firstOuput, _, err := runCommandWithOutput(cmd)
107
+	if err != nil {
108
+		t.Fatal(err)
109
+	}
110
+
111
+	chunks := strings.Split(strings.TrimSpace(firstOuput), " ")
112
+	inode := chunks[0]
113
+	found := false
114
+	for _, chunk := range chunks[1:] {
115
+		if chunk == inode {
116
+			found = true
117
+			break
118
+		}
119
+	}
120
+	if !found {
121
+		t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
122
+	}
123
+
124
+	cmd = exec.Command(dockerBinary, "commit", "hardlinks", "hardlinks")
125
+	imageID, _, err := runCommandWithOutput(cmd)
126
+	if err != nil {
127
+		t.Fatal(imageID, err)
128
+	}
129
+	imageID = strings.Trim(imageID, "\r\n")
130
+
131
+	cmd = exec.Command(dockerBinary, "run", "-t", "hardlinks", "ls", "-di", "file1", "file2")
132
+	secondOuput, _, err := runCommandWithOutput(cmd)
133
+	if err != nil {
134
+		t.Fatal(err)
135
+	}
136
+
137
+	chunks = strings.Split(strings.TrimSpace(secondOuput), " ")
138
+	inode = chunks[0]
139
+	found = false
140
+	for _, chunk := range chunks[1:] {
141
+		if chunk == inode {
142
+			found = true
143
+			break
144
+		}
145
+	}
146
+	if !found {
147
+		t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
148
+	}
149
+
150
+	deleteAllContainers()
151
+	deleteImages(imageID)
152
+
153
+	logDone("commit - commit hardlinks")
154
+}
155
+
104 156
 func TestCommitTTY(t *testing.T) {
105 157
 	cmd := exec.Command(dockerBinary, "run", "-t", "--name", "tty", "busybox", "/bin/ls")
106 158
 	if _, err := runCommand(cmd); err != nil {
... ...
@@ -153,7 +153,15 @@ func (compression *Compression) Extension() string {
153 153
 	return ""
154 154
 }
155 155
 
156
-func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
156
+type tarAppender struct {
157
+	TarWriter *tar.Writer
158
+	Buffer    *bufio.Writer
159
+
160
+	// for hardlink mapping
161
+	SeenFiles map[uint64]string
162
+}
163
+
164
+func (ta *tarAppender) addTarFile(path, name string) error {
157 165
 	fi, err := os.Lstat(path)
158 166
 	if err != nil {
159 167
 		return err
... ...
@@ -188,13 +196,28 @@ func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
188 188
 
189 189
 	}
190 190
 
191
+	// if it's a regular file and has more than 1 link,
192
+	// it's hardlinked, so set the type flag accordingly
193
+	if fi.Mode().IsRegular() && stat.Nlink > 1 {
194
+		// a link should have a name that it links too
195
+		// and that linked name should be first in the tar archive
196
+		ino := uint64(stat.Ino)
197
+		if oldpath, ok := ta.SeenFiles[ino]; ok {
198
+			hdr.Typeflag = tar.TypeLink
199
+			hdr.Linkname = oldpath
200
+			hdr.Size = 0 // This Must be here for the writer math to add up!
201
+		} else {
202
+			ta.SeenFiles[ino] = name
203
+		}
204
+	}
205
+
191 206
 	capability, _ := system.Lgetxattr(path, "security.capability")
192 207
 	if capability != nil {
193 208
 		hdr.Xattrs = make(map[string]string)
194 209
 		hdr.Xattrs["security.capability"] = string(capability)
195 210
 	}
196 211
 
197
-	if err := tw.WriteHeader(hdr); err != nil {
212
+	if err := ta.TarWriter.WriteHeader(hdr); err != nil {
198 213
 		return err
199 214
 	}
200 215
 
... ...
@@ -204,17 +227,17 @@ func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
204 204
 			return err
205 205
 		}
206 206
 
207
-		twBuf.Reset(tw)
208
-		_, err = io.Copy(twBuf, file)
207
+		ta.Buffer.Reset(ta.TarWriter)
208
+		_, err = io.Copy(ta.Buffer, file)
209 209
 		file.Close()
210 210
 		if err != nil {
211 211
 			return err
212 212
 		}
213
-		err = twBuf.Flush()
213
+		err = ta.Buffer.Flush()
214 214
 		if err != nil {
215 215
 			return err
216 216
 		}
217
-		twBuf.Reset(nil)
217
+		ta.Buffer.Reset(nil)
218 218
 	}
219 219
 
220 220
 	return nil
... ...
@@ -345,9 +368,15 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
345 345
 		return nil, err
346 346
 	}
347 347
 
348
-	tw := tar.NewWriter(compressWriter)
349
-
350 348
 	go func() {
349
+		ta := &tarAppender{
350
+			TarWriter: tar.NewWriter(compressWriter),
351
+			Buffer:    pools.BufioWriter32KPool.Get(nil),
352
+			SeenFiles: make(map[uint64]string),
353
+		}
354
+		// this buffer is needed for the duration of this piped stream
355
+		defer pools.BufioWriter32KPool.Put(ta.Buffer)
356
+
351 357
 		// In general we log errors here but ignore them because
352 358
 		// during e.g. a diff operation the container can continue
353 359
 		// mutating the filesystem and we can see transient errors
... ...
@@ -357,9 +386,6 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
357 357
 			options.Includes = []string{"."}
358 358
 		}
359 359
 
360
-		twBuf := pools.BufioWriter32KPool.Get(nil)
361
-		defer pools.BufioWriter32KPool.Put(twBuf)
362
-
363 360
 		var renamedRelFilePath string // For when tar.Options.Name is set
364 361
 		for _, include := range options.Includes {
365 362
 			filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error {
... ...
@@ -395,7 +421,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
395 395
 					relFilePath = strings.Replace(relFilePath, renamedRelFilePath, options.Name, 1)
396 396
 				}
397 397
 
398
-				if err := addTarFile(filePath, relFilePath, tw, twBuf); err != nil {
398
+				if err := ta.addTarFile(filePath, relFilePath); err != nil {
399 399
 					log.Debugf("Can't add file %s to tar: %s", srcPath, err)
400 400
 				}
401 401
 				return nil
... ...
@@ -403,7 +429,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
403 403
 		}
404 404
 
405 405
 		// Make sure to check the error on Close.
406
-		if err := tw.Close(); err != nil {
406
+		if err := ta.TarWriter.Close(); err != nil {
407 407
 			log.Debugf("Can't close tar writer: %s", err)
408 408
 		}
409 409
 		if err := compressWriter.Close(); err != nil {
... ...
@@ -249,6 +249,64 @@ func TestUntarUstarGnuConflict(t *testing.T) {
249 249
 	}
250 250
 }
251 251
 
252
+func TestTarWithHardLink(t *testing.T) {
253
+	origin, err := ioutil.TempDir("", "docker-test-tar-hardlink")
254
+	if err != nil {
255
+		t.Fatal(err)
256
+	}
257
+	defer os.RemoveAll(origin)
258
+	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
259
+		t.Fatal(err)
260
+	}
261
+	if err := os.Link(path.Join(origin, "1"), path.Join(origin, "2")); err != nil {
262
+		t.Fatal(err)
263
+	}
264
+
265
+	var i1, i2 uint64
266
+	if i1, err = getNlink(path.Join(origin, "1")); err != nil {
267
+		t.Fatal(err)
268
+	}
269
+	// sanity check that we can hardlink
270
+	if i1 != 2 {
271
+		t.Skipf("skipping since hardlinks don't work here; expected 2 links, got %d", i1)
272
+	}
273
+
274
+	dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest")
275
+	if err != nil {
276
+		t.Fatal(err)
277
+	}
278
+	defer os.RemoveAll(dest)
279
+
280
+	// we'll do this in two steps to separate failure
281
+	fh, err := Tar(origin, Uncompressed)
282
+	if err != nil {
283
+		t.Fatal(err)
284
+	}
285
+
286
+	// ensure we can read the whole thing with no error, before writing back out
287
+	buf, err := ioutil.ReadAll(fh)
288
+	if err != nil {
289
+		t.Fatal(err)
290
+	}
291
+
292
+	bRdr := bytes.NewReader(buf)
293
+	err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed})
294
+	if err != nil {
295
+		t.Fatal(err)
296
+	}
297
+
298
+	if i1, err = getInode(path.Join(dest, "1")); err != nil {
299
+		t.Fatal(err)
300
+	}
301
+	if i2, err = getInode(path.Join(dest, "2")); err != nil {
302
+		t.Fatal(err)
303
+	}
304
+
305
+	if i1 != i2 {
306
+		t.Errorf("expected matching inodes, but got %d and %d", i1, i2)
307
+	}
308
+}
309
+
252 310
 func getNlink(path string) (uint64, error) {
253 311
 	stat, err := os.Stat(path)
254 312
 	if err != nil {
... ...
@@ -368,11 +368,15 @@ func minor(device uint64) uint64 {
368 368
 // ExportChanges produces an Archive from the provided changes, relative to dir.
369 369
 func ExportChanges(dir string, changes []Change) (Archive, error) {
370 370
 	reader, writer := io.Pipe()
371
-	tw := tar.NewWriter(writer)
372
-
373 371
 	go func() {
374
-		twBuf := pools.BufioWriter32KPool.Get(nil)
375
-		defer pools.BufioWriter32KPool.Put(twBuf)
372
+		ta := &tarAppender{
373
+			TarWriter: tar.NewWriter(writer),
374
+			Buffer:    pools.BufioWriter32KPool.Get(nil),
375
+			SeenFiles: make(map[uint64]string),
376
+		}
377
+		// this buffer is needed for the duration of this piped stream
378
+		defer pools.BufioWriter32KPool.Put(ta.Buffer)
379
+
376 380
 		// In general we log errors here but ignore them because
377 381
 		// during e.g. a diff operation the container can continue
378 382
 		// mutating the filesystem and we can see transient errors
... ...
@@ -390,19 +394,19 @@ func ExportChanges(dir string, changes []Change) (Archive, error) {
390 390
 					AccessTime: timestamp,
391 391
 					ChangeTime: timestamp,
392 392
 				}
393
-				if err := tw.WriteHeader(hdr); err != nil {
393
+				if err := ta.TarWriter.WriteHeader(hdr); err != nil {
394 394
 					log.Debugf("Can't write whiteout header: %s", err)
395 395
 				}
396 396
 			} else {
397 397
 				path := filepath.Join(dir, change.Path)
398
-				if err := addTarFile(path, change.Path[1:], tw, twBuf); err != nil {
398
+				if err := ta.addTarFile(path, change.Path[1:]); err != nil {
399 399
 					log.Debugf("Can't add file %s to tar: %s", path, err)
400 400
 				}
401 401
 			}
402 402
 		}
403 403
 
404 404
 		// Make sure to check the error on Close.
405
-		if err := tw.Close(); err != nil {
405
+		if err := ta.TarWriter.Close(); err != nil {
406 406
 			log.Debugf("Can't close layer: %s", err)
407 407
 		}
408 408
 		writer.Close()