* integration test for preserving hardlinks
Signed-off-by: Vincent Batts <vbatts@redhat.com>
Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
| ... | ... |
@@ -101,6 +101,58 @@ func TestCommitNewFile(t *testing.T) {
|
| 101 | 101 |
logDone("commit - commit file and read")
|
| 102 | 102 |
} |
| 103 | 103 |
|
| 104 |
+func TestCommitHardlink(t *testing.T) {
|
|
| 105 |
+ cmd := exec.Command(dockerBinary, "run", "-t", "--name", "hardlinks", "busybox", "sh", "-c", "touch file1 && ln file1 file2 && ls -di file1 file2") |
|
| 106 |
+ firstOuput, _, err := runCommandWithOutput(cmd) |
|
| 107 |
+ if err != nil {
|
|
| 108 |
+ t.Fatal(err) |
|
| 109 |
+ } |
|
| 110 |
+ |
|
| 111 |
+ chunks := strings.Split(strings.TrimSpace(firstOuput), " ") |
|
| 112 |
+ inode := chunks[0] |
|
| 113 |
+ found := false |
|
| 114 |
+ for _, chunk := range chunks[1:] {
|
|
| 115 |
+ if chunk == inode {
|
|
| 116 |
+ found = true |
|
| 117 |
+ break |
|
| 118 |
+ } |
|
| 119 |
+ } |
|
| 120 |
+ if !found {
|
|
| 121 |
+ t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
|
|
| 122 |
+ } |
|
| 123 |
+ |
|
| 124 |
+ cmd = exec.Command(dockerBinary, "commit", "hardlinks", "hardlinks") |
|
| 125 |
+ imageID, _, err := runCommandWithOutput(cmd) |
|
| 126 |
+ if err != nil {
|
|
| 127 |
+ t.Fatal(imageID, err) |
|
| 128 |
+ } |
|
| 129 |
+ imageID = strings.Trim(imageID, "\r\n") |
|
| 130 |
+ |
|
| 131 |
+ cmd = exec.Command(dockerBinary, "run", "-t", "hardlinks", "ls", "-di", "file1", "file2") |
|
| 132 |
+ secondOuput, _, err := runCommandWithOutput(cmd) |
|
| 133 |
+ if err != nil {
|
|
| 134 |
+ t.Fatal(err) |
|
| 135 |
+ } |
|
| 136 |
+ |
|
| 137 |
+ chunks = strings.Split(strings.TrimSpace(secondOuput), " ") |
|
| 138 |
+ inode = chunks[0] |
|
| 139 |
+ found = false |
|
| 140 |
+ for _, chunk := range chunks[1:] {
|
|
| 141 |
+ if chunk == inode {
|
|
| 142 |
+ found = true |
|
| 143 |
+ break |
|
| 144 |
+ } |
|
| 145 |
+ } |
|
| 146 |
+ if !found {
|
|
| 147 |
+ t.Fatalf("Failed to create hardlink in a container. Expected to find %q in %q", inode, chunks[1:])
|
|
| 148 |
+ } |
|
| 149 |
+ |
|
| 150 |
+ deleteAllContainers() |
|
| 151 |
+ deleteImages(imageID) |
|
| 152 |
+ |
|
| 153 |
+ logDone("commit - commit hardlinks")
|
|
| 154 |
+} |
|
| 155 |
+ |
|
| 104 | 156 |
func TestCommitTTY(t *testing.T) {
|
| 105 | 157 |
cmd := exec.Command(dockerBinary, "run", "-t", "--name", "tty", "busybox", "/bin/ls") |
| 106 | 158 |
if _, err := runCommand(cmd); err != nil {
|
| ... | ... |
@@ -153,7 +153,15 @@ func (compression *Compression) Extension() string {
|
| 153 | 153 |
return "" |
| 154 | 154 |
} |
| 155 | 155 |
|
| 156 |
-func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
|
|
| 156 |
+type tarAppender struct {
|
|
| 157 |
+ TarWriter *tar.Writer |
|
| 158 |
+ Buffer *bufio.Writer |
|
| 159 |
+ |
|
| 160 |
+ // for hardlink mapping |
|
| 161 |
+ SeenFiles map[uint64]string |
|
| 162 |
+} |
|
| 163 |
+ |
|
| 164 |
+func (ta *tarAppender) addTarFile(path, name string) error {
|
|
| 157 | 165 |
fi, err := os.Lstat(path) |
| 158 | 166 |
if err != nil {
|
| 159 | 167 |
return err |
| ... | ... |
@@ -188,13 +196,28 @@ func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
|
| 188 | 188 |
|
| 189 | 189 |
} |
| 190 | 190 |
|
| 191 |
+ // if it's a regular file and has more than 1 link, |
|
| 192 |
+ // it's hardlinked, so set the type flag accordingly |
|
| 193 |
+ if fi.Mode().IsRegular() && stat.Nlink > 1 {
|
|
| 194 |
+ // a link should have a name that it links too |
|
| 195 |
+ // and that linked name should be first in the tar archive |
|
| 196 |
+ ino := uint64(stat.Ino) |
|
| 197 |
+ if oldpath, ok := ta.SeenFiles[ino]; ok {
|
|
| 198 |
+ hdr.Typeflag = tar.TypeLink |
|
| 199 |
+ hdr.Linkname = oldpath |
|
| 200 |
+ hdr.Size = 0 // This Must be here for the writer math to add up! |
|
| 201 |
+ } else {
|
|
| 202 |
+ ta.SeenFiles[ino] = name |
|
| 203 |
+ } |
|
| 204 |
+ } |
|
| 205 |
+ |
|
| 191 | 206 |
capability, _ := system.Lgetxattr(path, "security.capability") |
| 192 | 207 |
if capability != nil {
|
| 193 | 208 |
hdr.Xattrs = make(map[string]string) |
| 194 | 209 |
hdr.Xattrs["security.capability"] = string(capability) |
| 195 | 210 |
} |
| 196 | 211 |
|
| 197 |
- if err := tw.WriteHeader(hdr); err != nil {
|
|
| 212 |
+ if err := ta.TarWriter.WriteHeader(hdr); err != nil {
|
|
| 198 | 213 |
return err |
| 199 | 214 |
} |
| 200 | 215 |
|
| ... | ... |
@@ -204,17 +227,17 @@ func addTarFile(path, name string, tw *tar.Writer, twBuf *bufio.Writer) error {
|
| 204 | 204 |
return err |
| 205 | 205 |
} |
| 206 | 206 |
|
| 207 |
- twBuf.Reset(tw) |
|
| 208 |
- _, err = io.Copy(twBuf, file) |
|
| 207 |
+ ta.Buffer.Reset(ta.TarWriter) |
|
| 208 |
+ _, err = io.Copy(ta.Buffer, file) |
|
| 209 | 209 |
file.Close() |
| 210 | 210 |
if err != nil {
|
| 211 | 211 |
return err |
| 212 | 212 |
} |
| 213 |
- err = twBuf.Flush() |
|
| 213 |
+ err = ta.Buffer.Flush() |
|
| 214 | 214 |
if err != nil {
|
| 215 | 215 |
return err |
| 216 | 216 |
} |
| 217 |
- twBuf.Reset(nil) |
|
| 217 |
+ ta.Buffer.Reset(nil) |
|
| 218 | 218 |
} |
| 219 | 219 |
|
| 220 | 220 |
return nil |
| ... | ... |
@@ -345,9 +368,15 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) |
| 345 | 345 |
return nil, err |
| 346 | 346 |
} |
| 347 | 347 |
|
| 348 |
- tw := tar.NewWriter(compressWriter) |
|
| 349 |
- |
|
| 350 | 348 |
go func() {
|
| 349 |
+ ta := &tarAppender{
|
|
| 350 |
+ TarWriter: tar.NewWriter(compressWriter), |
|
| 351 |
+ Buffer: pools.BufioWriter32KPool.Get(nil), |
|
| 352 |
+ SeenFiles: make(map[uint64]string), |
|
| 353 |
+ } |
|
| 354 |
+ // this buffer is needed for the duration of this piped stream |
|
| 355 |
+ defer pools.BufioWriter32KPool.Put(ta.Buffer) |
|
| 356 |
+ |
|
| 351 | 357 |
// In general we log errors here but ignore them because |
| 352 | 358 |
// during e.g. a diff operation the container can continue |
| 353 | 359 |
// mutating the filesystem and we can see transient errors |
| ... | ... |
@@ -357,9 +386,6 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) |
| 357 | 357 |
options.Includes = []string{"."}
|
| 358 | 358 |
} |
| 359 | 359 |
|
| 360 |
- twBuf := pools.BufioWriter32KPool.Get(nil) |
|
| 361 |
- defer pools.BufioWriter32KPool.Put(twBuf) |
|
| 362 |
- |
|
| 363 | 360 |
var renamedRelFilePath string // For when tar.Options.Name is set |
| 364 | 361 |
for _, include := range options.Includes {
|
| 365 | 362 |
filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error {
|
| ... | ... |
@@ -395,7 +421,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) |
| 395 | 395 |
relFilePath = strings.Replace(relFilePath, renamedRelFilePath, options.Name, 1) |
| 396 | 396 |
} |
| 397 | 397 |
|
| 398 |
- if err := addTarFile(filePath, relFilePath, tw, twBuf); err != nil {
|
|
| 398 |
+ if err := ta.addTarFile(filePath, relFilePath); err != nil {
|
|
| 399 | 399 |
log.Debugf("Can't add file %s to tar: %s", srcPath, err)
|
| 400 | 400 |
} |
| 401 | 401 |
return nil |
| ... | ... |
@@ -403,7 +429,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) |
| 403 | 403 |
} |
| 404 | 404 |
|
| 405 | 405 |
// Make sure to check the error on Close. |
| 406 |
- if err := tw.Close(); err != nil {
|
|
| 406 |
+ if err := ta.TarWriter.Close(); err != nil {
|
|
| 407 | 407 |
log.Debugf("Can't close tar writer: %s", err)
|
| 408 | 408 |
} |
| 409 | 409 |
if err := compressWriter.Close(); err != nil {
|
| ... | ... |
@@ -249,6 +249,64 @@ func TestUntarUstarGnuConflict(t *testing.T) {
|
| 249 | 249 |
} |
| 250 | 250 |
} |
| 251 | 251 |
|
| 252 |
+func TestTarWithHardLink(t *testing.T) {
|
|
| 253 |
+ origin, err := ioutil.TempDir("", "docker-test-tar-hardlink")
|
|
| 254 |
+ if err != nil {
|
|
| 255 |
+ t.Fatal(err) |
|
| 256 |
+ } |
|
| 257 |
+ defer os.RemoveAll(origin) |
|
| 258 |
+ if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
|
|
| 259 |
+ t.Fatal(err) |
|
| 260 |
+ } |
|
| 261 |
+ if err := os.Link(path.Join(origin, "1"), path.Join(origin, "2")); err != nil {
|
|
| 262 |
+ t.Fatal(err) |
|
| 263 |
+ } |
|
| 264 |
+ |
|
| 265 |
+ var i1, i2 uint64 |
|
| 266 |
+ if i1, err = getNlink(path.Join(origin, "1")); err != nil {
|
|
| 267 |
+ t.Fatal(err) |
|
| 268 |
+ } |
|
| 269 |
+ // sanity check that we can hardlink |
|
| 270 |
+ if i1 != 2 {
|
|
| 271 |
+ t.Skipf("skipping since hardlinks don't work here; expected 2 links, got %d", i1)
|
|
| 272 |
+ } |
|
| 273 |
+ |
|
| 274 |
+ dest, err := ioutil.TempDir("", "docker-test-tar-hardlink-dest")
|
|
| 275 |
+ if err != nil {
|
|
| 276 |
+ t.Fatal(err) |
|
| 277 |
+ } |
|
| 278 |
+ defer os.RemoveAll(dest) |
|
| 279 |
+ |
|
| 280 |
+ // we'll do this in two steps to separate failure |
|
| 281 |
+ fh, err := Tar(origin, Uncompressed) |
|
| 282 |
+ if err != nil {
|
|
| 283 |
+ t.Fatal(err) |
|
| 284 |
+ } |
|
| 285 |
+ |
|
| 286 |
+ // ensure we can read the whole thing with no error, before writing back out |
|
| 287 |
+ buf, err := ioutil.ReadAll(fh) |
|
| 288 |
+ if err != nil {
|
|
| 289 |
+ t.Fatal(err) |
|
| 290 |
+ } |
|
| 291 |
+ |
|
| 292 |
+ bRdr := bytes.NewReader(buf) |
|
| 293 |
+ err = Untar(bRdr, dest, &TarOptions{Compression: Uncompressed})
|
|
| 294 |
+ if err != nil {
|
|
| 295 |
+ t.Fatal(err) |
|
| 296 |
+ } |
|
| 297 |
+ |
|
| 298 |
+ if i1, err = getInode(path.Join(dest, "1")); err != nil {
|
|
| 299 |
+ t.Fatal(err) |
|
| 300 |
+ } |
|
| 301 |
+ if i2, err = getInode(path.Join(dest, "2")); err != nil {
|
|
| 302 |
+ t.Fatal(err) |
|
| 303 |
+ } |
|
| 304 |
+ |
|
| 305 |
+ if i1 != i2 {
|
|
| 306 |
+ t.Errorf("expected matching inodes, but got %d and %d", i1, i2)
|
|
| 307 |
+ } |
|
| 308 |
+} |
|
| 309 |
+ |
|
| 252 | 310 |
func getNlink(path string) (uint64, error) {
|
| 253 | 311 |
stat, err := os.Stat(path) |
| 254 | 312 |
if err != nil {
|
| ... | ... |
@@ -368,11 +368,15 @@ func minor(device uint64) uint64 {
|
| 368 | 368 |
// ExportChanges produces an Archive from the provided changes, relative to dir. |
| 369 | 369 |
func ExportChanges(dir string, changes []Change) (Archive, error) {
|
| 370 | 370 |
reader, writer := io.Pipe() |
| 371 |
- tw := tar.NewWriter(writer) |
|
| 372 |
- |
|
| 373 | 371 |
go func() {
|
| 374 |
- twBuf := pools.BufioWriter32KPool.Get(nil) |
|
| 375 |
- defer pools.BufioWriter32KPool.Put(twBuf) |
|
| 372 |
+ ta := &tarAppender{
|
|
| 373 |
+ TarWriter: tar.NewWriter(writer), |
|
| 374 |
+ Buffer: pools.BufioWriter32KPool.Get(nil), |
|
| 375 |
+ SeenFiles: make(map[uint64]string), |
|
| 376 |
+ } |
|
| 377 |
+ // this buffer is needed for the duration of this piped stream |
|
| 378 |
+ defer pools.BufioWriter32KPool.Put(ta.Buffer) |
|
| 379 |
+ |
|
| 376 | 380 |
// In general we log errors here but ignore them because |
| 377 | 381 |
// during e.g. a diff operation the container can continue |
| 378 | 382 |
// mutating the filesystem and we can see transient errors |
| ... | ... |
@@ -390,19 +394,19 @@ func ExportChanges(dir string, changes []Change) (Archive, error) {
|
| 390 | 390 |
AccessTime: timestamp, |
| 391 | 391 |
ChangeTime: timestamp, |
| 392 | 392 |
} |
| 393 |
- if err := tw.WriteHeader(hdr); err != nil {
|
|
| 393 |
+ if err := ta.TarWriter.WriteHeader(hdr); err != nil {
|
|
| 394 | 394 |
log.Debugf("Can't write whiteout header: %s", err)
|
| 395 | 395 |
} |
| 396 | 396 |
} else {
|
| 397 | 397 |
path := filepath.Join(dir, change.Path) |
| 398 |
- if err := addTarFile(path, change.Path[1:], tw, twBuf); err != nil {
|
|
| 398 |
+ if err := ta.addTarFile(path, change.Path[1:]); err != nil {
|
|
| 399 | 399 |
log.Debugf("Can't add file %s to tar: %s", path, err)
|
| 400 | 400 |
} |
| 401 | 401 |
} |
| 402 | 402 |
} |
| 403 | 403 |
|
| 404 | 404 |
// Make sure to check the error on Close. |
| 405 |
- if err := tw.Close(); err != nil {
|
|
| 405 |
+ if err := ta.TarWriter.Close(); err != nil {
|
|
| 406 | 406 |
log.Debugf("Can't close layer: %s", err)
|
| 407 | 407 |
} |
| 408 | 408 |
writer.Close() |