Browse code

Merge pull request #34948 from euank/public-mounts

Fix EBUSY errors under overlayfs and v4.13+ kernels

Akihiro Suda authored on 2017/11/29 12:48:39
Showing 2 changed files
... ...
@@ -109,8 +109,10 @@ func init() {
109 109
 }
110 110
 
111 111
 // Init returns the NaiveDiffDriver, a native diff driver for overlay filesystem.
112
-// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
113
-// If an overlay filesystem is not supported over an existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
112
+// If overlay filesystem is not supported on the host, the error
113
+// graphdriver.ErrNotSupported is returned.
114
+// If an overlay filesystem is not supported over an existing filesystem then
115
+// error graphdriver.ErrIncompatibleFS is returned.
114 116
 func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
115 117
 
116 118
 	if err := supportsOverlay(); err != nil {
... ...
@@ -199,7 +201,8 @@ func (d *Driver) Status() [][2]string {
199 199
 	}
200 200
 }
201 201
 
202
-// GetMetadata returns meta data about the overlay driver such as root, LowerDir, UpperDir, WorkDir and MergeDir used to store data.
202
+// GetMetadata returns metadata about the overlay driver such as root,
203
+// LowerDir, UpperDir, WorkDir and MergeDir used to store data.
203 204
 func (d *Driver) GetMetadata(id string) (map[string]string, error) {
204 205
 	dir := d.dir(id)
205 206
 	if _, err := os.Stat(dir); err != nil {
... ...
@@ -293,9 +296,6 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
293 293
 		if err := idtools.MkdirAndChown(path.Join(dir, "work"), 0700, root); err != nil {
294 294
 			return err
295 295
 		}
296
-		if err := idtools.MkdirAndChown(path.Join(dir, "merged"), 0700, root); err != nil {
297
-			return err
298
-		}
299 296
 		if err := ioutil.WriteFile(path.Join(dir, "lower-id"), []byte(parent), 0666); err != nil {
300 297
 			return err
301 298
 		}
... ...
@@ -326,9 +326,6 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
326 326
 	if err := idtools.MkdirAndChown(path.Join(dir, "work"), 0700, root); err != nil {
327 327
 		return err
328 328
 	}
329
-	if err := idtools.MkdirAndChown(path.Join(dir, "merged"), 0700, root); err != nil {
330
-		return err
331
-	}
332 329
 
333 330
 	return copy.DirCopy(parentUpperDir, upperDir, copy.Content)
334 331
 }
... ...
@@ -357,6 +354,7 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, err erro
357 357
 	if _, err := os.Stat(rootDir); err == nil {
358 358
 		return containerfs.NewLocalContainerFS(rootDir), nil
359 359
 	}
360
+
360 361
 	mergedDir := path.Join(dir, "merged")
361 362
 	if count := d.ctr.Increment(mergedDir); count > 1 {
362 363
 		return containerfs.NewLocalContainerFS(mergedDir), nil
... ...
@@ -364,7 +362,13 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, err erro
364 364
 	defer func() {
365 365
 		if err != nil {
366 366
 			if c := d.ctr.Decrement(mergedDir); c <= 0 {
367
-				unix.Unmount(mergedDir, 0)
367
+				if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil {
368
+					logrus.Debugf("Failed to unmount %s: %v: %v", id, mntErr, err)
369
+				}
370
+				// Cleanup the created merged directory; see the comment in Put's rmdir
371
+				if rmErr := unix.Rmdir(mergedDir); rmErr != nil && !os.IsNotExist(rmErr) {
372
+					logrus.Warnf("Failed to remove %s: %v: %v", id, rmErr, err)
373
+				}
368 374
 			}
369 375
 		}
370 376
 	}()
... ...
@@ -372,6 +376,13 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, err erro
372 372
 	if err != nil {
373 373
 		return nil, err
374 374
 	}
375
+	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
376
+	if err != nil {
377
+		return nil, err
378
+	}
379
+	if err := idtools.MkdirAndChown(mergedDir, 0700, idtools.IDPair{rootUID, rootGID}); err != nil {
380
+		return nil, err
381
+	}
375 382
 	var (
376 383
 		lowerDir = path.Join(d.dir(string(lowerID)), "root")
377 384
 		upperDir = path.Join(dir, "upper")
... ...
@@ -383,10 +394,6 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, err erro
383 383
 	}
384 384
 	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
385 385
 	// user namespace requires this to move a directory from lower to upper.
386
-	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
387
-	if err != nil {
388
-		return nil, err
389
-	}
390 386
 	if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil {
391 387
 		return nil, err
392 388
 	}
... ...
@@ -394,6 +401,8 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, err erro
394 394
 }
395 395
 
396 396
 // Put unmounts the mount path created for the give id.
397
+// It also removes the 'merged' directory to force the kernel to unmount the
398
+// overlay mount in other namespaces.
397 399
 func (d *Driver) Put(id string) error {
398 400
 	d.locker.Lock(id)
399 401
 	defer d.locker.Unlock(id)
... ...
@@ -408,6 +417,17 @@ func (d *Driver) Put(id string) error {
408 408
 	if err := unix.Unmount(mountpoint, unix.MNT_DETACH); err != nil {
409 409
 		logrus.Debugf("Failed to unmount %s overlay: %v", id, err)
410 410
 	}
411
+
412
+	// Remove the mountpoint here. Removing the mountpoint (in newer kernels)
413
+	// will cause all other instances of this mount in other mount namespaces
414
+	// to be unmounted. This is necessary to avoid cases where an overlay mount
415
+	// that is present in another namespace will cause subsequent mounts
416
+	// operations to fail with ebusy.  We ignore any errors here because this may
417
+	// fail on older kernels which don't have
418
+	// torvalds/linux@8ed936b5671bfb33d89bc60bdcc7cf0470ba52fe applied.
419
+	if err := unix.Rmdir(mountpoint); err != nil {
420
+		logrus.Debugf("Failed to remove %s overlay: %v", id, err)
421
+	}
411 422
 	return nil
412 423
 }
413 424
 
... ...
@@ -77,7 +77,7 @@ const (
77 77
 	maxDepth   = 128
78 78
 
79 79
 	// idLength represents the number of random characters
80
-	// which can be used to create the unique link identifer
80
+	// which can be used to create the unique link identifier
81 81
 	// for every layer. If this value is too long then the
82 82
 	// page size limit for the mount command may be exceeded.
83 83
 	// The idLength should be selected such that following equation
... ...
@@ -91,7 +91,8 @@ type overlayOptions struct {
91 91
 	quota               quota.Quota
92 92
 }
93 93
 
94
-// Driver contains information about the home directory and the list of active mounts that are created using this driver.
94
+// Driver contains information about the home directory and the list of active
95
+// mounts that are created using this driver.
95 96
 type Driver struct {
96 97
 	home          string
97 98
 	uidMaps       []idtools.IDMap
... ...
@@ -116,9 +117,11 @@ func init() {
116 116
 	graphdriver.Register(driverName, Init)
117 117
 }
118 118
 
119
-// Init returns the a native diff driver for overlay filesystem.
120
-// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
121
-// If an overlay filesystem is not supported over an existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
119
+// Init returns the native diff driver for overlay filesystem.
120
+// If overlay filesystem is not supported on the host, the error
121
+// graphdriver.ErrNotSupported is returned.
122
+// If an overlay filesystem is not supported over an existing filesystem then
123
+// the error graphdriver.ErrIncompatibleFS is returned.
122 124
 func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
123 125
 	opts, err := parseOptions(options)
124 126
 	if err != nil {
... ...
@@ -289,8 +292,8 @@ func (d *Driver) Status() [][2]string {
289 289
 	}
290 290
 }
291 291
 
292
-// GetMetadata returns meta data about the overlay driver such as
293
-// LowerDir, UpperDir, WorkDir and MergeDir used to store data.
292
+// GetMetadata returns metadata about the overlay driver such as the LowerDir,
293
+// UpperDir, WorkDir, and MergeDir used to store data.
294 294
 func (d *Driver) GetMetadata(id string) (map[string]string, error) {
295 295
 	dir := d.dir(id)
296 296
 	if _, err := os.Stat(dir); err != nil {
... ...
@@ -414,9 +417,6 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr
414 414
 	if err := idtools.MkdirAndChown(path.Join(dir, "work"), 0700, root); err != nil {
415 415
 		return err
416 416
 	}
417
-	if err := idtools.MkdirAndChown(path.Join(dir, "merged"), 0700, root); err != nil {
418
-		return err
419
-	}
420 417
 
421 418
 	lower, err := d.getLower(parent)
422 419
 	if err != nil {
... ...
@@ -545,6 +545,10 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr e
545 545
 				if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil {
546 546
 					logrus.Errorf("error unmounting %v: %v", mergedDir, mntErr)
547 547
 				}
548
+				// Cleanup the created merged directory; see the comment in Put's rmdir
549
+				if rmErr := unix.Rmdir(mergedDir); rmErr != nil && !os.IsNotExist(rmErr) {
550
+					logrus.Debugf("Failed to remove %s: %v: %v", id, rmErr, err)
551
+				}
548 552
 			}
549 553
 		}
550 554
 	}()
... ...
@@ -560,6 +564,14 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr e
560 560
 	mount := unix.Mount
561 561
 	mountTarget := mergedDir
562 562
 
563
+	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
564
+	if err != nil {
565
+		return nil, err
566
+	}
567
+	if err := idtools.MkdirAndChown(mergedDir, 0700, idtools.IDPair{rootUID, rootGID}); err != nil {
568
+		return nil, err
569
+	}
570
+
563 571
 	pageSize := unix.Getpagesize()
564 572
 
565 573
 	// Go can return a larger page size than supported by the system
... ...
@@ -594,11 +606,6 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr e
594 594
 
595 595
 	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
596 596
 	// user namespace requires this to move a directory from lower to upper.
597
-	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
598
-	if err != nil {
599
-		return nil, err
600
-	}
601
-
602 597
 	if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil {
603 598
 		return nil, err
604 599
 	}
... ...
@@ -607,6 +614,8 @@ func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, retErr e
607 607
 }
608 608
 
609 609
 // Put unmounts the mount path created for the give id.
610
+// It also removes the 'merged' directory to force the kernel to unmount the
611
+// overlay mount in other namespaces.
610 612
 func (d *Driver) Put(id string) error {
611 613
 	d.locker.Lock(id)
612 614
 	defer d.locker.Unlock(id)
... ...
@@ -627,6 +636,16 @@ func (d *Driver) Put(id string) error {
627 627
 	if err := unix.Unmount(mountpoint, unix.MNT_DETACH); err != nil {
628 628
 		logrus.Debugf("Failed to unmount %s overlay: %s - %v", id, mountpoint, err)
629 629
 	}
630
+	// Remove the mountpoint here. Removing the mountpoint (in newer kernels)
631
+	// will cause all other instances of this mount in other mount namespaces
632
+	// to be unmounted. This is necessary to avoid cases where an overlay mount
633
+	// that is present in another namespace will cause subsequent mounts
634
+	// operations to fail with ebusy.  We ignore any errors here because this may
635
+	// fail on older kernels which don't have
636
+	// torvalds/linux@8ed936b5671bfb33d89bc60bdcc7cf0470ba52fe applied.
637
+	if err := unix.Rmdir(mountpoint); err != nil && !os.IsNotExist(err) {
638
+		logrus.Debugf("Failed to remove %s overlay: %v", id, err)
639
+	}
630 640
 	return nil
631 641
 }
632 642
 
... ...
@@ -636,7 +655,8 @@ func (d *Driver) Exists(id string) bool {
636 636
 	return err == nil
637 637
 }
638 638
 
639
-// isParent returns if the passed in parent is the direct parent of the passed in layer
639
+// isParent determines whether the given parent is the direct parent of the
640
+// given layer id
640 641
 func (d *Driver) isParent(id, parent string) bool {
641 642
 	lowers, err := d.getLowerDirs(id)
642 643
 	if err != nil {
... ...
@@ -711,8 +731,8 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
711 711
 	})
712 712
 }
713 713
 
714
-// Changes produces a list of changes between the specified layer
715
-// and its parent layer. If parent is "", then all changes will be ADD changes.
714
+// Changes produces a list of changes between the specified layer and its
715
+// parent layer. If parent is "", then all changes will be ADD changes.
716 716
 func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
717 717
 	if useNaiveDiff(d.home) || !d.isParent(id, parent) {
718 718
 		return d.naiveDiff.Changes(id, parent)