Browse code
Fix error handling with not-exist errors on remove
Specifically, none of the graphdrivers are supposed to return a
not-exist type of error on remove (or at least that's how they are
currently handled).
Found that AUFS still had one case where a not-exist error could escape,
when checking if the directory is mounted we call a `Statfs` on the
path.
This fixes AUFS to not return an error in this case, but also
double-checks at the daemon level on layer remove that the error is not
a `not-exist` type of error.
Signed-off-by: Brian Goff <cpuguy83@gmail.com>
Brian Goff authored on 2017/07/05 23:22:57Showing 2 changed files
| ... | ... |
@@ -119,7 +119,7 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo |
| 119 | 119 |
if container.RWLayer != nil {
|
| 120 | 120 |
metadata, err := daemon.stores[container.Platform].layerStore.ReleaseRWLayer(container.RWLayer) |
| 121 | 121 |
layer.LogReleaseMetadata(metadata) |
| 122 |
- if err != nil && err != layer.ErrMountDoesNotExist {
|
|
| 122 |
+ if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) {
|
|
| 123 | 123 |
return errors.Wrapf(err, "driver %q failed to remove root filesystem for %s", daemon.GraphDriverName(container.Platform), container.ID) |
| 124 | 124 |
} |
| 125 | 125 |
} |
| ... | ... |
@@ -46,6 +46,7 @@ import ( |
| 46 | 46 |
"github.com/docker/docker/pkg/system" |
| 47 | 47 |
rsystem "github.com/opencontainers/runc/libcontainer/system" |
| 48 | 48 |
"github.com/opencontainers/selinux/go-selinux/label" |
| 49 |
+ "github.com/pkg/errors" |
|
| 49 | 50 |
"github.com/vbatts/tar-split/tar/storage" |
| 50 | 51 |
"golang.org/x/sys/unix" |
| 51 | 52 |
) |
| ... | ... |
@@ -282,30 +283,41 @@ func (a *Driver) Remove(id string) error {
|
| 282 | 282 |
mountpoint = a.getMountpoint(id) |
| 283 | 283 |
} |
| 284 | 284 |
|
| 285 |
+ logger := logrus.WithFields(logrus.Fields{
|
|
| 286 |
+ "module": "graphdriver", |
|
| 287 |
+ "driver": "aufs", |
|
| 288 |
+ "layer": id, |
|
| 289 |
+ }) |
|
| 290 |
+ |
|
| 285 | 291 |
var retries int |
| 286 | 292 |
for {
|
| 287 | 293 |
mounted, err := a.mounted(mountpoint) |
| 288 | 294 |
if err != nil {
|
| 295 |
+ if os.IsNotExist(err) {
|
|
| 296 |
+ break |
|
| 297 |
+ } |
|
| 289 | 298 |
return err |
| 290 | 299 |
} |
| 291 | 300 |
if !mounted {
|
| 292 | 301 |
break |
| 293 | 302 |
} |
| 294 | 303 |
|
| 295 |
- if err := a.unmount(mountpoint); err != nil {
|
|
| 296 |
- if err != unix.EBUSY {
|
|
| 297 |
- return fmt.Errorf("aufs: unmount error: %s: %v", mountpoint, err)
|
|
| 298 |
- } |
|
| 299 |
- if retries >= 5 {
|
|
| 300 |
- return fmt.Errorf("aufs: unmount error after retries: %s: %v", mountpoint, err)
|
|
| 301 |
- } |
|
| 302 |
- // If unmount returns EBUSY, it could be a transient error. Sleep and retry. |
|
| 303 |
- retries++ |
|
| 304 |
- logrus.Warnf("unmount failed due to EBUSY: retry count: %d", retries)
|
|
| 305 |
- time.Sleep(100 * time.Millisecond) |
|
| 306 |
- continue |
|
| 304 |
+ err = a.unmount(mountpoint) |
|
| 305 |
+ if err == nil {
|
|
| 306 |
+ break |
|
| 307 | 307 |
} |
| 308 |
- break |
|
| 308 |
+ |
|
| 309 |
+ if err != unix.EBUSY {
|
|
| 310 |
+ return errors.Wrapf(err, "aufs: unmount error: %s", mountpoint) |
|
| 311 |
+ } |
|
| 312 |
+ if retries >= 5 {
|
|
| 313 |
+ return errors.Wrapf(err, "aufs: unmount error after retries: %s", mountpoint) |
|
| 314 |
+ } |
|
| 315 |
+ // If unmount returns EBUSY, it could be a transient error. Sleep and retry. |
|
| 316 |
+ retries++ |
|
| 317 |
+ logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries)
|
|
| 318 |
+ time.Sleep(100 * time.Millisecond) |
|
| 319 |
+ continue |
|
| 309 | 320 |
} |
| 310 | 321 |
|
| 311 | 322 |
// Atomically remove each directory in turn by first moving it out of the |
| ... | ... |
@@ -314,21 +326,22 @@ func (a *Driver) Remove(id string) error {
|
| 314 | 314 |
tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id))
|
| 315 | 315 |
if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) {
|
| 316 | 316 |
if err == unix.EBUSY {
|
| 317 |
- logrus.Warn("os.Rename err due to EBUSY")
|
|
| 317 |
+ logger.WithField("dir", mountpoint).WithError(err).Warn("os.Rename err due to EBUSY")
|
|
| 318 | 318 |
} |
| 319 |
- return err |
|
| 319 |
+ return errors.Wrapf(err, "error preparing atomic delete of aufs mountpoint for id: %s", id) |
|
| 320 |
+ } |
|
| 321 |
+ if err := system.EnsureRemoveAll(tmpMntPath); err != nil {
|
|
| 322 |
+ return errors.Wrapf(err, "error removing aufs layer %s", id) |
|
| 320 | 323 |
} |
| 321 |
- defer system.EnsureRemoveAll(tmpMntPath) |
|
| 322 | 324 |
|
| 323 | 325 |
tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id))
|
| 324 | 326 |
if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) {
|
| 325 |
- return err |
|
| 327 |
+ return errors.Wrapf(err, "error preparing atomic delete of aufs diff dir for id: %s", id) |
|
| 326 | 328 |
} |
| 327 |
- defer system.EnsureRemoveAll(tmpDiffpath) |
|
| 328 | 329 |
|
| 329 | 330 |
// Remove the layers file for the id |
| 330 | 331 |
if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) {
|
| 331 |
- return err |
|
| 332 |
+ return errors.Wrapf(err, "error removing layers dir for %s", id) |
|
| 332 | 333 |
} |
| 333 | 334 |
|
| 334 | 335 |
a.pathCacheLock.Lock() |