For some reason we seem to get transient EBUSY when removing
thinp devices, which prohibit removing containers. When
this happens we retry a few times which seems to fix the
issue for me.
Docker-DCO-1.1-Signed-off-by: Alexander Larsson <alexl@redhat.com> (github: alexlarsson)
| ... | ... |
@@ -12,6 +12,7 @@ import ( |
| 12 | 12 |
"path" |
| 13 | 13 |
"path/filepath" |
| 14 | 14 |
"strconv" |
| 15 |
+ "strings" |
|
| 15 | 16 |
"sync" |
| 16 | 17 |
"time" |
| 17 | 18 |
) |
| ... | ... |
@@ -52,6 +53,7 @@ type DeviceSet struct {
|
| 52 | 52 |
TransactionId uint64 |
| 53 | 53 |
NewTransactionId uint64 |
| 54 | 54 |
nextFreeDevice int |
| 55 |
+ sawBusy bool |
|
| 55 | 56 |
} |
| 56 | 57 |
|
| 57 | 58 |
type DiskUsage struct {
|
| ... | ... |
@@ -371,6 +373,10 @@ func (devices *DeviceSet) log(level int, file string, line int, dmError int, mes |
| 371 | 371 |
return // Ignore _LOG_DEBUG |
| 372 | 372 |
} |
| 373 | 373 |
|
| 374 |
+ if strings.Contains(message, "busy") {
|
|
| 375 |
+ devices.sawBusy = true |
|
| 376 |
+ } |
|
| 377 |
+ |
|
| 374 | 378 |
utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
|
| 375 | 379 |
} |
| 376 | 380 |
|
| ... | ... |
@@ -660,9 +666,26 @@ func (devices *DeviceSet) deactivateDevice(hash string) error {
|
| 660 | 660 |
// Issues the underlying dm remove operation and then waits |
| 661 | 661 |
// for it to finish. |
| 662 | 662 |
func (devices *DeviceSet) removeDeviceAndWait(devname string) error {
|
| 663 |
- if err := removeDevice(devname); err != nil {
|
|
| 663 |
+ var err error |
|
| 664 |
+ |
|
| 665 |
+ for i := 0; i < 10; i++ {
|
|
| 666 |
+ devices.sawBusy = false |
|
| 667 |
+ err = removeDevice(devname) |
|
| 668 |
+ if err == nil {
|
|
| 669 |
+ break |
|
| 670 |
+ } |
|
| 671 |
+ if !devices.sawBusy {
|
|
| 672 |
+ return err |
|
| 673 |
+ } |
|
| 674 |
+ |
|
| 675 |
+ // If we see EBUSY it may be a transient error, |
|
| 676 |
+ // sleep a bit a retry a few times. |
|
| 677 |
+ time.Sleep(5 * time.Millisecond) |
|
| 678 |
+ } |
|
| 679 |
+ if err != nil {
|
|
| 664 | 680 |
return err |
| 665 | 681 |
} |
| 682 |
+ |
|
| 666 | 683 |
if err := devices.waitRemove(devname); err != nil {
|
| 667 | 684 |
return err |
| 668 | 685 |
} |