@@ -17,7 +17,7 @@ import (

 	"github.com/docker/docker/container"

 	"github.com/docker/docker/daemon/config"

 	"github.com/docker/docker/errdefs"

-	"github.com/docker/docker/pkg/containerfs"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/opencontainers/selinux/go-selinux"

 	"github.com/pkg/errors"

 )

@@ -161,6 +161,8 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, config ba

 	// so that other goroutines don't attempt to concurrently open files

 	// within it. Having any file open on Windows (without the

 	// FILE_SHARE_DELETE flag) will block it from being deleted.

+	//

+	// TODO(thaJeztah): should this be moved to the "container" itself, or possibly be delegated to the graphdriver or snapshotter?

 	container.Lock()

 	err := containerfs.EnsureRemoveAll(container.Root)

 	container.Unlock()

@@ -37,7 +37,7 @@ import (

 	"github.com/containerd/containerd/pkg/userns"

 	"github.com/containerd/log"

 	"github.com/docker/docker/daemon/graphdriver"

-	"github.com/docker/docker/pkg/containerfs"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/docker/docker/pkg/idtools"

 	"github.com/docker/docker/pkg/parsers"

 	units "github.com/docker/go-units"

@@ -17,9 +17,9 @@ import (

 	"github.com/containerd/log"

 	"github.com/docker/docker/daemon/graphdriver"

 	"github.com/docker/docker/daemon/graphdriver/overlayutils"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/docker/docker/pkg/archive"

 	"github.com/docker/docker/pkg/chrootarchive"

-	"github.com/docker/docker/pkg/containerfs"

 	"github.com/docker/docker/pkg/directory"

 	"github.com/docker/docker/pkg/idtools"

 	"github.com/docker/docker/pkg/parsers/kernel"

@@ -19,9 +19,9 @@ import (

 	"github.com/containerd/log"

 	"github.com/docker/docker/daemon/graphdriver"

 	"github.com/docker/docker/daemon/graphdriver/overlayutils"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/docker/docker/pkg/archive"

 	"github.com/docker/docker/pkg/chrootarchive"

-	"github.com/docker/docker/pkg/containerfs"

 	"github.com/docker/docker/pkg/directory"

 	"github.com/docker/docker/pkg/idtools"

 	"github.com/docker/docker/pkg/ioutils"

@@ -7,7 +7,7 @@ import (

 	"github.com/docker/docker/daemon/graphdriver"

 	"github.com/docker/docker/errdefs"

-	"github.com/docker/docker/pkg/containerfs"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/docker/docker/pkg/idtools"

 	"github.com/docker/docker/pkg/parsers"

 	"github.com/docker/docker/quota"

new file mode 100644

@@ -0,0 +1,78 @@

+//go:build !darwin && !windows

+

+package containerfs

+

+import (

+	"os"

+	"syscall"

+	"time"

+

+	"github.com/moby/sys/mount"

+	"github.com/pkg/errors"

+)

+

+// EnsureRemoveAll wraps [os.RemoveAll] to check for specific errors that can

+// often be remedied.

+// Only use [EnsureRemoveAll] if you really want to make every effort to remove

+// a directory.

+//

+// Because of the way [os.Remove] (and by extension [os.RemoveAll]) works, there

+// can be a race between reading directory entries and then actually attempting

+// to remove everything in the directory.

+// These types of errors do not need to be returned since it's ok for the dir to

+// be gone we can just retry the remove operation.

+//

+// This should not return a [os.ErrNotExist] kind of error under any circumstances.

+func EnsureRemoveAll(dir string) error {

+	notExistErr := make(map[string]bool)

+

+	// track retries

+	exitOnErr := make(map[string]int)

+	maxRetry := 50

+

+	// Attempt to unmount anything beneath this dir first

+	mount.RecursiveUnmount(dir)

+

+	for {

+		err := os.RemoveAll(dir)

+		if err == nil {

+			return nil

+		}

+

+		pe, ok := err.(*os.PathError)

+		if !ok {

+			return err

+		}

+

+		if os.IsNotExist(err) {

+			if notExistErr[pe.Path] {

+				return err

+			}

+			notExistErr[pe.Path] = true

+

+			// There is a race where some subdir can be removed but after the parent

+			//   dir entries have been read.

+			// So the path could be from `os.Remove(subdir)`

+			// If the reported non-existent path is not the passed in `dir` we

+			// should just retry, but otherwise return with no error.

+			if pe.Path == dir {

+				return nil

+			}

+			continue

+		}

+

+		if pe.Err != syscall.EBUSY {

+			return err

+		}

+

+		if e := mount.Unmount(pe.Path); e != nil {

+			return errors.Wrapf(e, "error while removing %s", dir)

+		}

+

+		if exitOnErr[pe.Path] == maxRetry {

+			return err

+		}

+		exitOnErr[pe.Path]++

+		time.Sleep(100 * time.Millisecond)

+	}

+}

new file mode 100644

@@ -0,0 +1,36 @@

+//go:build !darwin

+

+package containerfs

+

+import (

+	"os"

+	"testing"

+)

+

+func TestEnsureRemoveAllNotExist(t *testing.T) {

+	// should never return an error for a non-existent path

+	if err := EnsureRemoveAll("/non/existent/path"); err != nil {

+		t.Fatal(err)

+	}

+}

+

+func TestEnsureRemoveAllWithDir(t *testing.T) {

+	dir, err := os.MkdirTemp("", "test-ensure-removeall-with-dir")

+	if err != nil {

+		t.Fatal(err)

+	}

+	if err := EnsureRemoveAll(dir); err != nil {

+		t.Fatal(err)

+	}

+}

+

+func TestEnsureRemoveAllWithFile(t *testing.T) {

+	tmp, err := os.CreateTemp("", "test-ensure-removeall-with-dir")

+	if err != nil {

+		t.Fatal(err)

+	}

+	tmp.Close()

+	if err := EnsureRemoveAll(tmp.Name()); err != nil {

+		t.Fatal(err)

+	}

+}

new file mode 100644

@@ -0,0 +1,56 @@

+//go:build !darwin && !windows

+

+package containerfs

+

+import (

+	"os"

+	"path/filepath"

+	"testing"

+	"time"

+

+	"github.com/moby/sys/mount"

+)

+

+func TestEnsureRemoveAllWithMount(t *testing.T) {

+	if os.Getuid() != 0 {

+		t.Skip("skipping test that requires root")

+	}

+

+	dir1, err := os.MkdirTemp("", "test-ensure-removeall-with-dir1")

+	if err != nil {

+		t.Fatal(err)

+	}

+	dir2, err := os.MkdirTemp("", "test-ensure-removeall-with-dir2")

+	if err != nil {

+		t.Fatal(err)

+	}

+	defer os.RemoveAll(dir2)

+

+	bindDir := filepath.Join(dir1, "bind")

+	if err := os.MkdirAll(bindDir, 0o755); err != nil {

+		t.Fatal(err)

+	}

+

+	if err := mount.Mount(dir2, bindDir, "none", "bind"); err != nil {

+		t.Fatal(err)

+	}

+

+	done := make(chan struct{}, 1)

+	go func() {

+		err = EnsureRemoveAll(dir1)

+		close(done)

+	}()

+

+	select {

+	case <-done:

+		if err != nil {

+			t.Fatal(err)

+		}

+	case <-time.After(5 * time.Second):

+		t.Fatal("timeout waiting for EnsureRemoveAll to finish")

+	}

+

+	if _, err := os.Stat(dir1); !os.IsNotExist(err) {

+		t.Fatalf("expected %q to not exist", dir1)

+	}

+}

new file mode 100644

@@ -0,0 +1,8 @@

+package containerfs

+

+import "os"

+

+// EnsureRemoveAll is an alias to [os.RemoveAll] on Windows.

+func EnsureRemoveAll(path string) error {

+	return os.RemoveAll(path)

+}

deleted file mode 100644

@@ -1,78 +0,0 @@

-//go:build !darwin && !windows

-

-package containerfs

-

-import (

-	"os"

-	"syscall"

-	"time"

-

-	"github.com/moby/sys/mount"

-	"github.com/pkg/errors"

-)

-

-// EnsureRemoveAll wraps [os.RemoveAll] to check for specific errors that can

-// often be remedied.

-// Only use [EnsureRemoveAll] if you really want to make every effort to remove

-// a directory.

-//

-// Because of the way [os.Remove] (and by extension [os.RemoveAll]) works, there

-// can be a race between reading directory entries and then actually attempting

-// to remove everything in the directory.

-// These types of errors do not need to be returned since it's ok for the dir to

-// be gone we can just retry the remove operation.

-//

-// This should not return a [os.ErrNotExist] kind of error under any circumstances.

-func EnsureRemoveAll(dir string) error {

-	notExistErr := make(map[string]bool)

-

-	// track retries

-	exitOnErr := make(map[string]int)

-	maxRetry := 50

-

-	// Attempt to unmount anything beneath this dir first

-	mount.RecursiveUnmount(dir)

-

-	for {

-		err := os.RemoveAll(dir)

-		if err == nil {

-			return nil

-		}

-

-		pe, ok := err.(*os.PathError)

-		if !ok {

-			return err

-		}

-

-		if os.IsNotExist(err) {

-			if notExistErr[pe.Path] {

-				return err

-			}

-			notExistErr[pe.Path] = true

-

-			// There is a race where some subdir can be removed but after the parent

-			//   dir entries have been read.

-			// So the path could be from `os.Remove(subdir)`

-			// If the reported non-existent path is not the passed in `dir` we

-			// should just retry, but otherwise return with no error.

-			if pe.Path == dir {

-				return nil

-			}

-			continue

-		}

-

-		if pe.Err != syscall.EBUSY {

-			return err

-		}

-

-		if e := mount.Unmount(pe.Path); e != nil {

-			return errors.Wrapf(e, "error while removing %s", dir)

-		}

-

-		if exitOnErr[pe.Path] == maxRetry {

-			return err

-		}

-		exitOnErr[pe.Path]++

-		time.Sleep(100 * time.Millisecond)

-	}

-}

deleted file mode 100644

@@ -1,36 +0,0 @@

-//go:build !darwin

-

-package containerfs

-

-import (

-	"os"

-	"testing"

-)

-

-func TestEnsureRemoveAllNotExist(t *testing.T) {

-	// should never return an error for a non-existent path

-	if err := EnsureRemoveAll("/non/existent/path"); err != nil {

-		t.Fatal(err)

-	}

-}

-

-func TestEnsureRemoveAllWithDir(t *testing.T) {

-	dir, err := os.MkdirTemp("", "test-ensure-removeall-with-dir")

-	if err != nil {

-		t.Fatal(err)

-	}

-	if err := EnsureRemoveAll(dir); err != nil {

-		t.Fatal(err)

-	}

-}

-

-func TestEnsureRemoveAllWithFile(t *testing.T) {

-	tmp, err := os.CreateTemp("", "test-ensure-removeall-with-dir")

-	if err != nil {

-		t.Fatal(err)

-	}

-	tmp.Close()

-	if err := EnsureRemoveAll(tmp.Name()); err != nil {

-		t.Fatal(err)

-	}

-}

deleted file mode 100644

@@ -1,56 +0,0 @@

-//go:build !darwin && !windows

-

-package containerfs

-

-import (

-	"os"

-	"path/filepath"

-	"testing"

-	"time"

-

-	"github.com/moby/sys/mount"

-)

-

-func TestEnsureRemoveAllWithMount(t *testing.T) {

-	if os.Getuid() != 0 {

-		t.Skip("skipping test that requires root")

-	}

-

-	dir1, err := os.MkdirTemp("", "test-ensure-removeall-with-dir1")

-	if err != nil {

-		t.Fatal(err)

-	}

-	dir2, err := os.MkdirTemp("", "test-ensure-removeall-with-dir2")

-	if err != nil {

-		t.Fatal(err)

-	}

-	defer os.RemoveAll(dir2)

-

-	bindDir := filepath.Join(dir1, "bind")

-	if err := os.MkdirAll(bindDir, 0o755); err != nil {

-		t.Fatal(err)

-	}

-

-	if err := mount.Mount(dir2, bindDir, "none", "bind"); err != nil {

-		t.Fatal(err)

-	}

-

-	done := make(chan struct{}, 1)

-	go func() {

-		err = EnsureRemoveAll(dir1)

-		close(done)

-	}()

-

-	select {

-	case <-done:

-		if err != nil {

-			t.Fatal(err)

-		}

-	case <-time.After(5 * time.Second):

-		t.Fatal("timeout waiting for EnsureRemoveAll to finish")

-	}

-

-	if _, err := os.Stat(dir1); !os.IsNotExist(err) {

-		t.Fatalf("expected %q to not exist", dir1)

-	}

-}

deleted file mode 100644

@@ -1,8 +0,0 @@

-package containerfs

-

-import "os"

-

-// EnsureRemoveAll is an alias to [os.RemoveAll] on Windows.

-func EnsureRemoveAll(path string) error {

-	return os.RemoveAll(path)

-}

@@ -29,9 +29,9 @@ import (

 	"github.com/docker/docker/api/types/registry"

 	"github.com/docker/docker/dockerversion"

 	"github.com/docker/docker/errdefs"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/docker/docker/pkg/authorization"

 	"github.com/docker/docker/pkg/chrootarchive"

-	"github.com/docker/docker/pkg/containerfs"

 	"github.com/docker/docker/pkg/pools"

 	"github.com/docker/docker/pkg/progress"

 	"github.com/docker/docker/pkg/stringid"

@@ -18,8 +18,8 @@ import (

 	"github.com/containerd/log"

 	"github.com/docker/docker/api/types"

 	"github.com/docker/docker/api/types/events"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/docker/docker/pkg/authorization"

-	"github.com/docker/docker/pkg/containerfs"

 	"github.com/docker/docker/pkg/ioutils"

 	v2 "github.com/docker/docker/plugin/v2"

 	"github.com/docker/docker/registry"

@@ -11,7 +11,7 @@ import (

 	"github.com/docker/docker/api/types"

 	"github.com/docker/docker/api/types/backend"

 	"github.com/docker/docker/api/types/events"

-	"github.com/docker/docker/pkg/containerfs"

+	"github.com/docker/docker/internal/containerfs"

 	"github.com/docker/docker/pkg/stringid"

 	v2 "github.com/docker/docker/plugin/v2"

 	"github.com/moby/sys/mount"