Browse code

Share logic to create-or-replace a container

The existing logic to handle container ID conflicts when attempting to
create a plugin container is not nearly as robust as the implementation
in daemon for user containers. Extract and refine the logic from daemon
and use it in the plugin executor.

Signed-off-by: Cory Snider <csnider@mirantis.com>

Cory Snider authored on 2022/05/06 02:00:45
Showing 3 changed files
... ...
@@ -9,6 +9,7 @@ import (
9 9
 	containertypes "github.com/docker/docker/api/types/container"
10 10
 	"github.com/docker/docker/container"
11 11
 	"github.com/docker/docker/errdefs"
12
+	"github.com/docker/docker/libcontainerd"
12 13
 	"github.com/pkg/errors"
13 14
 	"github.com/sirupsen/logrus"
14 15
 )
... ...
@@ -178,16 +179,9 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
178 178
 
179 179
 	ctx := context.TODO()
180 180
 
181
-	ctr, err := daemon.containerd.NewContainer(ctx, container.ID, spec, shim, createOptions)
181
+	ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions)
182 182
 	if err != nil {
183
-		if errdefs.IsConflict(err) {
184
-			logrus.WithError(err).WithField("container", container.ID).Error("Container not cleaned up from containerd from previous run")
185
-			daemon.cleanupStaleContainer(ctx, container.ID)
186
-			ctr, err = daemon.containerd.NewContainer(ctx, container.ID, spec, shim, createOptions)
187
-		}
188
-		if err != nil {
189
-			return translateContainerdStartErr(container.Path, container.SetExitCode, err)
190
-		}
183
+		return translateContainerdStartErr(container.Path, container.SetExitCode, err)
191 184
 	}
192 185
 
193 186
 	// TODO(mlaventure): we need to specify checkpoint options here
... ...
@@ -220,31 +214,6 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint
220 220
 	return nil
221 221
 }
222 222
 
223
-func (daemon *Daemon) cleanupStaleContainer(ctx context.Context, id string) {
224
-	// best effort to clean up old container object
225
-	log := logrus.WithContext(ctx).WithField("container", id)
226
-	ctr, err := daemon.containerd.LoadContainer(ctx, id)
227
-	if err != nil {
228
-		// Log an error no matter the kind. A container existed with the
229
-		// ID, so a NotFound error would be an exceptional situation
230
-		// worth logging.
231
-		log.WithError(err).Error("Error loading stale containerd container object")
232
-		return
233
-	}
234
-	if tsk, err := ctr.Task(ctx); err != nil {
235
-		if !errdefs.IsNotFound(err) {
236
-			log.WithError(err).Error("Error loading stale containerd task object")
237
-		}
238
-	} else {
239
-		if err := tsk.ForceDelete(ctx); err != nil {
240
-			log.WithError(err).Error("Error cleaning up stale containerd task object")
241
-		}
242
-	}
243
-	if err := ctr.Delete(ctx); err != nil && !errdefs.IsNotFound(err) {
244
-		log.WithError(err).Error("Error cleaning up stale containerd container object")
245
-	}
246
-}
247
-
248 223
 // Cleanup releases any network resources allocated to the container along with any rules
249 224
 // around how containers are linked together.  It also unmounts the container's root filesystem.
250 225
 func (daemon *Daemon) Cleanup(container *container.Container) {
251 226
new file mode 100644
... ...
@@ -0,0 +1,62 @@
0
+package libcontainerd // import "github.com/docker/docker/libcontainerd"
1
+
2
+import (
3
+	"context"
4
+
5
+	"github.com/containerd/containerd"
6
+	"github.com/opencontainers/runtime-spec/specs-go"
7
+	"github.com/pkg/errors"
8
+	"github.com/sirupsen/logrus"
9
+
10
+	"github.com/docker/docker/errdefs"
11
+	"github.com/docker/docker/libcontainerd/types"
12
+)
13
+
14
+// ReplaceContainer creates a new container, replacing any existing container
15
+// with the same id if necessary.
16
+func ReplaceContainer(ctx context.Context, client types.Client, id string, spec *specs.Spec, shim string, runtimeOptions interface{}, opts ...containerd.NewContainerOpts) (types.Container, error) {
17
+	newContainer := func() (types.Container, error) {
18
+		return client.NewContainer(ctx, id, spec, shim, runtimeOptions, opts...)
19
+	}
20
+	ctr, err := newContainer()
21
+	if err == nil || !errdefs.IsConflict(err) {
22
+		return ctr, err
23
+	}
24
+
25
+	log := logrus.WithContext(ctx).WithField("container", id)
26
+	log.Debug("A container already exists with the same ID. Attempting to clean up the old container.")
27
+	ctr, err = client.LoadContainer(ctx, id)
28
+	if err != nil {
29
+		if errdefs.IsNotFound(err) {
30
+			// Task failed successfully: the container no longer exists,
31
+			// despite us not doing anything. May as well try to create
32
+			// the container again. It might succeed.
33
+			return newContainer()
34
+		}
35
+		return nil, errors.Wrap(err, "could not load stale containerd container object")
36
+	}
37
+	tsk, err := ctr.Task(ctx)
38
+	if err != nil {
39
+		if errdefs.IsNotFound(err) {
40
+			goto deleteContainer
41
+		}
42
+		// There is no point in trying to delete the container if we
43
+		// cannot determine whether or not it has a task. The containerd
44
+		// client would just try to load the task itself, get the same
45
+		// error, and give up.
46
+		return nil, errors.Wrap(err, "could not load stale containerd task object")
47
+	}
48
+	if err := tsk.ForceDelete(ctx); err != nil {
49
+		if !errdefs.IsNotFound(err) {
50
+			return nil, errors.Wrap(err, "could not delete stale containerd task object")
51
+		}
52
+		// The task might have exited on its own. Proceed with
53
+		// attempting to delete the container.
54
+	}
55
+deleteContainer:
56
+	if err := ctr.Delete(ctx); err != nil && !errdefs.IsNotFound(err) {
57
+		return nil, errors.Wrap(err, "could not delete stale containerd container object")
58
+	}
59
+
60
+	return newContainer()
61
+}
... ...
@@ -75,39 +75,9 @@ func (p c8dPlugin) deleteTaskAndContainer(ctx context.Context) {
75 75
 func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error {
76 76
 	ctx := context.Background()
77 77
 	log := logrus.WithField("plugin", id)
78
-	ctr, err := e.client.NewContainer(ctx, id, &spec, e.runtime.Shim.Binary, e.runtime.Shim.Opts)
78
+	ctr, err := libcontainerd.ReplaceContainer(ctx, e.client, id, &spec, e.runtime.Shim.Binary, e.runtime.Shim.Opts)
79 79
 	if err != nil {
80
-		ctr2, err2 := e.client.LoadContainer(ctx, id)
81
-		if err2 != nil {
82
-			if !errdefs.IsNotFound(err2) {
83
-				log.WithError(err2).Warn("Received an error while attempting to load containerd container for plugin")
84
-			}
85
-		} else {
86
-			status := containerd.Unknown
87
-			t, err2 := ctr2.Task(ctx)
88
-			if err2 != nil {
89
-				if !errdefs.IsNotFound(err2) {
90
-					log.WithError(err2).Warn("Received an error while attempting to load containerd task for plugin")
91
-				}
92
-			} else {
93
-				s, err2 := t.Status(ctx)
94
-				if err2 != nil {
95
-					log.WithError(err2).Warn("Received an error while attempting to read plugin status")
96
-				} else {
97
-					status = s.Status
98
-				}
99
-			}
100
-			if status != containerd.Running && status != containerd.Unknown {
101
-				if err2 := ctr2.Delete(ctx); err2 != nil && !errdefs.IsNotFound(err2) {
102
-					log.WithError(err2).Error("Error cleaning up containerd container")
103
-				}
104
-				ctr, err = e.client.NewContainer(ctx, id, &spec, e.runtime.Shim.Binary, e.runtime.Shim.Opts)
105
-			}
106
-		}
107
-
108
-		if err != nil {
109
-			return errors.Wrap(err, "error creating containerd container")
110
-		}
80
+		return errors.Wrap(err, "error creating containerd container for plugin")
111 81
 	}
112 82
 
113 83
 	p := c8dPlugin{log: log, ctr: ctr}