The existing logic to handle container ID conflicts when attempting to
create a plugin container is not nearly as robust as the implementation
in daemon for user containers. Extract and refine the logic from daemon
and use it in the plugin executor.
Signed-off-by: Cory Snider <csnider@mirantis.com>
| ... | ... |
@@ -9,6 +9,7 @@ import ( |
| 9 | 9 |
containertypes "github.com/docker/docker/api/types/container" |
| 10 | 10 |
"github.com/docker/docker/container" |
| 11 | 11 |
"github.com/docker/docker/errdefs" |
| 12 |
+ "github.com/docker/docker/libcontainerd" |
|
| 12 | 13 |
"github.com/pkg/errors" |
| 13 | 14 |
"github.com/sirupsen/logrus" |
| 14 | 15 |
) |
| ... | ... |
@@ -178,16 +179,9 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint |
| 178 | 178 |
|
| 179 | 179 |
ctx := context.TODO() |
| 180 | 180 |
|
| 181 |
- ctr, err := daemon.containerd.NewContainer(ctx, container.ID, spec, shim, createOptions) |
|
| 181 |
+ ctr, err := libcontainerd.ReplaceContainer(ctx, daemon.containerd, container.ID, spec, shim, createOptions) |
|
| 182 | 182 |
if err != nil {
|
| 183 |
- if errdefs.IsConflict(err) {
|
|
| 184 |
- logrus.WithError(err).WithField("container", container.ID).Error("Container not cleaned up from containerd from previous run")
|
|
| 185 |
- daemon.cleanupStaleContainer(ctx, container.ID) |
|
| 186 |
- ctr, err = daemon.containerd.NewContainer(ctx, container.ID, spec, shim, createOptions) |
|
| 187 |
- } |
|
| 188 |
- if err != nil {
|
|
| 189 |
- return translateContainerdStartErr(container.Path, container.SetExitCode, err) |
|
| 190 |
- } |
|
| 183 |
+ return translateContainerdStartErr(container.Path, container.SetExitCode, err) |
|
| 191 | 184 |
} |
| 192 | 185 |
|
| 193 | 186 |
// TODO(mlaventure): we need to specify checkpoint options here |
| ... | ... |
@@ -220,31 +214,6 @@ func (daemon *Daemon) containerStart(container *container.Container, checkpoint |
| 220 | 220 |
return nil |
| 221 | 221 |
} |
| 222 | 222 |
|
| 223 |
-func (daemon *Daemon) cleanupStaleContainer(ctx context.Context, id string) {
|
|
| 224 |
- // best effort to clean up old container object |
|
| 225 |
- log := logrus.WithContext(ctx).WithField("container", id)
|
|
| 226 |
- ctr, err := daemon.containerd.LoadContainer(ctx, id) |
|
| 227 |
- if err != nil {
|
|
| 228 |
- // Log an error no matter the kind. A container existed with the |
|
| 229 |
- // ID, so a NotFound error would be an exceptional situation |
|
| 230 |
- // worth logging. |
|
| 231 |
- log.WithError(err).Error("Error loading stale containerd container object")
|
|
| 232 |
- return |
|
| 233 |
- } |
|
| 234 |
- if tsk, err := ctr.Task(ctx); err != nil {
|
|
| 235 |
- if !errdefs.IsNotFound(err) {
|
|
| 236 |
- log.WithError(err).Error("Error loading stale containerd task object")
|
|
| 237 |
- } |
|
| 238 |
- } else {
|
|
| 239 |
- if err := tsk.ForceDelete(ctx); err != nil {
|
|
| 240 |
- log.WithError(err).Error("Error cleaning up stale containerd task object")
|
|
| 241 |
- } |
|
| 242 |
- } |
|
| 243 |
- if err := ctr.Delete(ctx); err != nil && !errdefs.IsNotFound(err) {
|
|
| 244 |
- log.WithError(err).Error("Error cleaning up stale containerd container object")
|
|
| 245 |
- } |
|
| 246 |
-} |
|
| 247 |
- |
|
| 248 | 223 |
// Cleanup releases any network resources allocated to the container along with any rules |
| 249 | 224 |
// around how containers are linked together. It also unmounts the container's root filesystem. |
| 250 | 225 |
func (daemon *Daemon) Cleanup(container *container.Container) {
|
| 251 | 226 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,62 @@ |
| 0 |
+package libcontainerd // import "github.com/docker/docker/libcontainerd" |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/containerd/containerd" |
|
| 6 |
+ "github.com/opencontainers/runtime-spec/specs-go" |
|
| 7 |
+ "github.com/pkg/errors" |
|
| 8 |
+ "github.com/sirupsen/logrus" |
|
| 9 |
+ |
|
| 10 |
+ "github.com/docker/docker/errdefs" |
|
| 11 |
+ "github.com/docker/docker/libcontainerd/types" |
|
| 12 |
+) |
|
| 13 |
+ |
|
| 14 |
+// ReplaceContainer creates a new container, replacing any existing container |
|
| 15 |
+// with the same id if necessary. |
|
| 16 |
+func ReplaceContainer(ctx context.Context, client types.Client, id string, spec *specs.Spec, shim string, runtimeOptions interface{}, opts ...containerd.NewContainerOpts) (types.Container, error) {
|
|
| 17 |
+ newContainer := func() (types.Container, error) {
|
|
| 18 |
+ return client.NewContainer(ctx, id, spec, shim, runtimeOptions, opts...) |
|
| 19 |
+ } |
|
| 20 |
+ ctr, err := newContainer() |
|
| 21 |
+ if err == nil || !errdefs.IsConflict(err) {
|
|
| 22 |
+ return ctr, err |
|
| 23 |
+ } |
|
| 24 |
+ |
|
| 25 |
+ log := logrus.WithContext(ctx).WithField("container", id)
|
|
| 26 |
+ log.Debug("A container already exists with the same ID. Attempting to clean up the old container.")
|
|
| 27 |
+ ctr, err = client.LoadContainer(ctx, id) |
|
| 28 |
+ if err != nil {
|
|
| 29 |
+ if errdefs.IsNotFound(err) {
|
|
| 30 |
+ // Task failed successfully: the container no longer exists, |
|
| 31 |
+ // despite us not doing anything. May as well try to create |
|
| 32 |
+ // the container again. It might succeed. |
|
| 33 |
+ return newContainer() |
|
| 34 |
+ } |
|
| 35 |
+ return nil, errors.Wrap(err, "could not load stale containerd container object") |
|
| 36 |
+ } |
|
| 37 |
+ tsk, err := ctr.Task(ctx) |
|
| 38 |
+ if err != nil {
|
|
| 39 |
+ if errdefs.IsNotFound(err) {
|
|
| 40 |
+ goto deleteContainer |
|
| 41 |
+ } |
|
| 42 |
+ // There is no point in trying to delete the container if we |
|
| 43 |
+ // cannot determine whether or not it has a task. The containerd |
|
| 44 |
+ // client would just try to load the task itself, get the same |
|
| 45 |
+ // error, and give up. |
|
| 46 |
+ return nil, errors.Wrap(err, "could not load stale containerd task object") |
|
| 47 |
+ } |
|
| 48 |
+ if err := tsk.ForceDelete(ctx); err != nil {
|
|
| 49 |
+ if !errdefs.IsNotFound(err) {
|
|
| 50 |
+ return nil, errors.Wrap(err, "could not delete stale containerd task object") |
|
| 51 |
+ } |
|
| 52 |
+ // The task might have exited on its own. Proceed with |
|
| 53 |
+ // attempting to delete the container. |
|
| 54 |
+ } |
|
| 55 |
+deleteContainer: |
|
| 56 |
+ if err := ctr.Delete(ctx); err != nil && !errdefs.IsNotFound(err) {
|
|
| 57 |
+ return nil, errors.Wrap(err, "could not delete stale containerd container object") |
|
| 58 |
+ } |
|
| 59 |
+ |
|
| 60 |
+ return newContainer() |
|
| 61 |
+} |
| ... | ... |
@@ -75,39 +75,9 @@ func (p c8dPlugin) deleteTaskAndContainer(ctx context.Context) {
|
| 75 | 75 |
func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error {
|
| 76 | 76 |
ctx := context.Background() |
| 77 | 77 |
log := logrus.WithField("plugin", id)
|
| 78 |
- ctr, err := e.client.NewContainer(ctx, id, &spec, e.runtime.Shim.Binary, e.runtime.Shim.Opts) |
|
| 78 |
+ ctr, err := libcontainerd.ReplaceContainer(ctx, e.client, id, &spec, e.runtime.Shim.Binary, e.runtime.Shim.Opts) |
|
| 79 | 79 |
if err != nil {
|
| 80 |
- ctr2, err2 := e.client.LoadContainer(ctx, id) |
|
| 81 |
- if err2 != nil {
|
|
| 82 |
- if !errdefs.IsNotFound(err2) {
|
|
| 83 |
- log.WithError(err2).Warn("Received an error while attempting to load containerd container for plugin")
|
|
| 84 |
- } |
|
| 85 |
- } else {
|
|
| 86 |
- status := containerd.Unknown |
|
| 87 |
- t, err2 := ctr2.Task(ctx) |
|
| 88 |
- if err2 != nil {
|
|
| 89 |
- if !errdefs.IsNotFound(err2) {
|
|
| 90 |
- log.WithError(err2).Warn("Received an error while attempting to load containerd task for plugin")
|
|
| 91 |
- } |
|
| 92 |
- } else {
|
|
| 93 |
- s, err2 := t.Status(ctx) |
|
| 94 |
- if err2 != nil {
|
|
| 95 |
- log.WithError(err2).Warn("Received an error while attempting to read plugin status")
|
|
| 96 |
- } else {
|
|
| 97 |
- status = s.Status |
|
| 98 |
- } |
|
| 99 |
- } |
|
| 100 |
- if status != containerd.Running && status != containerd.Unknown {
|
|
| 101 |
- if err2 := ctr2.Delete(ctx); err2 != nil && !errdefs.IsNotFound(err2) {
|
|
| 102 |
- log.WithError(err2).Error("Error cleaning up containerd container")
|
|
| 103 |
- } |
|
| 104 |
- ctr, err = e.client.NewContainer(ctx, id, &spec, e.runtime.Shim.Binary, e.runtime.Shim.Opts) |
|
| 105 |
- } |
|
| 106 |
- } |
|
| 107 |
- |
|
| 108 |
- if err != nil {
|
|
| 109 |
- return errors.Wrap(err, "error creating containerd container") |
|
| 110 |
- } |
|
| 80 |
+ return errors.Wrap(err, "error creating containerd container for plugin") |
|
| 111 | 81 |
} |
| 112 | 82 |
|
| 113 | 83 |
p := c8dPlugin{log: log, ctr: ctr}
|