Browse code

Remove inmemory container map

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Michael Crosby authored on 2019/03/26 05:17:17
Showing 9 changed files
... ...
@@ -325,16 +325,17 @@ func (daemon *Daemon) restore() error {
325 325
 				alive    bool
326 326
 				ec       uint32
327 327
 				exitedAt time.Time
328
+				process  libcontainerdtypes.Process
328 329
 			)
329 330
 
330
-			alive, _, err = daemon.containerd.Restore(context.Background(), c.ID, c.InitializeStdio)
331
+			alive, _, process, err = daemon.containerd.Restore(context.Background(), c.ID, c.InitializeStdio)
331 332
 			if err != nil && !errdefs.IsNotFound(err) {
332 333
 				logrus.Errorf("Failed to restore container %s with containerd: %s", c.ID, err)
333 334
 				return
334 335
 			}
335
-			if !alive {
336
-				ec, exitedAt, err = daemon.containerd.DeleteTask(context.Background(), c.ID)
337
-				if err != nil && !errdefs.IsNotFound(err) {
336
+			if !alive && process != nil {
337
+				ec, exitedAt, err = process.Delete(context.Background())
338
+				if err != nil {
338 339
 					logrus.WithError(err).Errorf("Failed to delete container %s from containerd", c.ID)
339 340
 					return
340 341
 				}
... ...
@@ -11,6 +11,13 @@ import (
11 11
 	specs "github.com/opencontainers/runtime-spec/specs-go"
12 12
 )
13 13
 
14
+type mockProcess struct {
15
+}
16
+
17
+func (m *mockProcess) Delete(_ context.Context) (uint32, time.Time, error) {
18
+	return 0, time.Time{}, nil
19
+}
20
+
14 21
 // Mock containerd client implementation, for unit tests.
15 22
 type MockContainerdClient struct {
16 23
 }
... ...
@@ -18,8 +25,8 @@ type MockContainerdClient struct {
18 18
 func (c *MockContainerdClient) Version(ctx context.Context) (containerd.Version, error) {
19 19
 	return containerd.Version{}, nil
20 20
 }
21
-func (c *MockContainerdClient) Restore(ctx context.Context, containerID string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) {
22
-	return false, 0, nil
21
+func (c *MockContainerdClient) Restore(ctx context.Context, containerID string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, p libcontainerdtypes.Process, err error) {
22
+	return false, 0, &mockProcess{}, nil
23 23
 }
24 24
 func (c *MockContainerdClient) Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error {
25 25
 	return nil
... ...
@@ -1085,7 +1085,7 @@ func (c *client) Stats(_ context.Context, containerID string) (*libcontainerdtyp
1085 1085
 }
1086 1086
 
1087 1087
 // Restore is the handler for restoring a container
1088
-func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (bool, int, error) {
1088
+func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (bool, int, libcontainerdtypes.Process, error) {
1089 1089
 	c.logger.WithField("container", id).Debug("restore()")
1090 1090
 
1091 1091
 	// TODO Windows: On RS1, a re-attach isn't possible.
... ...
@@ -1107,10 +1107,13 @@ func (c *client) Restore(ctx context.Context, id string, attachStdio libcontaine
1107 1107
 
1108 1108
 		if err != nil {
1109 1109
 			c.logger.WithField("container", id).WithError(err).Debug("terminate failed on restore")
1110
-			return false, -1, err
1110
+			return false, -1, nil, err
1111 1111
 		}
1112 1112
 	}
1113
-	return false, -1, nil
1113
+	return false, -1, &restoredProcess{
1114
+		c:  c,
1115
+		id: id,
1116
+	}, nil
1114 1117
 }
1115 1118
 
1116 1119
 // GetPidsForContainer returns a list of process IDs running in a container.
... ...
@@ -1153,6 +1156,15 @@ func (c *client) Summary(_ context.Context, containerID string) ([]libcontainerd
1153 1153
 	return pl, nil
1154 1154
 }
1155 1155
 
1156
+type restoredProcess struct {
1157
+	id string
1158
+	c  *client
1159
+}
1160
+
1161
+func (p *restoredProcess) Delete(ctx context.Context) (uint32, time.Time, error) {
1162
+	return p.c.DeleteTask(ctx, p.id)
1163
+}
1164
+
1156 1165
 func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) {
1157 1166
 	ec := -1
1158 1167
 	ctr := c.getContainer(containerID)
... ...
@@ -38,86 +38,30 @@ import (
38 38
 	"google.golang.org/grpc/status"
39 39
 )
40 40
 
41
-type container struct {
42
-	mu sync.Mutex
43
-
44
-	bundleDir string
45
-	ctr       containerd.Container
46
-	task      containerd.Task
47
-	execs     map[string]containerd.Process
48
-	oomKilled bool
49
-}
50
-
51
-func (c *container) setTask(t containerd.Task) {
52
-	c.mu.Lock()
53
-	c.task = t
54
-	c.mu.Unlock()
55
-}
56
-
57
-func (c *container) getTask() containerd.Task {
58
-	c.mu.Lock()
59
-	t := c.task
60
-	c.mu.Unlock()
61
-	return t
62
-}
63
-
64
-func (c *container) addProcess(id string, p containerd.Process) {
65
-	c.mu.Lock()
66
-	if c.execs == nil {
67
-		c.execs = make(map[string]containerd.Process)
68
-	}
69
-	c.execs[id] = p
70
-	c.mu.Unlock()
71
-}
72
-
73
-func (c *container) deleteProcess(id string) {
74
-	c.mu.Lock()
75
-	delete(c.execs, id)
76
-	c.mu.Unlock()
77
-}
78
-
79
-func (c *container) getProcess(id string) containerd.Process {
80
-	c.mu.Lock()
81
-	p := c.execs[id]
82
-	c.mu.Unlock()
83
-	return p
84
-}
85
-
86
-func (c *container) setOOMKilled(killed bool) {
87
-	c.mu.Lock()
88
-	c.oomKilled = killed
89
-	c.mu.Unlock()
90
-}
91
-
92
-func (c *container) getOOMKilled() bool {
93
-	c.mu.Lock()
94
-	killed := c.oomKilled
95
-	c.mu.Unlock()
96
-	return killed
97
-}
41
+// DockerContainerBundlePath is the label key pointing to the container's bundle path
42
+const DockerContainerBundlePath = "com.docker/engine.bundle.path"
98 43
 
99 44
 type client struct {
100
-	sync.RWMutex // protects containers map
101
-
102 45
 	client   *containerd.Client
103 46
 	stateDir string
104 47
 	logger   *logrus.Entry
105 48
 	ns       string
106 49
 
107
-	backend    libcontainerdtypes.Backend
108
-	eventQ     queue.Queue
109
-	containers map[string]*container
50
+	backend libcontainerdtypes.Backend
51
+	eventQ  queue.Queue
52
+	oomMu   sync.Mutex
53
+	oom     map[string]bool
110 54
 }
111 55
 
112 56
 // NewClient creates a new libcontainerd client from a containerd client
113 57
 func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
114 58
 	c := &client{
115
-		client:     cli,
116
-		stateDir:   stateDir,
117
-		logger:     logrus.WithField("module", "libcontainerd").WithField("namespace", ns),
118
-		ns:         ns,
119
-		backend:    b,
120
-		containers: make(map[string]*container),
59
+		client:   cli,
60
+		stateDir: stateDir,
61
+		logger:   logrus.WithField("module", "libcontainerd").WithField("namespace", ns),
62
+		ns:       ns,
63
+		backend:  b,
64
+		oom:      make(map[string]bool),
121 65
 	}
122 66
 
123 67
 	go c.processEventStream(ctx, ns)
... ...
@@ -131,29 +75,7 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) {
131 131
 
132 132
 // Restore loads the containerd container.
133 133
 // It should not be called concurrently with any other operation for the given ID.
134
-func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) {
135
-	c.Lock()
136
-	_, ok := c.containers[id]
137
-	if ok {
138
-		c.Unlock()
139
-		return false, 0, errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
140
-	}
141
-
142
-	cntr := &container{}
143
-	c.containers[id] = cntr
144
-	cntr.mu.Lock()
145
-	defer cntr.mu.Unlock()
146
-
147
-	c.Unlock()
148
-
149
-	defer func() {
150
-		if err != nil {
151
-			c.Lock()
152
-			delete(c.containers, id)
153
-			c.Unlock()
154
-		}
155
-	}()
156
-
134
+func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, p libcontainerdtypes.Process, err error) {
157 135
 	var dio *cio.DirectIO
158 136
 	defer func() {
159 137
 		if err != nil && dio != nil {
... ...
@@ -165,13 +87,12 @@ func (c *client) Restore(ctx context.Context, id string, attachStdio libcontaine
165 165
 
166 166
 	ctr, err := c.client.LoadContainer(ctx, id)
167 167
 	if err != nil {
168
-		return false, -1, errors.WithStack(wrapError(err))
168
+		return false, -1, nil, errors.WithStack(wrapError(err))
169 169
 	}
170 170
 
171 171
 	attachIO := func(fifos *cio.FIFOSet) (cio.IO, error) {
172 172
 		// dio must be assigned to the previously defined dio for the defer above
173 173
 		// to handle cleanup
174
-
175 174
 		dio, err = c.newDirectIO(ctx, fifos)
176 175
 		if err != nil {
177 176
 			return nil, err
... ...
@@ -180,75 +101,57 @@ func (c *client) Restore(ctx context.Context, id string, attachStdio libcontaine
180 180
 	}
181 181
 	t, err := ctr.Task(ctx, attachIO)
182 182
 	if err != nil && !containerderrors.IsNotFound(err) {
183
-		return false, -1, errors.Wrap(wrapError(err), "error getting containerd task for container")
183
+		return false, -1, nil, errors.Wrap(wrapError(err), "error getting containerd task for container")
184 184
 	}
185 185
 
186 186
 	if t != nil {
187 187
 		s, err := t.Status(ctx)
188 188
 		if err != nil {
189
-			return false, -1, errors.Wrap(wrapError(err), "error getting task status")
189
+			return false, -1, nil, errors.Wrap(wrapError(err), "error getting task status")
190 190
 		}
191
-
192 191
 		alive = s.Status != containerd.Stopped
193 192
 		pid = int(t.Pid())
194 193
 	}
195 194
 
196
-	cntr.bundleDir = filepath.Join(c.stateDir, id)
197
-	cntr.ctr = ctr
198
-	cntr.task = t
199
-	// TODO(mlaventure): load execs
200
-
201 195
 	c.logger.WithFields(logrus.Fields{
202 196
 		"container": id,
203 197
 		"alive":     alive,
204 198
 		"pid":       pid,
205 199
 	}).Debug("restored container")
206 200
 
207
-	return alive, pid, nil
201
+	return alive, pid, &restoredProcess{
202
+		p: t,
203
+	}, nil
208 204
 }
209 205
 
210 206
 func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, runtimeOptions interface{}) error {
211
-	if ctr := c.getContainer(id); ctr != nil {
212
-		return errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
213
-	}
214
-
215
-	bdir := filepath.Join(c.stateDir, id)
207
+	bdir := c.bundleDir(id)
216 208
 	c.logger.WithField("bundle", bdir).WithField("root", ociSpec.Root.Path).Debug("bundle dir created")
217 209
 
218
-	cdCtr, err := c.client.NewContainer(ctx, id,
210
+	_, err := c.client.NewContainer(ctx, id,
219 211
 		containerd.WithSpec(ociSpec),
220 212
 		containerd.WithRuntime(runtimeName, runtimeOptions),
221 213
 		WithBundle(bdir, ociSpec),
222 214
 	)
223 215
 	if err != nil {
216
+		if containerderrors.IsAlreadyExists(err) {
217
+			return errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
218
+		}
224 219
 		return wrapError(err)
225 220
 	}
226
-
227
-	c.Lock()
228
-	c.containers[id] = &container{
229
-		bundleDir: bdir,
230
-		ctr:       cdCtr,
231
-	}
232
-	c.Unlock()
233
-
234 221
 	return nil
235 222
 }
236 223
 
237 224
 // Start create and start a task for the specified containerd id
238 225
 func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
239
-	ctr := c.getContainer(id)
240
-	if ctr == nil {
241
-		return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
242
-	}
243
-	if t := ctr.getTask(); t != nil {
244
-		return -1, errors.WithStack(errdefs.Conflict(errors.New("container already started")))
226
+	ctr, err := c.getContainer(ctx, id)
227
+	if err != nil {
228
+		return -1, err
245 229
 	}
246
-
247 230
 	var (
248 231
 		cp             *types.Descriptor
249 232
 		t              containerd.Task
250 233
 		rio            cio.IO
251
-		err            error
252 234
 		stdinCloseSync = make(chan struct{})
253 235
 	)
254 236
 
... ...
@@ -276,14 +179,19 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
276 276
 		}
277 277
 	}
278 278
 
279
-	spec, err := ctr.ctr.Spec(ctx)
279
+	spec, err := ctr.Spec(ctx)
280 280
 	if err != nil {
281 281
 		return -1, errors.Wrap(err, "failed to retrieve spec")
282 282
 	}
283
+	labels, err := ctr.Labels(ctx)
284
+	if err != nil {
285
+		return -1, errors.Wrap(err, "failed to retreive labels")
286
+	}
287
+	bundle := labels[DockerContainerBundlePath]
283 288
 	uid, gid := getSpecUser(spec)
284
-	t, err = ctr.ctr.NewTask(ctx,
289
+	t, err = ctr.NewTask(ctx,
285 290
 		func(id string) (cio.IO, error) {
286
-			fifos := newFIFOSet(ctr.bundleDir, libcontainerdtypes.InitProcessName, withStdin, spec.Process.Terminal)
291
+			fifos := newFIFOSet(bundle, libcontainerdtypes.InitProcessName, withStdin, spec.Process.Terminal)
287 292
 
288 293
 			rio, err = c.createIO(fifos, id, libcontainerdtypes.InitProcessName, stdinCloseSync, attachStdio)
289 294
 			return rio, err
... ...
@@ -313,8 +221,6 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
313 313
 		return -1, wrapError(err)
314 314
 	}
315 315
 
316
-	ctr.setTask(t)
317
-
318 316
 	// Signal c.createIO that it can call CloseIO
319 317
 	close(stdinCloseSync)
320 318
 
... ...
@@ -323,7 +229,6 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
323 323
 			c.logger.WithError(err).WithField("container", id).
324 324
 				Error("failed to delete task after fail start")
325 325
 		}
326
-		ctr.setTask(nil)
327 326
 		return -1, wrapError(err)
328 327
 	}
329 328
 
... ...
@@ -338,27 +243,30 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
338 338
 // the Start call. stdinCloseSync channel should be closed after Start exec
339 339
 // process.
340 340
 func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
341
-	ctr := c.getContainer(containerID)
342
-	if ctr == nil {
343
-		return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
344
-	}
345
-	t := ctr.getTask()
346
-	if t == nil {
347
-		return -1, errors.WithStack(errdefs.InvalidParameter(errors.New("container is not running")))
341
+	ctr, err := c.getContainer(ctx, containerID)
342
+	if err != nil {
343
+		return -1, err
348 344
 	}
349
-
350
-	if p := ctr.getProcess(processID); p != nil {
351
-		return -1, errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
345
+	t, err := ctr.Task(ctx, nil)
346
+	if err != nil {
347
+		if containerderrors.IsNotFound(err) {
348
+			return -1, errors.WithStack(errdefs.InvalidParameter(errors.New("container is not running")))
349
+		}
350
+		return -1, wrapError(err)
352 351
 	}
353 352
 
354 353
 	var (
355 354
 		p              containerd.Process
356 355
 		rio            cio.IO
357
-		err            error
358 356
 		stdinCloseSync = make(chan struct{})
359 357
 	)
360 358
 
361
-	fifos := newFIFOSet(ctr.bundleDir, processID, withStdin, spec.Terminal)
359
+	labels, err := ctr.Labels(ctx)
360
+	if err != nil {
361
+		return -1, wrapError(err)
362
+	}
363
+
364
+	fifos := newFIFOSet(labels[DockerContainerBundlePath], processID, withStdin, spec.Terminal)
362 365
 
363 366
 	defer func() {
364 367
 		if err != nil {
... ...
@@ -375,11 +283,12 @@ func (c *client) Exec(ctx context.Context, containerID, processID string, spec *
375 375
 	})
376 376
 	if err != nil {
377 377
 		close(stdinCloseSync)
378
+		if containerderrors.IsAlreadyExists(err) {
379
+			return -1, errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
380
+		}
378 381
 		return -1, wrapError(err)
379 382
 	}
380 383
 
381
-	ctr.addProcess(processID, p)
382
-
383 384
 	// Signal c.createIO that it can call CloseIO
384 385
 	//
385 386
 	// the stdin of exec process will be created after p.Start in containerd
... ...
@@ -392,15 +301,13 @@ func (c *client) Exec(ctx context.Context, containerID, processID string, spec *
392 392
 		ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
393 393
 		defer cancel()
394 394
 		p.Delete(ctx)
395
-		ctr.deleteProcess(processID)
396 395
 		return -1, wrapError(err)
397 396
 	}
398
-
399 397
 	return int(p.Pid()), nil
400 398
 }
401 399
 
402 400
 func (c *client) SignalProcess(ctx context.Context, containerID, processID string, signal int) error {
403
-	p, err := c.getProcess(containerID, processID)
401
+	p, err := c.getProcess(ctx, containerID, processID)
404 402
 	if err != nil {
405 403
 		return err
406 404
 	}
... ...
@@ -408,7 +315,7 @@ func (c *client) SignalProcess(ctx context.Context, containerID, processID strin
408 408
 }
409 409
 
410 410
 func (c *client) ResizeTerminal(ctx context.Context, containerID, processID string, width, height int) error {
411
-	p, err := c.getProcess(containerID, processID)
411
+	p, err := c.getProcess(ctx, containerID, processID)
412 412
 	if err != nil {
413 413
 		return err
414 414
 	}
... ...
@@ -417,7 +324,7 @@ func (c *client) ResizeTerminal(ctx context.Context, containerID, processID stri
417 417
 }
418 418
 
419 419
 func (c *client) CloseStdin(ctx context.Context, containerID, processID string) error {
420
-	p, err := c.getProcess(containerID, processID)
420
+	p, err := c.getProcess(ctx, containerID, processID)
421 421
 	if err != nil {
422 422
 		return err
423 423
 	}
... ...
@@ -426,7 +333,7 @@ func (c *client) CloseStdin(ctx context.Context, containerID, processID string)
426 426
 }
427 427
 
428 428
 func (c *client) Pause(ctx context.Context, containerID string) error {
429
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
429
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
430 430
 	if err != nil {
431 431
 		return err
432 432
 	}
... ...
@@ -435,7 +342,7 @@ func (c *client) Pause(ctx context.Context, containerID string) error {
435 435
 }
436 436
 
437 437
 func (c *client) Resume(ctx context.Context, containerID string) error {
438
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
438
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
439 439
 	if err != nil {
440 440
 		return err
441 441
 	}
... ...
@@ -444,7 +351,7 @@ func (c *client) Resume(ctx context.Context, containerID string) error {
444 444
 }
445 445
 
446 446
 func (c *client) Stats(ctx context.Context, containerID string) (*libcontainerdtypes.Stats, error) {
447
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
447
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
448 448
 	if err != nil {
449 449
 		return nil, err
450 450
 	}
... ...
@@ -462,7 +369,7 @@ func (c *client) Stats(ctx context.Context, containerID string) (*libcontainerdt
462 462
 }
463 463
 
464 464
 func (c *client) ListPids(ctx context.Context, containerID string) ([]uint32, error) {
465
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
465
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
466 466
 	if err != nil {
467 467
 		return nil, err
468 468
 	}
... ...
@@ -481,7 +388,7 @@ func (c *client) ListPids(ctx context.Context, containerID string) ([]uint32, er
481 481
 }
482 482
 
483 483
 func (c *client) Summary(ctx context.Context, containerID string) ([]libcontainerdtypes.Summary, error) {
484
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
484
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
485 485
 	if err != nil {
486 486
 		return nil, err
487 487
 	}
... ...
@@ -507,68 +414,75 @@ func (c *client) Summary(ctx context.Context, containerID string) ([]libcontaine
507 507
 	return infos, nil
508 508
 }
509 509
 
510
-func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) {
511
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
510
+type restoredProcess struct {
511
+	p containerd.Process
512
+}
513
+
514
+func (p *restoredProcess) Delete(ctx context.Context) (uint32, time.Time, error) {
515
+	if p.p == nil {
516
+		return 255, time.Now(), nil
517
+	}
518
+	status, err := p.p.Delete(ctx)
512 519
 	if err != nil {
513 520
 		return 255, time.Now(), nil
514 521
 	}
522
+	return status.ExitCode(), status.ExitTime(), nil
523
+}
515 524
 
516
-	status, err := p.(containerd.Task).Delete(ctx)
525
+func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) {
526
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
517 527
 	if err != nil {
518 528
 		return 255, time.Now(), nil
519 529
 	}
520 530
 
521
-	if ctr := c.getContainer(containerID); ctr != nil {
522
-		ctr.setTask(nil)
531
+	status, err := p.Delete(ctx)
532
+	if err != nil {
533
+		return 255, time.Now(), nil
523 534
 	}
524 535
 	return status.ExitCode(), status.ExitTime(), nil
525 536
 }
526 537
 
527 538
 func (c *client) Delete(ctx context.Context, containerID string) error {
528
-	ctr := c.getContainer(containerID)
529
-	if ctr == nil {
530
-		return errors.WithStack(errdefs.NotFound(errors.New("no such container")))
539
+	ctr, err := c.getContainer(ctx, containerID)
540
+	if err != nil {
541
+		return err
531 542
 	}
532
-
533
-	if err := ctr.ctr.Delete(ctx); err != nil {
543
+	labels, err := ctr.Labels(ctx)
544
+	if err != nil {
545
+		return err
546
+	}
547
+	bundle := labels[DockerContainerBundlePath]
548
+	if err := ctr.Delete(ctx); err != nil {
534 549
 		return wrapError(err)
535 550
 	}
536
-
551
+	c.oomMu.Lock()
552
+	delete(c.oom, containerID)
553
+	c.oomMu.Unlock()
537 554
 	if os.Getenv("LIBCONTAINERD_NOCLEAN") != "1" {
538
-		if err := os.RemoveAll(ctr.bundleDir); err != nil {
555
+		if err := os.RemoveAll(bundle); err != nil {
539 556
 			c.logger.WithError(err).WithFields(logrus.Fields{
540 557
 				"container": containerID,
541
-				"bundle":    ctr.bundleDir,
558
+				"bundle":    bundle,
542 559
 			}).Error("failed to remove state dir")
543 560
 		}
544 561
 	}
545
-
546
-	c.removeContainer(containerID)
547
-
548 562
 	return nil
549 563
 }
550 564
 
551 565
 func (c *client) Status(ctx context.Context, containerID string) (containerd.ProcessStatus, error) {
552
-	ctr := c.getContainer(containerID)
553
-	if ctr == nil {
554
-		return containerd.Unknown, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
555
-	}
556
-
557
-	t := ctr.getTask()
558
-	if t == nil {
559
-		return containerd.Unknown, errors.WithStack(errdefs.NotFound(errors.New("no such task")))
566
+	t, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
567
+	if err != nil {
568
+		return containerd.Unknown, err
560 569
 	}
561
-
562 570
 	s, err := t.Status(ctx)
563 571
 	if err != nil {
564 572
 		return containerd.Unknown, wrapError(err)
565 573
 	}
566
-
567 574
 	return s.Status, nil
568 575
 }
569 576
 
570 577
 func (c *client) CreateCheckpoint(ctx context.Context, containerID, checkpointDir string, exit bool) error {
571
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
578
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
572 579
 	if err != nil {
573 580
 		return err
574 581
 	}
... ...
@@ -633,37 +547,38 @@ func (c *client) CreateCheckpoint(ctx context.Context, containerID, checkpointDi
633 633
 	return err
634 634
 }
635 635
 
636
-func (c *client) getContainer(id string) *container {
637
-	c.RLock()
638
-	ctr := c.containers[id]
639
-	c.RUnlock()
640
-
641
-	return ctr
642
-}
643
-
644
-func (c *client) removeContainer(id string) {
645
-	c.Lock()
646
-	delete(c.containers, id)
647
-	c.Unlock()
636
+func (c *client) getContainer(ctx context.Context, id string) (containerd.Container, error) {
637
+	ctr, err := c.client.LoadContainer(ctx, id)
638
+	if err != nil {
639
+		if containerderrors.IsNotFound(err) {
640
+			return nil, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
641
+		}
642
+		return nil, wrapError(err)
643
+	}
644
+	return ctr, nil
648 645
 }
649 646
 
650
-func (c *client) getProcess(containerID, processID string) (containerd.Process, error) {
651
-	ctr := c.getContainer(containerID)
652
-	if ctr == nil {
653
-		return nil, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
647
+func (c *client) getProcess(ctx context.Context, containerID, processID string) (containerd.Process, error) {
648
+	ctr, err := c.getContainer(ctx, containerID)
649
+	if err != nil {
650
+		return nil, err
654 651
 	}
655
-
656
-	t := ctr.getTask()
657
-	if t == nil {
658
-		return nil, errors.WithStack(errdefs.NotFound(errors.New("container is not running")))
652
+	t, err := ctr.Task(ctx, nil)
653
+	if err != nil {
654
+		if containerderrors.IsNotFound(err) {
655
+			return nil, errors.WithStack(errdefs.NotFound(errors.New("container is not running")))
656
+		}
657
+		return nil, wrapError(err)
659 658
 	}
660 659
 	if processID == libcontainerdtypes.InitProcessName {
661 660
 		return t, nil
662 661
 	}
663
-
664
-	p := ctr.getProcess(processID)
665
-	if p == nil {
666
-		return nil, errors.WithStack(errdefs.NotFound(errors.New("no such exec")))
662
+	p, err := t.LoadProcess(ctx, processID, nil)
663
+	if err != nil {
664
+		if containerderrors.IsNotFound(err) {
665
+			return nil, errors.WithStack(errdefs.NotFound(errors.New("no such exec")))
666
+		}
667
+		return nil, wrapError(err)
667 668
 	}
668 669
 	return p, nil
669 670
 }
... ...
@@ -693,7 +608,7 @@ func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, std
693 693
 				// Exec/Start call failed.
694 694
 				go func() {
695 695
 					<-stdinCloseSync
696
-					p, err := c.getProcess(containerID, processID)
696
+					p, err := c.getProcess(context.Background(), containerID, processID)
697 697
 					if err == nil {
698 698
 						err = p.CloseIO(context.Background(), containerd.WithStdinCloser)
699 699
 						if err != nil && strings.Contains(err.Error(), "transport is closing") {
... ...
@@ -714,7 +629,7 @@ func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, std
714 714
 	return rio, err
715 715
 }
716 716
 
717
-func (c *client) processEvent(ctr *container, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) {
717
+func (c *client) processEvent(ctx context.Context, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) {
718 718
 	c.eventQ.Append(ei.ContainerID, func() {
719 719
 		err := c.backend.ProcessEvent(ei.ContainerID, et, ei)
720 720
 		if err != nil {
... ...
@@ -726,10 +641,12 @@ func (c *client) processEvent(ctr *container, et libcontainerdtypes.EventType, e
726 726
 		}
727 727
 
728 728
 		if et == libcontainerdtypes.EventExit && ei.ProcessID != ei.ContainerID {
729
-			p := ctr.getProcess(ei.ProcessID)
730
-			if p == nil {
729
+			p, err := c.getProcess(ctx, ei.ContainerID, ei.ProcessID)
730
+			if err != nil {
731
+
731 732
 				c.logger.WithError(errors.New("no such process")).
732 733
 					WithFields(logrus.Fields{
734
+						"error":     err,
733 735
 						"container": ei.ContainerID,
734 736
 						"process":   ei.ProcessID,
735 737
 					}).Error("exit event")
... ...
@@ -742,15 +659,23 @@ func (c *client) processEvent(ctr *container, et libcontainerdtypes.EventType, e
742 742
 					"process":   ei.ProcessID,
743 743
 				}).Warn("failed to delete process")
744 744
 			}
745
-			ctr.deleteProcess(ei.ProcessID)
746 745
 
747
-			ctr := c.getContainer(ei.ContainerID)
748
-			if ctr == nil {
746
+			ctr, err := c.getContainer(ctx, ei.ContainerID)
747
+			if err != nil {
749 748
 				c.logger.WithFields(logrus.Fields{
750 749
 					"container": ei.ContainerID,
750
+					"error":     err,
751 751
 				}).Error("failed to find container")
752 752
 			} else {
753
-				newFIFOSet(ctr.bundleDir, ei.ProcessID, true, false).Close()
753
+				labels, err := ctr.Labels(ctx)
754
+				if err != nil {
755
+					c.logger.WithFields(logrus.Fields{
756
+						"container": ei.ContainerID,
757
+						"error":     err,
758
+					}).Error("failed to find container")
759
+					return
760
+				}
761
+				newFIFOSet(labels[DockerContainerBundlePath], ei.ProcessID, true, false).Close()
754 762
 			}
755 763
 		}
756 764
 	})
... ...
@@ -762,7 +687,6 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
762 762
 		ev  *events.Envelope
763 763
 		et  libcontainerdtypes.EventType
764 764
 		ei  libcontainerdtypes.EventInfo
765
-		ctr *container
766 765
 	)
767 766
 
768 767
 	// Filter on both namespace *and* topic. To create an "and" filter,
... ...
@@ -771,8 +695,8 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
771 771
 
772 772
 	c.logger.Debug("processing event stream")
773 773
 
774
-	var oomKilled bool
775 774
 	for {
775
+		var oomKilled bool
776 776
 		select {
777 777
 		case err = <-errC:
778 778
 			if err != nil {
... ...
@@ -861,19 +785,14 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
861 861
 				continue
862 862
 			}
863 863
 
864
-			ctr = c.getContainer(ei.ContainerID)
865
-			if ctr == nil {
866
-				c.logger.WithField("container", ei.ContainerID).Warn("unknown container")
867
-				continue
868
-			}
869
-
864
+			c.oomMu.Lock()
870 865
 			if oomKilled {
871
-				ctr.setOOMKilled(true)
872
-				oomKilled = false
866
+				c.oom[ei.ContainerID] = true
873 867
 			}
874
-			ei.OOMKilled = ctr.getOOMKilled()
868
+			ei.OOMKilled = c.oom[ei.ContainerID]
869
+			c.oomMu.Unlock()
875 870
 
876
-			c.processEvent(ctr, et, ei)
871
+			c.processEvent(ctx, et, ei)
877 872
 		}
878 873
 	}
879 874
 }
... ...
@@ -901,6 +820,10 @@ func (c *client) writeContent(ctx context.Context, mediaType, ref string, r io.R
901 901
 	}, nil
902 902
 }
903 903
 
904
+func (c *client) bundleDir(id string) string {
905
+	return filepath.Join(c.stateDir, id)
906
+}
907
+
904 908
 func wrapError(err error) error {
905 909
 	switch {
906 910
 	case err == nil:
... ...
@@ -23,7 +23,7 @@ func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
23 23
 }
24 24
 
25 25
 func (c *client) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error {
26
-	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
26
+	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
27 27
 	if err != nil {
28 28
 		return err
29 29
 	}
... ...
@@ -62,8 +62,12 @@ func getSpecUser(ociSpec *specs.Spec) (int, int) {
62 62
 // WithBundle creates the bundle for the container
63 63
 func WithBundle(bundleDir string, ociSpec *specs.Spec) containerd.NewContainerOpts {
64 64
 	return func(ctx context.Context, client *containerd.Client, c *containers.Container) error {
65
+		if c.Labels == nil {
66
+			c.Labels = make(map[string]string)
67
+		}
65 68
 		uid, gid := getSpecUser(ociSpec)
66 69
 		if uid == 0 && gid == 0 {
70
+			c.Labels[DockerContainerBundlePath] = bundleDir
67 71
 			return idtools.MkdirAllAndChownNew(bundleDir, 0755, idtools.Identity{UID: 0, GID: 0})
68 72
 		}
69 73
 
... ...
@@ -82,6 +86,10 @@ func WithBundle(bundleDir string, ociSpec *specs.Spec) containerd.NewContainerOp
82 82
 				}
83 83
 			}
84 84
 		}
85
+		if c.Labels == nil {
86
+			c.Labels = make(map[string]string)
87
+		}
88
+		c.Labels[DockerContainerBundlePath] = p
85 89
 		return nil
86 90
 	}
87 91
 }
... ...
@@ -41,6 +41,10 @@ func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
41 41
 func WithBundle(bundleDir string, ociSpec *specs.Spec) containerd.NewContainerOpts {
42 42
 	return func(ctx context.Context, client *containerd.Client, c *containers.Container) error {
43 43
 		// TODO: (containerd) Determine if we need to use system.MkdirAllWithACL here
44
+		if c.Labels == nil {
45
+			c.Labels = make(map[string]string)
46
+		}
47
+		c.Labels[DockerContainerBundlePath] = bundleDir
44 48
 		return os.MkdirAll(bundleDir, 0755)
45 49
 	}
46 50
 }
... ...
@@ -41,11 +41,16 @@ type Backend interface {
41 41
 	ProcessEvent(containerID string, event EventType, ei EventInfo) error
42 42
 }
43 43
 
44
+// Process of a container
45
+type Process interface {
46
+	Delete(context.Context) (uint32, time.Time, error)
47
+}
48
+
44 49
 // Client provides access to containerd features.
45 50
 type Client interface {
46 51
 	Version(ctx context.Context) (containerd.Version, error)
47 52
 
48
-	Restore(ctx context.Context, containerID string, attachStdio StdioCallback) (alive bool, pid int, err error)
53
+	Restore(ctx context.Context, containerID string, attachStdio StdioCallback) (alive bool, pid int, p Process, err error)
49 54
 
50 55
 	Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error
51 56
 	Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio StdioCallback) (pid int, err error)
... ...
@@ -5,7 +5,6 @@ import (
5 5
 	"io"
6 6
 	"path/filepath"
7 7
 	"sync"
8
-	"time"
9 8
 
10 9
 	"github.com/containerd/containerd"
11 10
 	"github.com/containerd/containerd/cio"
... ...
@@ -26,19 +25,6 @@ type ExitHandler interface {
26 26
 	HandleExitEvent(id string) error
27 27
 }
28 28
 
29
-// Client is used by the exector to perform operations.
30
-// TODO(@cpuguy83): This should really just be based off the containerd client interface.
31
-// However right now this whole package is tied to github.com/docker/docker/libcontainerd
32
-type Client interface {
33
-	Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error
34
-	Restore(ctx context.Context, containerID string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error)
35
-	Status(ctx context.Context, containerID string) (containerd.ProcessStatus, error)
36
-	Delete(ctx context.Context, containerID string) error
37
-	DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error)
38
-	Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error)
39
-	SignalProcess(ctx context.Context, containerID, processID string, signal int) error
40
-}
41
-
42 29
 // New creates a new containerd plugin executor
43 30
 func New(ctx context.Context, rootDir string, cli *containerd.Client, exitHandler ExitHandler) (*Executor, error) {
44 31
 	e := &Executor{
... ...
@@ -57,19 +43,23 @@ func New(ctx context.Context, rootDir string, cli *containerd.Client, exitHandle
57 57
 // Executor is the containerd client implementation of a plugin executor
58 58
 type Executor struct {
59 59
 	rootDir     string
60
-	client      Client
60
+	client      libcontainerdtypes.Client
61 61
 	exitHandler ExitHandler
62 62
 }
63 63
 
64 64
 // deleteTaskAndContainer deletes plugin task and then plugin container from containerd
65
-func deleteTaskAndContainer(ctx context.Context, cli Client, id string) {
66
-	_, _, err := cli.DeleteTask(ctx, id)
67
-	if err != nil && !errdefs.IsNotFound(err) {
68
-		logrus.WithError(err).WithField("id", id).Error("failed to delete plugin task from containerd")
65
+func deleteTaskAndContainer(ctx context.Context, cli libcontainerdtypes.Client, id string, p libcontainerdtypes.Process) {
66
+	if p != nil {
67
+		if _, _, err := p.Delete(ctx); err != nil && !errdefs.IsNotFound(err) {
68
+			logrus.WithError(err).WithField("id", id).Error("failed to delete plugin task from containerd")
69
+		}
70
+	} else {
71
+		if _, _, err := cli.DeleteTask(ctx, id); err != nil && !errdefs.IsNotFound(err) {
72
+			logrus.WithError(err).WithField("id", id).Error("failed to delete plugin task from containerd")
73
+		}
69 74
 	}
70 75
 
71
-	err = cli.Delete(ctx, id)
72
-	if err != nil && !errdefs.IsNotFound(err) {
76
+	if err := cli.Delete(ctx, id); err != nil && !errdefs.IsNotFound(err) {
73 77
 		logrus.WithError(err).WithField("id", id).Error("failed to delete plugin container from containerd")
74 78
 	}
75 79
 }
... ...
@@ -103,19 +93,19 @@ func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteClo
103 103
 
104 104
 	_, err = e.client.Start(ctx, id, "", false, attachStreamsFunc(stdout, stderr))
105 105
 	if err != nil {
106
-		deleteTaskAndContainer(ctx, e.client, id)
106
+		deleteTaskAndContainer(ctx, e.client, id, nil)
107 107
 	}
108 108
 	return err
109 109
 }
110 110
 
111 111
 // Restore restores a container
112 112
 func (e *Executor) Restore(id string, stdout, stderr io.WriteCloser) (bool, error) {
113
-	alive, _, err := e.client.Restore(context.Background(), id, attachStreamsFunc(stdout, stderr))
113
+	alive, _, p, err := e.client.Restore(context.Background(), id, attachStreamsFunc(stdout, stderr))
114 114
 	if err != nil && !errdefs.IsNotFound(err) {
115 115
 		return false, err
116 116
 	}
117 117
 	if !alive {
118
-		deleteTaskAndContainer(context.Background(), e.client, id)
118
+		deleteTaskAndContainer(context.Background(), e.client, id, p)
119 119
 	}
120 120
 	return alive, nil
121 121
 }
... ...
@@ -136,7 +126,7 @@ func (e *Executor) Signal(id string, signal int) error {
136 136
 func (e *Executor) ProcessEvent(id string, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) error {
137 137
 	switch et {
138 138
 	case libcontainerdtypes.EventExit:
139
-		deleteTaskAndContainer(context.Background(), e.client, id)
139
+		deleteTaskAndContainer(context.Background(), e.client, id, nil)
140 140
 		return e.exitHandler.HandleExitEvent(ei.ContainerID)
141 141
 	}
142 142
 	return nil
143 143
deleted file mode 100644
... ...
@@ -1,149 +0,0 @@
1
-package containerd
2
-
3
-import (
4
-	"context"
5
-	"io/ioutil"
6
-	"os"
7
-	"sync"
8
-	"testing"
9
-	"time"
10
-
11
-	"github.com/containerd/containerd"
12
-	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
13
-	"github.com/opencontainers/runtime-spec/specs-go"
14
-	"github.com/pkg/errors"
15
-	"gotest.tools/assert"
16
-)
17
-
18
-func TestLifeCycle(t *testing.T) {
19
-	t.Parallel()
20
-
21
-	mock := newMockClient()
22
-	exec, cleanup := setupTest(t, mock, mock)
23
-	defer cleanup()
24
-
25
-	id := "test-create"
26
-	mock.simulateStartError(true, id)
27
-	err := exec.Create(id, specs.Spec{}, nil, nil)
28
-	assert.Assert(t, err != nil)
29
-	mock.simulateStartError(false, id)
30
-
31
-	err = exec.Create(id, specs.Spec{}, nil, nil)
32
-	assert.NilError(t, err)
33
-	running, _ := exec.IsRunning(id)
34
-	assert.Assert(t, running)
35
-
36
-	// create with the same ID
37
-	err = exec.Create(id, specs.Spec{}, nil, nil)
38
-	assert.Assert(t, err != nil)
39
-
40
-	mock.HandleExitEvent(id) // simulate a plugin that exits
41
-
42
-	err = exec.Create(id, specs.Spec{}, nil, nil)
43
-	assert.NilError(t, err)
44
-}
45
-
46
-func setupTest(t *testing.T, client Client, eh ExitHandler) (*Executor, func()) {
47
-	rootDir, err := ioutil.TempDir("", "test-daemon")
48
-	assert.NilError(t, err)
49
-	assert.Assert(t, client != nil)
50
-	assert.Assert(t, eh != nil)
51
-
52
-	return &Executor{
53
-			rootDir:     rootDir,
54
-			client:      client,
55
-			exitHandler: eh,
56
-		}, func() {
57
-			assert.Assert(t, os.RemoveAll(rootDir))
58
-		}
59
-}
60
-
61
-type mockClient struct {
62
-	mu           sync.Mutex
63
-	containers   map[string]bool
64
-	errorOnStart map[string]bool
65
-}
66
-
67
-func newMockClient() *mockClient {
68
-	return &mockClient{
69
-		containers:   make(map[string]bool),
70
-		errorOnStart: make(map[string]bool),
71
-	}
72
-}
73
-
74
-func (c *mockClient) Create(ctx context.Context, id string, _ *specs.Spec, _ interface{}) error {
75
-	c.mu.Lock()
76
-	defer c.mu.Unlock()
77
-
78
-	if _, ok := c.containers[id]; ok {
79
-		return errors.New("exists")
80
-	}
81
-
82
-	c.containers[id] = false
83
-	return nil
84
-}
85
-
86
-func (c *mockClient) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) {
87
-	return false, 0, nil
88
-}
89
-
90
-func (c *mockClient) Status(ctx context.Context, id string) (containerd.ProcessStatus, error) {
91
-	c.mu.Lock()
92
-	defer c.mu.Unlock()
93
-
94
-	running, ok := c.containers[id]
95
-	if !ok {
96
-		return containerd.Unknown, errors.New("not found")
97
-	}
98
-	if running {
99
-		return containerd.Running, nil
100
-	}
101
-	return containerd.Stopped, nil
102
-}
103
-
104
-func (c *mockClient) Delete(ctx context.Context, id string) error {
105
-	c.mu.Lock()
106
-	defer c.mu.Unlock()
107
-	delete(c.containers, id)
108
-	return nil
109
-}
110
-
111
-func (c *mockClient) DeleteTask(ctx context.Context, id string) (uint32, time.Time, error) {
112
-	return 0, time.Time{}, nil
113
-}
114
-
115
-func (c *mockClient) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error) {
116
-	c.mu.Lock()
117
-	defer c.mu.Unlock()
118
-
119
-	if _, ok := c.containers[id]; !ok {
120
-		return 0, errors.New("not found")
121
-	}
122
-
123
-	if c.errorOnStart[id] {
124
-		return 0, errors.New("some startup error")
125
-	}
126
-	c.containers[id] = true
127
-	return 1, nil
128
-}
129
-
130
-func (c *mockClient) SignalProcess(ctx context.Context, containerID, processID string, signal int) error {
131
-	return nil
132
-}
133
-
134
-func (c *mockClient) simulateStartError(sim bool, id string) {
135
-	c.mu.Lock()
136
-	defer c.mu.Unlock()
137
-	if sim {
138
-		c.errorOnStart[id] = sim
139
-		return
140
-	}
141
-	delete(c.errorOnStart, id)
142
-}
143
-
144
-func (c *mockClient) HandleExitEvent(id string) error {
145
-	c.mu.Lock()
146
-	defer c.mu.Unlock()
147
-	delete(c.containers, id)
148
-	return nil
149
-}