Browse code

container: Do not remove contianer if any of the resource failed cleanup

Do not remove container if any of the resource could not be cleaned up. We
don't want to leak resources.

Two new states have been created. RemovalInProgress and Dead. Once container
is Dead, it can not be started/restarted. Dead container signifies the
container where we tried to remove it but removal failed. User now needs to
figure out what went wrong, corrent the situation and try cleanup again.

RemovalInProgress signifies that container is already being removed. Only
one removal can be in progress.

Also, do not allow start of a container if it is already dead or removal is
in progress.

Also extend existing force option (-f) to docker rm to not return an error
and remove container from user view even if resource cleanup failed.
This will allow a user to get back to old behavior where resources
might leak but atleast user will be able to make progress.

Signed-off-by: Vivek Goyal <vgoyal@redhat.com>

Vivek Goyal authored on 2015/03/13 04:26:17
Showing 3 changed files
... ...
@@ -360,6 +360,10 @@ func (container *Container) Start() (err error) {
360 360
 		return nil
361 361
 	}
362 362
 
363
+	if container.removalInProgress || container.Dead {
364
+		return fmt.Errorf("Container is marked for removal and cannot be started.")
365
+	}
366
+
363 367
 	// if we encounter an error during start we need to ensure that any other
364 368
 	// setup has been cleaned up properly
365 369
 	defer func() {
... ...
@@ -63,8 +63,15 @@ func (daemon *Daemon) ContainerRm(job *engine.Job) error {
63 63
 				return fmt.Errorf("Conflict, You cannot remove a running container. Stop the container before attempting removal or use -f")
64 64
 			}
65 65
 		}
66
-		if err := daemon.Rm(container); err != nil {
67
-			return fmt.Errorf("Cannot destroy container %s: %s", name, err)
66
+
67
+		if forceRemove {
68
+			if err := daemon.ForceRm(container); err != nil {
69
+				logrus.Errorf("Cannot destroy container %s: %v", name, err)
70
+			}
71
+		} else {
72
+			if err := daemon.Rm(container); err != nil {
73
+				return fmt.Errorf("Cannot destroy container %s: %v", name, err)
74
+			}
68 75
 		}
69 76
 		container.LogEvent("destroy")
70 77
 		if removeVolume {
... ...
@@ -83,8 +90,16 @@ func (daemon *Daemon) DeleteVolumes(volumeIDs map[string]struct{}) {
83 83
 	}
84 84
 }
85 85
 
86
+func (daemon *Daemon) Rm(container *Container) (err error) {
87
+	return daemon.commonRm(container, false)
88
+}
89
+
90
+func (daemon *Daemon) ForceRm(container *Container) (err error) {
91
+	return daemon.commonRm(container, true)
92
+}
93
+
86 94
 // Destroy unregisters a container from the daemon and cleanly removes its contents from the filesystem.
87
-func (daemon *Daemon) Rm(container *Container) error {
95
+func (daemon *Daemon) commonRm(container *Container, forceRemove bool) (err error) {
88 96
 	if container == nil {
89 97
 		return fmt.Errorf("The given container is <nil>")
90 98
 	}
... ...
@@ -94,19 +109,40 @@ func (daemon *Daemon) Rm(container *Container) error {
94 94
 		return fmt.Errorf("Container %v not found - maybe it was already destroyed?", container.ID)
95 95
 	}
96 96
 
97
-	if err := container.Stop(3); err != nil {
97
+	// Container state RemovalInProgress should be used to avoid races.
98
+	if err = container.SetRemovalInProgress(); err != nil {
99
+		return fmt.Errorf("Failed to set container state to RemovalInProgress: %s", err)
100
+	}
101
+
102
+	defer container.ResetRemovalInProgress()
103
+
104
+	if err = container.Stop(3); err != nil {
98 105
 		return err
99 106
 	}
100 107
 
101
-	// Deregister the container before removing its directory, to avoid race conditions
102
-	daemon.idIndex.Delete(container.ID)
103
-	daemon.containers.Delete(container.ID)
108
+	// Mark container dead. We don't want anybody to be restarting it.
109
+	container.SetDead()
110
+
111
+	// Save container state to disk. So that if error happens before
112
+	// container meta file got removed from disk, then a restart of
113
+	// docker should not make a dead container alive.
114
+	container.ToDisk()
115
+
116
+	// If force removal is required, delete container from various
117
+	// indexes even if removal failed.
118
+	defer func() {
119
+		if err != nil && forceRemove {
120
+			daemon.idIndex.Delete(container.ID)
121
+			daemon.containers.Delete(container.ID)
122
+		}
123
+	}()
124
+
104 125
 	container.derefVolumes()
105 126
 	if _, err := daemon.containerGraph.Purge(container.ID); err != nil {
106 127
 		logrus.Debugf("Unable to remove container from link graph: %s", err)
107 128
 	}
108 129
 
109
-	if err := daemon.driver.Remove(container.ID); err != nil {
130
+	if err = daemon.driver.Remove(container.ID); err != nil {
110 131
 		return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", daemon.driver, container.ID, err)
111 132
 	}
112 133
 
... ...
@@ -115,15 +151,17 @@ func (daemon *Daemon) Rm(container *Container) error {
115 115
 		return fmt.Errorf("Driver %s failed to remove init filesystem %s: %s", daemon.driver, initID, err)
116 116
 	}
117 117
 
118
-	if err := os.RemoveAll(container.root); err != nil {
118
+	if err = os.RemoveAll(container.root); err != nil {
119 119
 		return fmt.Errorf("Unable to remove filesystem for %v: %v", container.ID, err)
120 120
 	}
121 121
 
122
-	if err := daemon.execDriver.Clean(container.ID); err != nil {
122
+	if err = daemon.execDriver.Clean(container.ID); err != nil {
123 123
 		return fmt.Errorf("Unable to remove execdriver data for %s: %s", container.ID, err)
124 124
 	}
125 125
 
126 126
 	selinuxFreeLxcContexts(container.ProcessLabel)
127
+	daemon.idIndex.Delete(container.ID)
128
+	daemon.containers.Delete(container.ID)
127 129
 
128 130
 	return nil
129 131
 }
... ...
@@ -11,16 +11,18 @@ import (
11 11
 
12 12
 type State struct {
13 13
 	sync.Mutex
14
-	Running    bool
15
-	Paused     bool
16
-	Restarting bool
17
-	OOMKilled  bool
18
-	Pid        int
19
-	ExitCode   int
20
-	Error      string // contains last known error when starting the container
21
-	StartedAt  time.Time
22
-	FinishedAt time.Time
23
-	waitChan   chan struct{}
14
+	Running           bool
15
+	Paused            bool
16
+	Restarting        bool
17
+	OOMKilled         bool
18
+	removalInProgress bool // Not need for this to be persistent on disk.
19
+	Dead              bool
20
+	Pid               int
21
+	ExitCode          int
22
+	Error             string // contains last known error when starting the container
23
+	StartedAt         time.Time
24
+	FinishedAt        time.Time
25
+	waitChan          chan struct{}
24 26
 }
25 27
 
26 28
 func NewState() *State {
... ...
@@ -42,6 +44,14 @@ func (s *State) String() string {
42 42
 		return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt)))
43 43
 	}
44 44
 
45
+	if s.removalInProgress {
46
+		return "Removal In Progress"
47
+	}
48
+
49
+	if s.Dead {
50
+		return "Dead"
51
+	}
52
+
45 53
 	if s.FinishedAt.IsZero() {
46 54
 		return ""
47 55
 	}
... ...
@@ -60,6 +70,11 @@ func (s *State) StateString() string {
60 60
 		}
61 61
 		return "running"
62 62
 	}
63
+
64
+	if s.Dead {
65
+		return "dead"
66
+	}
67
+
63 68
 	return "exited"
64 69
 }
65 70
 
... ...
@@ -217,3 +232,25 @@ func (s *State) IsPaused() bool {
217 217
 	s.Unlock()
218 218
 	return res
219 219
 }
220
+
221
+func (s *State) SetRemovalInProgress() error {
222
+	s.Lock()
223
+	defer s.Unlock()
224
+	if s.removalInProgress {
225
+		return fmt.Errorf("Status is already RemovalInProgress")
226
+	}
227
+	s.removalInProgress = true
228
+	return nil
229
+}
230
+
231
+func (s *State) ResetRemovalInProgress() {
232
+	s.Lock()
233
+	s.removalInProgress = false
234
+	s.Unlock()
235
+}
236
+
237
+func (s *State) SetDead() {
238
+	s.Lock()
239
+	s.Dead = true
240
+	s.Unlock()
241
+}