Browse code

Merge pull request #36224 from dnephin/refactor-commit

Refactor Daemon.Commit()

Akihiro Suda authored on 2018/02/08 21:02:30
Showing 10 changed files
... ...
@@ -21,7 +21,7 @@ type Backend interface {
21 21
 }
22 22
 
23 23
 type containerBackend interface {
24
-	Commit(name string, config *backend.ContainerCommitConfig) (imageID string, err error)
24
+	CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error)
25 25
 }
26 26
 
27 27
 type imageBackend interface {
... ...
@@ -34,33 +34,29 @@ func (s *imageRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *
34 34
 		return err
35 35
 	}
36 36
 
37
-	cname := r.Form.Get("container")
38
-
37
+	// TODO: remove pause arg, and always pause in backend
39 38
 	pause := httputils.BoolValue(r, "pause")
40 39
 	version := httputils.VersionFromContext(ctx)
41 40
 	if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") {
42 41
 		pause = true
43 42
 	}
44 43
 
45
-	c, _, _, err := s.decoder.DecodeConfig(r.Body)
44
+	config, _, _, err := s.decoder.DecodeConfig(r.Body)
46 45
 	if err != nil && err != io.EOF { //Do not fail if body is empty.
47 46
 		return err
48 47
 	}
49 48
 
50
-	commitCfg := &backend.ContainerCommitConfig{
51
-		ContainerCommitConfig: types.ContainerCommitConfig{
52
-			Pause:        pause,
53
-			Repo:         r.Form.Get("repo"),
54
-			Tag:          r.Form.Get("tag"),
55
-			Author:       r.Form.Get("author"),
56
-			Comment:      r.Form.Get("comment"),
57
-			Config:       c,
58
-			MergeConfigs: true,
59
-		},
49
+	commitCfg := &backend.CreateImageConfig{
50
+		Pause:   pause,
51
+		Repo:    r.Form.Get("repo"),
52
+		Tag:     r.Form.Get("tag"),
53
+		Author:  r.Form.Get("author"),
54
+		Comment: r.Form.Get("comment"),
55
+		Config:  config,
60 56
 		Changes: r.Form["changes"],
61 57
 	}
62 58
 
63
-	imgID, err := s.backend.Commit(cname, commitCfg)
59
+	imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg)
64 60
 	if err != nil {
65 61
 		return err
66 62
 	}
... ...
@@ -5,7 +5,6 @@ import (
5 5
 	"io"
6 6
 	"time"
7 7
 
8
-	"github.com/docker/docker/api/types"
9 8
 	"github.com/docker/docker/api/types/container"
10 9
 )
11 10
 
... ...
@@ -94,13 +93,26 @@ type ExecProcessConfig struct {
94 94
 	User       string   `json:"user,omitempty"`
95 95
 }
96 96
 
97
-// ContainerCommitConfig is a wrapper around
98
-// types.ContainerCommitConfig that also
99
-// transports configuration changes for a container.
100
-type ContainerCommitConfig struct {
101
-	types.ContainerCommitConfig
97
+// CreateImageConfig is the configuration for creating an image from a
98
+// container.
99
+type CreateImageConfig struct {
100
+	Repo    string
101
+	Tag     string
102
+	Pause   bool
103
+	Author  string
104
+	Comment string
105
+	Config  *container.Config
102 106
 	Changes []string
103
-	// TODO: ContainerConfig is only used by the dockerfile Builder, so remove it
104
-	// once the Builder has been updated to use a different interface
105
-	ContainerConfig *container.Config
107
+}
108
+
109
+// CommitConfig is the configuration for creating an image as part of a build.
110
+type CommitConfig struct {
111
+	Author              string
112
+	Comment             string
113
+	Config              *container.Config
114
+	ContainerConfig     *container.Config
115
+	ContainerID         string
116
+	ContainerMountLabel string
117
+	ContainerOS         string
118
+	ParentImageID       string
106 119
 }
... ...
@@ -25,19 +25,6 @@ type ContainerRmConfig struct {
25 25
 	ForceRemove, RemoveVolume, RemoveLink bool
26 26
 }
27 27
 
28
-// ContainerCommitConfig contains build configs for commit operation,
29
-// and is used when making a commit with the current state of the container.
30
-type ContainerCommitConfig struct {
31
-	Pause   bool
32
-	Repo    string
33
-	Tag     string
34
-	Author  string
35
-	Comment string
36
-	// merge container config into commit config before commit
37
-	MergeConfigs bool
38
-	Config       *container.Config
39
-}
40
-
41 28
 // ExecConfig is a small subset of the Config struct that holds the configuration
42 29
 // for the exec feature of docker.
43 30
 type ExecConfig struct {
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	"github.com/docker/docker/api/types/backend"
12 12
 	"github.com/docker/docker/api/types/container"
13 13
 	containerpkg "github.com/docker/docker/container"
14
+	"github.com/docker/docker/image"
14 15
 	"github.com/docker/docker/layer"
15 16
 	"github.com/docker/docker/pkg/containerfs"
16 17
 	"golang.org/x/net/context"
... ...
@@ -39,8 +40,9 @@ type Backend interface {
39 39
 	ImageBackend
40 40
 	ExecBackend
41 41
 
42
-	// Commit creates a new Docker image from an existing Docker container.
43
-	Commit(string, *backend.ContainerCommitConfig) (string, error)
42
+	// CommitBuildStep creates a new Docker image from the config generated by
43
+	// a build step.
44
+	CommitBuildStep(backend.CommitConfig) (image.ID, error)
44 45
 	// ContainerCreateWorkdir creates the workdir
45 46
 	ContainerCreateWorkdir(containerID string) error
46 47
 
... ...
@@ -13,6 +13,7 @@ import (
13 13
 	"github.com/docker/docker/builder"
14 14
 	"github.com/docker/docker/builder/dockerfile/instructions"
15 15
 	"github.com/docker/docker/builder/dockerfile/shell"
16
+	"github.com/docker/docker/image"
16 17
 	"github.com/docker/docker/pkg/system"
17 18
 	"github.com/docker/go-connections/nat"
18 19
 	"github.com/stretchr/testify/assert"
... ...
@@ -445,7 +446,7 @@ func TestRunWithBuildArgs(t *testing.T) {
445 445
 		assert.Equal(t, strslice.StrSlice{""}, config.Config.Entrypoint)
446 446
 		return container.ContainerCreateCreatedBody{ID: "12345"}, nil
447 447
 	}
448
-	mockBackend.commitFunc = func(cID string, cfg *backend.ContainerCommitConfig) (string, error) {
448
+	mockBackend.commitFunc = func(cfg backend.CommitConfig) (image.ID, error) {
449 449
 		// Check the runConfig.Cmd sent to commit()
450 450
 		assert.Equal(t, origCmd, cfg.Config.Cmd)
451 451
 		assert.Equal(t, cachedCmd, cfg.ContainerConfig.Cmd)
... ...
@@ -101,24 +101,17 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta
101 101
 		return nil
102 102
 	}
103 103
 
104
-	commitCfg := &backend.ContainerCommitConfig{
105
-		ContainerCommitConfig: types.ContainerCommitConfig{
106
-			Author: dispatchState.maintainer,
107
-			Pause:  true,
108
-			// TODO: this should be done by Commit()
109
-			Config: copyRunConfig(dispatchState.runConfig),
110
-		},
104
+	commitCfg := backend.CommitConfig{
105
+		Author: dispatchState.maintainer,
106
+		// TODO: this copy should be done by Commit()
107
+		Config:          copyRunConfig(dispatchState.runConfig),
111 108
 		ContainerConfig: containerConfig,
109
+		ContainerID:     id,
112 110
 	}
113 111
 
114
-	// Commit the container
115
-	imageID, err := b.docker.Commit(id, commitCfg)
116
-	if err != nil {
117
-		return err
118
-	}
119
-
120
-	dispatchState.imageID = imageID
121
-	return nil
112
+	imageID, err := b.docker.CommitBuildStep(commitCfg)
113
+	dispatchState.imageID = string(imageID)
114
+	return err
122 115
 }
123 116
 
124 117
 func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
... ...
@@ -10,6 +10,7 @@ import (
10 10
 	"github.com/docker/docker/api/types/container"
11 11
 	"github.com/docker/docker/builder"
12 12
 	containerpkg "github.com/docker/docker/container"
13
+	"github.com/docker/docker/image"
13 14
 	"github.com/docker/docker/layer"
14 15
 	"github.com/docker/docker/pkg/containerfs"
15 16
 	"golang.org/x/net/context"
... ...
@@ -18,7 +19,7 @@ import (
18 18
 // MockBackend implements the builder.Backend interface for unit testing
19 19
 type MockBackend struct {
20 20
 	containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
21
-	commitFunc          func(string, *backend.ContainerCommitConfig) (string, error)
21
+	commitFunc          func(backend.CommitConfig) (image.ID, error)
22 22
 	getImageFunc        func(string) (builder.Image, builder.ReleaseableLayer, error)
23 23
 	makeImageCacheFunc  func(cacheFrom []string) builder.ImageCache
24 24
 }
... ...
@@ -38,9 +39,9 @@ func (m *MockBackend) ContainerRm(name string, config *types.ContainerRmConfig)
38 38
 	return nil
39 39
 }
40 40
 
41
-func (m *MockBackend) Commit(cID string, cfg *backend.ContainerCommitConfig) (string, error) {
41
+func (m *MockBackend) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
42 42
 	if m.commitFunc != nil {
43
-		return m.commitFunc(cID, cfg)
43
+		return m.commitFunc(c)
44 44
 	}
45 45
 	return "", nil
46 46
 }
... ...
@@ -12,7 +12,6 @@ import (
12 12
 	"github.com/docker/docker/api/types/backend"
13 13
 	containertypes "github.com/docker/docker/api/types/container"
14 14
 	"github.com/docker/docker/builder/dockerfile"
15
-	"github.com/docker/docker/container"
16 15
 	"github.com/docker/docker/errdefs"
17 16
 	"github.com/docker/docker/image"
18 17
 	"github.com/docker/docker/layer"
... ...
@@ -122,9 +121,10 @@ func merge(userConf, imageConf *containertypes.Config) error {
122 122
 	return nil
123 123
 }
124 124
 
125
-// Commit creates a new filesystem image from the current state of a container.
126
-// The image can optionally be tagged into a repository.
127
-func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (string, error) {
125
+// CreateImageFromContainer creates a new image from a container. The container
126
+// config will be updated by applying the change set to the custom config, then
127
+// applying that config over the existing container config.
128
+func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateImageConfig) (string, error) {
128 129
 	start := time.Now()
129 130
 	container, err := daemon.GetContainer(name)
130 131
 	if err != nil {
... ...
@@ -150,26 +150,51 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
150 150
 		daemon.containerPause(container)
151 151
 		defer daemon.containerUnpause(container)
152 152
 	}
153
-	if !system.IsOSSupported(container.OS) {
154
-		return "", system.ErrNotSupportedOperatingSystem
155
-	}
156 153
 
157
-	if c.MergeConfigs && c.Config == nil {
154
+	if c.Config == nil {
158 155
 		c.Config = container.Config
159 156
 	}
160
-
161 157
 	newConfig, err := dockerfile.BuildFromConfig(c.Config, c.Changes, container.OS)
162 158
 	if err != nil {
163 159
 		return "", err
164 160
 	}
161
+	if err := merge(newConfig, container.Config); err != nil {
162
+		return "", err
163
+	}
165 164
 
166
-	if c.MergeConfigs {
167
-		if err := merge(newConfig, container.Config); err != nil {
168
-			return "", err
169
-		}
165
+	id, err := daemon.commitImage(backend.CommitConfig{
166
+		Author:              c.Author,
167
+		Comment:             c.Comment,
168
+		Config:              newConfig,
169
+		ContainerConfig:     container.Config,
170
+		ContainerID:         container.ID,
171
+		ContainerMountLabel: container.MountLabel,
172
+		ContainerOS:         container.OS,
173
+		ParentImageID:       string(container.ImageID),
174
+	})
175
+	if err != nil {
176
+		return "", err
170 177
 	}
171 178
 
172
-	rwTar, err := daemon.exportContainerRw(container)
179
+	imageRef, err := daemon.tagCommit(c.Repo, c.Tag, id)
180
+	if err != nil {
181
+		return "", err
182
+	}
183
+	daemon.LogContainerEventWithAttributes(container, "commit", map[string]string{
184
+		"comment":  c.Comment,
185
+		"imageID":  id.String(),
186
+		"imageRef": imageRef,
187
+	})
188
+	containerActions.WithValues("commit").UpdateSince(start)
189
+	return id.String(), nil
190
+}
191
+
192
+func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) {
193
+	layerStore, ok := daemon.layerStores[c.ContainerOS]
194
+	if !ok {
195
+		return "", system.ErrNotSupportedOperatingSystem
196
+	}
197
+	rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel)
173 198
 	if err != nil {
174 199
 		return "", err
175 200
 	}
... ...
@@ -180,35 +205,31 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
180 180
 	}()
181 181
 
182 182
 	var parent *image.Image
183
-	if container.ImageID == "" {
183
+	if c.ParentImageID == "" {
184 184
 		parent = new(image.Image)
185 185
 		parent.RootFS = image.NewRootFS()
186 186
 	} else {
187
-		parent, err = daemon.imageStore.Get(container.ImageID)
187
+		parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID))
188 188
 		if err != nil {
189 189
 			return "", err
190 190
 		}
191 191
 	}
192 192
 
193
-	l, err := daemon.layerStores[container.OS].Register(rwTar, parent.RootFS.ChainID())
193
+	l, err := layerStore.Register(rwTar, parent.RootFS.ChainID())
194 194
 	if err != nil {
195 195
 		return "", err
196 196
 	}
197
-	defer layer.ReleaseAndLog(daemon.layerStores[container.OS], l)
197
+	defer layer.ReleaseAndLog(layerStore, l)
198 198
 
199
-	containerConfig := c.ContainerConfig
200
-	if containerConfig == nil {
201
-		containerConfig = container.Config
202
-	}
203 199
 	cc := image.ChildConfig{
204
-		ContainerID:     container.ID,
200
+		ContainerID:     c.ContainerID,
205 201
 		Author:          c.Author,
206 202
 		Comment:         c.Comment,
207
-		ContainerConfig: containerConfig,
208
-		Config:          newConfig,
203
+		ContainerConfig: c.ContainerConfig,
204
+		Config:          c.Config,
209 205
 		DiffID:          l.DiffID(),
210 206
 	}
211
-	config, err := json.Marshal(image.NewChildImage(parent, cc, container.OS))
207
+	config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS))
212 208
 	if err != nil {
213 209
 		return "", err
214 210
 	}
... ...
@@ -218,23 +239,27 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
218 218
 		return "", err
219 219
 	}
220 220
 
221
-	if container.ImageID != "" {
222
-		if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
221
+	if c.ParentImageID != "" {
222
+		if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil {
223 223
 			return "", err
224 224
 		}
225 225
 	}
226
+	return id, nil
227
+}
226 228
 
229
+// TODO: remove from Daemon, move to api backend
230
+func (daemon *Daemon) tagCommit(repo string, tag string, id image.ID) (string, error) {
227 231
 	imageRef := ""
228
-	if c.Repo != "" {
229
-		newTag, err := reference.ParseNormalizedNamed(c.Repo) // todo: should move this to API layer
232
+	if repo != "" {
233
+		newTag, err := reference.ParseNormalizedNamed(repo) // todo: should move this to API layer
230 234
 		if err != nil {
231 235
 			return "", err
232 236
 		}
233 237
 		if !reference.IsNameOnly(newTag) {
234
-			return "", errors.Errorf("unexpected repository name: %s", c.Repo)
238
+			return "", errors.Errorf("unexpected repository name: %s", repo)
235 239
 		}
236
-		if c.Tag != "" {
237
-			if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
240
+		if tag != "" {
241
+			if newTag, err = reference.WithTag(newTag, tag); err != nil {
238 242
 				return "", err
239 243
 			}
240 244
 		}
... ...
@@ -243,26 +268,17 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
243 243
 		}
244 244
 		imageRef = reference.FamiliarString(newTag)
245 245
 	}
246
-
247
-	attributes := map[string]string{
248
-		"comment":  c.Comment,
249
-		"imageID":  id.String(),
250
-		"imageRef": imageRef,
251
-	}
252
-	daemon.LogContainerEventWithAttributes(container, "commit", attributes)
253
-	containerActions.WithValues("commit").UpdateSince(start)
254
-	return id.String(), nil
246
+	return imageRef, nil
255 247
 }
256 248
 
257
-func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io.ReadCloser, err error) {
258
-	// Note: Indexing by OS is safe as only called from `Commit` which has already performed validation
259
-	rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID)
249
+func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) {
250
+	rwlayer, err := layerStore.GetRWLayer(id)
260 251
 	if err != nil {
261 252
 		return nil, err
262 253
 	}
263 254
 	defer func() {
264 255
 		if err != nil {
265
-			daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
256
+			layerStore.ReleaseRWLayer(rwlayer)
266 257
 		}
267 258
 	}()
268 259
 
... ...
@@ -270,7 +286,7 @@ func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io
270 270
 	// mount the layer if needed. But the Diff() function for windows requests that
271 271
 	// the layer should be mounted when calling it. So we reserve this mount call
272 272
 	// until windows driver can implement Diff() interface correctly.
273
-	_, err = rwlayer.Mount(container.GetMountLabel())
273
+	_, err = rwlayer.Mount(mountLabel)
274 274
 	if err != nil {
275 275
 		return nil, err
276 276
 	}
... ...
@@ -283,8 +299,28 @@ func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io
283 283
 	return ioutils.NewReadCloserWrapper(archive, func() error {
284 284
 			archive.Close()
285 285
 			err = rwlayer.Unmount()
286
-			daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
286
+			layerStore.ReleaseRWLayer(rwlayer)
287 287
 			return err
288 288
 		}),
289 289
 		nil
290 290
 }
291
+
292
+// CommitBuildStep is used by the builder to create an image for each step in
293
+// the build.
294
+//
295
+// This method is different from CreateImageFromContainer:
296
+//   * it doesn't attempt to validate container state
297
+//   * it doesn't send a commit action to metrics
298
+//   * it doesn't log a container commit event
299
+//
300
+// This is a temporary shim. Should be removed when builder stops using commit.
301
+func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
302
+	container, err := daemon.GetContainer(c.ContainerID)
303
+	if err != nil {
304
+		return "", err
305
+	}
306
+	c.ContainerMountLabel = container.MountLabel
307
+	c.ContainerOS = container.OS
308
+	c.ParentImageID = string(container.ImageID)
309
+	return daemon.commitImage(c)
310
+}
... ...
@@ -601,7 +601,7 @@ func (s *DockerSuite) TestEventsFilterType(c *check.C) {
601 601
 	events = strings.Split(strings.TrimSpace(out), "\n")
602 602
 
603 603
 	// Events generated by the container that builds the image
604
-	c.Assert(events, checker.HasLen, 3, check.Commentf("Events == %s", events))
604
+	c.Assert(events, checker.HasLen, 2, check.Commentf("Events == %s", events))
605 605
 
606 606
 	out, _ = dockerCmd(
607 607
 		c,