Browse code

Refactor commit

The goal of this refactor is to make it easier to integrate buildkit
and containerd snapshotters.

Commit is used from two places (api and build), each calls it
with distinct arguments. Refactored to pull out the common commit
logic and provide different interfaces for each consumer.

Signed-off-by: Daniel Nephin <dnephin@docker.com>

Daniel Nephin authored on 2018/02/07 03:27:55
Showing 10 changed files
... ...
@@ -21,7 +21,7 @@ type Backend interface {
21 21
 }
22 22
 
23 23
 type containerBackend interface {
24
-	Commit(name string, config *backend.ContainerCommitConfig) (imageID string, err error)
24
+	CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error)
25 25
 }
26 26
 
27 27
 type imageBackend interface {
... ...
@@ -34,33 +34,29 @@ func (s *imageRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *
34 34
 		return err
35 35
 	}
36 36
 
37
-	cname := r.Form.Get("container")
38
-
37
+	// TODO: remove pause arg, and always pause in backend
39 38
 	pause := httputils.BoolValue(r, "pause")
40 39
 	version := httputils.VersionFromContext(ctx)
41 40
 	if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") {
42 41
 		pause = true
43 42
 	}
44 43
 
45
-	c, _, _, err := s.decoder.DecodeConfig(r.Body)
44
+	config, _, _, err := s.decoder.DecodeConfig(r.Body)
46 45
 	if err != nil && err != io.EOF { //Do not fail if body is empty.
47 46
 		return err
48 47
 	}
49 48
 
50
-	commitCfg := &backend.ContainerCommitConfig{
51
-		ContainerCommitConfig: types.ContainerCommitConfig{
52
-			Pause:        pause,
53
-			Repo:         r.Form.Get("repo"),
54
-			Tag:          r.Form.Get("tag"),
55
-			Author:       r.Form.Get("author"),
56
-			Comment:      r.Form.Get("comment"),
57
-			Config:       c,
58
-			MergeConfigs: true,
59
-		},
49
+	commitCfg := &backend.CreateImageConfig{
50
+		Pause:   pause,
51
+		Repo:    r.Form.Get("repo"),
52
+		Tag:     r.Form.Get("tag"),
53
+		Author:  r.Form.Get("author"),
54
+		Comment: r.Form.Get("comment"),
55
+		Config:  config,
60 56
 		Changes: r.Form["changes"],
61 57
 	}
62 58
 
63
-	imgID, err := s.backend.Commit(cname, commitCfg)
59
+	imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg)
64 60
 	if err != nil {
65 61
 		return err
66 62
 	}
... ...
@@ -5,7 +5,6 @@ import (
5 5
 	"io"
6 6
 	"time"
7 7
 
8
-	"github.com/docker/docker/api/types"
9 8
 	"github.com/docker/docker/api/types/container"
10 9
 )
11 10
 
... ...
@@ -94,13 +93,26 @@ type ExecProcessConfig struct {
94 94
 	User       string   `json:"user,omitempty"`
95 95
 }
96 96
 
97
-// ContainerCommitConfig is a wrapper around
98
-// types.ContainerCommitConfig that also
99
-// transports configuration changes for a container.
100
-type ContainerCommitConfig struct {
101
-	types.ContainerCommitConfig
97
+// CreateImageConfig is the configuration for creating an image from a
98
+// container.
99
+type CreateImageConfig struct {
100
+	Repo    string
101
+	Tag     string
102
+	Pause   bool
103
+	Author  string
104
+	Comment string
105
+	Config  *container.Config
102 106
 	Changes []string
103
-	// TODO: ContainerConfig is only used by the dockerfile Builder, so remove it
104
-	// once the Builder has been updated to use a different interface
105
-	ContainerConfig *container.Config
107
+}
108
+
109
+// CommitConfig is the configuration for creating an image as part of a build.
110
+type CommitConfig struct {
111
+	Author              string
112
+	Comment             string
113
+	Config              *container.Config
114
+	ContainerConfig     *container.Config
115
+	ContainerID         string
116
+	ContainerMountLabel string
117
+	ContainerOS         string
118
+	ParentImageID       string
106 119
 }
... ...
@@ -25,19 +25,6 @@ type ContainerRmConfig struct {
25 25
 	ForceRemove, RemoveVolume, RemoveLink bool
26 26
 }
27 27
 
28
-// ContainerCommitConfig contains build configs for commit operation,
29
-// and is used when making a commit with the current state of the container.
30
-type ContainerCommitConfig struct {
31
-	Pause   bool
32
-	Repo    string
33
-	Tag     string
34
-	Author  string
35
-	Comment string
36
-	// merge container config into commit config before commit
37
-	MergeConfigs bool
38
-	Config       *container.Config
39
-}
40
-
41 28
 // ExecConfig is a small subset of the Config struct that holds the configuration
42 29
 // for the exec feature of docker.
43 30
 type ExecConfig struct {
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	"github.com/docker/docker/api/types/backend"
12 12
 	"github.com/docker/docker/api/types/container"
13 13
 	containerpkg "github.com/docker/docker/container"
14
+	"github.com/docker/docker/image"
14 15
 	"github.com/docker/docker/layer"
15 16
 	"github.com/docker/docker/pkg/containerfs"
16 17
 	"golang.org/x/net/context"
... ...
@@ -39,8 +40,9 @@ type Backend interface {
39 39
 	ImageBackend
40 40
 	ExecBackend
41 41
 
42
-	// Commit creates a new Docker image from an existing Docker container.
43
-	Commit(string, *backend.ContainerCommitConfig) (string, error)
42
+	// CommitBuildStep creates a new Docker image from the config generated by
43
+	// a build step.
44
+	CommitBuildStep(backend.CommitConfig) (image.ID, error)
44 45
 	// ContainerCreateWorkdir creates the workdir
45 46
 	ContainerCreateWorkdir(containerID string) error
46 47
 
... ...
@@ -13,6 +13,7 @@ import (
13 13
 	"github.com/docker/docker/builder"
14 14
 	"github.com/docker/docker/builder/dockerfile/instructions"
15 15
 	"github.com/docker/docker/builder/dockerfile/shell"
16
+	"github.com/docker/docker/image"
16 17
 	"github.com/docker/docker/pkg/system"
17 18
 	"github.com/docker/go-connections/nat"
18 19
 	"github.com/stretchr/testify/assert"
... ...
@@ -445,7 +446,7 @@ func TestRunWithBuildArgs(t *testing.T) {
445 445
 		assert.Equal(t, strslice.StrSlice{""}, config.Config.Entrypoint)
446 446
 		return container.ContainerCreateCreatedBody{ID: "12345"}, nil
447 447
 	}
448
-	mockBackend.commitFunc = func(cID string, cfg *backend.ContainerCommitConfig) (string, error) {
448
+	mockBackend.commitFunc = func(cfg backend.CommitConfig) (image.ID, error) {
449 449
 		// Check the runConfig.Cmd sent to commit()
450 450
 		assert.Equal(t, origCmd, cfg.Config.Cmd)
451 451
 		assert.Equal(t, cachedCmd, cfg.ContainerConfig.Cmd)
... ...
@@ -101,24 +101,17 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta
101 101
 		return nil
102 102
 	}
103 103
 
104
-	commitCfg := &backend.ContainerCommitConfig{
105
-		ContainerCommitConfig: types.ContainerCommitConfig{
106
-			Author: dispatchState.maintainer,
107
-			Pause:  true,
108
-			// TODO: this should be done by Commit()
109
-			Config: copyRunConfig(dispatchState.runConfig),
110
-		},
104
+	commitCfg := backend.CommitConfig{
105
+		Author: dispatchState.maintainer,
106
+		// TODO: this copy should be done by Commit()
107
+		Config:          copyRunConfig(dispatchState.runConfig),
111 108
 		ContainerConfig: containerConfig,
109
+		ContainerID:     id,
112 110
 	}
113 111
 
114
-	// Commit the container
115
-	imageID, err := b.docker.Commit(id, commitCfg)
116
-	if err != nil {
117
-		return err
118
-	}
119
-
120
-	dispatchState.imageID = imageID
121
-	return nil
112
+	imageID, err := b.docker.CommitBuildStep(commitCfg)
113
+	dispatchState.imageID = string(imageID)
114
+	return err
122 115
 }
123 116
 
124 117
 func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
... ...
@@ -10,6 +10,7 @@ import (
10 10
 	"github.com/docker/docker/api/types/container"
11 11
 	"github.com/docker/docker/builder"
12 12
 	containerpkg "github.com/docker/docker/container"
13
+	"github.com/docker/docker/image"
13 14
 	"github.com/docker/docker/layer"
14 15
 	"github.com/docker/docker/pkg/containerfs"
15 16
 	"golang.org/x/net/context"
... ...
@@ -18,7 +19,7 @@ import (
18 18
 // MockBackend implements the builder.Backend interface for unit testing
19 19
 type MockBackend struct {
20 20
 	containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
21
-	commitFunc          func(string, *backend.ContainerCommitConfig) (string, error)
21
+	commitFunc          func(backend.CommitConfig) (image.ID, error)
22 22
 	getImageFunc        func(string) (builder.Image, builder.ReleaseableLayer, error)
23 23
 	makeImageCacheFunc  func(cacheFrom []string) builder.ImageCache
24 24
 }
... ...
@@ -38,9 +39,9 @@ func (m *MockBackend) ContainerRm(name string, config *types.ContainerRmConfig)
38 38
 	return nil
39 39
 }
40 40
 
41
-func (m *MockBackend) Commit(cID string, cfg *backend.ContainerCommitConfig) (string, error) {
41
+func (m *MockBackend) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
42 42
 	if m.commitFunc != nil {
43
-		return m.commitFunc(cID, cfg)
43
+		return m.commitFunc(c)
44 44
 	}
45 45
 	return "", nil
46 46
 }
... ...
@@ -12,7 +12,6 @@ import (
12 12
 	"github.com/docker/docker/api/types/backend"
13 13
 	containertypes "github.com/docker/docker/api/types/container"
14 14
 	"github.com/docker/docker/builder/dockerfile"
15
-	"github.com/docker/docker/container"
16 15
 	"github.com/docker/docker/errdefs"
17 16
 	"github.com/docker/docker/image"
18 17
 	"github.com/docker/docker/layer"
... ...
@@ -122,9 +121,10 @@ func merge(userConf, imageConf *containertypes.Config) error {
122 122
 	return nil
123 123
 }
124 124
 
125
-// Commit creates a new filesystem image from the current state of a container.
126
-// The image can optionally be tagged into a repository.
127
-func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (string, error) {
125
+// CreateImageFromContainer creates a new image from a container. The container
126
+// config will be updated by applying the change set to the custom config, then
127
+// applying that config over the existing container config.
128
+func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateImageConfig) (string, error) {
128 129
 	start := time.Now()
129 130
 	container, err := daemon.GetContainer(name)
130 131
 	if err != nil {
... ...
@@ -150,26 +150,51 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
150 150
 		daemon.containerPause(container)
151 151
 		defer daemon.containerUnpause(container)
152 152
 	}
153
-	if !system.IsOSSupported(container.OS) {
154
-		return "", system.ErrNotSupportedOperatingSystem
155
-	}
156 153
 
157
-	if c.MergeConfigs && c.Config == nil {
154
+	if c.Config == nil {
158 155
 		c.Config = container.Config
159 156
 	}
160
-
161 157
 	newConfig, err := dockerfile.BuildFromConfig(c.Config, c.Changes, container.OS)
162 158
 	if err != nil {
163 159
 		return "", err
164 160
 	}
161
+	if err := merge(newConfig, container.Config); err != nil {
162
+		return "", err
163
+	}
165 164
 
166
-	if c.MergeConfigs {
167
-		if err := merge(newConfig, container.Config); err != nil {
168
-			return "", err
169
-		}
165
+	id, err := daemon.commitImage(backend.CommitConfig{
166
+		Author:              c.Author,
167
+		Comment:             c.Comment,
168
+		Config:              newConfig,
169
+		ContainerConfig:     container.Config,
170
+		ContainerID:         container.ID,
171
+		ContainerMountLabel: container.MountLabel,
172
+		ContainerOS:         container.OS,
173
+		ParentImageID:       string(container.ImageID),
174
+	})
175
+	if err != nil {
176
+		return "", err
170 177
 	}
171 178
 
172
-	rwTar, err := daemon.exportContainerRw(container)
179
+	imageRef, err := daemon.tagCommit(c.Repo, c.Tag, id)
180
+	if err != nil {
181
+		return "", err
182
+	}
183
+	daemon.LogContainerEventWithAttributes(container, "commit", map[string]string{
184
+		"comment":  c.Comment,
185
+		"imageID":  id.String(),
186
+		"imageRef": imageRef,
187
+	})
188
+	containerActions.WithValues("commit").UpdateSince(start)
189
+	return id.String(), nil
190
+}
191
+
192
+func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) {
193
+	layerStore, ok := daemon.layerStores[c.ContainerOS]
194
+	if !ok {
195
+		return "", system.ErrNotSupportedOperatingSystem
196
+	}
197
+	rwTar, err := exportContainerRw(layerStore, c.ContainerID, c.ContainerMountLabel)
173 198
 	if err != nil {
174 199
 		return "", err
175 200
 	}
... ...
@@ -180,35 +205,31 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
180 180
 	}()
181 181
 
182 182
 	var parent *image.Image
183
-	if container.ImageID == "" {
183
+	if c.ParentImageID == "" {
184 184
 		parent = new(image.Image)
185 185
 		parent.RootFS = image.NewRootFS()
186 186
 	} else {
187
-		parent, err = daemon.imageStore.Get(container.ImageID)
187
+		parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID))
188 188
 		if err != nil {
189 189
 			return "", err
190 190
 		}
191 191
 	}
192 192
 
193
-	l, err := daemon.layerStores[container.OS].Register(rwTar, parent.RootFS.ChainID())
193
+	l, err := layerStore.Register(rwTar, parent.RootFS.ChainID())
194 194
 	if err != nil {
195 195
 		return "", err
196 196
 	}
197
-	defer layer.ReleaseAndLog(daemon.layerStores[container.OS], l)
197
+	defer layer.ReleaseAndLog(layerStore, l)
198 198
 
199
-	containerConfig := c.ContainerConfig
200
-	if containerConfig == nil {
201
-		containerConfig = container.Config
202
-	}
203 199
 	cc := image.ChildConfig{
204
-		ContainerID:     container.ID,
200
+		ContainerID:     c.ContainerID,
205 201
 		Author:          c.Author,
206 202
 		Comment:         c.Comment,
207
-		ContainerConfig: containerConfig,
208
-		Config:          newConfig,
203
+		ContainerConfig: c.ContainerConfig,
204
+		Config:          c.Config,
209 205
 		DiffID:          l.DiffID(),
210 206
 	}
211
-	config, err := json.Marshal(image.NewChildImage(parent, cc, container.OS))
207
+	config, err := json.Marshal(image.NewChildImage(parent, cc, c.ContainerOS))
212 208
 	if err != nil {
213 209
 		return "", err
214 210
 	}
... ...
@@ -218,23 +239,27 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
218 218
 		return "", err
219 219
 	}
220 220
 
221
-	if container.ImageID != "" {
222
-		if err := daemon.imageStore.SetParent(id, container.ImageID); err != nil {
221
+	if c.ParentImageID != "" {
222
+		if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil {
223 223
 			return "", err
224 224
 		}
225 225
 	}
226
+	return id, nil
227
+}
226 228
 
229
+// TODO: remove from Daemon, move to api backend
230
+func (daemon *Daemon) tagCommit(repo string, tag string, id image.ID) (string, error) {
227 231
 	imageRef := ""
228
-	if c.Repo != "" {
229
-		newTag, err := reference.ParseNormalizedNamed(c.Repo) // todo: should move this to API layer
232
+	if repo != "" {
233
+		newTag, err := reference.ParseNormalizedNamed(repo) // todo: should move this to API layer
230 234
 		if err != nil {
231 235
 			return "", err
232 236
 		}
233 237
 		if !reference.IsNameOnly(newTag) {
234
-			return "", errors.Errorf("unexpected repository name: %s", c.Repo)
238
+			return "", errors.Errorf("unexpected repository name: %s", repo)
235 239
 		}
236
-		if c.Tag != "" {
237
-			if newTag, err = reference.WithTag(newTag, c.Tag); err != nil {
240
+		if tag != "" {
241
+			if newTag, err = reference.WithTag(newTag, tag); err != nil {
238 242
 				return "", err
239 243
 			}
240 244
 		}
... ...
@@ -243,26 +268,17 @@ func (daemon *Daemon) Commit(name string, c *backend.ContainerCommitConfig) (str
243 243
 		}
244 244
 		imageRef = reference.FamiliarString(newTag)
245 245
 	}
246
-
247
-	attributes := map[string]string{
248
-		"comment":  c.Comment,
249
-		"imageID":  id.String(),
250
-		"imageRef": imageRef,
251
-	}
252
-	daemon.LogContainerEventWithAttributes(container, "commit", attributes)
253
-	containerActions.WithValues("commit").UpdateSince(start)
254
-	return id.String(), nil
246
+	return imageRef, nil
255 247
 }
256 248
 
257
-func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io.ReadCloser, err error) {
258
-	// Note: Indexing by OS is safe as only called from `Commit` which has already performed validation
259
-	rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID)
249
+func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) {
250
+	rwlayer, err := layerStore.GetRWLayer(id)
260 251
 	if err != nil {
261 252
 		return nil, err
262 253
 	}
263 254
 	defer func() {
264 255
 		if err != nil {
265
-			daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
256
+			layerStore.ReleaseRWLayer(rwlayer)
266 257
 		}
267 258
 	}()
268 259
 
... ...
@@ -270,7 +286,7 @@ func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io
270 270
 	// mount the layer if needed. But the Diff() function for windows requests that
271 271
 	// the layer should be mounted when calling it. So we reserve this mount call
272 272
 	// until windows driver can implement Diff() interface correctly.
273
-	_, err = rwlayer.Mount(container.GetMountLabel())
273
+	_, err = rwlayer.Mount(mountLabel)
274 274
 	if err != nil {
275 275
 		return nil, err
276 276
 	}
... ...
@@ -283,8 +299,28 @@ func (daemon *Daemon) exportContainerRw(container *container.Container) (arch io
283 283
 	return ioutils.NewReadCloserWrapper(archive, func() error {
284 284
 			archive.Close()
285 285
 			err = rwlayer.Unmount()
286
-			daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer)
286
+			layerStore.ReleaseRWLayer(rwlayer)
287 287
 			return err
288 288
 		}),
289 289
 		nil
290 290
 }
291
+
292
+// CommitBuildStep is used by the builder to create an image for each step in
293
+// the build.
294
+//
295
+// This method is different from CreateImageFromContainer:
296
+//   * it doesn't attempt to validate container state
297
+//   * it doesn't send a commit action to metrics
298
+//   * it doesn't log a container commit event
299
+//
300
+// This is a temporary shim. Should be removed when builder stops using commit.
301
+func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) {
302
+	container, err := daemon.GetContainer(c.ContainerID)
303
+	if err != nil {
304
+		return "", err
305
+	}
306
+	c.ContainerMountLabel = container.MountLabel
307
+	c.ContainerOS = container.OS
308
+	c.ParentImageID = string(container.ImageID)
309
+	return daemon.commitImage(c)
310
+}
... ...
@@ -601,7 +601,7 @@ func (s *DockerSuite) TestEventsFilterType(c *check.C) {
601 601
 	events = strings.Split(strings.TrimSpace(out), "\n")
602 602
 
603 603
 	// Events generated by the container that builds the image
604
-	c.Assert(events, checker.HasLen, 3, check.Commentf("Events == %s", events))
604
+	c.Assert(events, checker.HasLen, 2, check.Commentf("Events == %s", events))
605 605
 
606 606
 	out, _ = dockerCmd(
607 607
 		c,