Extract a common function for builder.createContainer
Extract imageCache for doing cache probes
Removes the cacheBuested field from Builder
Create a new containerManager class which reduces the interface between the
builder and managing containers to 3 functions (from 6)
Signed-off-by: Daniel Nephin <dnephin@docker.com>
... | ... |
@@ -7,12 +7,11 @@ package builder |
7 | 7 |
import ( |
8 | 8 |
"io" |
9 | 9 |
|
10 |
- "golang.org/x/net/context" |
|
11 |
- |
|
12 | 10 |
"github.com/docker/docker/api/types" |
13 | 11 |
"github.com/docker/docker/api/types/backend" |
14 | 12 |
"github.com/docker/docker/api/types/container" |
15 | 13 |
containerpkg "github.com/docker/docker/container" |
14 |
+ "golang.org/x/net/context" |
|
16 | 15 |
) |
17 | 16 |
|
18 | 17 |
const ( |
... | ... |
@@ -36,21 +35,10 @@ type Source interface { |
36 | 36 |
// Backend abstracts calls to a Docker Daemon. |
37 | 37 |
type Backend interface { |
38 | 38 |
ImageBackend |
39 |
+ ExecBackend |
|
39 | 40 |
|
40 |
- // ContainerAttachRaw attaches to container. |
|
41 |
- ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error |
|
42 |
- // ContainerCreate creates a new Docker container and returns potential warnings |
|
43 |
- ContainerCreate(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) |
|
44 |
- // ContainerRm removes a container specified by `id`. |
|
45 |
- ContainerRm(name string, config *types.ContainerRmConfig) error |
|
46 | 41 |
// Commit creates a new Docker image from an existing Docker container. |
47 | 42 |
Commit(string, *backend.ContainerCommitConfig) (string, error) |
48 |
- // ContainerKill stops the container execution abruptly. |
|
49 |
- ContainerKill(containerID string, sig uint64) error |
|
50 |
- // ContainerStart starts a new container |
|
51 |
- ContainerStart(containerID string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error |
|
52 |
- // ContainerWait stops processing until the given container is stopped. |
|
53 |
- ContainerWait(ctx context.Context, name string, condition containerpkg.WaitCondition) (<-chan containerpkg.StateStatus, error) |
|
54 | 43 |
// ContainerCreateWorkdir creates the workdir |
55 | 44 |
ContainerCreateWorkdir(containerID string) error |
56 | 45 |
|
... | ... |
@@ -59,6 +47,8 @@ type Backend interface { |
59 | 59 |
// TODO: extract in the builder instead of passing `decompress` |
60 | 60 |
// TODO: use containerd/fs.changestream instead as a source |
61 | 61 |
CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error |
62 |
+ |
|
63 |
+ ImageCacheBuilder |
|
62 | 64 |
} |
63 | 65 |
|
64 | 66 |
// ImageBackend are the interface methods required from an image component |
... | ... |
@@ -66,6 +56,22 @@ type ImageBackend interface { |
66 | 66 |
GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (Image, ReleaseableLayer, error) |
67 | 67 |
} |
68 | 68 |
|
69 |
+// ExecBackend contains the interface methods required for executing containers |
|
70 |
+type ExecBackend interface { |
|
71 |
+ // ContainerAttachRaw attaches to container. |
|
72 |
+ ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error |
|
73 |
+ // ContainerCreate creates a new Docker container and returns potential warnings |
|
74 |
+ ContainerCreate(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) |
|
75 |
+ // ContainerRm removes a container specified by `id`. |
|
76 |
+ ContainerRm(name string, config *types.ContainerRmConfig) error |
|
77 |
+ // ContainerKill stops the container execution abruptly. |
|
78 |
+ ContainerKill(containerID string, sig uint64) error |
|
79 |
+ // ContainerStart starts a new container |
|
80 |
+ ContainerStart(containerID string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error |
|
81 |
+ // ContainerWait stops processing until the given container is stopped. |
|
82 |
+ ContainerWait(ctx context.Context, name string, condition containerpkg.WaitCondition) (<-chan containerpkg.StateStatus, error) |
|
83 |
+} |
|
84 |
+ |
|
69 | 85 |
// Result is the output produced by a Builder |
70 | 86 |
type Result struct { |
71 | 87 |
ImageID string |
... | ... |
@@ -35,8 +35,6 @@ var validCommitCommands = map[string]bool{ |
35 | 35 |
"workdir": true, |
36 | 36 |
} |
37 | 37 |
|
38 |
-var defaultLogConfig = container.LogConfig{Type: "none"} |
|
39 |
- |
|
40 | 38 |
// BuildManager is shared across all Builder objects |
41 | 39 |
type BuildManager struct { |
42 | 40 |
backend builder.Backend |
... | ... |
@@ -100,14 +98,13 @@ type Builder struct { |
100 | 100 |
docker builder.Backend |
101 | 101 |
clientCtx context.Context |
102 | 102 |
|
103 |
- tmpContainers map[string]struct{} |
|
104 |
- buildStages *buildStages |
|
105 |
- disableCommit bool |
|
106 |
- cacheBusted bool |
|
107 |
- buildArgs *buildArgs |
|
108 |
- imageCache builder.ImageCache |
|
109 |
- imageSources *imageSources |
|
110 |
- pathCache pathCache |
|
103 |
+ buildStages *buildStages |
|
104 |
+ disableCommit bool |
|
105 |
+ buildArgs *buildArgs |
|
106 |
+ imageSources *imageSources |
|
107 |
+ pathCache pathCache |
|
108 |
+ containerManager *containerManager |
|
109 |
+ imageProber ImageProber |
|
111 | 110 |
} |
112 | 111 |
|
113 | 112 |
// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options. |
... | ... |
@@ -117,29 +114,23 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder { |
117 | 117 |
config = new(types.ImageBuildOptions) |
118 | 118 |
} |
119 | 119 |
b := &Builder{ |
120 |
- clientCtx: clientCtx, |
|
121 |
- options: config, |
|
122 |
- Stdout: options.ProgressWriter.StdoutFormatter, |
|
123 |
- Stderr: options.ProgressWriter.StderrFormatter, |
|
124 |
- Aux: options.ProgressWriter.AuxFormatter, |
|
125 |
- Output: options.ProgressWriter.Output, |
|
126 |
- docker: options.Backend, |
|
127 |
- tmpContainers: map[string]struct{}{}, |
|
128 |
- buildArgs: newBuildArgs(config.BuildArgs), |
|
129 |
- buildStages: newBuildStages(), |
|
130 |
- imageSources: newImageSources(clientCtx, options), |
|
131 |
- pathCache: options.PathCache, |
|
120 |
+ clientCtx: clientCtx, |
|
121 |
+ options: config, |
|
122 |
+ Stdout: options.ProgressWriter.StdoutFormatter, |
|
123 |
+ Stderr: options.ProgressWriter.StderrFormatter, |
|
124 |
+ Aux: options.ProgressWriter.AuxFormatter, |
|
125 |
+ Output: options.ProgressWriter.Output, |
|
126 |
+ docker: options.Backend, |
|
127 |
+ buildArgs: newBuildArgs(config.BuildArgs), |
|
128 |
+ buildStages: newBuildStages(), |
|
129 |
+ imageSources: newImageSources(clientCtx, options), |
|
130 |
+ pathCache: options.PathCache, |
|
131 |
+ imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache), |
|
132 |
+ containerManager: newContainerManager(options.Backend), |
|
132 | 133 |
} |
133 | 134 |
return b |
134 | 135 |
} |
135 | 136 |
|
136 |
-func (b *Builder) resetImageCache() { |
|
137 |
- if icb, ok := b.docker.(builder.ImageCacheBuilder); ok { |
|
138 |
- b.imageCache = icb.MakeImageCache(b.options.CacheFrom) |
|
139 |
- } |
|
140 |
- b.cacheBusted = false |
|
141 |
-} |
|
142 |
- |
|
143 | 137 |
// Build runs the Dockerfile builder by parsing the Dockerfile and executing |
144 | 138 |
// the instructions from the file. |
145 | 139 |
func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) { |
... | ... |
@@ -216,14 +207,14 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result, |
216 | 216 |
} |
217 | 217 |
if state, err = b.dispatch(opts); err != nil { |
218 | 218 |
if b.options.ForceRemove { |
219 |
- b.clearTmp() |
|
219 |
+ b.containerManager.RemoveAll(b.Stdout) |
|
220 | 220 |
} |
221 | 221 |
return nil, err |
222 | 222 |
} |
223 | 223 |
|
224 | 224 |
fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(state.imageID)) |
225 | 225 |
if b.options.Remove { |
226 |
- b.clearTmp() |
|
226 |
+ b.containerManager.RemoveAll(b.Stdout) |
|
227 | 227 |
} |
228 | 228 |
} |
229 | 229 |
|
... | ... |
@@ -258,7 +249,9 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con |
258 | 258 |
return config, nil |
259 | 259 |
} |
260 | 260 |
|
261 |
- b := newBuilder(context.Background(), builderOptions{}) |
|
261 |
+ b := newBuilder(context.Background(), builderOptions{ |
|
262 |
+ Options: &types.ImageBuildOptions{NoCache: true}, |
|
263 |
+ }) |
|
262 | 264 |
|
263 | 265 |
dockerfile, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n"))) |
264 | 266 |
if err != nil { |
265 | 267 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,143 @@ |
0 |
+package dockerfile |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "fmt" |
|
4 |
+ "io" |
|
5 |
+ |
|
6 |
+ "github.com/Sirupsen/logrus" |
|
7 |
+ "github.com/docker/docker/api/types" |
|
8 |
+ "github.com/docker/docker/api/types/container" |
|
9 |
+ "github.com/docker/docker/builder" |
|
10 |
+ containerpkg "github.com/docker/docker/container" |
|
11 |
+ "github.com/docker/docker/pkg/stringid" |
|
12 |
+ "github.com/pkg/errors" |
|
13 |
+ "golang.org/x/net/context" |
|
14 |
+) |
|
15 |
+ |
|
16 |
+type containerManager struct { |
|
17 |
+ tmpContainers map[string]struct{} |
|
18 |
+ backend builder.ExecBackend |
|
19 |
+} |
|
20 |
+ |
|
21 |
+// newContainerManager creates a new container backend |
|
22 |
+func newContainerManager(docker builder.ExecBackend) *containerManager { |
|
23 |
+ return &containerManager{ |
|
24 |
+ backend: docker, |
|
25 |
+ tmpContainers: make(map[string]struct{}), |
|
26 |
+ } |
|
27 |
+} |
|
28 |
+ |
|
29 |
+// Create a container |
|
30 |
+func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig) (container.ContainerCreateCreatedBody, error) { |
|
31 |
+ container, err := c.backend.ContainerCreate(types.ContainerCreateConfig{ |
|
32 |
+ Config: runConfig, |
|
33 |
+ HostConfig: hostConfig, |
|
34 |
+ }) |
|
35 |
+ if err != nil { |
|
36 |
+ return container, err |
|
37 |
+ } |
|
38 |
+ c.tmpContainers[container.ID] = struct{}{} |
|
39 |
+ return container, nil |
|
40 |
+} |
|
41 |
+ |
|
42 |
+var errCancelled = errors.New("build cancelled") |
|
43 |
+ |
|
44 |
+// Run a container by ID |
|
45 |
+func (c *containerManager) Run(ctx context.Context, cID string, stdout, stderr io.Writer) (err error) { |
|
46 |
+ attached := make(chan struct{}) |
|
47 |
+ errCh := make(chan error) |
|
48 |
+ go func() { |
|
49 |
+ errCh <- c.backend.ContainerAttachRaw(cID, nil, stdout, stderr, true, attached) |
|
50 |
+ }() |
|
51 |
+ select { |
|
52 |
+ case err := <-errCh: |
|
53 |
+ return err |
|
54 |
+ case <-attached: |
|
55 |
+ } |
|
56 |
+ |
|
57 |
+ finished := make(chan struct{}) |
|
58 |
+ cancelErrCh := make(chan error, 1) |
|
59 |
+ go func() { |
|
60 |
+ select { |
|
61 |
+ case <-ctx.Done(): |
|
62 |
+ logrus.Debugln("Build cancelled, killing and removing container:", cID) |
|
63 |
+ c.backend.ContainerKill(cID, 0) |
|
64 |
+ c.removeContainer(cID, stdout) |
|
65 |
+ cancelErrCh <- errCancelled |
|
66 |
+ case <-finished: |
|
67 |
+ cancelErrCh <- nil |
|
68 |
+ } |
|
69 |
+ }() |
|
70 |
+ |
|
71 |
+ if err := c.backend.ContainerStart(cID, nil, "", ""); err != nil { |
|
72 |
+ close(finished) |
|
73 |
+ logCancellationError(cancelErrCh, "error from ContainerStart: "+err.Error()) |
|
74 |
+ return err |
|
75 |
+ } |
|
76 |
+ |
|
77 |
+ // Block on reading output from container, stop on err or chan closed |
|
78 |
+ if err := <-errCh; err != nil { |
|
79 |
+ close(finished) |
|
80 |
+ logCancellationError(cancelErrCh, "error from errCh: "+err.Error()) |
|
81 |
+ return err |
|
82 |
+ } |
|
83 |
+ |
|
84 |
+ waitC, err := c.backend.ContainerWait(ctx, cID, containerpkg.WaitConditionNotRunning) |
|
85 |
+ if err != nil { |
|
86 |
+ close(finished) |
|
87 |
+ logCancellationError(cancelErrCh, fmt.Sprintf("unable to begin ContainerWait: %s", err)) |
|
88 |
+ return err |
|
89 |
+ } |
|
90 |
+ |
|
91 |
+ if status := <-waitC; status.ExitCode() != 0 { |
|
92 |
+ close(finished) |
|
93 |
+ logCancellationError(cancelErrCh, |
|
94 |
+ fmt.Sprintf("a non-zero code from ContainerWait: %d", status.ExitCode())) |
|
95 |
+ return &statusCodeError{code: status.ExitCode(), err: err} |
|
96 |
+ } |
|
97 |
+ |
|
98 |
+ close(finished) |
|
99 |
+ return <-cancelErrCh |
|
100 |
+} |
|
101 |
+ |
|
102 |
+func logCancellationError(cancelErrCh chan error, msg string) { |
|
103 |
+ if cancelErr := <-cancelErrCh; cancelErr != nil { |
|
104 |
+ logrus.Debugf("Build cancelled (%v): ", cancelErr, msg) |
|
105 |
+ } |
|
106 |
+} |
|
107 |
+ |
|
108 |
+type statusCodeError struct { |
|
109 |
+ code int |
|
110 |
+ err error |
|
111 |
+} |
|
112 |
+ |
|
113 |
+func (e *statusCodeError) Error() string { |
|
114 |
+ return e.err.Error() |
|
115 |
+} |
|
116 |
+ |
|
117 |
+func (e *statusCodeError) StatusCode() int { |
|
118 |
+ return e.code |
|
119 |
+} |
|
120 |
+ |
|
121 |
+func (c *containerManager) removeContainer(containerID string, stdout io.Writer) error { |
|
122 |
+ rmConfig := &types.ContainerRmConfig{ |
|
123 |
+ ForceRemove: true, |
|
124 |
+ RemoveVolume: true, |
|
125 |
+ } |
|
126 |
+ if err := c.backend.ContainerRm(containerID, rmConfig); err != nil { |
|
127 |
+ fmt.Fprintf(stdout, "Error removing intermediate container %s: %v\n", stringid.TruncateID(containerID), err) |
|
128 |
+ return err |
|
129 |
+ } |
|
130 |
+ return nil |
|
131 |
+} |
|
132 |
+ |
|
133 |
+// RemoveAll containers managed by this container manager |
|
134 |
+func (c *containerManager) RemoveAll(stdout io.Writer) { |
|
135 |
+ for containerID := range c.tmpContainers { |
|
136 |
+ if err := c.removeContainer(containerID, stdout); err != nil { |
|
137 |
+ return |
|
138 |
+ } |
|
139 |
+ delete(c.tmpContainers, containerID) |
|
140 |
+ fmt.Fprintf(stdout, "Removing intermediate container %s\n", stringid.TruncateID(containerID)) |
|
141 |
+ } |
|
142 |
+} |
... | ... |
@@ -19,11 +19,11 @@ import ( |
19 | 19 |
|
20 | 20 |
"github.com/Sirupsen/logrus" |
21 | 21 |
"github.com/docker/docker/api" |
22 |
- "github.com/docker/docker/api/types" |
|
23 | 22 |
"github.com/docker/docker/api/types/container" |
24 | 23 |
"github.com/docker/docker/api/types/strslice" |
25 | 24 |
"github.com/docker/docker/builder" |
26 | 25 |
"github.com/docker/docker/builder/dockerfile/parser" |
26 |
+ "github.com/docker/docker/pkg/jsonmessage" |
|
27 | 27 |
"github.com/docker/docker/pkg/signal" |
28 | 28 |
"github.com/docker/go-connections/nat" |
29 | 29 |
"github.com/pkg/errors" |
... | ... |
@@ -218,7 +218,7 @@ func from(req dispatchRequest) error { |
218 | 218 |
return err |
219 | 219 |
} |
220 | 220 |
|
221 |
- req.builder.resetImageCache() |
|
221 |
+ req.builder.imageProber.Reset() |
|
222 | 222 |
image, err := req.builder.getFromImage(req.shlex, req.args[0]) |
223 | 223 |
if err != nil { |
224 | 224 |
return err |
... | ... |
@@ -398,24 +398,15 @@ func workdir(req dispatchRequest) error { |
398 | 398 |
|
399 | 399 |
comment := "WORKDIR " + runConfig.WorkingDir |
400 | 400 |
runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment)) |
401 |
- if hit, err := req.builder.probeCache(req.state, runConfigWithCommentCmd); err != nil || hit { |
|
401 |
+ containerID, err := req.builder.probeAndCreate(req.state, runConfigWithCommentCmd) |
|
402 |
+ if err != nil || containerID == "" { |
|
402 | 403 |
return err |
403 | 404 |
} |
404 |
- |
|
405 |
- container, err := req.builder.docker.ContainerCreate(types.ContainerCreateConfig{ |
|
406 |
- Config: runConfigWithCommentCmd, |
|
407 |
- // Set a log config to override any default value set on the daemon |
|
408 |
- HostConfig: &container.HostConfig{LogConfig: defaultLogConfig}, |
|
409 |
- }) |
|
410 |
- if err != nil { |
|
411 |
- return err |
|
412 |
- } |
|
413 |
- req.builder.tmpContainers[container.ID] = struct{}{} |
|
414 |
- if err := req.builder.docker.ContainerCreateWorkdir(container.ID); err != nil { |
|
405 |
+ if err := req.builder.docker.ContainerCreateWorkdir(containerID); err != nil { |
|
415 | 406 |
return err |
416 | 407 |
} |
417 | 408 |
|
418 |
- return req.builder.commitContainer(req.state, container.ID, runConfigWithCommentCmd) |
|
409 |
+ return req.builder.commitContainer(req.state, containerID, runConfigWithCommentCmd) |
|
419 | 410 |
} |
420 | 411 |
|
421 | 412 |
// RUN some command yo |
... | ... |
@@ -471,7 +462,16 @@ func run(req dispatchRequest) error { |
471 | 471 |
if err != nil { |
472 | 472 |
return err |
473 | 473 |
} |
474 |
- if err := req.builder.run(cID, runConfig.Cmd); err != nil { |
|
474 |
+ if err := req.builder.containerManager.Run(req.builder.clientCtx, cID, req.builder.Stdout, req.builder.Stderr); err != nil { |
|
475 |
+ if err, ok := err.(*statusCodeError); ok { |
|
476 |
+ // TODO: change error type, because jsonmessage.JSONError assumes HTTP |
|
477 |
+ return &jsonmessage.JSONError{ |
|
478 |
+ Message: fmt.Sprintf( |
|
479 |
+ "The command '%s' returned a non-zero code: %d", |
|
480 |
+ strings.Join(runConfig.Cmd, " "), err.StatusCode()), |
|
481 |
+ Code: err.StatusCode(), |
|
482 |
+ } |
|
483 |
+ } |
|
475 | 484 |
return err |
476 | 485 |
} |
477 | 486 |
|
... | ... |
@@ -7,6 +7,7 @@ import ( |
7 | 7 |
|
8 | 8 |
"bytes" |
9 | 9 |
"context" |
10 |
+ |
|
10 | 11 |
"github.com/docker/docker/api/types" |
11 | 12 |
"github.com/docker/docker/api/types/backend" |
12 | 13 |
"github.com/docker/docker/api/types/container" |
... | ... |
@@ -54,7 +55,6 @@ func newBuilderWithMockBackend() *Builder { |
54 | 54 |
options: &types.ImageBuildOptions{}, |
55 | 55 |
docker: mockBackend, |
56 | 56 |
buildArgs: newBuildArgs(make(map[string]*string)), |
57 |
- tmpContainers: make(map[string]struct{}), |
|
58 | 57 |
Stdout: new(bytes.Buffer), |
59 | 58 |
clientCtx: ctx, |
60 | 59 |
disableCommit: true, |
... | ... |
@@ -62,7 +62,9 @@ func newBuilderWithMockBackend() *Builder { |
62 | 62 |
Options: &types.ImageBuildOptions{}, |
63 | 63 |
Backend: mockBackend, |
64 | 64 |
}), |
65 |
- buildStages: newBuildStages(), |
|
65 |
+ buildStages: newBuildStages(), |
|
66 |
+ imageProber: newImageProber(mockBackend, nil, false), |
|
67 |
+ containerManager: newContainerManager(mockBackend), |
|
66 | 68 |
} |
67 | 69 |
return b |
68 | 70 |
} |
... | ... |
@@ -479,9 +481,12 @@ func TestRunWithBuildArgs(t *testing.T) { |
479 | 479 |
return "", nil |
480 | 480 |
}, |
481 | 481 |
} |
482 |
- b.imageCache = imageCache |
|
483 | 482 |
|
484 | 483 |
mockBackend := b.docker.(*MockBackend) |
484 |
+ mockBackend.makeImageCacheFunc = func(_ []string) builder.ImageCache { |
|
485 |
+ return imageCache |
|
486 |
+ } |
|
487 |
+ b.imageProber = newImageProber(mockBackend, nil, false) |
|
485 | 488 |
mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ReleaseableLayer, error) { |
486 | 489 |
return &mockImage{ |
487 | 490 |
id: "abcdef", |
488 | 491 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,63 @@ |
0 |
+package dockerfile |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "github.com/Sirupsen/logrus" |
|
4 |
+ "github.com/docker/docker/api/types/container" |
|
5 |
+ "github.com/docker/docker/builder" |
|
6 |
+) |
|
7 |
+ |
|
8 |
+// ImageProber exposes an Image cache to the Builder. It supports resetting a |
|
9 |
+// cache. |
|
10 |
+type ImageProber interface { |
|
11 |
+ Reset() |
|
12 |
+ Probe(parentID string, runConfig *container.Config) (string, error) |
|
13 |
+} |
|
14 |
+ |
|
15 |
+type imageProber struct { |
|
16 |
+ cache builder.ImageCache |
|
17 |
+ reset func() builder.ImageCache |
|
18 |
+ cacheBusted bool |
|
19 |
+} |
|
20 |
+ |
|
21 |
+func newImageProber(cacheBuilder builder.ImageCacheBuilder, cacheFrom []string, noCache bool) ImageProber { |
|
22 |
+ if noCache { |
|
23 |
+ return &nopProber{} |
|
24 |
+ } |
|
25 |
+ |
|
26 |
+ reset := func() builder.ImageCache { |
|
27 |
+ return cacheBuilder.MakeImageCache(cacheFrom) |
|
28 |
+ } |
|
29 |
+ return &imageProber{cache: reset(), reset: reset} |
|
30 |
+} |
|
31 |
+ |
|
32 |
+func (c *imageProber) Reset() { |
|
33 |
+ c.cache = c.reset() |
|
34 |
+ c.cacheBusted = false |
|
35 |
+} |
|
36 |
+ |
|
37 |
+// Probe checks if cache match can be found for current build instruction. |
|
38 |
+// It returns the cachedID if there is a hit, and the empty string on miss |
|
39 |
+func (c *imageProber) Probe(parentID string, runConfig *container.Config) (string, error) { |
|
40 |
+ if c.cacheBusted { |
|
41 |
+ return "", nil |
|
42 |
+ } |
|
43 |
+ cacheID, err := c.cache.GetCache(parentID, runConfig) |
|
44 |
+ if err != nil { |
|
45 |
+ return "", err |
|
46 |
+ } |
|
47 |
+ if len(cacheID) == 0 { |
|
48 |
+ logrus.Debugf("[BUILDER] Cache miss: %s", runConfig.Cmd) |
|
49 |
+ c.cacheBusted = true |
|
50 |
+ return "", nil |
|
51 |
+ } |
|
52 |
+ logrus.Debugf("[BUILDER] Use cached version: %s", runConfig.Cmd) |
|
53 |
+ return cacheID, nil |
|
54 |
+} |
|
55 |
+ |
|
56 |
+type nopProber struct{} |
|
57 |
+ |
|
58 |
+func (c *nopProber) Reset() {} |
|
59 |
+ |
|
60 |
+func (c *nopProber) Probe(_ string, _ *container.Config) (string, error) { |
|
61 |
+ return "", nil |
|
62 |
+} |
... | ... |
@@ -9,12 +9,9 @@ import ( |
9 | 9 |
"fmt" |
10 | 10 |
"strings" |
11 | 11 |
|
12 |
- "github.com/Sirupsen/logrus" |
|
13 | 12 |
"github.com/docker/docker/api/types" |
14 | 13 |
"github.com/docker/docker/api/types/backend" |
15 | 14 |
"github.com/docker/docker/api/types/container" |
16 |
- containerpkg "github.com/docker/docker/container" |
|
17 |
- "github.com/docker/docker/pkg/jsonmessage" |
|
18 | 15 |
"github.com/docker/docker/pkg/stringid" |
19 | 16 |
"github.com/pkg/errors" |
20 | 17 |
) |
... | ... |
@@ -74,20 +71,11 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error |
74 | 74 |
runConfigWithCommentCmd := copyRunConfig( |
75 | 75 |
state.runConfig, |
76 | 76 |
withCmdCommentString(fmt.Sprintf("%s %s in %s ", inst.cmdName, srcHash, inst.dest))) |
77 |
- if hit, err := b.probeCache(state, runConfigWithCommentCmd); err != nil || hit { |
|
77 |
+ containerID, err := b.probeAndCreate(state, runConfigWithCommentCmd) |
|
78 |
+ if err != nil || containerID == "" { |
|
78 | 79 |
return err |
79 | 80 |
} |
80 | 81 |
|
81 |
- container, err := b.docker.ContainerCreate(types.ContainerCreateConfig{ |
|
82 |
- Config: runConfigWithCommentCmd, |
|
83 |
- // Set a log config to override any default value set on the daemon |
|
84 |
- HostConfig: &container.HostConfig{LogConfig: defaultLogConfig}, |
|
85 |
- }) |
|
86 |
- if err != nil { |
|
87 |
- return err |
|
88 |
- } |
|
89 |
- b.tmpContainers[container.ID] = struct{}{} |
|
90 |
- |
|
91 | 82 |
// Twiddle the destination when it's a relative path - meaning, make it |
92 | 83 |
// relative to the WORKINGDIR |
93 | 84 |
dest, err := normaliseDest(inst.cmdName, state.runConfig.WorkingDir, inst.dest) |
... | ... |
@@ -96,11 +84,11 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error |
96 | 96 |
} |
97 | 97 |
|
98 | 98 |
for _, info := range inst.infos { |
99 |
- if err := b.docker.CopyOnBuild(container.ID, dest, info.root, info.path, inst.allowLocalDecompression); err != nil { |
|
99 |
+ if err := b.docker.CopyOnBuild(containerID, dest, info.root, info.path, inst.allowLocalDecompression); err != nil { |
|
100 | 100 |
return err |
101 | 101 |
} |
102 | 102 |
} |
103 |
- return b.commitContainer(state, container.ID, runConfigWithCommentCmd) |
|
103 |
+ return b.commitContainer(state, containerID, runConfigWithCommentCmd) |
|
104 | 104 |
} |
105 | 105 |
|
106 | 106 |
// For backwards compat, if there's just one info then use it as the |
... | ... |
@@ -186,166 +174,65 @@ func getShell(c *container.Config) []string { |
186 | 186 |
return append([]string{}, c.Shell[:]...) |
187 | 187 |
} |
188 | 188 |
|
189 |
-// probeCache checks if cache match can be found for current build instruction. |
|
190 |
-// If an image is found, probeCache returns `(true, nil)`. |
|
191 |
-// If no image is found, it returns `(false, nil)`. |
|
192 |
-// If there is any error, it returns `(false, err)`. |
|
193 | 189 |
func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) { |
194 |
- c := b.imageCache |
|
195 |
- if c == nil || b.options.NoCache || b.cacheBusted { |
|
196 |
- return false, nil |
|
197 |
- } |
|
198 |
- cache, err := c.GetCache(dispatchState.imageID, runConfig) |
|
199 |
- if err != nil { |
|
190 |
+ cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig) |
|
191 |
+ if cachedID == "" || err != nil { |
|
200 | 192 |
return false, err |
201 | 193 |
} |
202 |
- if len(cache) == 0 { |
|
203 |
- logrus.Debugf("[BUILDER] Cache miss: %s", runConfig.Cmd) |
|
204 |
- b.cacheBusted = true |
|
205 |
- return false, nil |
|
206 |
- } |
|
207 |
- |
|
208 | 194 |
fmt.Fprint(b.Stdout, " ---> Using cache\n") |
209 |
- logrus.Debugf("[BUILDER] Use cached version: %s", runConfig.Cmd) |
|
210 |
- dispatchState.imageID = string(cache) |
|
211 |
- b.buildStages.update(dispatchState.imageID, runConfig) |
|
212 | 195 |
|
196 |
+ dispatchState.imageID = string(cachedID) |
|
197 |
+ b.buildStages.update(dispatchState.imageID, runConfig) |
|
213 | 198 |
return true, nil |
214 | 199 |
} |
215 | 200 |
|
216 |
-func (b *Builder) create(runConfig *container.Config) (string, error) { |
|
217 |
- resources := container.Resources{ |
|
218 |
- CgroupParent: b.options.CgroupParent, |
|
219 |
- CPUShares: b.options.CPUShares, |
|
220 |
- CPUPeriod: b.options.CPUPeriod, |
|
221 |
- CPUQuota: b.options.CPUQuota, |
|
222 |
- CpusetCpus: b.options.CPUSetCPUs, |
|
223 |
- CpusetMems: b.options.CPUSetMems, |
|
224 |
- Memory: b.options.Memory, |
|
225 |
- MemorySwap: b.options.MemorySwap, |
|
226 |
- Ulimits: b.options.Ulimits, |
|
227 |
- } |
|
201 |
+var defaultLogConfig = container.LogConfig{Type: "none"} |
|
228 | 202 |
|
229 |
- // TODO: why not embed a hostconfig in builder? |
|
230 |
- hostConfig := &container.HostConfig{ |
|
231 |
- SecurityOpt: b.options.SecurityOpt, |
|
232 |
- Isolation: b.options.Isolation, |
|
233 |
- ShmSize: b.options.ShmSize, |
|
234 |
- Resources: resources, |
|
235 |
- NetworkMode: container.NetworkMode(b.options.NetworkMode), |
|
236 |
- // Set a log config to override any default value set on the daemon |
|
237 |
- LogConfig: defaultLogConfig, |
|
238 |
- ExtraHosts: b.options.ExtraHosts, |
|
239 |
- } |
|
240 |
- |
|
241 |
- // Create the container |
|
242 |
- c, err := b.docker.ContainerCreate(types.ContainerCreateConfig{ |
|
243 |
- Config: runConfig, |
|
244 |
- HostConfig: hostConfig, |
|
245 |
- }) |
|
246 |
- if err != nil { |
|
203 |
+func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) { |
|
204 |
+ if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit { |
|
247 | 205 |
return "", err |
248 | 206 |
} |
249 |
- for _, warning := range c.Warnings { |
|
250 |
- fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning) |
|
251 |
- } |
|
252 |
- |
|
253 |
- b.tmpContainers[c.ID] = struct{}{} |
|
254 |
- fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(c.ID)) |
|
255 |
- return c.ID, nil |
|
207 |
+ // Set a log config to override any default value set on the daemon |
|
208 |
+ hostConfig := &container.HostConfig{LogConfig: defaultLogConfig} |
|
209 |
+ container, err := b.containerManager.Create(runConfig, hostConfig) |
|
210 |
+ return container.ID, err |
|
256 | 211 |
} |
257 | 212 |
|
258 |
-var errCancelled = errors.New("build cancelled") |
|
259 |
- |
|
260 |
-func (b *Builder) run(cID string, cmd []string) (err error) { |
|
261 |
- attached := make(chan struct{}) |
|
262 |
- errCh := make(chan error) |
|
263 |
- go func() { |
|
264 |
- errCh <- b.docker.ContainerAttachRaw(cID, nil, b.Stdout, b.Stderr, true, attached) |
|
265 |
- }() |
|
266 |
- |
|
267 |
- select { |
|
268 |
- case err := <-errCh: |
|
269 |
- return err |
|
270 |
- case <-attached: |
|
271 |
- } |
|
272 |
- |
|
273 |
- finished := make(chan struct{}) |
|
274 |
- cancelErrCh := make(chan error, 1) |
|
275 |
- go func() { |
|
276 |
- select { |
|
277 |
- case <-b.clientCtx.Done(): |
|
278 |
- logrus.Debugln("Build cancelled, killing and removing container:", cID) |
|
279 |
- b.docker.ContainerKill(cID, 0) |
|
280 |
- b.removeContainer(cID) |
|
281 |
- cancelErrCh <- errCancelled |
|
282 |
- case <-finished: |
|
283 |
- cancelErrCh <- nil |
|
284 |
- } |
|
285 |
- }() |
|
286 |
- |
|
287 |
- if err := b.docker.ContainerStart(cID, nil, "", ""); err != nil { |
|
288 |
- close(finished) |
|
289 |
- if cancelErr := <-cancelErrCh; cancelErr != nil { |
|
290 |
- logrus.Debugf("Build cancelled (%v) and got an error from ContainerStart: %v", |
|
291 |
- cancelErr, err) |
|
292 |
- } |
|
293 |
- return err |
|
294 |
- } |
|
295 |
- |
|
296 |
- // Block on reading output from container, stop on err or chan closed |
|
297 |
- if err := <-errCh; err != nil { |
|
298 |
- close(finished) |
|
299 |
- if cancelErr := <-cancelErrCh; cancelErr != nil { |
|
300 |
- logrus.Debugf("Build cancelled (%v) and got an error from errCh: %v", |
|
301 |
- cancelErr, err) |
|
302 |
- } |
|
303 |
- return err |
|
304 |
- } |
|
305 |
- |
|
306 |
- waitC, err := b.docker.ContainerWait(b.clientCtx, cID, containerpkg.WaitConditionNotRunning) |
|
213 |
+func (b *Builder) create(runConfig *container.Config) (string, error) { |
|
214 |
+ hostConfig := hostConfigFromOptions(b.options) |
|
215 |
+ container, err := b.containerManager.Create(runConfig, hostConfig) |
|
307 | 216 |
if err != nil { |
308 |
- // Unable to begin waiting for container. |
|
309 |
- close(finished) |
|
310 |
- if cancelErr := <-cancelErrCh; cancelErr != nil { |
|
311 |
- logrus.Debugf("Build cancelled (%v) and unable to begin ContainerWait: %d", cancelErr, err) |
|
312 |
- } |
|
313 |
- return err |
|
314 |
- } |
|
315 |
- |
|
316 |
- if status := <-waitC; status.ExitCode() != 0 { |
|
317 |
- close(finished) |
|
318 |
- if cancelErr := <-cancelErrCh; cancelErr != nil { |
|
319 |
- logrus.Debugf("Build cancelled (%v) and got a non-zero code from ContainerWait: %d", cancelErr, status.ExitCode()) |
|
320 |
- } |
|
321 |
- // TODO: change error type, because jsonmessage.JSONError assumes HTTP |
|
322 |
- return &jsonmessage.JSONError{ |
|
323 |
- Message: fmt.Sprintf("The command '%s' returned a non-zero code: %d", strings.Join(cmd, " "), status.ExitCode()), |
|
324 |
- Code: status.ExitCode(), |
|
325 |
- } |
|
326 |
- } |
|
327 |
- close(finished) |
|
328 |
- return <-cancelErrCh |
|
329 |
-} |
|
330 |
- |
|
331 |
-func (b *Builder) removeContainer(c string) error { |
|
332 |
- rmConfig := &types.ContainerRmConfig{ |
|
333 |
- ForceRemove: true, |
|
334 |
- RemoveVolume: true, |
|
217 |
+ return "", err |
|
335 | 218 |
} |
336 |
- if err := b.docker.ContainerRm(c, rmConfig); err != nil { |
|
337 |
- fmt.Fprintf(b.Stdout, "Error removing intermediate container %s: %v\n", stringid.TruncateID(c), err) |
|
338 |
- return err |
|
219 |
+ // TODO: could this be moved into containerManager.Create() ? |
|
220 |
+ for _, warning := range container.Warnings { |
|
221 |
+ fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning) |
|
339 | 222 |
} |
340 |
- return nil |
|
223 |
+ fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID)) |
|
224 |
+ return container.ID, nil |
|
341 | 225 |
} |
342 | 226 |
|
343 |
-func (b *Builder) clearTmp() { |
|
344 |
- for c := range b.tmpContainers { |
|
345 |
- if err := b.removeContainer(c); err != nil { |
|
346 |
- return |
|
347 |
- } |
|
348 |
- delete(b.tmpContainers, c) |
|
349 |
- fmt.Fprintf(b.Stdout, "Removing intermediate container %s\n", stringid.TruncateID(c)) |
|
227 |
+func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig { |
|
228 |
+ resources := container.Resources{ |
|
229 |
+ CgroupParent: options.CgroupParent, |
|
230 |
+ CPUShares: options.CPUShares, |
|
231 |
+ CPUPeriod: options.CPUPeriod, |
|
232 |
+ CPUQuota: options.CPUQuota, |
|
233 |
+ CpusetCpus: options.CPUSetCPUs, |
|
234 |
+ CpusetMems: options.CPUSetMems, |
|
235 |
+ Memory: options.Memory, |
|
236 |
+ MemorySwap: options.MemorySwap, |
|
237 |
+ Ulimits: options.Ulimits, |
|
238 |
+ } |
|
239 |
+ |
|
240 |
+ return &container.HostConfig{ |
|
241 |
+ SecurityOpt: options.SecurityOpt, |
|
242 |
+ Isolation: options.Isolation, |
|
243 |
+ ShmSize: options.ShmSize, |
|
244 |
+ Resources: resources, |
|
245 |
+ NetworkMode: container.NetworkMode(options.NetworkMode), |
|
246 |
+ // Set a log config to override any default value set on the daemon |
|
247 |
+ LogConfig: defaultLogConfig, |
|
248 |
+ ExtraHosts: options.ExtraHosts, |
|
350 | 249 |
} |
351 | 250 |
} |
... | ... |
@@ -3,13 +3,11 @@ package dockerfile |
3 | 3 |
import ( |
4 | 4 |
"io" |
5 | 5 |
|
6 |
- "github.com/docker/distribution/reference" |
|
7 | 6 |
"github.com/docker/docker/api/types" |
8 | 7 |
"github.com/docker/docker/api/types/backend" |
9 | 8 |
"github.com/docker/docker/api/types/container" |
10 | 9 |
"github.com/docker/docker/builder" |
11 | 10 |
containerpkg "github.com/docker/docker/container" |
12 |
- "github.com/docker/docker/image" |
|
13 | 11 |
"golang.org/x/net/context" |
14 | 12 |
) |
15 | 13 |
|
... | ... |
@@ -18,10 +16,7 @@ type MockBackend struct { |
18 | 18 |
containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) |
19 | 19 |
commitFunc func(string, *backend.ContainerCommitConfig) (string, error) |
20 | 20 |
getImageFunc func(string) (builder.Image, builder.ReleaseableLayer, error) |
21 |
-} |
|
22 |
- |
|
23 |
-func (m *MockBackend) TagImageWithReference(image.ID, reference.Named) error { |
|
24 |
- return nil |
|
21 |
+ makeImageCacheFunc func(cacheFrom []string) builder.ImageCache |
|
25 | 22 |
} |
26 | 23 |
|
27 | 24 |
func (m *MockBackend) ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error { |
... | ... |
@@ -74,6 +69,13 @@ func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID st |
74 | 74 |
return &mockImage{id: "theid"}, &mockLayer{}, nil |
75 | 75 |
} |
76 | 76 |
|
77 |
+func (m *MockBackend) MakeImageCache(cacheFrom []string) builder.ImageCache { |
|
78 |
+ if m.makeImageCacheFunc != nil { |
|
79 |
+ return m.makeImageCacheFunc(cacheFrom) |
|
80 |
+ } |
|
81 |
+ return nil |
|
82 |
+} |
|
83 |
+ |
|
77 | 84 |
type mockImage struct { |
78 | 85 |
id string |
79 | 86 |
config *container.Config |
... | ... |
@@ -100,7 +100,7 @@ func (s *DockerSuite) TestImagesFilterLabelMatch(c *check.C) { |
100 | 100 |
} |
101 | 101 |
|
102 | 102 |
// Regression : #15659 |
103 |
-func (s *DockerSuite) TestImagesFilterLabelWithCommit(c *check.C) { |
|
103 |
+func (s *DockerSuite) TestCommitWithFilterLabel(c *check.C) { |
|
104 | 104 |
// Create a container |
105 | 105 |
dockerCmd(c, "run", "--name", "bar", "busybox", "/bin/sh") |
106 | 106 |
// Commit with labels "using changes" |