Extract a common function for builder.createContainer
Extract imageCache for doing cache probes
Removes the cacheBuested field from Builder
Create a new containerManager class which reduces the interface between the
builder and managing containers to 3 functions (from 6)
Signed-off-by: Daniel Nephin <dnephin@docker.com>
| ... | ... |
@@ -7,12 +7,11 @@ package builder |
| 7 | 7 |
import ( |
| 8 | 8 |
"io" |
| 9 | 9 |
|
| 10 |
- "golang.org/x/net/context" |
|
| 11 |
- |
|
| 12 | 10 |
"github.com/docker/docker/api/types" |
| 13 | 11 |
"github.com/docker/docker/api/types/backend" |
| 14 | 12 |
"github.com/docker/docker/api/types/container" |
| 15 | 13 |
containerpkg "github.com/docker/docker/container" |
| 14 |
+ "golang.org/x/net/context" |
|
| 16 | 15 |
) |
| 17 | 16 |
|
| 18 | 17 |
const ( |
| ... | ... |
@@ -36,21 +35,10 @@ type Source interface {
|
| 36 | 36 |
// Backend abstracts calls to a Docker Daemon. |
| 37 | 37 |
type Backend interface {
|
| 38 | 38 |
ImageBackend |
| 39 |
+ ExecBackend |
|
| 39 | 40 |
|
| 40 |
- // ContainerAttachRaw attaches to container. |
|
| 41 |
- ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error
|
|
| 42 |
- // ContainerCreate creates a new Docker container and returns potential warnings |
|
| 43 |
- ContainerCreate(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) |
|
| 44 |
- // ContainerRm removes a container specified by `id`. |
|
| 45 |
- ContainerRm(name string, config *types.ContainerRmConfig) error |
|
| 46 | 41 |
// Commit creates a new Docker image from an existing Docker container. |
| 47 | 42 |
Commit(string, *backend.ContainerCommitConfig) (string, error) |
| 48 |
- // ContainerKill stops the container execution abruptly. |
|
| 49 |
- ContainerKill(containerID string, sig uint64) error |
|
| 50 |
- // ContainerStart starts a new container |
|
| 51 |
- ContainerStart(containerID string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error |
|
| 52 |
- // ContainerWait stops processing until the given container is stopped. |
|
| 53 |
- ContainerWait(ctx context.Context, name string, condition containerpkg.WaitCondition) (<-chan containerpkg.StateStatus, error) |
|
| 54 | 43 |
// ContainerCreateWorkdir creates the workdir |
| 55 | 44 |
ContainerCreateWorkdir(containerID string) error |
| 56 | 45 |
|
| ... | ... |
@@ -59,6 +47,8 @@ type Backend interface {
|
| 59 | 59 |
// TODO: extract in the builder instead of passing `decompress` |
| 60 | 60 |
// TODO: use containerd/fs.changestream instead as a source |
| 61 | 61 |
CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error |
| 62 |
+ |
|
| 63 |
+ ImageCacheBuilder |
|
| 62 | 64 |
} |
| 63 | 65 |
|
| 64 | 66 |
// ImageBackend are the interface methods required from an image component |
| ... | ... |
@@ -66,6 +56,22 @@ type ImageBackend interface {
|
| 66 | 66 |
GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (Image, ReleaseableLayer, error) |
| 67 | 67 |
} |
| 68 | 68 |
|
| 69 |
+// ExecBackend contains the interface methods required for executing containers |
|
| 70 |
+type ExecBackend interface {
|
|
| 71 |
+ // ContainerAttachRaw attaches to container. |
|
| 72 |
+ ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error
|
|
| 73 |
+ // ContainerCreate creates a new Docker container and returns potential warnings |
|
| 74 |
+ ContainerCreate(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) |
|
| 75 |
+ // ContainerRm removes a container specified by `id`. |
|
| 76 |
+ ContainerRm(name string, config *types.ContainerRmConfig) error |
|
| 77 |
+ // ContainerKill stops the container execution abruptly. |
|
| 78 |
+ ContainerKill(containerID string, sig uint64) error |
|
| 79 |
+ // ContainerStart starts a new container |
|
| 80 |
+ ContainerStart(containerID string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error |
|
| 81 |
+ // ContainerWait stops processing until the given container is stopped. |
|
| 82 |
+ ContainerWait(ctx context.Context, name string, condition containerpkg.WaitCondition) (<-chan containerpkg.StateStatus, error) |
|
| 83 |
+} |
|
| 84 |
+ |
|
| 69 | 85 |
// Result is the output produced by a Builder |
| 70 | 86 |
type Result struct {
|
| 71 | 87 |
ImageID string |
| ... | ... |
@@ -35,8 +35,6 @@ var validCommitCommands = map[string]bool{
|
| 35 | 35 |
"workdir": true, |
| 36 | 36 |
} |
| 37 | 37 |
|
| 38 |
-var defaultLogConfig = container.LogConfig{Type: "none"}
|
|
| 39 |
- |
|
| 40 | 38 |
// BuildManager is shared across all Builder objects |
| 41 | 39 |
type BuildManager struct {
|
| 42 | 40 |
backend builder.Backend |
| ... | ... |
@@ -100,14 +98,13 @@ type Builder struct {
|
| 100 | 100 |
docker builder.Backend |
| 101 | 101 |
clientCtx context.Context |
| 102 | 102 |
|
| 103 |
- tmpContainers map[string]struct{}
|
|
| 104 |
- buildStages *buildStages |
|
| 105 |
- disableCommit bool |
|
| 106 |
- cacheBusted bool |
|
| 107 |
- buildArgs *buildArgs |
|
| 108 |
- imageCache builder.ImageCache |
|
| 109 |
- imageSources *imageSources |
|
| 110 |
- pathCache pathCache |
|
| 103 |
+ buildStages *buildStages |
|
| 104 |
+ disableCommit bool |
|
| 105 |
+ buildArgs *buildArgs |
|
| 106 |
+ imageSources *imageSources |
|
| 107 |
+ pathCache pathCache |
|
| 108 |
+ containerManager *containerManager |
|
| 109 |
+ imageProber ImageProber |
|
| 111 | 110 |
} |
| 112 | 111 |
|
| 113 | 112 |
// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options. |
| ... | ... |
@@ -117,29 +114,23 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
|
| 117 | 117 |
config = new(types.ImageBuildOptions) |
| 118 | 118 |
} |
| 119 | 119 |
b := &Builder{
|
| 120 |
- clientCtx: clientCtx, |
|
| 121 |
- options: config, |
|
| 122 |
- Stdout: options.ProgressWriter.StdoutFormatter, |
|
| 123 |
- Stderr: options.ProgressWriter.StderrFormatter, |
|
| 124 |
- Aux: options.ProgressWriter.AuxFormatter, |
|
| 125 |
- Output: options.ProgressWriter.Output, |
|
| 126 |
- docker: options.Backend, |
|
| 127 |
- tmpContainers: map[string]struct{}{},
|
|
| 128 |
- buildArgs: newBuildArgs(config.BuildArgs), |
|
| 129 |
- buildStages: newBuildStages(), |
|
| 130 |
- imageSources: newImageSources(clientCtx, options), |
|
| 131 |
- pathCache: options.PathCache, |
|
| 120 |
+ clientCtx: clientCtx, |
|
| 121 |
+ options: config, |
|
| 122 |
+ Stdout: options.ProgressWriter.StdoutFormatter, |
|
| 123 |
+ Stderr: options.ProgressWriter.StderrFormatter, |
|
| 124 |
+ Aux: options.ProgressWriter.AuxFormatter, |
|
| 125 |
+ Output: options.ProgressWriter.Output, |
|
| 126 |
+ docker: options.Backend, |
|
| 127 |
+ buildArgs: newBuildArgs(config.BuildArgs), |
|
| 128 |
+ buildStages: newBuildStages(), |
|
| 129 |
+ imageSources: newImageSources(clientCtx, options), |
|
| 130 |
+ pathCache: options.PathCache, |
|
| 131 |
+ imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache), |
|
| 132 |
+ containerManager: newContainerManager(options.Backend), |
|
| 132 | 133 |
} |
| 133 | 134 |
return b |
| 134 | 135 |
} |
| 135 | 136 |
|
| 136 |
-func (b *Builder) resetImageCache() {
|
|
| 137 |
- if icb, ok := b.docker.(builder.ImageCacheBuilder); ok {
|
|
| 138 |
- b.imageCache = icb.MakeImageCache(b.options.CacheFrom) |
|
| 139 |
- } |
|
| 140 |
- b.cacheBusted = false |
|
| 141 |
-} |
|
| 142 |
- |
|
| 143 | 137 |
// Build runs the Dockerfile builder by parsing the Dockerfile and executing |
| 144 | 138 |
// the instructions from the file. |
| 145 | 139 |
func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) {
|
| ... | ... |
@@ -216,14 +207,14 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result, |
| 216 | 216 |
} |
| 217 | 217 |
if state, err = b.dispatch(opts); err != nil {
|
| 218 | 218 |
if b.options.ForceRemove {
|
| 219 |
- b.clearTmp() |
|
| 219 |
+ b.containerManager.RemoveAll(b.Stdout) |
|
| 220 | 220 |
} |
| 221 | 221 |
return nil, err |
| 222 | 222 |
} |
| 223 | 223 |
|
| 224 | 224 |
fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(state.imageID)) |
| 225 | 225 |
if b.options.Remove {
|
| 226 |
- b.clearTmp() |
|
| 226 |
+ b.containerManager.RemoveAll(b.Stdout) |
|
| 227 | 227 |
} |
| 228 | 228 |
} |
| 229 | 229 |
|
| ... | ... |
@@ -258,7 +249,9 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con |
| 258 | 258 |
return config, nil |
| 259 | 259 |
} |
| 260 | 260 |
|
| 261 |
- b := newBuilder(context.Background(), builderOptions{})
|
|
| 261 |
+ b := newBuilder(context.Background(), builderOptions{
|
|
| 262 |
+ Options: &types.ImageBuildOptions{NoCache: true},
|
|
| 263 |
+ }) |
|
| 262 | 264 |
|
| 263 | 265 |
dockerfile, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n"))) |
| 264 | 266 |
if err != nil {
|
| 265 | 267 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,143 @@ |
| 0 |
+package dockerfile |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "fmt" |
|
| 4 |
+ "io" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/Sirupsen/logrus" |
|
| 7 |
+ "github.com/docker/docker/api/types" |
|
| 8 |
+ "github.com/docker/docker/api/types/container" |
|
| 9 |
+ "github.com/docker/docker/builder" |
|
| 10 |
+ containerpkg "github.com/docker/docker/container" |
|
| 11 |
+ "github.com/docker/docker/pkg/stringid" |
|
| 12 |
+ "github.com/pkg/errors" |
|
| 13 |
+ "golang.org/x/net/context" |
|
| 14 |
+) |
|
| 15 |
+ |
|
| 16 |
+type containerManager struct {
|
|
| 17 |
+ tmpContainers map[string]struct{}
|
|
| 18 |
+ backend builder.ExecBackend |
|
| 19 |
+} |
|
| 20 |
+ |
|
| 21 |
+// newContainerManager creates a new container backend |
|
| 22 |
+func newContainerManager(docker builder.ExecBackend) *containerManager {
|
|
| 23 |
+ return &containerManager{
|
|
| 24 |
+ backend: docker, |
|
| 25 |
+ tmpContainers: make(map[string]struct{}),
|
|
| 26 |
+ } |
|
| 27 |
+} |
|
| 28 |
+ |
|
| 29 |
+// Create a container |
|
| 30 |
+func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig) (container.ContainerCreateCreatedBody, error) {
|
|
| 31 |
+ container, err := c.backend.ContainerCreate(types.ContainerCreateConfig{
|
|
| 32 |
+ Config: runConfig, |
|
| 33 |
+ HostConfig: hostConfig, |
|
| 34 |
+ }) |
|
| 35 |
+ if err != nil {
|
|
| 36 |
+ return container, err |
|
| 37 |
+ } |
|
| 38 |
+ c.tmpContainers[container.ID] = struct{}{}
|
|
| 39 |
+ return container, nil |
|
| 40 |
+} |
|
| 41 |
+ |
|
| 42 |
+var errCancelled = errors.New("build cancelled")
|
|
| 43 |
+ |
|
| 44 |
+// Run a container by ID |
|
| 45 |
+func (c *containerManager) Run(ctx context.Context, cID string, stdout, stderr io.Writer) (err error) {
|
|
| 46 |
+ attached := make(chan struct{})
|
|
| 47 |
+ errCh := make(chan error) |
|
| 48 |
+ go func() {
|
|
| 49 |
+ errCh <- c.backend.ContainerAttachRaw(cID, nil, stdout, stderr, true, attached) |
|
| 50 |
+ }() |
|
| 51 |
+ select {
|
|
| 52 |
+ case err := <-errCh: |
|
| 53 |
+ return err |
|
| 54 |
+ case <-attached: |
|
| 55 |
+ } |
|
| 56 |
+ |
|
| 57 |
+ finished := make(chan struct{})
|
|
| 58 |
+ cancelErrCh := make(chan error, 1) |
|
| 59 |
+ go func() {
|
|
| 60 |
+ select {
|
|
| 61 |
+ case <-ctx.Done(): |
|
| 62 |
+ logrus.Debugln("Build cancelled, killing and removing container:", cID)
|
|
| 63 |
+ c.backend.ContainerKill(cID, 0) |
|
| 64 |
+ c.removeContainer(cID, stdout) |
|
| 65 |
+ cancelErrCh <- errCancelled |
|
| 66 |
+ case <-finished: |
|
| 67 |
+ cancelErrCh <- nil |
|
| 68 |
+ } |
|
| 69 |
+ }() |
|
| 70 |
+ |
|
| 71 |
+ if err := c.backend.ContainerStart(cID, nil, "", ""); err != nil {
|
|
| 72 |
+ close(finished) |
|
| 73 |
+ logCancellationError(cancelErrCh, "error from ContainerStart: "+err.Error()) |
|
| 74 |
+ return err |
|
| 75 |
+ } |
|
| 76 |
+ |
|
| 77 |
+ // Block on reading output from container, stop on err or chan closed |
|
| 78 |
+ if err := <-errCh; err != nil {
|
|
| 79 |
+ close(finished) |
|
| 80 |
+ logCancellationError(cancelErrCh, "error from errCh: "+err.Error()) |
|
| 81 |
+ return err |
|
| 82 |
+ } |
|
| 83 |
+ |
|
| 84 |
+ waitC, err := c.backend.ContainerWait(ctx, cID, containerpkg.WaitConditionNotRunning) |
|
| 85 |
+ if err != nil {
|
|
| 86 |
+ close(finished) |
|
| 87 |
+ logCancellationError(cancelErrCh, fmt.Sprintf("unable to begin ContainerWait: %s", err))
|
|
| 88 |
+ return err |
|
| 89 |
+ } |
|
| 90 |
+ |
|
| 91 |
+ if status := <-waitC; status.ExitCode() != 0 {
|
|
| 92 |
+ close(finished) |
|
| 93 |
+ logCancellationError(cancelErrCh, |
|
| 94 |
+ fmt.Sprintf("a non-zero code from ContainerWait: %d", status.ExitCode()))
|
|
| 95 |
+ return &statusCodeError{code: status.ExitCode(), err: err}
|
|
| 96 |
+ } |
|
| 97 |
+ |
|
| 98 |
+ close(finished) |
|
| 99 |
+ return <-cancelErrCh |
|
| 100 |
+} |
|
| 101 |
+ |
|
| 102 |
+func logCancellationError(cancelErrCh chan error, msg string) {
|
|
| 103 |
+ if cancelErr := <-cancelErrCh; cancelErr != nil {
|
|
| 104 |
+ logrus.Debugf("Build cancelled (%v): ", cancelErr, msg)
|
|
| 105 |
+ } |
|
| 106 |
+} |
|
| 107 |
+ |
|
| 108 |
+type statusCodeError struct {
|
|
| 109 |
+ code int |
|
| 110 |
+ err error |
|
| 111 |
+} |
|
| 112 |
+ |
|
| 113 |
+func (e *statusCodeError) Error() string {
|
|
| 114 |
+ return e.err.Error() |
|
| 115 |
+} |
|
| 116 |
+ |
|
| 117 |
+func (e *statusCodeError) StatusCode() int {
|
|
| 118 |
+ return e.code |
|
| 119 |
+} |
|
| 120 |
+ |
|
| 121 |
+func (c *containerManager) removeContainer(containerID string, stdout io.Writer) error {
|
|
| 122 |
+ rmConfig := &types.ContainerRmConfig{
|
|
| 123 |
+ ForceRemove: true, |
|
| 124 |
+ RemoveVolume: true, |
|
| 125 |
+ } |
|
| 126 |
+ if err := c.backend.ContainerRm(containerID, rmConfig); err != nil {
|
|
| 127 |
+ fmt.Fprintf(stdout, "Error removing intermediate container %s: %v\n", stringid.TruncateID(containerID), err) |
|
| 128 |
+ return err |
|
| 129 |
+ } |
|
| 130 |
+ return nil |
|
| 131 |
+} |
|
| 132 |
+ |
|
| 133 |
+// RemoveAll containers managed by this container manager |
|
| 134 |
+func (c *containerManager) RemoveAll(stdout io.Writer) {
|
|
| 135 |
+ for containerID := range c.tmpContainers {
|
|
| 136 |
+ if err := c.removeContainer(containerID, stdout); err != nil {
|
|
| 137 |
+ return |
|
| 138 |
+ } |
|
| 139 |
+ delete(c.tmpContainers, containerID) |
|
| 140 |
+ fmt.Fprintf(stdout, "Removing intermediate container %s\n", stringid.TruncateID(containerID)) |
|
| 141 |
+ } |
|
| 142 |
+} |
| ... | ... |
@@ -19,11 +19,11 @@ import ( |
| 19 | 19 |
|
| 20 | 20 |
"github.com/Sirupsen/logrus" |
| 21 | 21 |
"github.com/docker/docker/api" |
| 22 |
- "github.com/docker/docker/api/types" |
|
| 23 | 22 |
"github.com/docker/docker/api/types/container" |
| 24 | 23 |
"github.com/docker/docker/api/types/strslice" |
| 25 | 24 |
"github.com/docker/docker/builder" |
| 26 | 25 |
"github.com/docker/docker/builder/dockerfile/parser" |
| 26 |
+ "github.com/docker/docker/pkg/jsonmessage" |
|
| 27 | 27 |
"github.com/docker/docker/pkg/signal" |
| 28 | 28 |
"github.com/docker/go-connections/nat" |
| 29 | 29 |
"github.com/pkg/errors" |
| ... | ... |
@@ -218,7 +218,7 @@ func from(req dispatchRequest) error {
|
| 218 | 218 |
return err |
| 219 | 219 |
} |
| 220 | 220 |
|
| 221 |
- req.builder.resetImageCache() |
|
| 221 |
+ req.builder.imageProber.Reset() |
|
| 222 | 222 |
image, err := req.builder.getFromImage(req.shlex, req.args[0]) |
| 223 | 223 |
if err != nil {
|
| 224 | 224 |
return err |
| ... | ... |
@@ -398,24 +398,15 @@ func workdir(req dispatchRequest) error {
|
| 398 | 398 |
|
| 399 | 399 |
comment := "WORKDIR " + runConfig.WorkingDir |
| 400 | 400 |
runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment)) |
| 401 |
- if hit, err := req.builder.probeCache(req.state, runConfigWithCommentCmd); err != nil || hit {
|
|
| 401 |
+ containerID, err := req.builder.probeAndCreate(req.state, runConfigWithCommentCmd) |
|
| 402 |
+ if err != nil || containerID == "" {
|
|
| 402 | 403 |
return err |
| 403 | 404 |
} |
| 404 |
- |
|
| 405 |
- container, err := req.builder.docker.ContainerCreate(types.ContainerCreateConfig{
|
|
| 406 |
- Config: runConfigWithCommentCmd, |
|
| 407 |
- // Set a log config to override any default value set on the daemon |
|
| 408 |
- HostConfig: &container.HostConfig{LogConfig: defaultLogConfig},
|
|
| 409 |
- }) |
|
| 410 |
- if err != nil {
|
|
| 411 |
- return err |
|
| 412 |
- } |
|
| 413 |
- req.builder.tmpContainers[container.ID] = struct{}{}
|
|
| 414 |
- if err := req.builder.docker.ContainerCreateWorkdir(container.ID); err != nil {
|
|
| 405 |
+ if err := req.builder.docker.ContainerCreateWorkdir(containerID); err != nil {
|
|
| 415 | 406 |
return err |
| 416 | 407 |
} |
| 417 | 408 |
|
| 418 |
- return req.builder.commitContainer(req.state, container.ID, runConfigWithCommentCmd) |
|
| 409 |
+ return req.builder.commitContainer(req.state, containerID, runConfigWithCommentCmd) |
|
| 419 | 410 |
} |
| 420 | 411 |
|
| 421 | 412 |
// RUN some command yo |
| ... | ... |
@@ -471,7 +462,16 @@ func run(req dispatchRequest) error {
|
| 471 | 471 |
if err != nil {
|
| 472 | 472 |
return err |
| 473 | 473 |
} |
| 474 |
- if err := req.builder.run(cID, runConfig.Cmd); err != nil {
|
|
| 474 |
+ if err := req.builder.containerManager.Run(req.builder.clientCtx, cID, req.builder.Stdout, req.builder.Stderr); err != nil {
|
|
| 475 |
+ if err, ok := err.(*statusCodeError); ok {
|
|
| 476 |
+ // TODO: change error type, because jsonmessage.JSONError assumes HTTP |
|
| 477 |
+ return &jsonmessage.JSONError{
|
|
| 478 |
+ Message: fmt.Sprintf( |
|
| 479 |
+ "The command '%s' returned a non-zero code: %d", |
|
| 480 |
+ strings.Join(runConfig.Cmd, " "), err.StatusCode()), |
|
| 481 |
+ Code: err.StatusCode(), |
|
| 482 |
+ } |
|
| 483 |
+ } |
|
| 475 | 484 |
return err |
| 476 | 485 |
} |
| 477 | 486 |
|
| ... | ... |
@@ -7,6 +7,7 @@ import ( |
| 7 | 7 |
|
| 8 | 8 |
"bytes" |
| 9 | 9 |
"context" |
| 10 |
+ |
|
| 10 | 11 |
"github.com/docker/docker/api/types" |
| 11 | 12 |
"github.com/docker/docker/api/types/backend" |
| 12 | 13 |
"github.com/docker/docker/api/types/container" |
| ... | ... |
@@ -54,7 +55,6 @@ func newBuilderWithMockBackend() *Builder {
|
| 54 | 54 |
options: &types.ImageBuildOptions{},
|
| 55 | 55 |
docker: mockBackend, |
| 56 | 56 |
buildArgs: newBuildArgs(make(map[string]*string)), |
| 57 |
- tmpContainers: make(map[string]struct{}),
|
|
| 58 | 57 |
Stdout: new(bytes.Buffer), |
| 59 | 58 |
clientCtx: ctx, |
| 60 | 59 |
disableCommit: true, |
| ... | ... |
@@ -62,7 +62,9 @@ func newBuilderWithMockBackend() *Builder {
|
| 62 | 62 |
Options: &types.ImageBuildOptions{},
|
| 63 | 63 |
Backend: mockBackend, |
| 64 | 64 |
}), |
| 65 |
- buildStages: newBuildStages(), |
|
| 65 |
+ buildStages: newBuildStages(), |
|
| 66 |
+ imageProber: newImageProber(mockBackend, nil, false), |
|
| 67 |
+ containerManager: newContainerManager(mockBackend), |
|
| 66 | 68 |
} |
| 67 | 69 |
return b |
| 68 | 70 |
} |
| ... | ... |
@@ -479,9 +481,12 @@ func TestRunWithBuildArgs(t *testing.T) {
|
| 479 | 479 |
return "", nil |
| 480 | 480 |
}, |
| 481 | 481 |
} |
| 482 |
- b.imageCache = imageCache |
|
| 483 | 482 |
|
| 484 | 483 |
mockBackend := b.docker.(*MockBackend) |
| 484 |
+ mockBackend.makeImageCacheFunc = func(_ []string) builder.ImageCache {
|
|
| 485 |
+ return imageCache |
|
| 486 |
+ } |
|
| 487 |
+ b.imageProber = newImageProber(mockBackend, nil, false) |
|
| 485 | 488 |
mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ReleaseableLayer, error) {
|
| 486 | 489 |
return &mockImage{
|
| 487 | 490 |
id: "abcdef", |
| 488 | 491 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,63 @@ |
| 0 |
+package dockerfile |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "github.com/Sirupsen/logrus" |
|
| 4 |
+ "github.com/docker/docker/api/types/container" |
|
| 5 |
+ "github.com/docker/docker/builder" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+// ImageProber exposes an Image cache to the Builder. It supports resetting a |
|
| 9 |
+// cache. |
|
| 10 |
+type ImageProber interface {
|
|
| 11 |
+ Reset() |
|
| 12 |
+ Probe(parentID string, runConfig *container.Config) (string, error) |
|
| 13 |
+} |
|
| 14 |
+ |
|
| 15 |
+type imageProber struct {
|
|
| 16 |
+ cache builder.ImageCache |
|
| 17 |
+ reset func() builder.ImageCache |
|
| 18 |
+ cacheBusted bool |
|
| 19 |
+} |
|
| 20 |
+ |
|
| 21 |
+func newImageProber(cacheBuilder builder.ImageCacheBuilder, cacheFrom []string, noCache bool) ImageProber {
|
|
| 22 |
+ if noCache {
|
|
| 23 |
+ return &nopProber{}
|
|
| 24 |
+ } |
|
| 25 |
+ |
|
| 26 |
+ reset := func() builder.ImageCache {
|
|
| 27 |
+ return cacheBuilder.MakeImageCache(cacheFrom) |
|
| 28 |
+ } |
|
| 29 |
+ return &imageProber{cache: reset(), reset: reset}
|
|
| 30 |
+} |
|
| 31 |
+ |
|
| 32 |
+func (c *imageProber) Reset() {
|
|
| 33 |
+ c.cache = c.reset() |
|
| 34 |
+ c.cacheBusted = false |
|
| 35 |
+} |
|
| 36 |
+ |
|
| 37 |
+// Probe checks if cache match can be found for current build instruction. |
|
| 38 |
+// It returns the cachedID if there is a hit, and the empty string on miss |
|
| 39 |
+func (c *imageProber) Probe(parentID string, runConfig *container.Config) (string, error) {
|
|
| 40 |
+ if c.cacheBusted {
|
|
| 41 |
+ return "", nil |
|
| 42 |
+ } |
|
| 43 |
+ cacheID, err := c.cache.GetCache(parentID, runConfig) |
|
| 44 |
+ if err != nil {
|
|
| 45 |
+ return "", err |
|
| 46 |
+ } |
|
| 47 |
+ if len(cacheID) == 0 {
|
|
| 48 |
+ logrus.Debugf("[BUILDER] Cache miss: %s", runConfig.Cmd)
|
|
| 49 |
+ c.cacheBusted = true |
|
| 50 |
+ return "", nil |
|
| 51 |
+ } |
|
| 52 |
+ logrus.Debugf("[BUILDER] Use cached version: %s", runConfig.Cmd)
|
|
| 53 |
+ return cacheID, nil |
|
| 54 |
+} |
|
| 55 |
+ |
|
| 56 |
+type nopProber struct{}
|
|
| 57 |
+ |
|
| 58 |
+func (c *nopProber) Reset() {}
|
|
| 59 |
+ |
|
| 60 |
+func (c *nopProber) Probe(_ string, _ *container.Config) (string, error) {
|
|
| 61 |
+ return "", nil |
|
| 62 |
+} |
| ... | ... |
@@ -9,12 +9,9 @@ import ( |
| 9 | 9 |
"fmt" |
| 10 | 10 |
"strings" |
| 11 | 11 |
|
| 12 |
- "github.com/Sirupsen/logrus" |
|
| 13 | 12 |
"github.com/docker/docker/api/types" |
| 14 | 13 |
"github.com/docker/docker/api/types/backend" |
| 15 | 14 |
"github.com/docker/docker/api/types/container" |
| 16 |
- containerpkg "github.com/docker/docker/container" |
|
| 17 |
- "github.com/docker/docker/pkg/jsonmessage" |
|
| 18 | 15 |
"github.com/docker/docker/pkg/stringid" |
| 19 | 16 |
"github.com/pkg/errors" |
| 20 | 17 |
) |
| ... | ... |
@@ -74,20 +71,11 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error |
| 74 | 74 |
runConfigWithCommentCmd := copyRunConfig( |
| 75 | 75 |
state.runConfig, |
| 76 | 76 |
withCmdCommentString(fmt.Sprintf("%s %s in %s ", inst.cmdName, srcHash, inst.dest)))
|
| 77 |
- if hit, err := b.probeCache(state, runConfigWithCommentCmd); err != nil || hit {
|
|
| 77 |
+ containerID, err := b.probeAndCreate(state, runConfigWithCommentCmd) |
|
| 78 |
+ if err != nil || containerID == "" {
|
|
| 78 | 79 |
return err |
| 79 | 80 |
} |
| 80 | 81 |
|
| 81 |
- container, err := b.docker.ContainerCreate(types.ContainerCreateConfig{
|
|
| 82 |
- Config: runConfigWithCommentCmd, |
|
| 83 |
- // Set a log config to override any default value set on the daemon |
|
| 84 |
- HostConfig: &container.HostConfig{LogConfig: defaultLogConfig},
|
|
| 85 |
- }) |
|
| 86 |
- if err != nil {
|
|
| 87 |
- return err |
|
| 88 |
- } |
|
| 89 |
- b.tmpContainers[container.ID] = struct{}{}
|
|
| 90 |
- |
|
| 91 | 82 |
// Twiddle the destination when it's a relative path - meaning, make it |
| 92 | 83 |
// relative to the WORKINGDIR |
| 93 | 84 |
dest, err := normaliseDest(inst.cmdName, state.runConfig.WorkingDir, inst.dest) |
| ... | ... |
@@ -96,11 +84,11 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error |
| 96 | 96 |
} |
| 97 | 97 |
|
| 98 | 98 |
for _, info := range inst.infos {
|
| 99 |
- if err := b.docker.CopyOnBuild(container.ID, dest, info.root, info.path, inst.allowLocalDecompression); err != nil {
|
|
| 99 |
+ if err := b.docker.CopyOnBuild(containerID, dest, info.root, info.path, inst.allowLocalDecompression); err != nil {
|
|
| 100 | 100 |
return err |
| 101 | 101 |
} |
| 102 | 102 |
} |
| 103 |
- return b.commitContainer(state, container.ID, runConfigWithCommentCmd) |
|
| 103 |
+ return b.commitContainer(state, containerID, runConfigWithCommentCmd) |
|
| 104 | 104 |
} |
| 105 | 105 |
|
| 106 | 106 |
// For backwards compat, if there's just one info then use it as the |
| ... | ... |
@@ -186,166 +174,65 @@ func getShell(c *container.Config) []string {
|
| 186 | 186 |
return append([]string{}, c.Shell[:]...)
|
| 187 | 187 |
} |
| 188 | 188 |
|
| 189 |
-// probeCache checks if cache match can be found for current build instruction. |
|
| 190 |
-// If an image is found, probeCache returns `(true, nil)`. |
|
| 191 |
-// If no image is found, it returns `(false, nil)`. |
|
| 192 |
-// If there is any error, it returns `(false, err)`. |
|
| 193 | 189 |
func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) {
|
| 194 |
- c := b.imageCache |
|
| 195 |
- if c == nil || b.options.NoCache || b.cacheBusted {
|
|
| 196 |
- return false, nil |
|
| 197 |
- } |
|
| 198 |
- cache, err := c.GetCache(dispatchState.imageID, runConfig) |
|
| 199 |
- if err != nil {
|
|
| 190 |
+ cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig) |
|
| 191 |
+ if cachedID == "" || err != nil {
|
|
| 200 | 192 |
return false, err |
| 201 | 193 |
} |
| 202 |
- if len(cache) == 0 {
|
|
| 203 |
- logrus.Debugf("[BUILDER] Cache miss: %s", runConfig.Cmd)
|
|
| 204 |
- b.cacheBusted = true |
|
| 205 |
- return false, nil |
|
| 206 |
- } |
|
| 207 |
- |
|
| 208 | 194 |
fmt.Fprint(b.Stdout, " ---> Using cache\n") |
| 209 |
- logrus.Debugf("[BUILDER] Use cached version: %s", runConfig.Cmd)
|
|
| 210 |
- dispatchState.imageID = string(cache) |
|
| 211 |
- b.buildStages.update(dispatchState.imageID, runConfig) |
|
| 212 | 195 |
|
| 196 |
+ dispatchState.imageID = string(cachedID) |
|
| 197 |
+ b.buildStages.update(dispatchState.imageID, runConfig) |
|
| 213 | 198 |
return true, nil |
| 214 | 199 |
} |
| 215 | 200 |
|
| 216 |
-func (b *Builder) create(runConfig *container.Config) (string, error) {
|
|
| 217 |
- resources := container.Resources{
|
|
| 218 |
- CgroupParent: b.options.CgroupParent, |
|
| 219 |
- CPUShares: b.options.CPUShares, |
|
| 220 |
- CPUPeriod: b.options.CPUPeriod, |
|
| 221 |
- CPUQuota: b.options.CPUQuota, |
|
| 222 |
- CpusetCpus: b.options.CPUSetCPUs, |
|
| 223 |
- CpusetMems: b.options.CPUSetMems, |
|
| 224 |
- Memory: b.options.Memory, |
|
| 225 |
- MemorySwap: b.options.MemorySwap, |
|
| 226 |
- Ulimits: b.options.Ulimits, |
|
| 227 |
- } |
|
| 201 |
+var defaultLogConfig = container.LogConfig{Type: "none"}
|
|
| 228 | 202 |
|
| 229 |
- // TODO: why not embed a hostconfig in builder? |
|
| 230 |
- hostConfig := &container.HostConfig{
|
|
| 231 |
- SecurityOpt: b.options.SecurityOpt, |
|
| 232 |
- Isolation: b.options.Isolation, |
|
| 233 |
- ShmSize: b.options.ShmSize, |
|
| 234 |
- Resources: resources, |
|
| 235 |
- NetworkMode: container.NetworkMode(b.options.NetworkMode), |
|
| 236 |
- // Set a log config to override any default value set on the daemon |
|
| 237 |
- LogConfig: defaultLogConfig, |
|
| 238 |
- ExtraHosts: b.options.ExtraHosts, |
|
| 239 |
- } |
|
| 240 |
- |
|
| 241 |
- // Create the container |
|
| 242 |
- c, err := b.docker.ContainerCreate(types.ContainerCreateConfig{
|
|
| 243 |
- Config: runConfig, |
|
| 244 |
- HostConfig: hostConfig, |
|
| 245 |
- }) |
|
| 246 |
- if err != nil {
|
|
| 203 |
+func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) {
|
|
| 204 |
+ if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit {
|
|
| 247 | 205 |
return "", err |
| 248 | 206 |
} |
| 249 |
- for _, warning := range c.Warnings {
|
|
| 250 |
- fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning) |
|
| 251 |
- } |
|
| 252 |
- |
|
| 253 |
- b.tmpContainers[c.ID] = struct{}{}
|
|
| 254 |
- fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(c.ID)) |
|
| 255 |
- return c.ID, nil |
|
| 207 |
+ // Set a log config to override any default value set on the daemon |
|
| 208 |
+ hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
|
|
| 209 |
+ container, err := b.containerManager.Create(runConfig, hostConfig) |
|
| 210 |
+ return container.ID, err |
|
| 256 | 211 |
} |
| 257 | 212 |
|
| 258 |
-var errCancelled = errors.New("build cancelled")
|
|
| 259 |
- |
|
| 260 |
-func (b *Builder) run(cID string, cmd []string) (err error) {
|
|
| 261 |
- attached := make(chan struct{})
|
|
| 262 |
- errCh := make(chan error) |
|
| 263 |
- go func() {
|
|
| 264 |
- errCh <- b.docker.ContainerAttachRaw(cID, nil, b.Stdout, b.Stderr, true, attached) |
|
| 265 |
- }() |
|
| 266 |
- |
|
| 267 |
- select {
|
|
| 268 |
- case err := <-errCh: |
|
| 269 |
- return err |
|
| 270 |
- case <-attached: |
|
| 271 |
- } |
|
| 272 |
- |
|
| 273 |
- finished := make(chan struct{})
|
|
| 274 |
- cancelErrCh := make(chan error, 1) |
|
| 275 |
- go func() {
|
|
| 276 |
- select {
|
|
| 277 |
- case <-b.clientCtx.Done(): |
|
| 278 |
- logrus.Debugln("Build cancelled, killing and removing container:", cID)
|
|
| 279 |
- b.docker.ContainerKill(cID, 0) |
|
| 280 |
- b.removeContainer(cID) |
|
| 281 |
- cancelErrCh <- errCancelled |
|
| 282 |
- case <-finished: |
|
| 283 |
- cancelErrCh <- nil |
|
| 284 |
- } |
|
| 285 |
- }() |
|
| 286 |
- |
|
| 287 |
- if err := b.docker.ContainerStart(cID, nil, "", ""); err != nil {
|
|
| 288 |
- close(finished) |
|
| 289 |
- if cancelErr := <-cancelErrCh; cancelErr != nil {
|
|
| 290 |
- logrus.Debugf("Build cancelled (%v) and got an error from ContainerStart: %v",
|
|
| 291 |
- cancelErr, err) |
|
| 292 |
- } |
|
| 293 |
- return err |
|
| 294 |
- } |
|
| 295 |
- |
|
| 296 |
- // Block on reading output from container, stop on err or chan closed |
|
| 297 |
- if err := <-errCh; err != nil {
|
|
| 298 |
- close(finished) |
|
| 299 |
- if cancelErr := <-cancelErrCh; cancelErr != nil {
|
|
| 300 |
- logrus.Debugf("Build cancelled (%v) and got an error from errCh: %v",
|
|
| 301 |
- cancelErr, err) |
|
| 302 |
- } |
|
| 303 |
- return err |
|
| 304 |
- } |
|
| 305 |
- |
|
| 306 |
- waitC, err := b.docker.ContainerWait(b.clientCtx, cID, containerpkg.WaitConditionNotRunning) |
|
| 213 |
+func (b *Builder) create(runConfig *container.Config) (string, error) {
|
|
| 214 |
+ hostConfig := hostConfigFromOptions(b.options) |
|
| 215 |
+ container, err := b.containerManager.Create(runConfig, hostConfig) |
|
| 307 | 216 |
if err != nil {
|
| 308 |
- // Unable to begin waiting for container. |
|
| 309 |
- close(finished) |
|
| 310 |
- if cancelErr := <-cancelErrCh; cancelErr != nil {
|
|
| 311 |
- logrus.Debugf("Build cancelled (%v) and unable to begin ContainerWait: %d", cancelErr, err)
|
|
| 312 |
- } |
|
| 313 |
- return err |
|
| 314 |
- } |
|
| 315 |
- |
|
| 316 |
- if status := <-waitC; status.ExitCode() != 0 {
|
|
| 317 |
- close(finished) |
|
| 318 |
- if cancelErr := <-cancelErrCh; cancelErr != nil {
|
|
| 319 |
- logrus.Debugf("Build cancelled (%v) and got a non-zero code from ContainerWait: %d", cancelErr, status.ExitCode())
|
|
| 320 |
- } |
|
| 321 |
- // TODO: change error type, because jsonmessage.JSONError assumes HTTP |
|
| 322 |
- return &jsonmessage.JSONError{
|
|
| 323 |
- Message: fmt.Sprintf("The command '%s' returned a non-zero code: %d", strings.Join(cmd, " "), status.ExitCode()),
|
|
| 324 |
- Code: status.ExitCode(), |
|
| 325 |
- } |
|
| 326 |
- } |
|
| 327 |
- close(finished) |
|
| 328 |
- return <-cancelErrCh |
|
| 329 |
-} |
|
| 330 |
- |
|
| 331 |
-func (b *Builder) removeContainer(c string) error {
|
|
| 332 |
- rmConfig := &types.ContainerRmConfig{
|
|
| 333 |
- ForceRemove: true, |
|
| 334 |
- RemoveVolume: true, |
|
| 217 |
+ return "", err |
|
| 335 | 218 |
} |
| 336 |
- if err := b.docker.ContainerRm(c, rmConfig); err != nil {
|
|
| 337 |
- fmt.Fprintf(b.Stdout, "Error removing intermediate container %s: %v\n", stringid.TruncateID(c), err) |
|
| 338 |
- return err |
|
| 219 |
+ // TODO: could this be moved into containerManager.Create() ? |
|
| 220 |
+ for _, warning := range container.Warnings {
|
|
| 221 |
+ fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning) |
|
| 339 | 222 |
} |
| 340 |
- return nil |
|
| 223 |
+ fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID)) |
|
| 224 |
+ return container.ID, nil |
|
| 341 | 225 |
} |
| 342 | 226 |
|
| 343 |
-func (b *Builder) clearTmp() {
|
|
| 344 |
- for c := range b.tmpContainers {
|
|
| 345 |
- if err := b.removeContainer(c); err != nil {
|
|
| 346 |
- return |
|
| 347 |
- } |
|
| 348 |
- delete(b.tmpContainers, c) |
|
| 349 |
- fmt.Fprintf(b.Stdout, "Removing intermediate container %s\n", stringid.TruncateID(c)) |
|
| 227 |
+func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig {
|
|
| 228 |
+ resources := container.Resources{
|
|
| 229 |
+ CgroupParent: options.CgroupParent, |
|
| 230 |
+ CPUShares: options.CPUShares, |
|
| 231 |
+ CPUPeriod: options.CPUPeriod, |
|
| 232 |
+ CPUQuota: options.CPUQuota, |
|
| 233 |
+ CpusetCpus: options.CPUSetCPUs, |
|
| 234 |
+ CpusetMems: options.CPUSetMems, |
|
| 235 |
+ Memory: options.Memory, |
|
| 236 |
+ MemorySwap: options.MemorySwap, |
|
| 237 |
+ Ulimits: options.Ulimits, |
|
| 238 |
+ } |
|
| 239 |
+ |
|
| 240 |
+ return &container.HostConfig{
|
|
| 241 |
+ SecurityOpt: options.SecurityOpt, |
|
| 242 |
+ Isolation: options.Isolation, |
|
| 243 |
+ ShmSize: options.ShmSize, |
|
| 244 |
+ Resources: resources, |
|
| 245 |
+ NetworkMode: container.NetworkMode(options.NetworkMode), |
|
| 246 |
+ // Set a log config to override any default value set on the daemon |
|
| 247 |
+ LogConfig: defaultLogConfig, |
|
| 248 |
+ ExtraHosts: options.ExtraHosts, |
|
| 350 | 249 |
} |
| 351 | 250 |
} |
| ... | ... |
@@ -3,13 +3,11 @@ package dockerfile |
| 3 | 3 |
import ( |
| 4 | 4 |
"io" |
| 5 | 5 |
|
| 6 |
- "github.com/docker/distribution/reference" |
|
| 7 | 6 |
"github.com/docker/docker/api/types" |
| 8 | 7 |
"github.com/docker/docker/api/types/backend" |
| 9 | 8 |
"github.com/docker/docker/api/types/container" |
| 10 | 9 |
"github.com/docker/docker/builder" |
| 11 | 10 |
containerpkg "github.com/docker/docker/container" |
| 12 |
- "github.com/docker/docker/image" |
|
| 13 | 11 |
"golang.org/x/net/context" |
| 14 | 12 |
) |
| 15 | 13 |
|
| ... | ... |
@@ -18,10 +16,7 @@ type MockBackend struct {
|
| 18 | 18 |
containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) |
| 19 | 19 |
commitFunc func(string, *backend.ContainerCommitConfig) (string, error) |
| 20 | 20 |
getImageFunc func(string) (builder.Image, builder.ReleaseableLayer, error) |
| 21 |
-} |
|
| 22 |
- |
|
| 23 |
-func (m *MockBackend) TagImageWithReference(image.ID, reference.Named) error {
|
|
| 24 |
- return nil |
|
| 21 |
+ makeImageCacheFunc func(cacheFrom []string) builder.ImageCache |
|
| 25 | 22 |
} |
| 26 | 23 |
|
| 27 | 24 |
func (m *MockBackend) ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error {
|
| ... | ... |
@@ -74,6 +69,13 @@ func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID st |
| 74 | 74 |
return &mockImage{id: "theid"}, &mockLayer{}, nil
|
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 |
+func (m *MockBackend) MakeImageCache(cacheFrom []string) builder.ImageCache {
|
|
| 78 |
+ if m.makeImageCacheFunc != nil {
|
|
| 79 |
+ return m.makeImageCacheFunc(cacheFrom) |
|
| 80 |
+ } |
|
| 81 |
+ return nil |
|
| 82 |
+} |
|
| 83 |
+ |
|
| 77 | 84 |
type mockImage struct {
|
| 78 | 85 |
id string |
| 79 | 86 |
config *container.Config |
| ... | ... |
@@ -100,7 +100,7 @@ func (s *DockerSuite) TestImagesFilterLabelMatch(c *check.C) {
|
| 100 | 100 |
} |
| 101 | 101 |
|
| 102 | 102 |
// Regression : #15659 |
| 103 |
-func (s *DockerSuite) TestImagesFilterLabelWithCommit(c *check.C) {
|
|
| 103 |
+func (s *DockerSuite) TestCommitWithFilterLabel(c *check.C) {
|
|
| 104 | 104 |
// Create a container |
| 105 | 105 |
dockerCmd(c, "run", "--name", "bar", "busybox", "/bin/sh") |
| 106 | 106 |
// Commit with labels "using changes" |