builder/dockerfile/internals.go
f41230b9
 package dockerfile
22c46af4
 
3f5f6b03
 // internals for handling commands. Covers many areas and a lot of
 // non-contiguous functionality. Please read the comments.
 
d6c0bbc3
 import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
7a7357da
 	"io"
 	"os"
 	"path"
19a29f6f
 	"path/filepath"
deb335d0
 	"runtime"
d6c0bbc3
 	"strings"
 
91e197d6
 	"github.com/docker/docker/api/types"
93e02efa
 	"github.com/docker/docker/api/types/backend"
91e197d6
 	"github.com/docker/docker/api/types/container"
bd5f92d2
 	"github.com/docker/docker/image"
7a7357da
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/containerfs"
19a29f6f
 	"github.com/docker/docker/pkg/idtools"
b80fae73
 	"github.com/docker/docker/pkg/stringid"
7a7357da
 	"github.com/docker/docker/pkg/system"
9bcd5d25
 	"github.com/docker/go-connections/nat"
f95f5828
 	"github.com/pkg/errors"
d6c0bbc3
 )
 
7a7357da
 // Archiver defines an interface for copying files from one destination to
 // another using Tar/Untar.
 type Archiver interface {
 	TarUntar(src, dst string) error
 	UntarPath(src, dst string) error
 	CopyWithTar(src, dst string) error
 	CopyFileWithTar(src, dst string) error
 	IDMappings() *idtools.IDMappings
 }
 
 // The builder will use the following interfaces if the container fs implements
 // these for optimized copies to and from the container.
 type extractor interface {
 	ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error
 }
 
 type archiver interface {
 	ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error)
 }
 
 // helper functions to get tar/untar func
 func untarFunc(i interface{}) containerfs.UntarFunc {
 	if ea, ok := i.(extractor); ok {
 		return ea.ExtractArchive
 	}
 	return chrootarchive.Untar
 }
 
 func tarFunc(i interface{}) containerfs.TarFunc {
 	if ap, ok := i.(archiver); ok {
 		return ap.ArchivePath
 	}
 	return archive.TarWithOptions
 }
 
 func (b *Builder) getArchiver(src, dst containerfs.Driver) Archiver {
 	t, u := tarFunc(src), untarFunc(dst)
 	return &containerfs.Archiver{
 		SrcDriver:     src,
 		DstDriver:     dst,
 		Tar:           t,
 		Untar:         u,
 		IDMappingsVar: b.idMappings,
 	}
 }
 
2f0ebba0
 func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
7f091eca
 	if b.disableCommit {
 		return nil
 	}
2f0ebba0
 	if !dispatchState.hasFromImage() {
514adcf4
 		return errors.New("Please provide a source image with `from` prior to commit")
22c46af4
 	}
fc214b44
 
d98ecf2d
 	optionsPlatform := system.ParsePlatform(b.options.Platform)
 	runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, optionsPlatform.OS))
2f0ebba0
 	hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd)
0d9e66b9
 	if err != nil || hit {
 		return err
 	}
 	id, err := b.create(runConfigWithCommentCmd)
 	if err != nil {
 		return err
22c46af4
 	}
e0ef11a4
 
2f0ebba0
 	return b.commitContainer(dispatchState, id, runConfigWithCommentCmd)
0d9e66b9
 }
 
2f0ebba0
 func (b *Builder) commitContainer(dispatchState *dispatchState, id string, containerConfig *container.Config) error {
0d9e66b9
 	if b.disableCommit {
 		return nil
 	}
ac107995
 
93e02efa
 	commitCfg := &backend.ContainerCommitConfig{
 		ContainerCommitConfig: types.ContainerCommitConfig{
2f0ebba0
 			Author: dispatchState.maintainer,
93e02efa
 			Pause:  true,
d9371ee8
 			// TODO: this should be done by Commit()
2f0ebba0
 			Config: copyRunConfig(dispatchState.runConfig),
93e02efa
 		},
9f738cc5
 		ContainerConfig: containerConfig,
7046651b
 	}
 
22c46af4
 	// Commit the container
4352da78
 	imageID, err := b.docker.Commit(id, commitCfg)
22c46af4
 	if err != nil {
 		return err
 	}
fc214b44
 
2f0ebba0
 	dispatchState.imageID = imageID
22c46af4
 	return nil
 }
 
51360965
 func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
afd305c4
 	newLayer, err := imageMount.Layer().Commit()
51360965
 	if err != nil {
 		return err
 	}
 
 	// add an image mount without an image so the layer is properly unmounted
 	// if there is an error before we can add the full mount with image
 	b.imageSources.Add(newImageMount(nil, newLayer))
 
 	parentImage, ok := imageMount.Image().(*image.Image)
 	if !ok {
 		return errors.Errorf("unexpected image type")
 	}
 
 	newImage := image.NewChildImage(parentImage, image.ChildConfig{
 		Author:          state.maintainer,
 		ContainerConfig: runConfig,
 		DiffID:          newLayer.DiffID(),
 		Config:          copyRunConfig(state.runConfig),
8f537806
 	}, parentImage.OS)
51360965
 
 	// TODO: it seems strange to marshal this here instead of just passing in the
 	// image struct
 	config, err := newImage.MarshalJSON()
bd5f92d2
 	if err != nil {
 		return errors.Wrap(err, "failed to encode image config")
 	}
 
ce8e529e
 	exportedImage, err := b.docker.CreateImage(config, state.imageID)
51360965
 	if err != nil {
 		return errors.Wrapf(err, "failed to export image")
 	}
 
 	state.imageID = exportedImage.ImageID()
 	b.imageSources.Add(newImageMount(exportedImage, newLayer))
 	return nil
bd5f92d2
 }
 
213ed02e
 func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error {
 	srcHash := getSourceHashFromInfos(inst.infos)
22c46af4
 
19a29f6f
 	var chownComment string
 	if inst.chownStr != "" {
 		chownComment = fmt.Sprintf("--chown=%s", inst.chownStr)
 	}
 	commentStr := fmt.Sprintf("%s %s%s in %s ", inst.cmdName, chownComment, srcHash, inst.dest)
 
0d9e66b9
 	// TODO: should this have been using origPaths instead of srcHash in the comment?
d98ecf2d
 	optionsPlatform := system.ParsePlatform(b.options.Platform)
9f738cc5
 	runConfigWithCommentCmd := copyRunConfig(
213ed02e
 		state.runConfig,
d98ecf2d
 		withCmdCommentString(commentStr, optionsPlatform.OS))
51360965
 	hit, err := b.probeCache(state, runConfigWithCommentCmd)
 	if err != nil || hit {
faab7170
 		return err
05b8a1eb
 	}
 
c268d9da
 	imageMount, err := b.imageSources.Get(state.imageID, true)
bd5f92d2
 	if err != nil {
51360965
 		return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
bd5f92d2
 	}
7a7357da
 
d98ecf2d
 	destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount, b.options.Platform)
bd5f92d2
 	if err != nil {
51360965
 		return err
bd5f92d2
 	}
 
7a7357da
 	chownPair := b.idMappings.RootPair()
19a29f6f
 	// if a chown was requested, perform the steps to get the uid, gid
 	// translated (if necessary because of user namespaces), and replace
 	// the root pair with the chown pair for copy operations
 	if inst.chownStr != "" {
7a7357da
 		chownPair, err = parseChownFlag(inst.chownStr, destInfo.root.Path(), b.idMappings)
19a29f6f
 		if err != nil {
 			return errors.Wrapf(err, "unable to convert uid/gid chown string to host mapping")
 		}
 	}
 
213ed02e
 	for _, info := range inst.infos {
7a7357da
 		opts := copyFileOptions{
 			decompress: inst.allowLocalDecompression,
 			archiver:   b.getArchiver(info.root, destInfo.root),
 			chownPair:  chownPair,
 		}
51360965
 		if err := performCopyForInfo(destInfo, info, opts); err != nil {
 			return errors.Wrapf(err, "failed to copy files")
05b8a1eb
 		}
 	}
51360965
 	return b.exportImage(state, imageMount, runConfigWithCommentCmd)
 }
bd5f92d2
 
7a7357da
 func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount, platform string) (copyInfo, error) {
51360965
 	// Twiddle the destination when it's a relative path - meaning, make it
 	// relative to the WORKINGDIR
7a7357da
 	dest, err := normalizeDest(workingDir, inst.dest, platform)
51360965
 	if err != nil {
 		return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName)
 	}
 
 	destMount, err := imageMount.Source()
 	if err != nil {
 		return copyInfo{}, errors.Wrapf(err, "failed to mount copy source")
 	}
 
 	return newCopyInfoFromSource(destMount, dest, ""), nil
213ed02e
 }
05b8a1eb
 
7a7357da
 // normalizeDest normalises the destination of a COPY/ADD command in a
 // platform semantically consistent way.
 func normalizeDest(workingDir, requested string, platform string) (string, error) {
 	dest := fromSlash(requested, platform)
 	endsInSlash := strings.HasSuffix(dest, string(separator(platform)))
 
 	if platform != "windows" {
 		if !path.IsAbs(requested) {
 			dest = path.Join("/", filepath.ToSlash(workingDir), dest)
 			// Make sure we preserve any trailing slash
 			if endsInSlash {
 				dest += "/"
 			}
 		}
 		return dest, nil
 	}
 
 	// We are guaranteed that the working directory is already consistent,
 	// However, Windows also has, for now, the limitation that ADD/COPY can
 	// only be done to the system drive, not any drives that might be present
 	// as a result of a bind mount.
 	//
 	// So... if the path requested is Linux-style absolute (/foo or \\foo),
 	// we assume it is the system drive. If it is a Windows-style absolute
 	// (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we
 	// strip any configured working directories drive letter so that it
 	// can be subsequently legitimately converted to a Windows volume-style
 	// pathname.
 
 	// Not a typo - filepath.IsAbs, not system.IsAbs on this next check as
 	// we only want to validate where the DriveColon part has been supplied.
 	if filepath.IsAbs(dest) {
 		if strings.ToUpper(string(dest[0])) != "C" {
 			return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
 		}
 		dest = dest[2:] // Strip the drive letter
 	}
 
 	// Cannot handle relative where WorkingDir is not the system drive.
 	if len(workingDir) > 0 {
 		if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) {
 			return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir)
 		}
 		if !system.IsAbs(dest) {
 			if string(workingDir[0]) != "C" {
 				return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
 			}
 			dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
 			// Make sure we preserve any trailing slash
 			if endsInSlash {
 				dest += string(os.PathSeparator)
 			}
 		}
 	}
 	return dest, nil
 }
 
213ed02e
 // For backwards compat, if there's just one info then use it as the
 // cache look-up string, otherwise hash 'em all into one
 func getSourceHashFromInfos(infos []copyInfo) string {
 	if len(infos) == 1 {
 		return infos[0].hash
 	}
 	var hashs []string
 	for _, info := range infos {
 		hashs = append(hashs, info.hash)
 	}
 	return hashStringSlice("multi", hashs)
 }
 
 func hashStringSlice(prefix string, slice []string) string {
 	hasher := sha256.New()
 	hasher.Write([]byte(strings.Join(slice, ",")))
 	return prefix + ":" + hex.EncodeToString(hasher.Sum(nil))
0d9e66b9
 }
 
 type runConfigModifier func(*container.Config)
 
 func withCmd(cmd []string) runConfigModifier {
 	return func(runConfig *container.Config) {
 		runConfig.Cmd = cmd
 	}
 }
 
9f738cc5
 // withCmdComment sets Cmd to a nop comment string. See withCmdCommentString for
 // why there are two almost identical versions of this.
fe7b4d8f
 func withCmdComment(comment string, platform string) runConfigModifier {
0d9e66b9
 	return func(runConfig *container.Config) {
fe7b4d8f
 		runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) ", comment)
0d9e66b9
 	}
 }
 
9f738cc5
 // withCmdCommentString exists to maintain compatibility with older versions.
 // A few instructions (workdir, copy, add) used a nop comment that is a single arg
 // where as all the other instructions used a two arg comment string. This
 // function implements the single arg version.
fe7b4d8f
 func withCmdCommentString(comment string, platform string) runConfigModifier {
9f738cc5
 	return func(runConfig *container.Config) {
fe7b4d8f
 		runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) "+comment)
9f738cc5
 	}
 }
 
0d9e66b9
 func withEnv(env []string) runConfigModifier {
 	return func(runConfig *container.Config) {
 		runConfig.Env = env
 	}
 }
 
d9371ee8
 // withEntrypointOverride sets an entrypoint on runConfig if the command is
 // not empty. The entrypoint is left unmodified if command is empty.
 //
 // The dockerfile RUN instruction expect to run without an entrypoint
 // so the runConfig entrypoint needs to be modified accordingly. ContainerCreate
 // will change a []string{""} entrypoint to nil, so we probe the cache with the
 // nil entrypoint.
 func withEntrypointOverride(cmd []string, entrypoint []string) runConfigModifier {
 	return func(runConfig *container.Config) {
 		if len(cmd) > 0 {
 			runConfig.Entrypoint = entrypoint
 		}
 	}
 }
 
9bcd5d25
 func copyRunConfig(runConfig *container.Config, modifiers ...runConfigModifier) *container.Config {
 	copy := *runConfig
 	copy.Cmd = copyStringSlice(runConfig.Cmd)
 	copy.Env = copyStringSlice(runConfig.Env)
 	copy.Entrypoint = copyStringSlice(runConfig.Entrypoint)
 	copy.OnBuild = copyStringSlice(runConfig.OnBuild)
 	copy.Shell = copyStringSlice(runConfig.Shell)
 
 	if copy.Volumes != nil {
 		copy.Volumes = make(map[string]struct{}, len(runConfig.Volumes))
 		for k, v := range runConfig.Volumes {
 			copy.Volumes[k] = v
 		}
 	}
 
 	if copy.ExposedPorts != nil {
 		copy.ExposedPorts = make(nat.PortSet, len(runConfig.ExposedPorts))
 		for k, v := range runConfig.ExposedPorts {
 			copy.ExposedPorts[k] = v
 		}
 	}
 
 	if copy.Labels != nil {
 		copy.Labels = make(map[string]string, len(runConfig.Labels))
 		for k, v := range runConfig.Labels {
 			copy.Labels[k] = v
 		}
 	}
 
 	for _, modifier := range modifiers {
 		modifier(&copy)
 	}
 	return &copy
 }
 
 func copyStringSlice(orig []string) []string {
 	if orig == nil {
 		return nil
 	}
 	return append([]string{}, orig...)
 }
 
0d9e66b9
 // getShell is a helper function which gets the right shell for prefixing the
 // shell-form of RUN, ENTRYPOINT and CMD instructions
0380fbff
 func getShell(c *container.Config, os string) []string {
0d9e66b9
 	if 0 == len(c.Shell) {
0380fbff
 		return append([]string{}, defaultShellForOS(os)[:]...)
0d9e66b9
 	}
 	return append([]string{}, c.Shell[:]...)
05b8a1eb
 }
 
2f0ebba0
 func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) {
19f3b071
 	cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig)
 	if cachedID == "" || err != nil {
2420c1f0
 		return false, err
 	}
514adcf4
 	fmt.Fprint(b.Stdout, " ---> Using cache\n")
e0ef11a4
 
2f5f0af3
 	dispatchState.imageID = cachedID
2420c1f0
 	return true, nil
d6c0bbc3
 }
 
19f3b071
 var defaultLogConfig = container.LogConfig{Type: "none"}
1a0b483e
 
19f3b071
 func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) {
 	if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit {
c70f8b3c
 		return "", err
d6c0bbc3
 	}
19f3b071
 	// Set a log config to override any default value set on the daemon
 	hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
afd305c4
 	container, err := b.containerManager.Create(runConfig, hostConfig)
19f3b071
 	return container.ID, err
d6c0bbc3
 }
 
19f3b071
 func (b *Builder) create(runConfig *container.Config) (string, error) {
 	hostConfig := hostConfigFromOptions(b.options)
afd305c4
 	container, err := b.containerManager.Create(runConfig, hostConfig)
cfdf84d5
 	if err != nil {
19f3b071
 		return "", err
9392de9d
 	}
19f3b071
 	// TODO: could this be moved into containerManager.Create() ?
 	for _, warning := range container.Warnings {
 		fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning)
9392de9d
 	}
19f3b071
 	fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID))
 	return container.ID, nil
9392de9d
 }
 
19f3b071
 func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig {
 	resources := container.Resources{
 		CgroupParent: options.CgroupParent,
 		CPUShares:    options.CPUShares,
 		CPUPeriod:    options.CPUPeriod,
 		CPUQuota:     options.CPUQuota,
 		CpusetCpus:   options.CPUSetCPUs,
 		CpusetMems:   options.CPUSetMems,
 		Memory:       options.Memory,
 		MemorySwap:   options.MemorySwap,
 		Ulimits:      options.Ulimits,
 	}
 
deb335d0
 	hc := &container.HostConfig{
19f3b071
 		SecurityOpt: options.SecurityOpt,
 		Isolation:   options.Isolation,
 		ShmSize:     options.ShmSize,
 		Resources:   resources,
 		NetworkMode: container.NetworkMode(options.NetworkMode),
 		// Set a log config to override any default value set on the daemon
 		LogConfig:  defaultLogConfig,
 		ExtraHosts: options.ExtraHosts,
d6c0bbc3
 	}
deb335d0
 
 	// For WCOW, the default of 20GB hard-coded in the platform
 	// is too small for builder scenarios where many users are
 	// using RUN statements to install large amounts of data.
 	// Use 127GB as that's the default size of a VHD in Hyper-V.
 	if runtime.GOOS == "windows" && options.Platform == "windows" {
 		hc.StorageOpt = make(map[string]string)
 		hc.StorageOpt["size"] = "127GB"
 	}
 
 	return hc
d6c0bbc3
 }
7a7357da
 
 // fromSlash works like filepath.FromSlash but with a given OS platform field
 func fromSlash(path, platform string) string {
 	if platform == "windows" {
 		return strings.Replace(path, "/", "\\", -1)
 	}
 	return path
 }
 
 // separator returns a OS path separator for the given OS platform
 func separator(platform string) byte {
 	if platform == "windows" {
 		return '\\'
 	}
 	return '/'
 }