Browse code

build: accept -f - to read Dockerfile from stdin

Heavily based on implementation by David Sheets

Signed-off-by: David Sheets <sheets@alum.mit.edu>
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

David Sheets authored on 2017/02/22 05:07:45
Showing 5 changed files
... ...
@@ -6,10 +6,12 @@ import (
6 6
 	"bytes"
7 7
 	"fmt"
8 8
 	"io"
9
+	"io/ioutil"
9 10
 	"os"
10 11
 	"path/filepath"
11 12
 	"regexp"
12 13
 	"runtime"
14
+	"time"
13 15
 
14 16
 	"github.com/docker/distribution/reference"
15 17
 	"github.com/docker/docker/api"
... ...
@@ -25,6 +27,7 @@ import (
25 25
 	"github.com/docker/docker/pkg/jsonmessage"
26 26
 	"github.com/docker/docker/pkg/progress"
27 27
 	"github.com/docker/docker/pkg/streamformatter"
28
+	"github.com/docker/docker/pkg/stringid"
28 29
 	"github.com/docker/docker/pkg/urlutil"
29 30
 	runconfigopts "github.com/docker/docker/runconfig/opts"
30 31
 	units "github.com/docker/go-units"
... ...
@@ -141,6 +144,7 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error {
141 141
 func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
142 142
 	var (
143 143
 		buildCtx      io.ReadCloser
144
+		dockerfileCtx io.ReadCloser
144 145
 		err           error
145 146
 		contextDir    string
146 147
 		tempDir       string
... ...
@@ -157,6 +161,13 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
157 157
 		buildBuff = bytes.NewBuffer(nil)
158 158
 	}
159 159
 
160
+	if options.dockerfileName == "-" {
161
+		if specifiedContext == "-" {
162
+			return errors.New("invalid argument: can't use stdin for both build context and dockerfile")
163
+		}
164
+		dockerfileCtx = dockerCli.In()
165
+	}
166
+
160 167
 	switch {
161 168
 	case specifiedContext == "-":
162 169
 		buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName)
... ...
@@ -214,11 +225,11 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
214 214
 		// removed. The daemon will remove them for us, if needed, after it
215 215
 		// parses the Dockerfile. Ignore errors here, as they will have been
216 216
 		// caught by validateContextDirectory above.
217
-		var includes = []string{"."}
218
-		keepThem1, _ := fileutils.Matches(".dockerignore", excludes)
219
-		keepThem2, _ := fileutils.Matches(relDockerfile, excludes)
220
-		if keepThem1 || keepThem2 {
221
-			includes = append(includes, ".dockerignore", relDockerfile)
217
+		if keep, _ := fileutils.Matches(".dockerignore", excludes); keep {
218
+			excludes = append(excludes, "!.dockerignore")
219
+		}
220
+		if keep, _ := fileutils.Matches(relDockerfile, excludes); keep && dockerfileCtx == nil {
221
+			excludes = append(excludes, "!"+relDockerfile)
222 222
 		}
223 223
 
224 224
 		compression := archive.Uncompressed
... ...
@@ -228,13 +239,56 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
228 228
 		buildCtx, err = archive.TarWithOptions(contextDir, &archive.TarOptions{
229 229
 			Compression:     compression,
230 230
 			ExcludePatterns: excludes,
231
-			IncludeFiles:    includes,
232 231
 		})
233 232
 		if err != nil {
234 233
 			return err
235 234
 		}
236 235
 	}
237 236
 
237
+	// replace Dockerfile if added dynamically
238
+	if dockerfileCtx != nil {
239
+		file, err := ioutil.ReadAll(dockerfileCtx)
240
+		dockerfileCtx.Close()
241
+		if err != nil {
242
+			return err
243
+		}
244
+		now := time.Now()
245
+		hdrTmpl := &tar.Header{
246
+			Mode:       0600,
247
+			Uid:        0,
248
+			Gid:        0,
249
+			ModTime:    now,
250
+			Typeflag:   tar.TypeReg,
251
+			AccessTime: now,
252
+			ChangeTime: now,
253
+		}
254
+		randomName := ".dockerfile." + stringid.GenerateRandomID()[:20]
255
+
256
+		buildCtx = archive.ReplaceFileTarWrapper(buildCtx, map[string]archive.TarModifierFunc{
257
+			randomName: func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
258
+				return hdrTmpl, file, nil
259
+			},
260
+			".dockerignore": func(_ string, h *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
261
+				if h == nil {
262
+					h = hdrTmpl
263
+				}
264
+				extraIgnore := randomName + "\n"
265
+				b := &bytes.Buffer{}
266
+				if content != nil {
267
+					_, err := b.ReadFrom(content)
268
+					if err != nil {
269
+						return nil, nil, err
270
+					}
271
+				} else {
272
+					extraIgnore += ".dockerignore\n"
273
+				}
274
+				b.Write([]byte("\n" + extraIgnore))
275
+				return h, b.Bytes(), nil
276
+			},
277
+		})
278
+		relDockerfile = randomName
279
+	}
280
+
238 281
 	ctx := context.Background()
239 282
 
240 283
 	var resolvedTags []*resolvedTag
... ...
@@ -89,6 +89,10 @@ func GetContextFromReader(r io.ReadCloser, dockerfileName string) (out io.ReadCl
89 89
 		return ioutils.NewReadCloserWrapper(buf, func() error { return r.Close() }), dockerfileName, nil
90 90
 	}
91 91
 
92
+	if dockerfileName == "-" {
93
+		return nil, "", errors.New("build context is not an archive")
94
+	}
95
+
92 96
 	// Input should be read as a Dockerfile.
93 97
 	tmpDir, err := ioutil.TempDir("", "docker-build-context-")
94 98
 	if err != nil {
... ...
@@ -166,7 +170,7 @@ func GetContextFromLocalDir(localDir, dockerfileName string) (absContextDir, rel
166 166
 	// When using a local context directory, when the Dockerfile is specified
167 167
 	// with the `-f/--file` option then it is considered relative to the
168 168
 	// current directory and not the context directory.
169
-	if dockerfileName != "" {
169
+	if dockerfileName != "" && dockerfileName != "-" {
170 170
 		if dockerfileName, err = filepath.Abs(dockerfileName); err != nil {
171 171
 			return "", "", errors.Errorf("unable to get absolute path to Dockerfile: %v", err)
172 172
 		}
... ...
@@ -220,6 +224,8 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi
220 220
 				absDockerfile = altPath
221 221
 			}
222 222
 		}
223
+	} else if absDockerfile == "-" {
224
+		absDockerfile = filepath.Join(absContextDir, DefaultDockerfileName)
223 225
 	}
224 226
 
225 227
 	// If not already an absolute path, the Dockerfile path should be joined to
... ...
@@ -234,18 +240,21 @@ func getDockerfileRelPath(givenContextDir, givenDockerfile string) (absContextDi
234 234
 	// an issue in golang. On Windows, EvalSymLinks does not work on UNC file
235 235
 	// paths (those starting with \\). This hack means that when using links
236 236
 	// on UNC paths, they will not be followed.
237
-	if !isUNC(absDockerfile) {
238
-		absDockerfile, err = filepath.EvalSymlinks(absDockerfile)
239
-		if err != nil {
240
-			return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err)
237
+	if givenDockerfile != "-" {
238
+		if !isUNC(absDockerfile) {
239
+			absDockerfile, err = filepath.EvalSymlinks(absDockerfile)
240
+			if err != nil {
241
+				return "", "", errors.Errorf("unable to evaluate symlinks in Dockerfile path: %v", err)
242
+
243
+			}
241 244
 		}
242
-	}
243 245
 
244
-	if _, err := os.Lstat(absDockerfile); err != nil {
245
-		if os.IsNotExist(err) {
246
-			return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile)
246
+		if _, err := os.Lstat(absDockerfile); err != nil {
247
+			if os.IsNotExist(err) {
248
+				return "", "", errors.Errorf("Cannot locate Dockerfile: %q", absDockerfile)
249
+			}
250
+			return "", "", errors.Errorf("unable to stat Dockerfile: %v", err)
247 251
 		}
248
-		return "", "", errors.Errorf("unable to stat Dockerfile: %v", err)
249 252
 	}
250 253
 
251 254
 	if relDockerfile, err = filepath.Rel(absContextDir, absDockerfile); err != nil {
... ...
@@ -2024,6 +2024,81 @@ func (s *DockerSuite) TestBuildNoContext(c *check.C) {
2024 2024
 	}
2025 2025
 }
2026 2026
 
2027
+func (s *DockerSuite) TestBuildDockerfileStdin(c *check.C) {
2028
+	name := "stdindockerfile"
2029
+	tmpDir, err := ioutil.TempDir("", "fake-context")
2030
+	c.Assert(err, check.IsNil)
2031
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("bar"), 0600)
2032
+	c.Assert(err, check.IsNil)
2033
+
2034
+	icmd.RunCmd(icmd.Cmd{
2035
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", tmpDir},
2036
+		Stdin: strings.NewReader(
2037
+			`FROM busybox
2038
+ADD foo /foo
2039
+CMD ["cat", "/foo"]`),
2040
+	}).Assert(c, icmd.Success)
2041
+
2042
+	res := inspectField(c, name, "Config.Cmd")
2043
+	c.Assert(strings.TrimSpace(string(res)), checker.Equals, `[cat /foo]`)
2044
+}
2045
+
2046
+func (s *DockerSuite) TestBuildDockerfileStdinConflict(c *check.C) {
2047
+	name := "stdindockerfiletarcontext"
2048
+	icmd.RunCmd(icmd.Cmd{
2049
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", "-"},
2050
+	}).Assert(c, icmd.Expected{
2051
+		ExitCode: 1,
2052
+		Err:      "use stdin for both build context and dockerfile",
2053
+	})
2054
+}
2055
+
2056
+func (s *DockerSuite) TestBuildDockerfileStdinNoExtraFiles(c *check.C) {
2057
+	s.testBuildDockerfileStdinNoExtraFiles(c, false, false)
2058
+}
2059
+
2060
+func (s *DockerSuite) TestBuildDockerfileStdinDockerignore(c *check.C) {
2061
+	s.testBuildDockerfileStdinNoExtraFiles(c, true, false)
2062
+}
2063
+
2064
+func (s *DockerSuite) TestBuildDockerfileStdinDockerignoreIgnored(c *check.C) {
2065
+	s.testBuildDockerfileStdinNoExtraFiles(c, true, true)
2066
+}
2067
+
2068
+func (s *DockerSuite) testBuildDockerfileStdinNoExtraFiles(c *check.C, hasDockerignore, ignoreDockerignore bool) {
2069
+	name := "stdindockerfilenoextra"
2070
+	tmpDir, err := ioutil.TempDir("", "fake-context")
2071
+	c.Assert(err, check.IsNil)
2072
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "foo"), []byte("bar"), 0600)
2073
+	c.Assert(err, check.IsNil)
2074
+	if hasDockerignore {
2075
+		// test that this file is removed
2076
+		err = ioutil.WriteFile(filepath.Join(tmpDir, "Dockerfile"), []byte(""), 0600)
2077
+		c.Assert(err, check.IsNil)
2078
+		ignores := "Dockerfile\n"
2079
+		if ignoreDockerignore {
2080
+			ignores += ".dockerignore\n"
2081
+		}
2082
+		err = ioutil.WriteFile(filepath.Join(tmpDir, ".dockerignore"), []byte(ignores), 0600)
2083
+		c.Assert(err, check.IsNil)
2084
+	}
2085
+
2086
+	icmd.RunCmd(icmd.Cmd{
2087
+		Command: []string{dockerBinary, "build", "-t", name, "-f", "-", tmpDir},
2088
+		Stdin: strings.NewReader(
2089
+			`FROM busybox
2090
+COPY . /baz`),
2091
+	}).Assert(c, icmd.Success)
2092
+
2093
+	out, _ := dockerCmd(c, "run", "--rm", name, "ls", "-A", "/baz")
2094
+	if hasDockerignore && !ignoreDockerignore {
2095
+		c.Assert(strings.TrimSpace(string(out)), checker.Equals, ".dockerignore\nfoo")
2096
+	} else {
2097
+		c.Assert(strings.TrimSpace(string(out)), checker.Equals, "foo")
2098
+	}
2099
+
2100
+}
2101
+
2027 2102
 func (s *DockerSuite) TestBuildWithVolumeOwnership(c *check.C) {
2028 2103
 	testRequires(c, DaemonIsLinux)
2029 2104
 	name := "testbuildimg"
... ...
@@ -14,6 +14,7 @@ import (
14 14
 	"os/exec"
15 15
 	"path/filepath"
16 16
 	"runtime"
17
+	"sort"
17 18
 	"strings"
18 19
 	"syscall"
19 20
 
... ...
@@ -225,6 +226,91 @@ func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, er
225 225
 	}
226 226
 }
227 227
 
228
+// TarModifierFunc is a function that can be passed to ReplaceFileTarWrapper to
229
+// define a modification step for a single path
230
+type TarModifierFunc func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error)
231
+
232
+// ReplaceFileTarWrapper converts inputTarStream to a new tar stream
233
+// while replacing a single file called header.Name with new contents.
234
+// If the file with header.Name does not exist it is added to the tar stream.
235
+// TODO: make this into a generic tar conversion function with walkFn argument
236
+func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModifierFunc) io.ReadCloser {
237
+	pipeReader, pipeWriter := io.Pipe()
238
+
239
+	modKeys := make([]string, 0, len(mods))
240
+	for key := range mods {
241
+		modKeys = append(modKeys, key)
242
+	}
243
+	sort.Strings(modKeys)
244
+
245
+	go func() {
246
+		tarReader := tar.NewReader(inputTarStream)
247
+		tarWriter := tar.NewWriter(pipeWriter)
248
+
249
+		defer inputTarStream.Close()
250
+
251
+	loop0:
252
+		for {
253
+			hdr, err := tarReader.Next()
254
+			for len(modKeys) > 0 && (err == io.EOF || err == nil && hdr.Name >= modKeys[0]) {
255
+				var h *tar.Header
256
+				var rdr io.Reader
257
+				if hdr != nil && hdr.Name == modKeys[0] {
258
+					h = hdr
259
+					rdr = tarReader
260
+				}
261
+
262
+				h2, dt, err := mods[modKeys[0]](modKeys[0], h, rdr)
263
+				if err != nil {
264
+					pipeWriter.CloseWithError(err)
265
+					return
266
+				}
267
+				if h2 != nil {
268
+					h2.Name = modKeys[0]
269
+					h2.Size = int64(len(dt))
270
+					if err := tarWriter.WriteHeader(h2); err != nil {
271
+						pipeWriter.CloseWithError(err)
272
+						return
273
+					}
274
+					if len(dt) != 0 {
275
+						if _, err := tarWriter.Write(dt); err != nil {
276
+							pipeWriter.CloseWithError(err)
277
+							return
278
+						}
279
+					}
280
+				}
281
+				modKeys = modKeys[1:]
282
+				if h != nil {
283
+					continue loop0
284
+				}
285
+			}
286
+
287
+			if err == io.EOF {
288
+				tarWriter.Close()
289
+				pipeWriter.Close()
290
+				return
291
+			}
292
+
293
+			if err != nil {
294
+				pipeWriter.CloseWithError(err)
295
+				return
296
+			}
297
+
298
+			if err := tarWriter.WriteHeader(hdr); err != nil {
299
+				pipeWriter.CloseWithError(err)
300
+				return
301
+			}
302
+
303
+			if _, err := pools.Copy(tarWriter, tarReader); err != nil {
304
+				pipeWriter.CloseWithError(err)
305
+				return
306
+			}
307
+
308
+		}
309
+	}()
310
+	return pipeReader
311
+}
312
+
228 313
 // Extension returns the extension of a file that uses the specified compression algorithm.
229 314
 func (compression *Compression) Extension() string {
230 315
 	switch *compression {
... ...
@@ -1160,3 +1160,59 @@ func TestTempArchiveCloseMultipleTimes(t *testing.T) {
1160 1160
 		}
1161 1161
 	}
1162 1162
 }
1163
+
1164
+func testReplaceFileTarWrapper(t *testing.T, name string) {
1165
+	srcDir, err := ioutil.TempDir("", "docker-test-srcDir")
1166
+	if err != nil {
1167
+		t.Fatal(err)
1168
+	}
1169
+	defer os.RemoveAll(srcDir)
1170
+
1171
+	destDir, err := ioutil.TempDir("", "docker-test-destDir")
1172
+	if err != nil {
1173
+		t.Fatal(err)
1174
+	}
1175
+	defer os.RemoveAll(destDir)
1176
+
1177
+	_, err = prepareUntarSourceDirectory(20, srcDir, false)
1178
+	if err != nil {
1179
+		t.Fatal(err)
1180
+	}
1181
+
1182
+	archive, err := TarWithOptions(srcDir, &TarOptions{})
1183
+	if err != nil {
1184
+		t.Fatal(err)
1185
+	}
1186
+	defer archive.Close()
1187
+
1188
+	archive2 := ReplaceFileTarWrapper(archive, map[string]TarModifierFunc{name: func(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
1189
+		return &tar.Header{
1190
+			Mode:     0600,
1191
+			Typeflag: tar.TypeReg,
1192
+		}, []byte("foobar"), nil
1193
+	}})
1194
+
1195
+	if err := Untar(archive2, destDir, nil); err != nil {
1196
+		t.Fatal(err)
1197
+	}
1198
+
1199
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, name))
1200
+	if err != nil {
1201
+		t.Fatal(err)
1202
+	}
1203
+	if expected, actual := "foobar", string(dt); actual != expected {
1204
+		t.Fatalf("file contents mismatch, expected: %q, got %q", expected, actual)
1205
+	}
1206
+}
1207
+
1208
+func TestReplaceFileTarWrapperNewFile(t *testing.T) {
1209
+	testReplaceFileTarWrapper(t, "abc")
1210
+}
1211
+
1212
+func TestReplaceFileTarWrapperReplaceFile(t *testing.T) {
1213
+	testReplaceFileTarWrapper(t, "file-2")
1214
+}
1215
+
1216
+func TestReplaceFileTarWrapperLastFile(t *testing.T) {
1217
+	testReplaceFileTarWrapper(t, "file-999")
1218
+}