Browse code

Fix remote build target as Dockerfile

The test was passing previously because the preamble was already buffered. After
the change to return Scanner.Err() the final read error on the buffer was no
longer being ignored.

Signed-off-by: Daniel Nephin <dnephin@docker.com>

Daniel Nephin authored on 2017/11/09 03:06:57
Showing 4 changed files
... ...
@@ -79,7 +79,6 @@ func FromArchive(tarStream io.Reader) (builder.Source, error) {
79 79
 	}
80 80
 
81 81
 	tsc.sums = sum.GetSums()
82
-
83 82
 	return tsc, nil
84 83
 }
85 84
 
... ...
@@ -97,26 +97,23 @@ func newGitRemote(gitURL string, dockerfilePath string) (builder.Source, *parser
97 97
 }
98 98
 
99 99
 func newURLRemote(url string, dockerfilePath string, progressReader func(in io.ReadCloser) io.ReadCloser) (builder.Source, *parser.Result, error) {
100
-	var dockerfile io.ReadCloser
101
-	dockerfileFoundErr := errors.New("found-dockerfile")
102
-	c, err := MakeRemoteContext(url, map[string]func(io.ReadCloser) (io.ReadCloser, error){
103
-		mimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
104
-			dockerfile = rc
105
-			return nil, dockerfileFoundErr
106
-		},
107
-		// fallback handler (tar context)
108
-		"": func(rc io.ReadCloser) (io.ReadCloser, error) {
109
-			return progressReader(rc), nil
110
-		},
111
-	})
112
-	switch {
113
-	case err == dockerfileFoundErr:
114
-		res, err := parser.Parse(dockerfile)
115
-		return nil, res, err
116
-	case err != nil:
100
+	contentType, content, err := downloadRemote(url)
101
+	if err != nil {
117 102
 		return nil, nil, err
118 103
 	}
119
-	return withDockerfileFromContext(c.(modifiableContext), dockerfilePath)
104
+	defer content.Close()
105
+
106
+	switch contentType {
107
+	case mimeTypes.TextPlain:
108
+		res, err := parser.Parse(progressReader(content))
109
+		return nil, res, err
110
+	default:
111
+		source, err := FromArchive(progressReader(content))
112
+		if err != nil {
113
+			return nil, nil, err
114
+		}
115
+		return withDockerfileFromContext(source.(modifiableContext), dockerfilePath)
116
+	}
120 117
 }
121 118
 
122 119
 func removeDockerfile(c modifiableContext, filesToRemove ...string) error {
... ...
@@ -10,7 +10,7 @@ import (
10 10
 	"net/url"
11 11
 	"regexp"
12 12
 
13
-	"github.com/docker/docker/builder"
13
+	"github.com/docker/docker/pkg/ioutils"
14 14
 	"github.com/pkg/errors"
15 15
 )
16 16
 
... ...
@@ -22,50 +22,23 @@ const acceptableRemoteMIME = `(?:application/(?:(?:x\-)?tar|octet\-stream|((?:x\
22 22
 
23 23
 var mimeRe = regexp.MustCompile(acceptableRemoteMIME)
24 24
 
25
-// MakeRemoteContext downloads a context from remoteURL and returns it.
26
-//
27
-// If contentTypeHandlers is non-nil, then the Content-Type header is read along with a maximum of
28
-// maxPreambleLength bytes from the body to help detecting the MIME type.
29
-// Look at acceptableRemoteMIME for more details.
30
-//
31
-// If a match is found, then the body is sent to the contentType handler and a (potentially compressed) tar stream is expected
32
-// to be returned. If no match is found, it is assumed the body is a tar stream (compressed or not).
33
-// In either case, an (assumed) tar stream is passed to FromArchive whose result is returned.
34
-func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io.ReadCloser) (io.ReadCloser, error)) (builder.Source, error) {
35
-	f, err := GetWithStatusError(remoteURL)
25
+// downloadRemote context from a url and returns it, along with the parsed content type
26
+func downloadRemote(remoteURL string) (string, io.ReadCloser, error) {
27
+	response, err := GetWithStatusError(remoteURL)
36 28
 	if err != nil {
37
-		return nil, fmt.Errorf("error downloading remote context %s: %v", remoteURL, err)
29
+		return "", nil, fmt.Errorf("error downloading remote context %s: %v", remoteURL, err)
38 30
 	}
39
-	defer f.Body.Close()
40 31
 
41
-	var contextReader io.ReadCloser
42
-	if contentTypeHandlers != nil {
43
-		contentType := f.Header.Get("Content-Type")
44
-		clen := f.ContentLength
45
-
46
-		contentType, contextReader, err = inspectResponse(contentType, f.Body, clen)
47
-		if err != nil {
48
-			return nil, fmt.Errorf("error detecting content type for remote %s: %v", remoteURL, err)
49
-		}
50
-		defer contextReader.Close()
51
-
52
-		// This loop tries to find a content-type handler for the detected content-type.
53
-		// If it could not find one from the caller-supplied map, it tries the empty content-type `""`
54
-		// which is interpreted as a fallback handler (usually used for raw tar contexts).
55
-		for _, ct := range []string{contentType, ""} {
56
-			if fn, ok := contentTypeHandlers[ct]; ok {
57
-				defer contextReader.Close()
58
-				if contextReader, err = fn(contextReader); err != nil {
59
-					return nil, err
60
-				}
61
-				break
62
-			}
63
-		}
32
+	contentType, contextReader, err := inspectResponse(
33
+		response.Header.Get("Content-Type"),
34
+		response.Body,
35
+		response.ContentLength)
36
+	if err != nil {
37
+		response.Body.Close()
38
+		return "", nil, fmt.Errorf("error detecting content type for remote %s: %v", remoteURL, err)
64 39
 	}
65 40
 
66
-	// Pass through - this is a pre-packaged context, presumably
67
-	// with a Dockerfile with the right name inside it.
68
-	return FromArchive(contextReader)
41
+	return contentType, ioutils.NewReadCloserWrapper(contextReader, response.Body.Close), nil
69 42
 }
70 43
 
71 44
 // GetWithStatusError does an http.Get() and returns an error if the
... ...
@@ -110,7 +83,7 @@ func GetWithStatusError(address string) (resp *http.Response, err error) {
110 110
 //    - an io.Reader for the response body
111 111
 //    - an error value which will be non-nil either when something goes wrong while
112 112
 //      reading bytes from r or when the detected content-type is not acceptable.
113
-func inspectResponse(ct string, r io.Reader, clen int64) (string, io.ReadCloser, error) {
113
+func inspectResponse(ct string, r io.Reader, clen int64) (string, io.Reader, error) {
114 114
 	plen := clen
115 115
 	if plen <= 0 || plen > maxPreambleLength {
116 116
 		plen = maxPreambleLength
... ...
@@ -119,14 +92,14 @@ func inspectResponse(ct string, r io.Reader, clen int64) (string, io.ReadCloser,
119 119
 	preamble := make([]byte, plen)
120 120
 	rlen, err := r.Read(preamble)
121 121
 	if rlen == 0 {
122
-		return ct, ioutil.NopCloser(r), errors.New("empty response")
122
+		return ct, r, errors.New("empty response")
123 123
 	}
124 124
 	if err != nil && err != io.EOF {
125
-		return ct, ioutil.NopCloser(r), err
125
+		return ct, r, err
126 126
 	}
127 127
 
128 128
 	preambleR := bytes.NewReader(preamble[:rlen])
129
-	bodyReader := ioutil.NopCloser(io.MultiReader(preambleR, r))
129
+	bodyReader := io.MultiReader(preambleR, r)
130 130
 	// Some web servers will use application/octet-stream as the default
131 131
 	// content type for files without an extension (e.g. 'Dockerfile')
132 132
 	// so if we receive this value we better check for text content
... ...
@@ -11,7 +11,7 @@ import (
11 11
 
12 12
 	"github.com/docker/docker/builder"
13 13
 	"github.com/docker/docker/internal/testutil"
14
-	"github.com/docker/docker/pkg/archive"
14
+	"github.com/gotestyourself/gotestyourself/fs"
15 15
 	"github.com/stretchr/testify/assert"
16 16
 	"github.com/stretchr/testify/require"
17 17
 )
... ...
@@ -174,11 +174,10 @@ func TestUnknownContentLength(t *testing.T) {
174 174
 	}
175 175
 }
176 176
 
177
-func TestMakeRemoteContext(t *testing.T) {
178
-	contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
179
-	defer cleanup()
180
-
181
-	createTestTempFile(t, contextDir, builder.DefaultDockerfileName, dockerfileContents, 0777)
177
+func TestDownloadRemote(t *testing.T) {
178
+	contextDir := fs.NewDir(t, "test-builder-download-remote",
179
+		fs.WithFile(builder.DefaultDockerfileName, dockerfileContents))
180
+	defer contextDir.Remove()
182 181
 
183 182
 	mux := http.NewServeMux()
184 183
 	server := httptest.NewServer(mux)
... ...
@@ -187,39 +186,15 @@ func TestMakeRemoteContext(t *testing.T) {
187 187
 	serverURL.Path = "/" + builder.DefaultDockerfileName
188 188
 	remoteURL := serverURL.String()
189 189
 
190
-	mux.Handle("/", http.FileServer(http.Dir(contextDir)))
191
-
192
-	remoteContext, err := MakeRemoteContext(remoteURL, map[string]func(io.ReadCloser) (io.ReadCloser, error){
193
-		mimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
194
-			dockerfile, err := ioutil.ReadAll(rc)
195
-			if err != nil {
196
-				return nil, err
197
-			}
198
-
199
-			r, err := archive.Generate(builder.DefaultDockerfileName, string(dockerfile))
200
-			if err != nil {
201
-				return nil, err
202
-			}
203
-			return ioutil.NopCloser(r), nil
204
-		},
205
-	})
206
-
207
-	if err != nil {
208
-		t.Fatalf("Error when executing DetectContextFromRemoteURL: %s", err)
209
-	}
210
-
211
-	if remoteContext == nil {
212
-		t.Fatal("Remote context should not be nil")
213
-	}
190
+	mux.Handle("/", http.FileServer(http.Dir(contextDir.Path())))
214 191
 
215
-	h, err := remoteContext.Hash(builder.DefaultDockerfileName)
216
-	if err != nil {
217
-		t.Fatalf("failed to compute hash %s", err)
218
-	}
192
+	contentType, content, err := downloadRemote(remoteURL)
193
+	require.NoError(t, err)
219 194
 
220
-	if expected, actual := "7b6b6b66bee9e2102fbdc2228be6c980a2a23adf371962a37286a49f7de0f7cc", h; expected != actual {
221
-		t.Fatalf("There should be file named %s %s in fileInfoSums", expected, actual)
222
-	}
195
+	assert.Equal(t, mimeTypes.TextPlain, contentType)
196
+	raw, err := ioutil.ReadAll(content)
197
+	require.NoError(t, err)
198
+	assert.Equal(t, dockerfileContents, string(raw))
223 199
 }
224 200
 
225 201
 func TestGetWithStatusError(t *testing.T) {