The test was passing previously because the preamble was already buffered. After
the change to return Scanner.Err() the final read error on the buffer was no
longer being ignored.
Signed-off-by: Daniel Nephin <dnephin@docker.com>
| ... | ... |
@@ -97,26 +97,23 @@ func newGitRemote(gitURL string, dockerfilePath string) (builder.Source, *parser |
| 97 | 97 |
} |
| 98 | 98 |
|
| 99 | 99 |
func newURLRemote(url string, dockerfilePath string, progressReader func(in io.ReadCloser) io.ReadCloser) (builder.Source, *parser.Result, error) {
|
| 100 |
- var dockerfile io.ReadCloser |
|
| 101 |
- dockerfileFoundErr := errors.New("found-dockerfile")
|
|
| 102 |
- c, err := MakeRemoteContext(url, map[string]func(io.ReadCloser) (io.ReadCloser, error){
|
|
| 103 |
- mimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
|
|
| 104 |
- dockerfile = rc |
|
| 105 |
- return nil, dockerfileFoundErr |
|
| 106 |
- }, |
|
| 107 |
- // fallback handler (tar context) |
|
| 108 |
- "": func(rc io.ReadCloser) (io.ReadCloser, error) {
|
|
| 109 |
- return progressReader(rc), nil |
|
| 110 |
- }, |
|
| 111 |
- }) |
|
| 112 |
- switch {
|
|
| 113 |
- case err == dockerfileFoundErr: |
|
| 114 |
- res, err := parser.Parse(dockerfile) |
|
| 115 |
- return nil, res, err |
|
| 116 |
- case err != nil: |
|
| 100 |
+ contentType, content, err := downloadRemote(url) |
|
| 101 |
+ if err != nil {
|
|
| 117 | 102 |
return nil, nil, err |
| 118 | 103 |
} |
| 119 |
- return withDockerfileFromContext(c.(modifiableContext), dockerfilePath) |
|
| 104 |
+ defer content.Close() |
|
| 105 |
+ |
|
| 106 |
+ switch contentType {
|
|
| 107 |
+ case mimeTypes.TextPlain: |
|
| 108 |
+ res, err := parser.Parse(progressReader(content)) |
|
| 109 |
+ return nil, res, err |
|
| 110 |
+ default: |
|
| 111 |
+ source, err := FromArchive(progressReader(content)) |
|
| 112 |
+ if err != nil {
|
|
| 113 |
+ return nil, nil, err |
|
| 114 |
+ } |
|
| 115 |
+ return withDockerfileFromContext(source.(modifiableContext), dockerfilePath) |
|
| 116 |
+ } |
|
| 120 | 117 |
} |
| 121 | 118 |
|
| 122 | 119 |
func removeDockerfile(c modifiableContext, filesToRemove ...string) error {
|
| ... | ... |
@@ -10,7 +10,7 @@ import ( |
| 10 | 10 |
"net/url" |
| 11 | 11 |
"regexp" |
| 12 | 12 |
|
| 13 |
- "github.com/docker/docker/builder" |
|
| 13 |
+ "github.com/docker/docker/pkg/ioutils" |
|
| 14 | 14 |
"github.com/pkg/errors" |
| 15 | 15 |
) |
| 16 | 16 |
|
| ... | ... |
@@ -22,50 +22,23 @@ const acceptableRemoteMIME = `(?:application/(?:(?:x\-)?tar|octet\-stream|((?:x\ |
| 22 | 22 |
|
| 23 | 23 |
var mimeRe = regexp.MustCompile(acceptableRemoteMIME) |
| 24 | 24 |
|
| 25 |
-// MakeRemoteContext downloads a context from remoteURL and returns it. |
|
| 26 |
-// |
|
| 27 |
-// If contentTypeHandlers is non-nil, then the Content-Type header is read along with a maximum of |
|
| 28 |
-// maxPreambleLength bytes from the body to help detecting the MIME type. |
|
| 29 |
-// Look at acceptableRemoteMIME for more details. |
|
| 30 |
-// |
|
| 31 |
-// If a match is found, then the body is sent to the contentType handler and a (potentially compressed) tar stream is expected |
|
| 32 |
-// to be returned. If no match is found, it is assumed the body is a tar stream (compressed or not). |
|
| 33 |
-// In either case, an (assumed) tar stream is passed to FromArchive whose result is returned. |
|
| 34 |
-func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io.ReadCloser) (io.ReadCloser, error)) (builder.Source, error) {
|
|
| 35 |
- f, err := GetWithStatusError(remoteURL) |
|
| 25 |
+// downloadRemote context from a url and returns it, along with the parsed content type |
|
| 26 |
+func downloadRemote(remoteURL string) (string, io.ReadCloser, error) {
|
|
| 27 |
+ response, err := GetWithStatusError(remoteURL) |
|
| 36 | 28 |
if err != nil {
|
| 37 |
- return nil, fmt.Errorf("error downloading remote context %s: %v", remoteURL, err)
|
|
| 29 |
+ return "", nil, fmt.Errorf("error downloading remote context %s: %v", remoteURL, err)
|
|
| 38 | 30 |
} |
| 39 |
- defer f.Body.Close() |
|
| 40 | 31 |
|
| 41 |
- var contextReader io.ReadCloser |
|
| 42 |
- if contentTypeHandlers != nil {
|
|
| 43 |
- contentType := f.Header.Get("Content-Type")
|
|
| 44 |
- clen := f.ContentLength |
|
| 45 |
- |
|
| 46 |
- contentType, contextReader, err = inspectResponse(contentType, f.Body, clen) |
|
| 47 |
- if err != nil {
|
|
| 48 |
- return nil, fmt.Errorf("error detecting content type for remote %s: %v", remoteURL, err)
|
|
| 49 |
- } |
|
| 50 |
- defer contextReader.Close() |
|
| 51 |
- |
|
| 52 |
- // This loop tries to find a content-type handler for the detected content-type. |
|
| 53 |
- // If it could not find one from the caller-supplied map, it tries the empty content-type `""` |
|
| 54 |
- // which is interpreted as a fallback handler (usually used for raw tar contexts). |
|
| 55 |
- for _, ct := range []string{contentType, ""} {
|
|
| 56 |
- if fn, ok := contentTypeHandlers[ct]; ok {
|
|
| 57 |
- defer contextReader.Close() |
|
| 58 |
- if contextReader, err = fn(contextReader); err != nil {
|
|
| 59 |
- return nil, err |
|
| 60 |
- } |
|
| 61 |
- break |
|
| 62 |
- } |
|
| 63 |
- } |
|
| 32 |
+ contentType, contextReader, err := inspectResponse( |
|
| 33 |
+ response.Header.Get("Content-Type"),
|
|
| 34 |
+ response.Body, |
|
| 35 |
+ response.ContentLength) |
|
| 36 |
+ if err != nil {
|
|
| 37 |
+ response.Body.Close() |
|
| 38 |
+ return "", nil, fmt.Errorf("error detecting content type for remote %s: %v", remoteURL, err)
|
|
| 64 | 39 |
} |
| 65 | 40 |
|
| 66 |
- // Pass through - this is a pre-packaged context, presumably |
|
| 67 |
- // with a Dockerfile with the right name inside it. |
|
| 68 |
- return FromArchive(contextReader) |
|
| 41 |
+ return contentType, ioutils.NewReadCloserWrapper(contextReader, response.Body.Close), nil |
|
| 69 | 42 |
} |
| 70 | 43 |
|
| 71 | 44 |
// GetWithStatusError does an http.Get() and returns an error if the |
| ... | ... |
@@ -110,7 +83,7 @@ func GetWithStatusError(address string) (resp *http.Response, err error) {
|
| 110 | 110 |
// - an io.Reader for the response body |
| 111 | 111 |
// - an error value which will be non-nil either when something goes wrong while |
| 112 | 112 |
// reading bytes from r or when the detected content-type is not acceptable. |
| 113 |
-func inspectResponse(ct string, r io.Reader, clen int64) (string, io.ReadCloser, error) {
|
|
| 113 |
+func inspectResponse(ct string, r io.Reader, clen int64) (string, io.Reader, error) {
|
|
| 114 | 114 |
plen := clen |
| 115 | 115 |
if plen <= 0 || plen > maxPreambleLength {
|
| 116 | 116 |
plen = maxPreambleLength |
| ... | ... |
@@ -119,14 +92,14 @@ func inspectResponse(ct string, r io.Reader, clen int64) (string, io.ReadCloser, |
| 119 | 119 |
preamble := make([]byte, plen) |
| 120 | 120 |
rlen, err := r.Read(preamble) |
| 121 | 121 |
if rlen == 0 {
|
| 122 |
- return ct, ioutil.NopCloser(r), errors.New("empty response")
|
|
| 122 |
+ return ct, r, errors.New("empty response")
|
|
| 123 | 123 |
} |
| 124 | 124 |
if err != nil && err != io.EOF {
|
| 125 |
- return ct, ioutil.NopCloser(r), err |
|
| 125 |
+ return ct, r, err |
|
| 126 | 126 |
} |
| 127 | 127 |
|
| 128 | 128 |
preambleR := bytes.NewReader(preamble[:rlen]) |
| 129 |
- bodyReader := ioutil.NopCloser(io.MultiReader(preambleR, r)) |
|
| 129 |
+ bodyReader := io.MultiReader(preambleR, r) |
|
| 130 | 130 |
// Some web servers will use application/octet-stream as the default |
| 131 | 131 |
// content type for files without an extension (e.g. 'Dockerfile') |
| 132 | 132 |
// so if we receive this value we better check for text content |
| ... | ... |
@@ -11,7 +11,7 @@ import ( |
| 11 | 11 |
|
| 12 | 12 |
"github.com/docker/docker/builder" |
| 13 | 13 |
"github.com/docker/docker/internal/testutil" |
| 14 |
- "github.com/docker/docker/pkg/archive" |
|
| 14 |
+ "github.com/gotestyourself/gotestyourself/fs" |
|
| 15 | 15 |
"github.com/stretchr/testify/assert" |
| 16 | 16 |
"github.com/stretchr/testify/require" |
| 17 | 17 |
) |
| ... | ... |
@@ -174,11 +174,10 @@ func TestUnknownContentLength(t *testing.T) {
|
| 174 | 174 |
} |
| 175 | 175 |
} |
| 176 | 176 |
|
| 177 |
-func TestMakeRemoteContext(t *testing.T) {
|
|
| 178 |
- contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test") |
|
| 179 |
- defer cleanup() |
|
| 180 |
- |
|
| 181 |
- createTestTempFile(t, contextDir, builder.DefaultDockerfileName, dockerfileContents, 0777) |
|
| 177 |
+func TestDownloadRemote(t *testing.T) {
|
|
| 178 |
+ contextDir := fs.NewDir(t, "test-builder-download-remote", |
|
| 179 |
+ fs.WithFile(builder.DefaultDockerfileName, dockerfileContents)) |
|
| 180 |
+ defer contextDir.Remove() |
|
| 182 | 181 |
|
| 183 | 182 |
mux := http.NewServeMux() |
| 184 | 183 |
server := httptest.NewServer(mux) |
| ... | ... |
@@ -187,39 +186,15 @@ func TestMakeRemoteContext(t *testing.T) {
|
| 187 | 187 |
serverURL.Path = "/" + builder.DefaultDockerfileName |
| 188 | 188 |
remoteURL := serverURL.String() |
| 189 | 189 |
|
| 190 |
- mux.Handle("/", http.FileServer(http.Dir(contextDir)))
|
|
| 191 |
- |
|
| 192 |
- remoteContext, err := MakeRemoteContext(remoteURL, map[string]func(io.ReadCloser) (io.ReadCloser, error){
|
|
| 193 |
- mimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
|
|
| 194 |
- dockerfile, err := ioutil.ReadAll(rc) |
|
| 195 |
- if err != nil {
|
|
| 196 |
- return nil, err |
|
| 197 |
- } |
|
| 198 |
- |
|
| 199 |
- r, err := archive.Generate(builder.DefaultDockerfileName, string(dockerfile)) |
|
| 200 |
- if err != nil {
|
|
| 201 |
- return nil, err |
|
| 202 |
- } |
|
| 203 |
- return ioutil.NopCloser(r), nil |
|
| 204 |
- }, |
|
| 205 |
- }) |
|
| 206 |
- |
|
| 207 |
- if err != nil {
|
|
| 208 |
- t.Fatalf("Error when executing DetectContextFromRemoteURL: %s", err)
|
|
| 209 |
- } |
|
| 210 |
- |
|
| 211 |
- if remoteContext == nil {
|
|
| 212 |
- t.Fatal("Remote context should not be nil")
|
|
| 213 |
- } |
|
| 190 |
+ mux.Handle("/", http.FileServer(http.Dir(contextDir.Path())))
|
|
| 214 | 191 |
|
| 215 |
- h, err := remoteContext.Hash(builder.DefaultDockerfileName) |
|
| 216 |
- if err != nil {
|
|
| 217 |
- t.Fatalf("failed to compute hash %s", err)
|
|
| 218 |
- } |
|
| 192 |
+ contentType, content, err := downloadRemote(remoteURL) |
|
| 193 |
+ require.NoError(t, err) |
|
| 219 | 194 |
|
| 220 |
- if expected, actual := "7b6b6b66bee9e2102fbdc2228be6c980a2a23adf371962a37286a49f7de0f7cc", h; expected != actual {
|
|
| 221 |
- t.Fatalf("There should be file named %s %s in fileInfoSums", expected, actual)
|
|
| 222 |
- } |
|
| 195 |
+ assert.Equal(t, mimeTypes.TextPlain, contentType) |
|
| 196 |
+ raw, err := ioutil.ReadAll(content) |
|
| 197 |
+ require.NoError(t, err) |
|
| 198 |
+ assert.Equal(t, dockerfileContents, string(raw)) |
|
| 223 | 199 |
} |
| 224 | 200 |
|
| 225 | 201 |
func TestGetWithStatusError(t *testing.T) {
|