archive: add test for prefix header
With docker-17.06.0 some images pulled do not extract properly. Some files don't appear in correct directories. This may or may not cause the pull to fail. These images can't be pushed or saved. 17.06 is the first version of Docker built with go1.8.
Cause
There are multiple updates to the tar package in go1.8.
https://go-review.googlesource.com/c/32234/ disables using "prefix" field when new tar archives are being written. Prefix field was previously set when a record in the archive used a path longer than 100 bytes.
Another change https://go-review.googlesource.com/c/31444/ makes the reader ignore the "prefix" field value if the record is in GNU format. GNU format defines that same area should be used for access and modified times. If the "prefix" field is not read, a file will only be extracted by the basename.
The problem is that with a previous version of the golang archive package headers could be written, that use the prefix field while at the same time setting the header format to GNU. This happens when numeric fields are big enough that they can not be written as octal strings and need to be written in binary. Usually, this shouldn't happen: uid, gid, devmajor, devminor can use up to 7 bytes, size and timestamp can use 11. If one of the records does overflow it switches the whole writer to GNU mode and all next files will be saved in GNU format.
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Tonis Tiigi authored on 2017/07/14 10:56:00Showing 6 changed files
- Dockerfile.aarch64
- Dockerfile.armhf
- Dockerfile.ppc64le
- Dockerfile.s390x
- Dockerfile.simple
- pkg/archive/archive_test.go
| ... | ... |
@@ -97,6 +97,7 @@ RUN set -x \ |
| 97 | 97 |
# bootstrap, so we use golang-go (1.6) as bootstrap to build Go from source code. |
| 98 | 98 |
# We don't use the official ARMv6 released binaries as a GOROOT_BOOTSTRAP, because |
| 99 | 99 |
# not all ARM64 platforms support 32-bit mode. 32-bit mode is optional for ARMv8. |
| 100 |
+# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored |
|
| 100 | 101 |
ENV GO_VERSION 1.8.3 |
| 101 | 102 |
RUN mkdir /usr/src/go && curl -fsSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/src/go -xz --strip-components=1 \
|
| 102 | 103 |
&& cd /usr/src/go/src \ |
| ... | ... |
@@ -70,6 +70,7 @@ RUN cd /usr/local/lvm2 \ |
| 70 | 70 |
# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL |
| 71 | 71 |
|
| 72 | 72 |
# Install Go |
| 73 |
+# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored |
|
| 73 | 74 |
ENV GO_VERSION 1.8.3 |
| 74 | 75 |
RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-armv6l.tar.gz" \
|
| 75 | 76 |
| tar -xzC /usr/local |
| ... | ... |
@@ -94,6 +94,7 @@ RUN set -x \ |
| 94 | 94 |
|
| 95 | 95 |
# Install Go |
| 96 | 96 |
# NOTE: official ppc64le go binaries weren't available until go 1.6.4 and 1.7.4 |
| 97 |
+# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored |
|
| 97 | 98 |
ENV GO_VERSION 1.8.3 |
| 98 | 99 |
RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-ppc64le.tar.gz" \
|
| 99 | 100 |
| tar -xzC /usr/local |
| ... | ... |
@@ -87,6 +87,7 @@ RUN cd /usr/local/lvm2 \ |
| 87 | 87 |
&& make install_device-mapper |
| 88 | 88 |
# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL |
| 89 | 89 |
|
| 90 |
+# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored |
|
| 90 | 91 |
ENV GO_VERSION 1.8.3 |
| 91 | 92 |
RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-s390x.tar.gz" \
|
| 92 | 93 |
| tar -xzC /usr/local |
| ... | ... |
@@ -53,6 +53,7 @@ RUN set -x \ |
| 53 | 53 |
# IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines |
| 54 | 54 |
# will need updating, to avoid errors. Ping #docker-maintainers on IRC |
| 55 | 55 |
# with a heads-up. |
| 56 |
+# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored |
|
| 56 | 57 |
ENV GO_VERSION 1.8.3 |
| 57 | 58 |
RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
|
| 58 | 59 |
| tar -xzC /usr/local |
| ... | ... |
@@ -1203,6 +1203,25 @@ func TestReplaceFileTarWrapper(t *testing.T) {
|
| 1203 | 1203 |
} |
| 1204 | 1204 |
} |
| 1205 | 1205 |
|
| 1206 |
+// TestPrefixHeaderReadable tests that files that could be created with the |
|
| 1207 |
+// version of this package that was built with <=go17 are still readable. |
|
| 1208 |
+func TestPrefixHeaderReadable(t *testing.T) {
|
|
| 1209 |
+ // https://gist.github.com/stevvooe/e2a790ad4e97425896206c0816e1a882#file-out-go |
|
| 1210 |
+ var testFile = []byte("\x1f\x8b\x08\x08\x44\x21\x68\x59\x00\x03\x74\x2e\x74\x61\x72\x00\x4b\xcb\xcf\x67\xa0\x35\x30\x80\x00\x86\x06\x10\x47\x01\xc1\x37\x40\x00\x54\xb6\xb1\xa1\xa9\x99\x09\x48\x25\x1d\x40\x69\x71\x49\x62\x91\x02\xe5\x76\xa1\x79\x84\x21\x91\xd6\x80\x72\xaf\x8f\x82\x51\x30\x0a\x46\x36\x00\x00\xf0\x1c\x1e\x95\x00\x06\x00\x00")
|
|
| 1211 |
+ |
|
| 1212 |
+ tmpDir, err := ioutil.TempDir("", "prefix-test")
|
|
| 1213 |
+ require.NoError(t, err) |
|
| 1214 |
+ defer os.RemoveAll(tmpDir) |
|
| 1215 |
+ err = Untar(bytes.NewReader(testFile), tmpDir, nil) |
|
| 1216 |
+ require.NoError(t, err) |
|
| 1217 |
+ |
|
| 1218 |
+ baseName := "foo" |
|
| 1219 |
+ pth := strings.Repeat("a", 100-len(baseName)) + "/" + baseName
|
|
| 1220 |
+ |
|
| 1221 |
+ _, err = os.Lstat(filepath.Join(tmpDir, pth)) |
|
| 1222 |
+ require.NoError(t, err) |
|
| 1223 |
+} |
|
| 1224 |
+ |
|
| 1206 | 1225 |
func buildSourceArchive(t *testing.T, numberOfFiles int) (io.ReadCloser, func()) {
|
| 1207 | 1226 |
srcDir, err := ioutil.TempDir("", "docker-test-srcDir")
|
| 1208 | 1227 |
require.NoError(t, err) |