After finding our initial thinking on env. space versus arg list space
was wrong, we need to solve this by using a pipe between the caller and
child to marshall the (potentially very large) options array to the
archiver.
Docker-DCO-1.1-Signed-off-by: Phil Estes <estesp@linux.vnet.ibm.com> (github: estesp)
| ... | ... |
@@ -1,6 +1,7 @@ |
| 1 | 1 |
package chrootarchive |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "bytes" |
|
| 4 | 5 |
"encoding/json" |
| 5 | 6 |
"flag" |
| 6 | 7 |
"fmt" |
| ... | ... |
@@ -29,7 +30,8 @@ func untar() {
|
| 29 | 29 |
|
| 30 | 30 |
var options *archive.TarOptions |
| 31 | 31 |
|
| 32 |
- if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil {
|
|
| 32 |
+ //read the options from the pipe "ExtraFiles" |
|
| 33 |
+ if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
|
|
| 33 | 34 |
fatal(err) |
| 34 | 35 |
} |
| 35 | 36 |
|
| ... | ... |
@@ -62,28 +64,39 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error |
| 62 | 62 |
} |
| 63 | 63 |
} |
| 64 | 64 |
|
| 65 |
- // We can't pass the exclude list directly via cmd line |
|
| 66 |
- // because we easily overrun the shell max argument list length |
|
| 67 |
- // when the full image list is passed (e.g. when this is used |
|
| 68 |
- // by `docker load`). Instead we will add the JSON marshalled |
|
| 69 |
- // and placed in the env, which has significantly larger |
|
| 70 |
- // max size |
|
| 71 |
- data, err := json.Marshal(options) |
|
| 72 |
- if err != nil {
|
|
| 73 |
- return fmt.Errorf("Untar json encode: %v", err)
|
|
| 74 |
- } |
|
| 75 | 65 |
decompressedArchive, err := archive.DecompressStream(tarArchive) |
| 76 | 66 |
if err != nil {
|
| 77 | 67 |
return err |
| 78 | 68 |
} |
| 79 | 69 |
defer decompressedArchive.Close() |
| 80 | 70 |
|
| 71 |
+ // We can't pass a potentially large exclude list directly via cmd line |
|
| 72 |
+ // because we easily overrun the kernel's max argument/environment size |
|
| 73 |
+ // when the full image list is passed (e.g. when this is used by |
|
| 74 |
+ // `docker load`). We will marshall the options via a pipe to the |
|
| 75 |
+ // child |
|
| 76 |
+ r, w, err := os.Pipe() |
|
| 77 |
+ if err != nil {
|
|
| 78 |
+ return fmt.Errorf("Untar pipe failure: %v", err)
|
|
| 79 |
+ } |
|
| 81 | 80 |
cmd := reexec.Command("docker-untar", dest)
|
| 82 | 81 |
cmd.Stdin = decompressedArchive |
| 83 |
- cmd.Env = append(cmd.Env, fmt.Sprintf("OPT=%s", data))
|
|
| 84 |
- out, err := cmd.CombinedOutput() |
|
| 85 |
- if err != nil {
|
|
| 86 |
- return fmt.Errorf("Untar %s %s", err, out)
|
|
| 82 |
+ cmd.ExtraFiles = append(cmd.ExtraFiles, r) |
|
| 83 |
+ var output bytes.Buffer |
|
| 84 |
+ cmd.Stdout = &output |
|
| 85 |
+ cmd.Stderr = &output |
|
| 86 |
+ |
|
| 87 |
+ if err := cmd.Start(); err != nil {
|
|
| 88 |
+ return fmt.Errorf("Untar error on re-exec cmd: %v", err)
|
|
| 89 |
+ } |
|
| 90 |
+ //write the options to the pipe for the untar exec to read |
|
| 91 |
+ if err := json.NewEncoder(w).Encode(options); err != nil {
|
|
| 92 |
+ return fmt.Errorf("Untar json encode to pipe failed: %v", err)
|
|
| 93 |
+ } |
|
| 94 |
+ w.Close() |
|
| 95 |
+ |
|
| 96 |
+ if err := cmd.Wait(); err != nil {
|
|
| 97 |
+ return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
|
|
| 87 | 98 |
} |
| 88 | 99 |
return nil |
| 89 | 100 |
} |
| ... | ... |
@@ -8,6 +8,7 @@ import ( |
| 8 | 8 |
"os" |
| 9 | 9 |
"path" |
| 10 | 10 |
"path/filepath" |
| 11 |
+ "strings" |
|
| 11 | 12 |
"testing" |
| 12 | 13 |
"time" |
| 13 | 14 |
|
| ... | ... |
@@ -48,6 +49,42 @@ func TestChrootTarUntar(t *testing.T) {
|
| 48 | 48 |
} |
| 49 | 49 |
} |
| 50 | 50 |
|
| 51 |
+// gh#10426: Verify the fix for having a huge excludes list (like on `docker load` with large # of |
|
| 52 |
+// local images) |
|
| 53 |
+func TestChrootUntarWithHugeExcludesList(t *testing.T) {
|
|
| 54 |
+ tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarHugeExcludes")
|
|
| 55 |
+ if err != nil {
|
|
| 56 |
+ t.Fatal(err) |
|
| 57 |
+ } |
|
| 58 |
+ defer os.RemoveAll(tmpdir) |
|
| 59 |
+ src := filepath.Join(tmpdir, "src") |
|
| 60 |
+ if err := os.MkdirAll(src, 0700); err != nil {
|
|
| 61 |
+ t.Fatal(err) |
|
| 62 |
+ } |
|
| 63 |
+ if err := ioutil.WriteFile(filepath.Join(src, "toto"), []byte("hello toto"), 0644); err != nil {
|
|
| 64 |
+ t.Fatal(err) |
|
| 65 |
+ } |
|
| 66 |
+ stream, err := archive.Tar(src, archive.Uncompressed) |
|
| 67 |
+ if err != nil {
|
|
| 68 |
+ t.Fatal(err) |
|
| 69 |
+ } |
|
| 70 |
+ dest := filepath.Join(tmpdir, "dest") |
|
| 71 |
+ if err := os.MkdirAll(dest, 0700); err != nil {
|
|
| 72 |
+ t.Fatal(err) |
|
| 73 |
+ } |
|
| 74 |
+ options := &archive.TarOptions{}
|
|
| 75 |
+ //65534 entries of 64-byte strings ~= 4MB of environment space which should overflow |
|
| 76 |
+ //on most systems when passed via environment or command line arguments |
|
| 77 |
+ excludes := make([]string, 65534, 65534) |
|
| 78 |
+ for i := 0; i < 65534; i++ {
|
|
| 79 |
+ excludes[i] = strings.Repeat(string(i), 64) |
|
| 80 |
+ } |
|
| 81 |
+ options.ExcludePatterns = excludes |
|
| 82 |
+ if err := Untar(stream, dest, options); err != nil {
|
|
| 83 |
+ t.Fatal(err) |
|
| 84 |
+ } |
|
| 85 |
+} |
|
| 86 |
+ |
|
| 51 | 87 |
func TestChrootUntarEmptyArchive(t *testing.T) {
|
| 52 | 88 |
tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarEmptyArchive")
|
| 53 | 89 |
if err != nil {
|