Browse code

Send archive options via pipe in chrootarchive

After finding our initial thinking on env. space versus arg list space
was wrong, we need to solve this by using a pipe between the caller and
child to marshall the (potentially very large) options array to the
archiver.

Docker-DCO-1.1-Signed-off-by: Phil Estes <estesp@linux.vnet.ibm.com> (github: estesp)

Phil Estes authored on 2015/04/11 03:23:09
Showing 2 changed files
... ...
@@ -1,6 +1,7 @@
1 1
 package chrootarchive
2 2
 
3 3
 import (
4
+	"bytes"
4 5
 	"encoding/json"
5 6
 	"flag"
6 7
 	"fmt"
... ...
@@ -29,7 +30,8 @@ func untar() {
29 29
 
30 30
 	var options *archive.TarOptions
31 31
 
32
-	if err := json.Unmarshal([]byte(os.Getenv("OPT")), &options); err != nil {
32
+	//read the options from the pipe "ExtraFiles"
33
+	if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil {
33 34
 		fatal(err)
34 35
 	}
35 36
 
... ...
@@ -62,28 +64,39 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error
62 62
 		}
63 63
 	}
64 64
 
65
-	// We can't pass the exclude list directly via cmd line
66
-	// because we easily overrun the shell max argument list length
67
-	// when the full image list is passed (e.g. when this is used
68
-	// by `docker load`). Instead we will add the JSON marshalled
69
-	// and placed in the env, which has significantly larger
70
-	// max size
71
-	data, err := json.Marshal(options)
72
-	if err != nil {
73
-		return fmt.Errorf("Untar json encode: %v", err)
74
-	}
75 65
 	decompressedArchive, err := archive.DecompressStream(tarArchive)
76 66
 	if err != nil {
77 67
 		return err
78 68
 	}
79 69
 	defer decompressedArchive.Close()
80 70
 
71
+	// We can't pass a potentially large exclude list directly via cmd line
72
+	// because we easily overrun the kernel's max argument/environment size
73
+	// when the full image list is passed (e.g. when this is used by
74
+	// `docker load`). We will marshall the options via a pipe to the
75
+	// child
76
+	r, w, err := os.Pipe()
77
+	if err != nil {
78
+		return fmt.Errorf("Untar pipe failure: %v", err)
79
+	}
81 80
 	cmd := reexec.Command("docker-untar", dest)
82 81
 	cmd.Stdin = decompressedArchive
83
-	cmd.Env = append(cmd.Env, fmt.Sprintf("OPT=%s", data))
84
-	out, err := cmd.CombinedOutput()
85
-	if err != nil {
86
-		return fmt.Errorf("Untar %s %s", err, out)
82
+	cmd.ExtraFiles = append(cmd.ExtraFiles, r)
83
+	var output bytes.Buffer
84
+	cmd.Stdout = &output
85
+	cmd.Stderr = &output
86
+
87
+	if err := cmd.Start(); err != nil {
88
+		return fmt.Errorf("Untar error on re-exec cmd: %v", err)
89
+	}
90
+	//write the options to the pipe for the untar exec to read
91
+	if err := json.NewEncoder(w).Encode(options); err != nil {
92
+		return fmt.Errorf("Untar json encode to pipe failed: %v", err)
93
+	}
94
+	w.Close()
95
+
96
+	if err := cmd.Wait(); err != nil {
97
+		return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
87 98
 	}
88 99
 	return nil
89 100
 }
... ...
@@ -8,6 +8,7 @@ import (
8 8
 	"os"
9 9
 	"path"
10 10
 	"path/filepath"
11
+	"strings"
11 12
 	"testing"
12 13
 	"time"
13 14
 
... ...
@@ -48,6 +49,42 @@ func TestChrootTarUntar(t *testing.T) {
48 48
 	}
49 49
 }
50 50
 
51
+// gh#10426: Verify the fix for having a huge excludes list (like on `docker load` with large # of
52
+// local images)
53
+func TestChrootUntarWithHugeExcludesList(t *testing.T) {
54
+	tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarHugeExcludes")
55
+	if err != nil {
56
+		t.Fatal(err)
57
+	}
58
+	defer os.RemoveAll(tmpdir)
59
+	src := filepath.Join(tmpdir, "src")
60
+	if err := os.MkdirAll(src, 0700); err != nil {
61
+		t.Fatal(err)
62
+	}
63
+	if err := ioutil.WriteFile(filepath.Join(src, "toto"), []byte("hello toto"), 0644); err != nil {
64
+		t.Fatal(err)
65
+	}
66
+	stream, err := archive.Tar(src, archive.Uncompressed)
67
+	if err != nil {
68
+		t.Fatal(err)
69
+	}
70
+	dest := filepath.Join(tmpdir, "dest")
71
+	if err := os.MkdirAll(dest, 0700); err != nil {
72
+		t.Fatal(err)
73
+	}
74
+	options := &archive.TarOptions{}
75
+	//65534 entries of 64-byte strings ~= 4MB of environment space which should overflow
76
+	//on most systems when passed via environment or command line arguments
77
+	excludes := make([]string, 65534, 65534)
78
+	for i := 0; i < 65534; i++ {
79
+		excludes[i] = strings.Repeat(string(i), 64)
80
+	}
81
+	options.ExcludePatterns = excludes
82
+	if err := Untar(stream, dest, options); err != nil {
83
+		t.Fatal(err)
84
+	}
85
+}
86
+
51 87
 func TestChrootUntarEmptyArchive(t *testing.T) {
52 88
 	tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarEmptyArchive")
53 89
 	if err != nil {