Browse code

Move archive.go to sub package

Michael Crosby authored on 2013/11/01 08:57:45
Showing 14 changed files
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	"encoding/base64"
6 6
 	"encoding/json"
7 7
 	"fmt"
8
+	"github.com/dotcloud/docker/archive"
8 9
 	"github.com/dotcloud/docker/auth"
9 10
 	"github.com/dotcloud/docker/utils"
10 11
 	"github.com/gorilla/mux"
... ...
@@ -905,7 +906,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ
905 905
 			return fmt.Errorf("Error trying to use git: %s (%s)", err, output)
906 906
 		}
907 907
 
908
-		c, err := Tar(root, Bzip2)
908
+		c, err := archive.Tar(root, archive.Bzip2)
909 909
 		if err != nil {
910 910
 			return err
911 911
 		}
912 912
deleted file mode 100644
... ...
@@ -1,302 +0,0 @@
1
-package docker
2
-
3
-import (
4
-	"archive/tar"
5
-	"bytes"
6
-	"fmt"
7
-	"github.com/dotcloud/docker/utils"
8
-	"io"
9
-	"io/ioutil"
10
-	"os"
11
-	"os/exec"
12
-	"path"
13
-	"path/filepath"
14
-)
15
-
16
-type Archive io.Reader
17
-
18
-type Compression uint32
19
-
20
-const (
21
-	Uncompressed Compression = iota
22
-	Bzip2
23
-	Gzip
24
-	Xz
25
-)
26
-
27
-func DetectCompression(source []byte) Compression {
28
-	sourceLen := len(source)
29
-	for compression, m := range map[Compression][]byte{
30
-		Bzip2: {0x42, 0x5A, 0x68},
31
-		Gzip:  {0x1F, 0x8B, 0x08},
32
-		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
33
-	} {
34
-		fail := false
35
-		if len(m) > sourceLen {
36
-			utils.Debugf("Len too short")
37
-			continue
38
-		}
39
-		i := 0
40
-		for _, b := range m {
41
-			if b != source[i] {
42
-				fail = true
43
-				break
44
-			}
45
-			i++
46
-		}
47
-		if !fail {
48
-			return compression
49
-		}
50
-	}
51
-	return Uncompressed
52
-}
53
-
54
-func (compression *Compression) Flag() string {
55
-	switch *compression {
56
-	case Bzip2:
57
-		return "j"
58
-	case Gzip:
59
-		return "z"
60
-	case Xz:
61
-		return "J"
62
-	}
63
-	return ""
64
-}
65
-
66
-func (compression *Compression) Extension() string {
67
-	switch *compression {
68
-	case Uncompressed:
69
-		return "tar"
70
-	case Bzip2:
71
-		return "tar.bz2"
72
-	case Gzip:
73
-		return "tar.gz"
74
-	case Xz:
75
-		return "tar.xz"
76
-	}
77
-	return ""
78
-}
79
-
80
-// Tar creates an archive from the directory at `path`, and returns it as a
81
-// stream of bytes.
82
-func Tar(path string, compression Compression) (io.Reader, error) {
83
-	return TarFilter(path, compression, nil)
84
-}
85
-
86
-// Tar creates an archive from the directory at `path`, only including files whose relative
87
-// paths are included in `filter`. If `filter` is nil, then all files are included.
88
-func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
89
-	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
90
-	if filter == nil {
91
-		filter = []string{"."}
92
-	}
93
-	for _, f := range filter {
94
-		args = append(args, "-c"+compression.Flag(), f)
95
-	}
96
-	return CmdStream(exec.Command(args[0], args[1:]...))
97
-}
98
-
99
-// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
100
-// and unpacks it into the directory at `path`.
101
-// The archive may be compressed with one of the following algorithms:
102
-//  identity (uncompressed), gzip, bzip2, xz.
103
-// FIXME: specify behavior when target path exists vs. doesn't exist.
104
-func Untar(archive io.Reader, path string) error {
105
-	if archive == nil {
106
-		return fmt.Errorf("Empty archive")
107
-	}
108
-
109
-	buf := make([]byte, 10)
110
-	totalN := 0
111
-	for totalN < 10 {
112
-		if n, err := archive.Read(buf[totalN:]); err != nil {
113
-			if err == io.EOF {
114
-				return fmt.Errorf("Tarball too short")
115
-			}
116
-			return err
117
-		} else {
118
-			totalN += n
119
-			utils.Debugf("[tar autodetect] n: %d", n)
120
-		}
121
-	}
122
-	compression := DetectCompression(buf)
123
-
124
-	utils.Debugf("Archive compression detected: %s", compression.Extension())
125
-
126
-	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
127
-	cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive)
128
-	// Hardcode locale environment for predictable outcome regardless of host configuration.
129
-	//   (see https://github.com/dotcloud/docker/issues/355)
130
-	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
131
-	output, err := cmd.CombinedOutput()
132
-	if err != nil {
133
-		return fmt.Errorf("%s: %s", err, output)
134
-	}
135
-	return nil
136
-}
137
-
138
-// TarUntar is a convenience function which calls Tar and Untar, with
139
-// the output of one piped into the other. If either Tar or Untar fails,
140
-// TarUntar aborts and returns the error.
141
-func TarUntar(src string, filter []string, dst string) error {
142
-	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
143
-	archive, err := TarFilter(src, Uncompressed, filter)
144
-	if err != nil {
145
-		return err
146
-	}
147
-	return Untar(archive, dst)
148
-}
149
-
150
-// UntarPath is a convenience function which looks for an archive
151
-// at filesystem path `src`, and unpacks it at `dst`.
152
-func UntarPath(src, dst string) error {
153
-	if archive, err := os.Open(src); err != nil {
154
-		return err
155
-	} else if err := Untar(archive, dst); err != nil {
156
-		return err
157
-	}
158
-	return nil
159
-}
160
-
161
-// CopyWithTar creates a tar archive of filesystem path `src`, and
162
-// unpacks it at filesystem path `dst`.
163
-// The archive is streamed directly with fixed buffering and no
164
-// intermediary disk IO.
165
-//
166
-func CopyWithTar(src, dst string) error {
167
-	srcSt, err := os.Stat(src)
168
-	if err != nil {
169
-		return err
170
-	}
171
-	if !srcSt.IsDir() {
172
-		return CopyFileWithTar(src, dst)
173
-	}
174
-	// Create dst, copy src's content into it
175
-	utils.Debugf("Creating dest directory: %s", dst)
176
-	if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) {
177
-		return err
178
-	}
179
-	utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
180
-	return TarUntar(src, nil, dst)
181
-}
182
-
183
-// CopyFileWithTar emulates the behavior of the 'cp' command-line
184
-// for a single file. It copies a regular file from path `src` to
185
-// path `dst`, and preserves all its metadata.
186
-//
187
-// If `dst` ends with a trailing slash '/', the final destination path
188
-// will be `dst/base(src)`.
189
-func CopyFileWithTar(src, dst string) error {
190
-	utils.Debugf("CopyFileWithTar(%s, %s)", src, dst)
191
-	srcSt, err := os.Stat(src)
192
-	if err != nil {
193
-		return err
194
-	}
195
-	if srcSt.IsDir() {
196
-		return fmt.Errorf("Can't copy a directory")
197
-	}
198
-	// Clean up the trailing /
199
-	if dst[len(dst)-1] == '/' {
200
-		dst = path.Join(dst, filepath.Base(src))
201
-	}
202
-	// Create the holding directory if necessary
203
-	if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) {
204
-		return err
205
-	}
206
-	buf := new(bytes.Buffer)
207
-	tw := tar.NewWriter(buf)
208
-	hdr, err := tar.FileInfoHeader(srcSt, "")
209
-	if err != nil {
210
-		return err
211
-	}
212
-	hdr.Name = filepath.Base(dst)
213
-	if err := tw.WriteHeader(hdr); err != nil {
214
-		return err
215
-	}
216
-	srcF, err := os.Open(src)
217
-	if err != nil {
218
-		return err
219
-	}
220
-	if _, err := io.Copy(tw, srcF); err != nil {
221
-		return err
222
-	}
223
-	tw.Close()
224
-	return Untar(buf, filepath.Dir(dst))
225
-}
226
-
227
-// CmdStream executes a command, and returns its stdout as a stream.
228
-// If the command fails to run or doesn't complete successfully, an error
229
-// will be returned, including anything written on stderr.
230
-func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
231
-	stdout, err := cmd.StdoutPipe()
232
-	if err != nil {
233
-		return nil, err
234
-	}
235
-	stderr, err := cmd.StderrPipe()
236
-	if err != nil {
237
-		return nil, err
238
-	}
239
-	pipeR, pipeW := io.Pipe()
240
-	errChan := make(chan []byte)
241
-	// Collect stderr, we will use it in case of an error
242
-	go func() {
243
-		errText, e := ioutil.ReadAll(stderr)
244
-		if e != nil {
245
-			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
246
-		}
247
-		errChan <- errText
248
-	}()
249
-	// Copy stdout to the returned pipe
250
-	go func() {
251
-		_, err := io.Copy(pipeW, stdout)
252
-		if err != nil {
253
-			pipeW.CloseWithError(err)
254
-		}
255
-		errText := <-errChan
256
-		if err := cmd.Wait(); err != nil {
257
-			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText))
258
-		} else {
259
-			pipeW.Close()
260
-		}
261
-	}()
262
-	// Run the command and return the pipe
263
-	if err := cmd.Start(); err != nil {
264
-		return nil, err
265
-	}
266
-	return pipeR, nil
267
-}
268
-
269
-// NewTempArchive reads the content of src into a temporary file, and returns the contents
270
-// of that file as an archive. The archive can only be read once - as soon as reading completes,
271
-// the file will be deleted.
272
-func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
273
-	f, err := ioutil.TempFile(dir, "")
274
-	if err != nil {
275
-		return nil, err
276
-	}
277
-	if _, err := io.Copy(f, src); err != nil {
278
-		return nil, err
279
-	}
280
-	if _, err := f.Seek(0, 0); err != nil {
281
-		return nil, err
282
-	}
283
-	st, err := f.Stat()
284
-	if err != nil {
285
-		return nil, err
286
-	}
287
-	size := st.Size()
288
-	return &TempArchive{f, size}, nil
289
-}
290
-
291
-type TempArchive struct {
292
-	*os.File
293
-	Size int64 // Pre-computed from Stat().Size() as a convenience
294
-}
295
-
296
-func (archive *TempArchive) Read(data []byte) (int, error) {
297
-	n, err := archive.File.Read(data)
298
-	if err != nil {
299
-		os.Remove(archive.File.Name())
300
-	}
301
-	return n, err
302
-}
303 1
new file mode 100644
... ...
@@ -0,0 +1 @@
0
+Michael Crosby <michael@crosbymichael.com> (@crosbymichael)
0 1
new file mode 100644
... ...
@@ -0,0 +1,302 @@
0
+package archive
1
+
2
+import (
3
+	"archive/tar"
4
+	"bytes"
5
+	"fmt"
6
+	"github.com/dotcloud/docker/utils"
7
+	"io"
8
+	"io/ioutil"
9
+	"os"
10
+	"os/exec"
11
+	"path"
12
+	"path/filepath"
13
+)
14
+
15
+type Archive io.Reader
16
+
17
+type Compression uint32
18
+
19
+const (
20
+	Uncompressed Compression = iota
21
+	Bzip2
22
+	Gzip
23
+	Xz
24
+)
25
+
26
+func DetectCompression(source []byte) Compression {
27
+	sourceLen := len(source)
28
+	for compression, m := range map[Compression][]byte{
29
+		Bzip2: {0x42, 0x5A, 0x68},
30
+		Gzip:  {0x1F, 0x8B, 0x08},
31
+		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
32
+	} {
33
+		fail := false
34
+		if len(m) > sourceLen {
35
+			utils.Debugf("Len too short")
36
+			continue
37
+		}
38
+		i := 0
39
+		for _, b := range m {
40
+			if b != source[i] {
41
+				fail = true
42
+				break
43
+			}
44
+			i++
45
+		}
46
+		if !fail {
47
+			return compression
48
+		}
49
+	}
50
+	return Uncompressed
51
+}
52
+
53
+func (compression *Compression) Flag() string {
54
+	switch *compression {
55
+	case Bzip2:
56
+		return "j"
57
+	case Gzip:
58
+		return "z"
59
+	case Xz:
60
+		return "J"
61
+	}
62
+	return ""
63
+}
64
+
65
+func (compression *Compression) Extension() string {
66
+	switch *compression {
67
+	case Uncompressed:
68
+		return "tar"
69
+	case Bzip2:
70
+		return "tar.bz2"
71
+	case Gzip:
72
+		return "tar.gz"
73
+	case Xz:
74
+		return "tar.xz"
75
+	}
76
+	return ""
77
+}
78
+
79
+// Tar creates an archive from the directory at `path`, and returns it as a
80
+// stream of bytes.
81
+func Tar(path string, compression Compression) (io.Reader, error) {
82
+	return TarFilter(path, compression, nil)
83
+}
84
+
85
+// Tar creates an archive from the directory at `path`, only including files whose relative
86
+// paths are included in `filter`. If `filter` is nil, then all files are included.
87
+func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
88
+	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
89
+	if filter == nil {
90
+		filter = []string{"."}
91
+	}
92
+	for _, f := range filter {
93
+		args = append(args, "-c"+compression.Flag(), f)
94
+	}
95
+	return CmdStream(exec.Command(args[0], args[1:]...))
96
+}
97
+
98
+// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
99
+// and unpacks it into the directory at `path`.
100
+// The archive may be compressed with one of the following algorithms:
101
+//  identity (uncompressed), gzip, bzip2, xz.
102
+// FIXME: specify behavior when target path exists vs. doesn't exist.
103
+func Untar(archive io.Reader, path string) error {
104
+	if archive == nil {
105
+		return fmt.Errorf("Empty archive")
106
+	}
107
+
108
+	buf := make([]byte, 10)
109
+	totalN := 0
110
+	for totalN < 10 {
111
+		if n, err := archive.Read(buf[totalN:]); err != nil {
112
+			if err == io.EOF {
113
+				return fmt.Errorf("Tarball too short")
114
+			}
115
+			return err
116
+		} else {
117
+			totalN += n
118
+			utils.Debugf("[tar autodetect] n: %d", n)
119
+		}
120
+	}
121
+	compression := DetectCompression(buf)
122
+
123
+	utils.Debugf("Archive compression detected: %s", compression.Extension())
124
+
125
+	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
126
+	cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive)
127
+	// Hardcode locale environment for predictable outcome regardless of host configuration.
128
+	//   (see https://github.com/dotcloud/docker/issues/355)
129
+	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
130
+	output, err := cmd.CombinedOutput()
131
+	if err != nil {
132
+		return fmt.Errorf("%s: %s", err, output)
133
+	}
134
+	return nil
135
+}
136
+
137
+// TarUntar is a convenience function which calls Tar and Untar, with
138
+// the output of one piped into the other. If either Tar or Untar fails,
139
+// TarUntar aborts and returns the error.
140
+func TarUntar(src string, filter []string, dst string) error {
141
+	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
142
+	archive, err := TarFilter(src, Uncompressed, filter)
143
+	if err != nil {
144
+		return err
145
+	}
146
+	return Untar(archive, dst)
147
+}
148
+
149
+// UntarPath is a convenience function which looks for an archive
150
+// at filesystem path `src`, and unpacks it at `dst`.
151
+func UntarPath(src, dst string) error {
152
+	if archive, err := os.Open(src); err != nil {
153
+		return err
154
+	} else if err := Untar(archive, dst); err != nil {
155
+		return err
156
+	}
157
+	return nil
158
+}
159
+
160
+// CopyWithTar creates a tar archive of filesystem path `src`, and
161
+// unpacks it at filesystem path `dst`.
162
+// The archive is streamed directly with fixed buffering and no
163
+// intermediary disk IO.
164
+//
165
+func CopyWithTar(src, dst string) error {
166
+	srcSt, err := os.Stat(src)
167
+	if err != nil {
168
+		return err
169
+	}
170
+	if !srcSt.IsDir() {
171
+		return CopyFileWithTar(src, dst)
172
+	}
173
+	// Create dst, copy src's content into it
174
+	utils.Debugf("Creating dest directory: %s", dst)
175
+	if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) {
176
+		return err
177
+	}
178
+	utils.Debugf("Calling TarUntar(%s, %s)", src, dst)
179
+	return TarUntar(src, nil, dst)
180
+}
181
+
182
+// CopyFileWithTar emulates the behavior of the 'cp' command-line
183
+// for a single file. It copies a regular file from path `src` to
184
+// path `dst`, and preserves all its metadata.
185
+//
186
+// If `dst` ends with a trailing slash '/', the final destination path
187
+// will be `dst/base(src)`.
188
+func CopyFileWithTar(src, dst string) error {
189
+	utils.Debugf("CopyFileWithTar(%s, %s)", src, dst)
190
+	srcSt, err := os.Stat(src)
191
+	if err != nil {
192
+		return err
193
+	}
194
+	if srcSt.IsDir() {
195
+		return fmt.Errorf("Can't copy a directory")
196
+	}
197
+	// Clean up the trailing /
198
+	if dst[len(dst)-1] == '/' {
199
+		dst = path.Join(dst, filepath.Base(src))
200
+	}
201
+	// Create the holding directory if necessary
202
+	if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) {
203
+		return err
204
+	}
205
+	buf := new(bytes.Buffer)
206
+	tw := tar.NewWriter(buf)
207
+	hdr, err := tar.FileInfoHeader(srcSt, "")
208
+	if err != nil {
209
+		return err
210
+	}
211
+	hdr.Name = filepath.Base(dst)
212
+	if err := tw.WriteHeader(hdr); err != nil {
213
+		return err
214
+	}
215
+	srcF, err := os.Open(src)
216
+	if err != nil {
217
+		return err
218
+	}
219
+	if _, err := io.Copy(tw, srcF); err != nil {
220
+		return err
221
+	}
222
+	tw.Close()
223
+	return Untar(buf, filepath.Dir(dst))
224
+}
225
+
226
+// CmdStream executes a command, and returns its stdout as a stream.
227
+// If the command fails to run or doesn't complete successfully, an error
228
+// will be returned, including anything written on stderr.
229
+func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
230
+	stdout, err := cmd.StdoutPipe()
231
+	if err != nil {
232
+		return nil, err
233
+	}
234
+	stderr, err := cmd.StderrPipe()
235
+	if err != nil {
236
+		return nil, err
237
+	}
238
+	pipeR, pipeW := io.Pipe()
239
+	errChan := make(chan []byte)
240
+	// Collect stderr, we will use it in case of an error
241
+	go func() {
242
+		errText, e := ioutil.ReadAll(stderr)
243
+		if e != nil {
244
+			errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")")
245
+		}
246
+		errChan <- errText
247
+	}()
248
+	// Copy stdout to the returned pipe
249
+	go func() {
250
+		_, err := io.Copy(pipeW, stdout)
251
+		if err != nil {
252
+			pipeW.CloseWithError(err)
253
+		}
254
+		errText := <-errChan
255
+		if err := cmd.Wait(); err != nil {
256
+			pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText))
257
+		} else {
258
+			pipeW.Close()
259
+		}
260
+	}()
261
+	// Run the command and return the pipe
262
+	if err := cmd.Start(); err != nil {
263
+		return nil, err
264
+	}
265
+	return pipeR, nil
266
+}
267
+
268
+// NewTempArchive reads the content of src into a temporary file, and returns the contents
269
+// of that file as an archive. The archive can only be read once - as soon as reading completes,
270
+// the file will be deleted.
271
+func NewTempArchive(src Archive, dir string) (*TempArchive, error) {
272
+	f, err := ioutil.TempFile(dir, "")
273
+	if err != nil {
274
+		return nil, err
275
+	}
276
+	if _, err := io.Copy(f, src); err != nil {
277
+		return nil, err
278
+	}
279
+	if _, err := f.Seek(0, 0); err != nil {
280
+		return nil, err
281
+	}
282
+	st, err := f.Stat()
283
+	if err != nil {
284
+		return nil, err
285
+	}
286
+	size := st.Size()
287
+	return &TempArchive{f, size}, nil
288
+}
289
+
290
+type TempArchive struct {
291
+	*os.File
292
+	Size int64 // Pre-computed from Stat().Size() as a convenience
293
+}
294
+
295
+func (archive *TempArchive) Read(data []byte) (int, error) {
296
+	n, err := archive.File.Read(data)
297
+	if err != nil {
298
+		os.Remove(archive.File.Name())
299
+	}
300
+	return n, err
301
+}
0 302
new file mode 100644
... ...
@@ -0,0 +1,118 @@
0
+package archive
1
+
2
+import (
3
+	"bytes"
4
+	"fmt"
5
+	"io"
6
+	"io/ioutil"
7
+	"os"
8
+	"os/exec"
9
+	"path"
10
+	"testing"
11
+	"time"
12
+)
13
+
14
+func TestCmdStreamLargeStderr(t *testing.T) {
15
+	cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
16
+	out, err := CmdStream(cmd)
17
+	if err != nil {
18
+		t.Fatalf("Failed to start command: %s", err)
19
+	}
20
+	errCh := make(chan error)
21
+	go func() {
22
+		_, err := io.Copy(ioutil.Discard, out)
23
+		errCh <- err
24
+	}()
25
+	select {
26
+	case err := <-errCh:
27
+		if err != nil {
28
+			t.Fatalf("Command should not have failed (err=%.100s...)", err)
29
+		}
30
+	case <-time.After(5 * time.Second):
31
+		t.Fatalf("Command did not complete in 5 seconds; probable deadlock")
32
+	}
33
+}
34
+
35
+func TestCmdStreamBad(t *testing.T) {
36
+	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
37
+	out, err := CmdStream(badCmd)
38
+	if err != nil {
39
+		t.Fatalf("Failed to start command: %s", err)
40
+	}
41
+	if output, err := ioutil.ReadAll(out); err == nil {
42
+		t.Fatalf("Command should have failed")
43
+	} else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" {
44
+		t.Fatalf("Wrong error value (%s)", err)
45
+	} else if s := string(output); s != "hello\n" {
46
+		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
47
+	}
48
+}
49
+
50
+func TestCmdStreamGood(t *testing.T) {
51
+	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
52
+	out, err := CmdStream(cmd)
53
+	if err != nil {
54
+		t.Fatal(err)
55
+	}
56
+	if output, err := ioutil.ReadAll(out); err != nil {
57
+		t.Fatalf("Command should not have failed (err=%s)", err)
58
+	} else if s := string(output); s != "hello\n" {
59
+		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
60
+	}
61
+}
62
+
63
+func tarUntar(t *testing.T, origin string, compression Compression) error {
64
+	archive, err := Tar(origin, compression)
65
+	if err != nil {
66
+		t.Fatal(err)
67
+	}
68
+
69
+	buf := make([]byte, 10)
70
+	if _, err := archive.Read(buf); err != nil {
71
+		return err
72
+	}
73
+	archive = io.MultiReader(bytes.NewReader(buf), archive)
74
+
75
+	detectedCompression := DetectCompression(buf)
76
+	if detectedCompression.Extension() != compression.Extension() {
77
+		return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension())
78
+	}
79
+
80
+	tmp, err := ioutil.TempDir("", "docker-test-untar")
81
+	if err != nil {
82
+		return err
83
+	}
84
+	defer os.RemoveAll(tmp)
85
+	if err := Untar(archive, tmp); err != nil {
86
+		return err
87
+	}
88
+	if _, err := os.Stat(tmp); err != nil {
89
+		return err
90
+	}
91
+	return nil
92
+}
93
+
94
+func TestTarUntar(t *testing.T) {
95
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
96
+	if err != nil {
97
+		t.Fatal(err)
98
+	}
99
+	defer os.RemoveAll(origin)
100
+	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
101
+		t.Fatal(err)
102
+	}
103
+	if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil {
104
+		t.Fatal(err)
105
+	}
106
+
107
+	for _, c := range []Compression{
108
+		Uncompressed,
109
+		Gzip,
110
+		Bzip2,
111
+		Xz,
112
+	} {
113
+		if err := tarUntar(t, origin, c); err != nil {
114
+			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)
115
+		}
116
+	}
117
+}
0 118
deleted file mode 100644
... ...
@@ -1,118 +0,0 @@
1
-package docker
2
-
3
-import (
4
-	"bytes"
5
-	"fmt"
6
-	"io"
7
-	"io/ioutil"
8
-	"os"
9
-	"os/exec"
10
-	"path"
11
-	"testing"
12
-	"time"
13
-)
14
-
15
-func TestCmdStreamLargeStderr(t *testing.T) {
16
-	cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
17
-	out, err := CmdStream(cmd)
18
-	if err != nil {
19
-		t.Fatalf("Failed to start command: %s", err)
20
-	}
21
-	errCh := make(chan error)
22
-	go func() {
23
-		_, err := io.Copy(ioutil.Discard, out)
24
-		errCh <- err
25
-	}()
26
-	select {
27
-	case err := <-errCh:
28
-		if err != nil {
29
-			t.Fatalf("Command should not have failed (err=%.100s...)", err)
30
-		}
31
-	case <-time.After(5 * time.Second):
32
-		t.Fatalf("Command did not complete in 5 seconds; probable deadlock")
33
-	}
34
-}
35
-
36
-func TestCmdStreamBad(t *testing.T) {
37
-	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
38
-	out, err := CmdStream(badCmd)
39
-	if err != nil {
40
-		t.Fatalf("Failed to start command: %s", err)
41
-	}
42
-	if output, err := ioutil.ReadAll(out); err == nil {
43
-		t.Fatalf("Command should have failed")
44
-	} else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" {
45
-		t.Fatalf("Wrong error value (%s)", err)
46
-	} else if s := string(output); s != "hello\n" {
47
-		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
48
-	}
49
-}
50
-
51
-func TestCmdStreamGood(t *testing.T) {
52
-	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
53
-	out, err := CmdStream(cmd)
54
-	if err != nil {
55
-		t.Fatal(err)
56
-	}
57
-	if output, err := ioutil.ReadAll(out); err != nil {
58
-		t.Fatalf("Command should not have failed (err=%s)", err)
59
-	} else if s := string(output); s != "hello\n" {
60
-		t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output)
61
-	}
62
-}
63
-
64
-func tarUntar(t *testing.T, origin string, compression Compression) error {
65
-	archive, err := Tar(origin, compression)
66
-	if err != nil {
67
-		t.Fatal(err)
68
-	}
69
-
70
-	buf := make([]byte, 10)
71
-	if _, err := archive.Read(buf); err != nil {
72
-		return err
73
-	}
74
-	archive = io.MultiReader(bytes.NewReader(buf), archive)
75
-
76
-	detectedCompression := DetectCompression(buf)
77
-	if detectedCompression.Extension() != compression.Extension() {
78
-		return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension())
79
-	}
80
-
81
-	tmp, err := ioutil.TempDir("", "docker-test-untar")
82
-	if err != nil {
83
-		return err
84
-	}
85
-	defer os.RemoveAll(tmp)
86
-	if err := Untar(archive, tmp); err != nil {
87
-		return err
88
-	}
89
-	if _, err := os.Stat(tmp); err != nil {
90
-		return err
91
-	}
92
-	return nil
93
-}
94
-
95
-func TestTarUntar(t *testing.T) {
96
-	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
97
-	if err != nil {
98
-		t.Fatal(err)
99
-	}
100
-	defer os.RemoveAll(origin)
101
-	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
102
-		t.Fatal(err)
103
-	}
104
-	if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil {
105
-		t.Fatal(err)
106
-	}
107
-
108
-	for _, c := range []Compression{
109
-		Uncompressed,
110
-		Gzip,
111
-		Bzip2,
112
-		Xz,
113
-	} {
114
-		if err := tarUntar(t, origin, c); err != nil {
115
-			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)
116
-		}
117
-	}
118
-}
... ...
@@ -3,6 +3,7 @@ package docker
3 3
 import (
4 4
 	"encoding/json"
5 5
 	"fmt"
6
+	"github.com/dotcloud/docker/archive"
6 7
 	"github.com/dotcloud/docker/utils"
7 8
 	"io"
8 9
 	"io/ioutil"
... ...
@@ -291,17 +292,17 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error {
291 291
 		return fmt.Errorf("%s: no such file or directory", orig)
292 292
 	}
293 293
 	if fi.IsDir() {
294
-		if err := CopyWithTar(origPath, destPath); err != nil {
294
+		if err := archive.CopyWithTar(origPath, destPath); err != nil {
295 295
 			return err
296 296
 		}
297 297
 		// First try to unpack the source as an archive
298
-	} else if err := UntarPath(origPath, destPath); err != nil {
298
+	} else if err := archive.UntarPath(origPath, destPath); err != nil {
299 299
 		utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err)
300 300
 		// If that fails, just copy it as a regular file
301 301
 		if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil {
302 302
 			return err
303 303
 		}
304
-		if err := CopyWithTar(origPath, destPath); err != nil {
304
+		if err := archive.CopyWithTar(origPath, destPath); err != nil {
305 305
 			return err
306 306
 		}
307 307
 	}
... ...
@@ -473,7 +474,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) {
473 473
 	if err != nil {
474 474
 		return "", err
475 475
 	}
476
-	if err := Untar(context, name); err != nil {
476
+	if err := archive.Untar(context, name); err != nil {
477 477
 		return "", err
478 478
 	}
479 479
 	defer os.RemoveAll(name)
... ...
@@ -2,6 +2,7 @@ package docker
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"github.com/dotcloud/docker/archive"
5 6
 	"io/ioutil"
6 7
 	"net"
7 8
 	"net/http"
... ...
@@ -12,7 +13,7 @@ import (
12 12
 
13 13
 // mkTestContext generates a build context from the contents of the provided dockerfile.
14 14
 // This context is suitable for use as an argument to BuildFile.Build()
15
-func mkTestContext(dockerfile string, files [][2]string, t *testing.T) Archive {
15
+func mkTestContext(dockerfile string, files [][2]string, t *testing.T) archive.Archive {
16 16
 	context, err := mkBuildContext(dockerfile, files)
17 17
 	if err != nil {
18 18
 		t.Fatal(err)
... ...
@@ -9,6 +9,7 @@ import (
9 9
 	"errors"
10 10
 	"flag"
11 11
 	"fmt"
12
+	"github.com/dotcloud/docker/archive"
12 13
 	"github.com/dotcloud/docker/auth"
13 14
 	"github.com/dotcloud/docker/registry"
14 15
 	"github.com/dotcloud/docker/term"
... ...
@@ -137,7 +138,7 @@ func (cli *DockerCli) CmdInsert(args ...string) error {
137 137
 
138 138
 // mkBuildContext returns an archive of an empty context with the contents
139 139
 // of `dockerfile` at the path ./Dockerfile
140
-func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) {
140
+func mkBuildContext(dockerfile string, files [][2]string) (archive.Archive, error) {
141 141
 	buf := new(bytes.Buffer)
142 142
 	tw := tar.NewWriter(buf)
143 143
 	files = append(files, [2]string{"Dockerfile", dockerfile})
... ...
@@ -175,7 +176,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
175 175
 	}
176 176
 
177 177
 	var (
178
-		context  Archive
178
+		context  archive.Archive
179 179
 		isRemote bool
180 180
 		err      error
181 181
 	)
... ...
@@ -194,7 +195,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
194 194
 		if _, err := os.Stat(cmd.Arg(0)); err != nil {
195 195
 			return err
196 196
 		}
197
-		context, err = Tar(cmd.Arg(0), Uncompressed)
197
+		context, err = archive.Tar(cmd.Arg(0), archive.Uncompressed)
198 198
 	}
199 199
 	var body io.Reader
200 200
 	// Setup an upload progress bar
... ...
@@ -1773,7 +1774,7 @@ func (cli *DockerCli) CmdCp(args ...string) error {
1773 1773
 
1774 1774
 	if statusCode == 200 {
1775 1775
 		r := bytes.NewReader(data)
1776
-		if err := Untar(r, copyData.HostPath); err != nil {
1776
+		if err := archive.Untar(r, copyData.HostPath); err != nil {
1777 1777
 			return err
1778 1778
 		}
1779 1779
 	}
... ...
@@ -6,6 +6,7 @@ import (
6 6
 	"errors"
7 7
 	"flag"
8 8
 	"fmt"
9
+	"github.com/dotcloud/docker/archive"
9 10
 	"github.com/dotcloud/docker/term"
10 11
 	"github.com/dotcloud/docker/utils"
11 12
 	"github.com/kr/pty"
... ...
@@ -817,7 +818,7 @@ func (container *Container) Start(hostConfig *HostConfig) (err error) {
817 817
 				}
818 818
 				if len(srcList) == 0 {
819 819
 					// If the source volume is empty copy files from the root into the volume
820
-					if err := CopyWithTar(rootVolPath, srcPath); err != nil {
820
+					if err := archive.CopyWithTar(rootVolPath, srcPath); err != nil {
821 821
 						return err
822 822
 					}
823 823
 
... ...
@@ -1337,23 +1338,23 @@ func (container *Container) Resize(h, w int) error {
1337 1337
 	return term.SetWinsize(pty.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)})
1338 1338
 }
1339 1339
 
1340
-func (container *Container) ExportRw() (Archive, error) {
1341
-	return Tar(container.rwPath(), Uncompressed)
1340
+func (container *Container) ExportRw() (archive.Archive, error) {
1341
+	return archive.Tar(container.rwPath(), archive.Uncompressed)
1342 1342
 }
1343 1343
 
1344 1344
 func (container *Container) RwChecksum() (string, error) {
1345
-	rwData, err := Tar(container.rwPath(), Xz)
1345
+	rwData, err := archive.Tar(container.rwPath(), archive.Xz)
1346 1346
 	if err != nil {
1347 1347
 		return "", err
1348 1348
 	}
1349 1349
 	return utils.HashData(rwData)
1350 1350
 }
1351 1351
 
1352
-func (container *Container) Export() (Archive, error) {
1352
+func (container *Container) Export() (archive.Archive, error) {
1353 1353
 	if err := container.EnsureMounted(); err != nil {
1354 1354
 		return nil, err
1355 1355
 	}
1356
-	return Tar(container.RootfsPath(), Uncompressed)
1356
+	return archive.Tar(container.RootfsPath(), archive.Uncompressed)
1357 1357
 }
1358 1358
 
1359 1359
 func (container *Container) WaitTimeout(timeout time.Duration) error {
... ...
@@ -1488,7 +1489,7 @@ func (container *Container) GetSize() (int64, int64) {
1488 1488
 	return sizeRw, sizeRootfs
1489 1489
 }
1490 1490
 
1491
-func (container *Container) Copy(resource string) (Archive, error) {
1491
+func (container *Container) Copy(resource string) (archive.Archive, error) {
1492 1492
 	if err := container.EnsureMounted(); err != nil {
1493 1493
 		return nil, err
1494 1494
 	}
... ...
@@ -1506,7 +1507,7 @@ func (container *Container) Copy(resource string) (Archive, error) {
1506 1506
 		filter = []string{path.Base(basePath)}
1507 1507
 		basePath = path.Dir(basePath)
1508 1508
 	}
1509
-	return TarFilter(basePath, Uncompressed, filter)
1509
+	return archive.TarFilter(basePath, archive.Uncompressed, filter)
1510 1510
 }
1511 1511
 
1512 1512
 // Returns true if the container exposes a certain port
... ...
@@ -2,6 +2,7 @@ package docker
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"github.com/dotcloud/docker/archive"
5 6
 	"github.com/dotcloud/docker/utils"
6 7
 	"io"
7 8
 	"io/ioutil"
... ...
@@ -94,7 +95,7 @@ func (graph *Graph) Get(name string) (*Image, error) {
94 94
 }
95 95
 
96 96
 // Create creates a new image and registers it in the graph.
97
-func (graph *Graph) Create(layerData Archive, container *Container, comment, author string, config *Config) (*Image, error) {
97
+func (graph *Graph) Create(layerData archive.Archive, container *Container, comment, author string, config *Config) (*Image, error) {
98 98
 	img := &Image{
99 99
 		ID:            GenerateID(),
100 100
 		Comment:       comment,
... ...
@@ -117,7 +118,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut
117 117
 
118 118
 // Register imports a pre-existing image into the graph.
119 119
 // FIXME: pass img as first argument
120
-func (graph *Graph) Register(jsonData []byte, layerData Archive, img *Image) error {
120
+func (graph *Graph) Register(jsonData []byte, layerData archive.Archive, img *Image) error {
121 121
 	if err := ValidateID(img.ID); err != nil {
122 122
 		return err
123 123
 	}
... ...
@@ -146,7 +147,7 @@ func (graph *Graph) Register(jsonData []byte, layerData Archive, img *Image) err
146 146
 //   The archive is stored on disk and will be automatically deleted as soon as has been read.
147 147
 //   If output is not nil, a human-readable progress bar will be written to it.
148 148
 //   FIXME: does this belong in Graph? How about MktempFile, let the caller use it for archives?
149
-func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *utils.StreamFormatter, output io.Writer) (*TempArchive, error) {
149
+func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, sf *utils.StreamFormatter, output io.Writer) (*archive.TempArchive, error) {
150 150
 	image, err := graph.Get(id)
151 151
 	if err != nil {
152 152
 		return nil, err
... ...
@@ -155,11 +156,11 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *uti
155 155
 	if err != nil {
156 156
 		return nil, err
157 157
 	}
158
-	archive, err := image.TarLayer(compression)
158
+	a, err := image.TarLayer(compression)
159 159
 	if err != nil {
160 160
 		return nil, err
161 161
 	}
162
-	return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root)
162
+	return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root)
163 163
 }
164 164
 
165 165
 // Mktemp creates a temporary sub-directory inside the graph's filesystem.
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"archive/tar"
5 5
 	"bytes"
6 6
 	"errors"
7
+	"github.com/dotcloud/docker/archive"
7 8
 	"github.com/dotcloud/docker/utils"
8 9
 	"io"
9 10
 	"io/ioutil"
... ...
@@ -301,7 +302,7 @@ func tempGraph(t *testing.T) *Graph {
301 301
 	return graph
302 302
 }
303 303
 
304
-func testArchive(t *testing.T) Archive {
304
+func testArchive(t *testing.T) archive.Archive {
305 305
 	archive, err := fakeTar()
306 306
 	if err != nil {
307 307
 		t.Fatal(err)
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	"encoding/hex"
6 6
 	"encoding/json"
7 7
 	"fmt"
8
+	"github.com/dotcloud/docker/archive"
8 9
 	"github.com/dotcloud/docker/utils"
9 10
 	"io"
10 11
 	"io/ioutil"
... ...
@@ -72,7 +73,7 @@ func LoadImage(root string) (*Image, error) {
72 72
 	return img, nil
73 73
 }
74 74
 
75
-func StoreImage(img *Image, jsonData []byte, layerData Archive, root string) error {
75
+func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root string) error {
76 76
 	// Check that root doesn't already exist
77 77
 	if _, err := os.Stat(root); err == nil {
78 78
 		return fmt.Errorf("Image %s already exists", img.ID)
... ...
@@ -89,7 +90,7 @@ func StoreImage(img *Image, jsonData []byte, layerData Archive, root string) err
89 89
 	if layerData != nil {
90 90
 		start := time.Now()
91 91
 		utils.Debugf("Start untar layer")
92
-		if err := Untar(layerData, layer); err != nil {
92
+		if err := archive.Untar(layerData, layer); err != nil {
93 93
 			return err
94 94
 		}
95 95
 		utils.Debugf("Untar time: %vs", time.Now().Sub(start).Seconds())
... ...
@@ -162,12 +163,12 @@ func MountAUFS(ro []string, rw string, target string) error {
162 162
 }
163 163
 
164 164
 // TarLayer returns a tar archive of the image's filesystem layer.
165
-func (image *Image) TarLayer(compression Compression) (Archive, error) {
165
+func (image *Image) TarLayer(compression archive.Compression) (archive.Archive, error) {
166 166
 	layerPath, err := image.layer()
167 167
 	if err != nil {
168 168
 		return nil, err
169 169
 	}
170
-	return Tar(layerPath, compression)
170
+	return archive.Tar(layerPath, compression)
171 171
 }
172 172
 
173 173
 func (image *Image) Mount(root, rw string) error {
... ...
@@ -5,11 +5,12 @@ import (
5 5
 	"encoding/json"
6 6
 	"errors"
7 7
 	"fmt"
8
+	"github.com/dotcloud/docker/archive"
8 9
 	"github.com/dotcloud/docker/auth"
10
+	"github.com/dotcloud/docker/engine"
9 11
 	"github.com/dotcloud/docker/gograph"
10 12
 	"github.com/dotcloud/docker/registry"
11 13
 	"github.com/dotcloud/docker/utils"
12
-	"github.com/dotcloud/docker/engine"
13 14
 	"io"
14 15
 	"io/ioutil"
15 16
 	"log"
... ...
@@ -17,14 +18,14 @@ import (
17 17
 	"net/url"
18 18
 	"os"
19 19
 	"os/exec"
20
+	"os/signal"
20 21
 	"path"
21 22
 	"path/filepath"
22 23
 	"runtime"
23 24
 	"strings"
24 25
 	"sync"
25
-	"time"
26 26
 	"syscall"
27
-	"os/signal"
27
+	"time"
28 28
 )
29 29
 
30 30
 func (srv *Server) Close() error {
... ...
@@ -92,7 +93,6 @@ func (srv *Server) Daemon() error {
92 92
 	return nil
93 93
 }
94 94
 
95
-
96 95
 func (srv *Server) DockerVersion() APIVersion {
97 96
 	return APIVersion{
98 97
 		Version:   VERSION,
... ...
@@ -902,7 +902,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID,
902 902
 		return "", err
903 903
 	}
904 904
 
905
-	layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out)
905
+	layerData, err := srv.runtime.graph.TempLayerArchive(imgID, archive.Uncompressed, sf, out)
906 906
 	if err != nil {
907 907
 		return "", fmt.Errorf("Failed to generate layer archive: %s", err)
908 908
 	}