* Runtime: Remove bsdtar dependency
Guillaume J. Charmes authored on 2013/06/19 09:24:26... | ... |
@@ -262,7 +262,7 @@ Setting up a dev environment |
262 | 262 |
Instructions that have been verified to work on Ubuntu 12.10, |
263 | 263 |
|
264 | 264 |
```bash |
265 |
-sudo apt-get -y install lxc wget bsdtar curl golang git |
|
265 |
+sudo apt-get -y install lxc curl xz-utils golang git |
|
266 | 266 |
|
267 | 267 |
export GOPATH=~/go/ |
268 | 268 |
export PATH=$GOPATH/bin:$PATH |
... | ... |
@@ -1,8 +1,10 @@ |
1 | 1 |
package docker |
2 | 2 |
|
3 | 3 |
import ( |
4 |
+ "bufio" |
|
4 | 5 |
"errors" |
5 | 6 |
"fmt" |
7 |
+ "github.com/dotcloud/docker/utils" |
|
6 | 8 |
"io" |
7 | 9 |
"io/ioutil" |
8 | 10 |
"os" |
... | ... |
@@ -20,6 +22,37 @@ const ( |
20 | 20 |
Xz |
21 | 21 |
) |
22 | 22 |
|
23 |
+func DetectCompression(source []byte) Compression { |
|
24 |
+ for _, c := range source[:10] { |
|
25 |
+ utils.Debugf("%x", c) |
|
26 |
+ } |
|
27 |
+ |
|
28 |
+ sourceLen := len(source) |
|
29 |
+ for compression, m := range map[Compression][]byte{ |
|
30 |
+ Bzip2: {0x42, 0x5A, 0x68}, |
|
31 |
+ Gzip: {0x1F, 0x8B, 0x08}, |
|
32 |
+ Xz: {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, |
|
33 |
+ } { |
|
34 |
+ fail := false |
|
35 |
+ if len(m) > sourceLen { |
|
36 |
+ utils.Debugf("Len too short") |
|
37 |
+ continue |
|
38 |
+ } |
|
39 |
+ i := 0 |
|
40 |
+ for _, b := range m { |
|
41 |
+ if b != source[i] { |
|
42 |
+ fail = true |
|
43 |
+ break |
|
44 |
+ } |
|
45 |
+ i++ |
|
46 |
+ } |
|
47 |
+ if !fail { |
|
48 |
+ return compression |
|
49 |
+ } |
|
50 |
+ } |
|
51 |
+ return Uncompressed |
|
52 |
+} |
|
53 |
+ |
|
23 | 54 |
func (compression *Compression) Flag() string { |
24 | 55 |
switch *compression { |
25 | 56 |
case Bzip2: |
... | ... |
@@ -47,14 +80,23 @@ func (compression *Compression) Extension() string { |
47 | 47 |
} |
48 | 48 |
|
49 | 49 |
func Tar(path string, compression Compression) (io.Reader, error) { |
50 |
- cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".") |
|
51 |
- return CmdStream(cmd) |
|
50 |
+ return CmdStream(exec.Command("tar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".")) |
|
52 | 51 |
} |
53 | 52 |
|
54 | 53 |
// FIXME: specify behavior when target path exists vs. doesn't exist. |
55 | 54 |
func Untar(archive io.Reader, path string) error { |
56 |
- cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x") |
|
57 |
- cmd.Stdin = archive |
|
55 |
+ |
|
56 |
+ bufferedArchive := bufio.NewReaderSize(archive, 10) |
|
57 |
+ buf, err := bufferedArchive.Peek(10) |
|
58 |
+ if err != nil { |
|
59 |
+ return err |
|
60 |
+ } |
|
61 |
+ compression := DetectCompression(buf) |
|
62 |
+ |
|
63 |
+ utils.Debugf("Archive compression detected: %s", compression.Extension()) |
|
64 |
+ |
|
65 |
+ cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag()) |
|
66 |
+ cmd.Stdin = bufferedArchive |
|
58 | 67 |
// Hardcode locale environment for predictable outcome regardless of host configuration. |
59 | 68 |
// (see https://github.com/dotcloud/docker/issues/355) |
60 | 69 |
cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"} |
... | ... |
@@ -1,10 +1,13 @@ |
1 | 1 |
package docker |
2 | 2 |
|
3 | 3 |
import ( |
4 |
+ "bytes" |
|
5 |
+ "fmt" |
|
4 | 6 |
"io" |
5 | 7 |
"io/ioutil" |
6 | 8 |
"os" |
7 | 9 |
"os/exec" |
10 |
+ "path" |
|
8 | 11 |
"testing" |
9 | 12 |
"time" |
10 | 13 |
) |
... | ... |
@@ -58,20 +61,58 @@ func TestCmdStreamGood(t *testing.T) { |
58 | 58 |
} |
59 | 59 |
} |
60 | 60 |
|
61 |
-func TestTarUntar(t *testing.T) { |
|
62 |
- archive, err := Tar(".", Uncompressed) |
|
61 |
+func tarUntar(t *testing.T, origin string, compression Compression) error { |
|
62 |
+ archive, err := Tar(origin, compression) |
|
63 | 63 |
if err != nil { |
64 | 64 |
t.Fatal(err) |
65 | 65 |
} |
66 |
+ |
|
67 |
+ buf := make([]byte, 10) |
|
68 |
+ if _, err := archive.Read(buf); err != nil { |
|
69 |
+ return err |
|
70 |
+ } |
|
71 |
+ archive = io.MultiReader(bytes.NewReader(buf), archive) |
|
72 |
+ |
|
73 |
+ detectedCompression := DetectCompression(buf) |
|
74 |
+ if detectedCompression.Extension() != compression.Extension() { |
|
75 |
+ return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension()) |
|
76 |
+ } |
|
77 |
+ |
|
66 | 78 |
tmp, err := ioutil.TempDir("", "docker-test-untar") |
67 | 79 |
if err != nil { |
68 |
- t.Fatal(err) |
|
80 |
+ return err |
|
69 | 81 |
} |
70 | 82 |
defer os.RemoveAll(tmp) |
71 | 83 |
if err := Untar(archive, tmp); err != nil { |
72 |
- t.Fatal(err) |
|
84 |
+ return err |
|
73 | 85 |
} |
74 | 86 |
if _, err := os.Stat(tmp); err != nil { |
75 |
- t.Fatalf("Error stating %s: %s", tmp, err.Error()) |
|
87 |
+ return err |
|
88 |
+ } |
|
89 |
+ return nil |
|
90 |
+} |
|
91 |
+ |
|
92 |
+func TestTarUntar(t *testing.T) { |
|
93 |
+ origin, err := ioutil.TempDir("", "docker-test-untar-origin") |
|
94 |
+ if err != nil { |
|
95 |
+ t.Fatal(err) |
|
96 |
+ } |
|
97 |
+ defer os.RemoveAll(origin) |
|
98 |
+ if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil { |
|
99 |
+ t.Fatal(err) |
|
100 |
+ } |
|
101 |
+ if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil { |
|
102 |
+ t.Fatal(err) |
|
103 |
+ } |
|
104 |
+ |
|
105 |
+ for _, c := range []Compression{ |
|
106 |
+ Uncompressed, |
|
107 |
+ Gzip, |
|
108 |
+ Bzip2, |
|
109 |
+ Xz, |
|
110 |
+ } { |
|
111 |
+ if err := tarUntar(t, origin, c); err != nil { |
|
112 |
+ t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err) |
|
113 |
+ } |
|
76 | 114 |
} |
77 | 115 |
} |
... | ... |
@@ -8,7 +8,7 @@ |
8 | 8 |
|
9 | 9 |
echo "Ensuring basic dependencies are installed..." |
10 | 10 |
apt-get -qq update |
11 |
-apt-get -qq install lxc wget bsdtar |
|
11 |
+apt-get -qq install lxc wget |
|
12 | 12 |
|
13 | 13 |
echo "Looking in /proc/filesystems to see if we have AUFS support..." |
14 | 14 |
if grep -q aufs /proc/filesystems |
... | ... |
@@ -33,7 +33,7 @@ Installation |
33 | 33 |
sudo apt-get install python-software-properties |
34 | 34 |
sudo add-apt-repository ppa:gophers/go |
35 | 35 |
sudo apt-get update |
36 |
- sudo apt-get -y install lxc wget bsdtar curl golang-stable git aufs-tools |
|
36 |
+ sudo apt-get -y install lxc xz-utils curl golang-stable git aufs-tools |
|
37 | 37 |
|
38 | 38 |
export GOPATH=~/go/ |
39 | 39 |
export PATH=$GOPATH/bin:$PATH |
... | ... |
@@ -192,11 +192,19 @@ func TestDelete(t *testing.T) { |
192 | 192 |
} |
193 | 193 |
assertNImages(graph, t, 0) |
194 | 194 |
|
195 |
+ archive, err = fakeTar() |
|
196 |
+ if err != nil { |
|
197 |
+ t.Fatal(err) |
|
198 |
+ } |
|
195 | 199 |
// Test 2 create (same name) / 1 delete |
196 | 200 |
img1, err := graph.Create(archive, nil, "Testing", "", nil) |
197 | 201 |
if err != nil { |
198 | 202 |
t.Fatal(err) |
199 | 203 |
} |
204 |
+ archive, err = fakeTar() |
|
205 |
+ if err != nil { |
|
206 |
+ t.Fatal(err) |
|
207 |
+ } |
|
200 | 208 |
if _, err = graph.Create(archive, nil, "Testing", "", nil); err != nil { |
201 | 209 |
t.Fatal(err) |
202 | 210 |
} |
... | ... |
@@ -212,6 +220,10 @@ func TestDelete(t *testing.T) { |
212 | 212 |
} |
213 | 213 |
assertNImages(graph, t, 1) |
214 | 214 |
|
215 |
+ archive, err = fakeTar() |
|
216 |
+ if err != nil { |
|
217 |
+ t.Fatal(err) |
|
218 |
+ } |
|
215 | 219 |
// Test delete twice (pull -> rm -> pull -> rm) |
216 | 220 |
if err := graph.Register(archive, false, img1); err != nil { |
217 | 221 |
t.Fatal(err) |
... | ... |
@@ -22,7 +22,7 @@ Vagrant::Config.run do |config| |
22 | 22 |
pkg_cmd = "touch #{DOCKER_PATH}; " |
23 | 23 |
# Install docker dependencies |
24 | 24 |
pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \ |
25 |
- "apt-get install -q -y lxc bsdtar git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \ |
|
25 |
+ "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \ |
|
26 | 26 |
"chown -R #{USER}.#{USER} #{GOPATH}; " \ |
27 | 27 |
"install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile" |
28 | 28 |
config.vm.provision :shell, :inline => pkg_cmd |
... | ... |
@@ -30,7 +30,7 @@ Vagrant::Config.run do |config| |
30 | 30 |
# Install docker dependencies |
31 | 31 |
pkg_cmd << "apt-get install -q -y python-software-properties; " \ |
32 | 32 |
"add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \ |
33 |
- "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable aufs-tools make; " |
|
33 |
+ "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; " |
|
34 | 34 |
# Activate new kernel |
35 | 35 |
pkg_cmd << "shutdown -r +1; " |
36 | 36 |
config.vm.provision :shell, :inline => pkg_cmd |