Browse code

Merge pull request #885 from dotcloud/remove_bsdtar

* Runtime: Remove bsdtar dependency

Guillaume J. Charmes authored on 2013/06/19 09:24:26
Showing 9 changed files
... ...
@@ -262,7 +262,7 @@ Setting up a dev environment
262 262
 Instructions that have been verified to work on Ubuntu 12.10,
263 263
 
264 264
 ```bash
265
-sudo apt-get -y install lxc wget bsdtar curl golang git
265
+sudo apt-get -y install lxc curl xz-utils golang git
266 266
 
267 267
 export GOPATH=~/go/
268 268
 export PATH=$GOPATH/bin:$PATH
... ...
@@ -1,8 +1,10 @@
1 1
 package docker
2 2
 
3 3
 import (
4
+	"bufio"
4 5
 	"errors"
5 6
 	"fmt"
7
+	"github.com/dotcloud/docker/utils"
6 8
 	"io"
7 9
 	"io/ioutil"
8 10
 	"os"
... ...
@@ -20,6 +22,37 @@ const (
20 20
 	Xz
21 21
 )
22 22
 
23
+func DetectCompression(source []byte) Compression {
24
+	for _, c := range source[:10] {
25
+		utils.Debugf("%x", c)
26
+	}
27
+
28
+	sourceLen := len(source)
29
+	for compression, m := range map[Compression][]byte{
30
+		Bzip2: {0x42, 0x5A, 0x68},
31
+		Gzip:  {0x1F, 0x8B, 0x08},
32
+		Xz:    {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
33
+	} {
34
+		fail := false
35
+		if len(m) > sourceLen {
36
+			utils.Debugf("Len too short")
37
+			continue
38
+		}
39
+		i := 0
40
+		for _, b := range m {
41
+			if b != source[i] {
42
+				fail = true
43
+				break
44
+			}
45
+			i++
46
+		}
47
+		if !fail {
48
+			return compression
49
+		}
50
+	}
51
+	return Uncompressed
52
+}
53
+
23 54
 func (compression *Compression) Flag() string {
24 55
 	switch *compression {
25 56
 	case Bzip2:
... ...
@@ -47,14 +80,23 @@ func (compression *Compression) Extension() string {
47 47
 }
48 48
 
49 49
 func Tar(path string, compression Compression) (io.Reader, error) {
50
-	cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".")
51
-	return CmdStream(cmd)
50
+	return CmdStream(exec.Command("tar", "-f", "-", "-C", path, "-c"+compression.Flag(), "."))
52 51
 }
53 52
 
54 53
 // FIXME: specify behavior when target path exists vs. doesn't exist.
55 54
 func Untar(archive io.Reader, path string) error {
56
-	cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x")
57
-	cmd.Stdin = archive
55
+
56
+	bufferedArchive := bufio.NewReaderSize(archive, 10)
57
+	buf, err := bufferedArchive.Peek(10)
58
+	if err != nil {
59
+		return err
60
+	}
61
+	compression := DetectCompression(buf)
62
+
63
+	utils.Debugf("Archive compression detected: %s", compression.Extension())
64
+
65
+	cmd := exec.Command("tar", "-f", "-", "-C", path, "-x"+compression.Flag())
66
+	cmd.Stdin = bufferedArchive
58 67
 	// Hardcode locale environment for predictable outcome regardless of host configuration.
59 68
 	//   (see https://github.com/dotcloud/docker/issues/355)
60 69
 	cmd.Env = []string{"LANG=en_US.utf-8", "LC_ALL=en_US.utf-8"}
... ...
@@ -1,10 +1,13 @@
1 1
 package docker
2 2
 
3 3
 import (
4
+	"bytes"
5
+	"fmt"
4 6
 	"io"
5 7
 	"io/ioutil"
6 8
 	"os"
7 9
 	"os/exec"
10
+	"path"
8 11
 	"testing"
9 12
 	"time"
10 13
 )
... ...
@@ -58,20 +61,58 @@ func TestCmdStreamGood(t *testing.T) {
58 58
 	}
59 59
 }
60 60
 
61
-func TestTarUntar(t *testing.T) {
62
-	archive, err := Tar(".", Uncompressed)
61
+func tarUntar(t *testing.T, origin string, compression Compression) error {
62
+	archive, err := Tar(origin, compression)
63 63
 	if err != nil {
64 64
 		t.Fatal(err)
65 65
 	}
66
+
67
+	buf := make([]byte, 10)
68
+	if _, err := archive.Read(buf); err != nil {
69
+		return err
70
+	}
71
+	archive = io.MultiReader(bytes.NewReader(buf), archive)
72
+
73
+	detectedCompression := DetectCompression(buf)
74
+	if detectedCompression.Extension() != compression.Extension() {
75
+		return fmt.Errorf("Wrong compression detected. Actual compression: %s, found %s", compression.Extension(), detectedCompression.Extension())
76
+	}
77
+
66 78
 	tmp, err := ioutil.TempDir("", "docker-test-untar")
67 79
 	if err != nil {
68
-		t.Fatal(err)
80
+		return err
69 81
 	}
70 82
 	defer os.RemoveAll(tmp)
71 83
 	if err := Untar(archive, tmp); err != nil {
72
-		t.Fatal(err)
84
+		return err
73 85
 	}
74 86
 	if _, err := os.Stat(tmp); err != nil {
75
-		t.Fatalf("Error stating %s: %s", tmp, err.Error())
87
+		return err
88
+	}
89
+	return nil
90
+}
91
+
92
+func TestTarUntar(t *testing.T) {
93
+	origin, err := ioutil.TempDir("", "docker-test-untar-origin")
94
+	if err != nil {
95
+		t.Fatal(err)
96
+	}
97
+	defer os.RemoveAll(origin)
98
+	if err := ioutil.WriteFile(path.Join(origin, "1"), []byte("hello world"), 0700); err != nil {
99
+		t.Fatal(err)
100
+	}
101
+	if err := ioutil.WriteFile(path.Join(origin, "2"), []byte("welcome!"), 0700); err != nil {
102
+		t.Fatal(err)
103
+	}
104
+
105
+	for _, c := range []Compression{
106
+		Uncompressed,
107
+		Gzip,
108
+		Bzip2,
109
+		Xz,
110
+	} {
111
+		if err := tarUntar(t, origin, c); err != nil {
112
+			t.Fatalf("Error tar/untar for compression %s: %s", c.Extension(), err)
113
+		}
76 114
 	}
77 115
 }
... ...
@@ -8,7 +8,7 @@
8 8
 
9 9
 echo "Ensuring basic dependencies are installed..."
10 10
 apt-get -qq update
11
-apt-get -qq install lxc wget bsdtar
11
+apt-get -qq install lxc wget
12 12
 
13 13
 echo "Looking in /proc/filesystems to see if we have AUFS support..."
14 14
 if grep -q aufs /proc/filesystems
... ...
@@ -33,7 +33,7 @@ Installation
33 33
     sudo apt-get install python-software-properties
34 34
     sudo add-apt-repository ppa:gophers/go
35 35
     sudo apt-get update
36
-    sudo apt-get -y install lxc wget bsdtar curl golang-stable git aufs-tools
36
+    sudo apt-get -y install lxc xz-utils curl golang-stable git aufs-tools
37 37
 
38 38
     export GOPATH=~/go/
39 39
     export PATH=$GOPATH/bin:$PATH
... ...
@@ -30,8 +30,7 @@ Dependencies:
30 30
 * 3.8 Kernel (read more about :ref:`kernel`)
31 31
 * AUFS filesystem support
32 32
 * lxc
33
-* bsdtar
34
-
33
+* xz-utils
35 34
 
36 35
 Get the docker binary:
37 36
 ----------------------
... ...
@@ -192,11 +192,19 @@ func TestDelete(t *testing.T) {
192 192
 	}
193 193
 	assertNImages(graph, t, 0)
194 194
 
195
+	archive, err = fakeTar()
196
+	if err != nil {
197
+		t.Fatal(err)
198
+	}
195 199
 	// Test 2 create (same name) / 1 delete
196 200
 	img1, err := graph.Create(archive, nil, "Testing", "", nil)
197 201
 	if err != nil {
198 202
 		t.Fatal(err)
199 203
 	}
204
+	archive, err = fakeTar()
205
+	if err != nil {
206
+		t.Fatal(err)
207
+	}
200 208
 	if _, err = graph.Create(archive, nil, "Testing", "", nil); err != nil {
201 209
 		t.Fatal(err)
202 210
 	}
... ...
@@ -212,6 +220,10 @@ func TestDelete(t *testing.T) {
212 212
 	}
213 213
 	assertNImages(graph, t, 1)
214 214
 
215
+	archive, err = fakeTar()
216
+	if err != nil {
217
+		t.Fatal(err)
218
+	}
215 219
 	// Test delete twice (pull -> rm -> pull -> rm)
216 220
 	if err := graph.Register(archive, false, img1); err != nil {
217 221
 		t.Fatal(err)
... ...
@@ -22,7 +22,7 @@ Vagrant::Config.run do |config|
22 22
   pkg_cmd = "touch #{DOCKER_PATH}; "
23 23
   # Install docker dependencies
24 24
   pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \
25
-    "apt-get install -q -y lxc bsdtar git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \
25
+    "apt-get install -q -y lxc git aufs-tools golang make linux-image-extra-3.8.0-19-generic; " \
26 26
     "chown -R #{USER}.#{USER} #{GOPATH}; " \
27 27
     "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile"
28 28
   config.vm.provision :shell, :inline => pkg_cmd
... ...
@@ -30,7 +30,7 @@ Vagrant::Config.run do |config|
30 30
     # Install docker dependencies
31 31
     pkg_cmd << "apt-get install -q -y python-software-properties; " \
32 32
       "add-apt-repository -y ppa:gophers/go/ubuntu; apt-get update -qq; " \
33
-      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc bsdtar git golang-stable aufs-tools make; "
33
+      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git golang-stable aufs-tools make; "
34 34
     # Activate new kernel
35 35
     pkg_cmd << "shutdown -r +1; "
36 36
     config.vm.provision :shell, :inline => pkg_cmd