1. docker
  2. Commit 7fb061832a0ebf197abb41c576887a1d2aca18be
Browse code

vendor: adding tar-split dependency for graph

tar-split is a facility to disassemble and reassemble tar archives

Signed-off-by: Vincent Batts <vbatts@redhat.com>

Vincent Batts authored on 2015/06/20 23:33:14
Showing 45 changed files
hack/vendor.sh
History View file @ 7fb0618
... ... 
@@ -33,8 +33,9 @@ clone git github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc37
33 33 
 clone git github.com/coreos/go-etcd v2.0.0
34 34 
 clone git github.com/hashicorp/consul v0.5.2
35 35
36 
-# get distribution packages
36 
+# get graph and distribution packages
37 37 
 clone git github.com/docker/distribution 419bbc2da637d9b2a812be78ef8436df7caac70d
38 
+clone git github.com/vbatts/tar-split v0.9.4
38 39
39 40 
 clone git github.com/opencontainers/runc v0.0.2 # libcontainer
40 41 
 # libcontainer deps (see src/github.com/docker/libcontainer/update-vendor.sh)
vendor/src/github.com/vbatts/tar-split/.travis.yml
History View file @ 7fb0618
41 42 
new file mode 100644
... ... 
@@ -0,0 +1,13 @@
0 
+language: go
1 
+go:
2 
+  - 1.4.2
3 
+  - 1.3.3
4 
+
5 
+# let us have pretty, fast Docker-based Travis workers!
6 
+sudo: false
7 
+
8 
+# we don't need "go get" here <3
9 
+install: go get -d ./...
10 
+
11 
+script:
12 
+  - go test -v ./...
vendor/src/github.com/vbatts/tar-split/DESIGN.md
History View file @ 7fb0618
0 13 
new file mode 100644
... ... 
@@ -0,0 +1,36 @@
0 
+Flow of TAR stream
1 
+==================
2 
+
3 
+The underlying use of `github.com/vbatts/tar-split/archive/tar` is most similar
4 
+to stdlib.
5 
+
6 
+
7 
+Packer interface
8 
+----------------
9 
+
10 
+For ease of storage and usage of the raw bytes, there will be a storage
11 
+interface, that accepts an io.Writer (This way you could pass it an in memory
12 
+buffer or a file handle).
13 
+
14 
+Having a Packer interface can allow configuration of hash.Hash for file payloads
15 
+and providing your own io.Writer.
16 
+
17 
+Instead of having a state directory to store all the header information for all
18 
+Readers, we will leave that up to user of Reader. Because we can not assume an
19 
+ID for each Reader, and keeping that information differentiated.
20 
+
21 
+
22 
+
23 
+State Directory
24 
+---------------
25 
+
26 
+Perhaps we could deduplicate the header info, by hashing the rawbytes and
27 
+storing them in a directory tree like:
28 
+
29 
+	./ac/dc/beef
30 
+
31 
+Then reference the hash of the header info, in the positional records for the
32 
+tar stream. Though this could be a future feature, and not required for an
33 
+initial implementation. Also, this would imply an owned state directory, rather
34 
+than just writing storage info to an io.Writer.
35 
+
vendor/src/github.com/vbatts/tar-split/LICENSE
History View file @ 7fb0618
0 36 
new file mode 100644
... ... 
@@ -0,0 +1,19 @@
0 
+Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA
1 
+
2 
+Permission is hereby granted, free of charge, to any person obtaining a copy
3 
+of this software and associated documentation files (the "Software"), to deal
4 
+in the Software without restriction, including without limitation the rights
5 
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
6 
+copies of the Software, and to permit persons to whom the Software is
7 
+furnished to do so, subject to the following conditions:
8 
+
9 
+The above copyright notice and this permission notice shall be included in
10 
+all copies or substantial portions of the Software.
11 
+
12 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13 
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15 
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
18 
+THE SOFTWARE.
vendor/src/github.com/vbatts/tar-split/README.md
History View file @ 7fb0618
0 19 
new file mode 100644
... ... 
@@ -0,0 +1,181 @@
0 
+tar-split
1 
+========
2 
+
3 
+[![Build Status](https://travis-ci.org/vbatts/tar-split.svg?branch=master)](https://travis-ci.org/vbatts/tar-split)
4 
+
5 
+Extend the upstream golang stdlib `archive/tar` library, to expose the raw
6 
+bytes of the TAR, rather than just the marshalled headers and file stream.
7 
+
8 
+The goal being that by preserving the raw bytes of each header, padding bytes,
9 
+and the raw file payload, one could reassemble the original archive.
10 
+
11 
+
12 
+Docs
13 
+----
14 
+
15 
+* https://godoc.org/github.com/vbatts/tar-split/tar/asm
16 
+* https://godoc.org/github.com/vbatts/tar-split/tar/storage
17 
+* https://godoc.org/github.com/vbatts/tar-split/archive/tar
18 
+
19 
+
20 
+Caveat
21 
+------
22 
+
23 
+Eventually this should detect TARs that this is not possible with.
24 
+
25 
+For example stored sparse files that have "holes" in them, will be read as a
26 
+contiguous file, though the archive contents may be recorded in sparse format.
27 
+Therefore when adding the file payload to a reassembled tar, to achieve
28 
+identical output, the file payload would need be precisely re-sparsified. This
29 
+is not something I seek to fix imediately, but would rather have an alert that
30 
+precise reassembly is not possible.
31 
+(see more http://www.gnu.org/software/tar/manual/html_node/Sparse-Formats.html)
32 
+
33 
+
34 
+Other caveat, while tar archives support having multiple file entries for the
35 
+same path, we will not support this feature. If there are more than one entries
36 
+with the same path, expect an err (like `ErrDuplicatePath`) or a resulting tar
37 
+stream that does not validate your original checksum/signature.
38 
+
39 
+
40 
+Contract
41 
+--------
42 
+
43 
+Do not break the API of stdlib `archive/tar` in our fork (ideally find an
44 
+upstream mergeable solution)
45 
+
46 
+
47 
+Std Version
48 
+-----------
49 
+
50 
+The version of golang stdlib `archive/tar` is from go1.4.1, and their master branch around [a9dddb53f](https://github.com/golang/go/tree/a9dddb53f)
51 
+
52 
+
53 
+Example
54 
+-------
55 
+
56 
+First we'll get an archive to work with. For repeatability, we'll make an
57 
+archive from what you've just cloned:
58 
+
59 
+```
60 
+git archive --format=tar -o tar-split.tar HEAD .
61 
+```
62 
+
63 
+Then build the example main.go:
64 
+
65 
+```
66 
+go build ./main.go
67 
+```
68 
+
69 
+Now run the example over the archive:
70 
+
71 
+```
72 
+$ ./main tar-split.tar
73 
+2015/02/20 15:00:58 writing "tar-split.tar" to "tar-split.tar.out"
74 
+pax_global_header pre: 512 read: 52
75 
+.travis.yml pre: 972 read: 374
76 
+DESIGN.md pre: 650 read: 1131
77 
+LICENSE pre: 917 read: 1075
78 
+README.md pre: 973 read: 4289
79 
+archive/ pre: 831 read: 0
80 
+archive/tar/ pre: 512 read: 0
81 
+archive/tar/common.go pre: 512 read: 7790
82 
+[...]
83 
+tar/storage/entry_test.go pre: 667 read: 1137
84 
+tar/storage/getter.go pre: 911 read: 2741
85 
+tar/storage/getter_test.go pre: 843 read: 1491
86 
+tar/storage/packer.go pre: 557 read: 3141
87 
+tar/storage/packer_test.go pre: 955 read: 3096
88 
+EOF padding: 1512
89 
+Remainder: 512
90 
+Size: 215040; Sum: 215040
91 
+```
92 
+
93 
+*What are we seeing here?*
94 
+
95 
+* `pre` is the header of a file entry, and potentially the padding from the
96 
+  end of the prior file's payload. Also with particular tar extensions and pax
97 
+  attributes, the header can exceed 512 bytes.
98 
+* `read` is the size of the file payload from the entry
99 
+* `EOF padding` is the expected 1024 null bytes on the end of a tar archive,
100 
+  plus potential padding from the end of the prior file entry's payload
101 
+* `Remainder` is the remaining bytes of an archive. This is typically deadspace
102 
+  as most tar implmentations will return after having reached the end of the
103 
+  1024 null bytes. Though various implementations will include some amount of
104 
+  bytes here, which will affect the checksum of the resulting tar archive,
105 
+  therefore this must be accounted for as well.
106 
+
107 
+Ideally the input tar and output `*.out`, will match:
108 
+
109 
+```
110 
+$ sha1sum tar-split.tar*
111 
+ca9e19966b892d9ad5960414abac01ef585a1e22  tar-split.tar
112 
+ca9e19966b892d9ad5960414abac01ef585a1e22  tar-split.tar.out
113 
+```
114 
+
115 
+
116 
+Stored Metadata
117 
+---------------
118 
+
119 
+Since the raw bytes of the headers and padding are stored, you may be wondering
120 
+what the size implications are. The headers are at least 512 bytes per
121 
+file (sometimes more), at least 1024 null bytes on the end, and then various
122 
+padding. This makes for a constant linear growth in the stored metadata, with a
123 
+naive storage implementation.
124 
+
125 
+Reusing our prior example's `tar-split.tar`, let's build the checksize.go example:
126 
+
127 
+```
128 
+go build ./checksize.go
129 
+```
130 
+
131 
+```
132 
+$ ./checksize ./tar-split.tar
133 
+inspecting "tar-split.tar" (size 210k)
134 
+ -- number of files: 50
135 
+ -- size of metadata uncompressed: 53k
136 
+ -- size of gzip compressed metadata: 3k
137 
+```
138 
+
139 
+So assuming you've managed the extraction of the archive yourself, for reuse of
140 
+the file payloads from a relative path, then the only additional storage
141 
+implications are as little as 3kb.
142 
+
143 
+But let's look at a larger archive, with many files.
144 
+
145 
+```
146 
+$ ls -sh ./d.tar
147 
+1.4G ./d.tar
148 
+$ ./checksize ~/d.tar
149 
+inspecting "/home/vbatts/d.tar" (size 1420749k)
150 
+ -- number of files: 38718
151 
+ -- size of metadata uncompressed: 43261k
152 
+ -- size of gzip compressed metadata: 2251k
153 
+```
154 
+
155 
+Here, an archive with 38,718 files has a compressed footprint of about 2mb.
156 
+
157 
+Rolling the null bytes on the end of the archive, we will assume a
158 
+bytes-per-file rate for the storage implications.
159 
+
160 
+| uncompressed | compressed |
161 
+| :----------: | :--------: |
162 
+| ~ 1kb per/file | 0.06kb per/file |
163 
+
164 
+
165 
+What's Next?
166 
+------------
167 
+
168 
+* More implementations of storage Packer and Unpacker
169 
+ - could be a redis or mongo backend
170 
+* More implementations of FileGetter and FilePutter
171 
+ - could be a redis or mongo backend
172 
+* cli tooling to assemble/disassemble a provided tar archive
173 
+* would be interesting to have an assembler stream that implements `io.Seeker`
174 
+
175 
+License
176 
+-------
177 
+
178 
+See LICENSE
179 
+
180 
+
vendor/src/github.com/vbatts/tar-split/archive/tar/common.go
History View file @ 7fb0618
0 181 
new file mode 100644
... ... 
@@ -0,0 +1,305 @@
0 
+// Copyright 2009 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+// Package tar implements access to tar archives.
5 
+// It aims to cover most of the variations, including those produced
6 
+// by GNU and BSD tars.
7 
+//
8 
+// References:
9 
+//   http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5
10 
+//   http://www.gnu.org/software/tar/manual/html_node/Standard.html
11 
+//   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html
12 
+package tar
13 
+
14 
+import (
15 
+	"bytes"
16 
+	"errors"
17 
+	"fmt"
18 
+	"os"
19 
+	"path"
20 
+	"time"
21 
+)
22 
+
23 
+const (
24 
+	blockSize = 512
25 
+
26 
+	// Types
27 
+	TypeReg           = '0'    // regular file
28 
+	TypeRegA          = '\x00' // regular file
29 
+	TypeLink          = '1'    // hard link
30 
+	TypeSymlink       = '2'    // symbolic link
31 
+	TypeChar          = '3'    // character device node
32 
+	TypeBlock         = '4'    // block device node
33 
+	TypeDir           = '5'    // directory
34 
+	TypeFifo          = '6'    // fifo node
35 
+	TypeCont          = '7'    // reserved
36 
+	TypeXHeader       = 'x'    // extended header
37 
+	TypeXGlobalHeader = 'g'    // global extended header
38 
+	TypeGNULongName   = 'L'    // Next file has a long name
39 
+	TypeGNULongLink   = 'K'    // Next file symlinks to a file w/ a long name
40 
+	TypeGNUSparse     = 'S'    // sparse file
41 
+)
42 
+
43 
+// A Header represents a single header in a tar archive.
44 
+// Some fields may not be populated.
45 
+type Header struct {
46 
+	Name       string    // name of header file entry
47 
+	Mode       int64     // permission and mode bits
48 
+	Uid        int       // user id of owner
49 
+	Gid        int       // group id of owner
50 
+	Size       int64     // length in bytes
51 
+	ModTime    time.Time // modified time
52 
+	Typeflag   byte      // type of header entry
53 
+	Linkname   string    // target name of link
54 
+	Uname      string    // user name of owner
55 
+	Gname      string    // group name of owner
56 
+	Devmajor   int64     // major number of character or block device
57 
+	Devminor   int64     // minor number of character or block device
58 
+	AccessTime time.Time // access time
59 
+	ChangeTime time.Time // status change time
60 
+	Xattrs     map[string]string
61 
+}
62 
+
63 
+// File name constants from the tar spec.
64 
+const (
65 
+	fileNameSize       = 100 // Maximum number of bytes in a standard tar name.
66 
+	fileNamePrefixSize = 155 // Maximum number of ustar extension bytes.
67 
+)
68 
+
69 
+// FileInfo returns an os.FileInfo for the Header.
70 
+func (h *Header) FileInfo() os.FileInfo {
71 
+	return headerFileInfo{h}
72 
+}
73 
+
74 
+// headerFileInfo implements os.FileInfo.
75 
+type headerFileInfo struct {
76 
+	h *Header
77 
+}
78 
+
79 
+func (fi headerFileInfo) Size() int64        { return fi.h.Size }
80 
+func (fi headerFileInfo) IsDir() bool        { return fi.Mode().IsDir() }
81 
+func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime }
82 
+func (fi headerFileInfo) Sys() interface{}   { return fi.h }
83 
+
84 
+// Name returns the base name of the file.
85 
+func (fi headerFileInfo) Name() string {
86 
+	if fi.IsDir() {
87 
+		return path.Base(path.Clean(fi.h.Name))
88 
+	}
89 
+	return path.Base(fi.h.Name)
90 
+}
91 
+
92 
+// Mode returns the permission and mode bits for the headerFileInfo.
93 
+func (fi headerFileInfo) Mode() (mode os.FileMode) {
94 
+	// Set file permission bits.
95 
+	mode = os.FileMode(fi.h.Mode).Perm()
96 
+
97 
+	// Set setuid, setgid and sticky bits.
98 
+	if fi.h.Mode&c_ISUID != 0 {
99 
+		// setuid
100 
+		mode |= os.ModeSetuid
101 
+	}
102 
+	if fi.h.Mode&c_ISGID != 0 {
103 
+		// setgid
104 
+		mode |= os.ModeSetgid
105 
+	}
106 
+	if fi.h.Mode&c_ISVTX != 0 {
107 
+		// sticky
108 
+		mode |= os.ModeSticky
109 
+	}
110 
+
111 
+	// Set file mode bits.
112 
+	// clear perm, setuid, setgid and sticky bits.
113 
+	m := os.FileMode(fi.h.Mode) &^ 07777
114 
+	if m == c_ISDIR {
115 
+		// directory
116 
+		mode |= os.ModeDir
117 
+	}
118 
+	if m == c_ISFIFO {
119 
+		// named pipe (FIFO)
120 
+		mode |= os.ModeNamedPipe
121 
+	}
122 
+	if m == c_ISLNK {
123 
+		// symbolic link
124 
+		mode |= os.ModeSymlink
125 
+	}
126 
+	if m == c_ISBLK {
127 
+		// device file
128 
+		mode |= os.ModeDevice
129 
+	}
130 
+	if m == c_ISCHR {
131 
+		// Unix character device
132 
+		mode |= os.ModeDevice
133 
+		mode |= os.ModeCharDevice
134 
+	}
135 
+	if m == c_ISSOCK {
136 
+		// Unix domain socket
137 
+		mode |= os.ModeSocket
138 
+	}
139 
+
140 
+	switch fi.h.Typeflag {
141 
+	case TypeLink, TypeSymlink:
142 
+		// hard link, symbolic link
143 
+		mode |= os.ModeSymlink
144 
+	case TypeChar:
145 
+		// character device node
146 
+		mode |= os.ModeDevice
147 
+		mode |= os.ModeCharDevice
148 
+	case TypeBlock:
149 
+		// block device node
150 
+		mode |= os.ModeDevice
151 
+	case TypeDir:
152 
+		// directory
153 
+		mode |= os.ModeDir
154 
+	case TypeFifo:
155 
+		// fifo node
156 
+		mode |= os.ModeNamedPipe
157 
+	}
158 
+
159 
+	return mode
160 
+}
161 
+
162 
+// sysStat, if non-nil, populates h from system-dependent fields of fi.
163 
+var sysStat func(fi os.FileInfo, h *Header) error
164 
+
165 
+// Mode constants from the tar spec.
166 
+const (
167 
+	c_ISUID  = 04000   // Set uid
168 
+	c_ISGID  = 02000   // Set gid
169 
+	c_ISVTX  = 01000   // Save text (sticky bit)
170 
+	c_ISDIR  = 040000  // Directory
171 
+	c_ISFIFO = 010000  // FIFO
172 
+	c_ISREG  = 0100000 // Regular file
173 
+	c_ISLNK  = 0120000 // Symbolic link
174 
+	c_ISBLK  = 060000  // Block special file
175 
+	c_ISCHR  = 020000  // Character special file
176 
+	c_ISSOCK = 0140000 // Socket
177 
+)
178 
+
179 
+// Keywords for the PAX Extended Header
180 
+const (
181 
+	paxAtime    = "atime"
182 
+	paxCharset  = "charset"
183 
+	paxComment  = "comment"
184 
+	paxCtime    = "ctime" // please note that ctime is not a valid pax header.
185 
+	paxGid      = "gid"
186 
+	paxGname    = "gname"
187 
+	paxLinkpath = "linkpath"
188 
+	paxMtime    = "mtime"
189 
+	paxPath     = "path"
190 
+	paxSize     = "size"
191 
+	paxUid      = "uid"
192 
+	paxUname    = "uname"
193 
+	paxXattr    = "SCHILY.xattr."
194 
+	paxNone     = ""
195 
+)
196 
+
197 
+// FileInfoHeader creates a partially-populated Header from fi.
198 
+// If fi describes a symlink, FileInfoHeader records link as the link target.
199 
+// If fi describes a directory, a slash is appended to the name.
200 
+// Because os.FileInfo's Name method returns only the base name of
201 
+// the file it describes, it may be necessary to modify the Name field
202 
+// of the returned header to provide the full path name of the file.
203 
+func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
204 
+	if fi == nil {
205 
+		return nil, errors.New("tar: FileInfo is nil")
206 
+	}
207 
+	fm := fi.Mode()
208 
+	h := &Header{
209 
+		Name:    fi.Name(),
210 
+		ModTime: fi.ModTime(),
211 
+		Mode:    int64(fm.Perm()), // or'd with c_IS* constants later
212 
+	}
213 
+	switch {
214 
+	case fm.IsRegular():
215 
+		h.Mode |= c_ISREG
216 
+		h.Typeflag = TypeReg
217 
+		h.Size = fi.Size()
218 
+	case fi.IsDir():
219 
+		h.Typeflag = TypeDir
220 
+		h.Mode |= c_ISDIR
221 
+		h.Name += "/"
222 
+	case fm&os.ModeSymlink != 0:
223 
+		h.Typeflag = TypeSymlink
224 
+		h.Mode |= c_ISLNK
225 
+		h.Linkname = link
226 
+	case fm&os.ModeDevice != 0:
227 
+		if fm&os.ModeCharDevice != 0 {
228 
+			h.Mode |= c_ISCHR
229 
+			h.Typeflag = TypeChar
230 
+		} else {
231 
+			h.Mode |= c_ISBLK
232 
+			h.Typeflag = TypeBlock
233 
+		}
234 
+	case fm&os.ModeNamedPipe != 0:
235 
+		h.Typeflag = TypeFifo
236 
+		h.Mode |= c_ISFIFO
237 
+	case fm&os.ModeSocket != 0:
238 
+		h.Mode |= c_ISSOCK
239 
+	default:
240 
+		return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm)
241 
+	}
242 
+	if fm&os.ModeSetuid != 0 {
243 
+		h.Mode |= c_ISUID
244 
+	}
245 
+	if fm&os.ModeSetgid != 0 {
246 
+		h.Mode |= c_ISGID
247 
+	}
248 
+	if fm&os.ModeSticky != 0 {
249 
+		h.Mode |= c_ISVTX
250 
+	}
251 
+	if sysStat != nil {
252 
+		return h, sysStat(fi, h)
253 
+	}
254 
+	return h, nil
255 
+}
256 
+
257 
+var zeroBlock = make([]byte, blockSize)
258 
+
259 
+// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values.
260 
+// We compute and return both.
261 
+func checksum(header []byte) (unsigned int64, signed int64) {
262 
+	for i := 0; i < len(header); i++ {
263 
+		if i == 148 {
264 
+			// The chksum field (header[148:156]) is special: it should be treated as space bytes.
265 
+			unsigned += ' ' * 8
266 
+			signed += ' ' * 8
267 
+			i += 7
268 
+			continue
269 
+		}
270 
+		unsigned += int64(header[i])
271 
+		signed += int64(int8(header[i]))
272 
+	}
273 
+	return
274 
+}
275 
+
276 
+type slicer []byte
277 
+
278 
+func (sp *slicer) next(n int) (b []byte) {
279 
+	s := *sp
280 
+	b, *sp = s[0:n], s[n:]
281 
+	return
282 
+}
283 
+
284 
+func isASCII(s string) bool {
285 
+	for _, c := range s {
286 
+		if c >= 0x80 {
287 
+			return false
288 
+		}
289 
+	}
290 
+	return true
291 
+}
292 
+
293 
+func toASCII(s string) string {
294 
+	if isASCII(s) {
295 
+		return s
296 
+	}
297 
+	var buf bytes.Buffer
298 
+	for _, c := range s {
299 
+		if c < 0x80 {
300 
+			buf.WriteByte(byte(c))
301 
+		}
302 
+	}
303 
+	return buf.String()
304 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/example_test.go
History View file @ 7fb0618
0 305 
new file mode 100644
... ... 
@@ -0,0 +1,80 @@
0 
+// Copyright 2013 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+package tar_test
5 
+
6 
+import (
7 
+	"archive/tar"
8 
+	"bytes"
9 
+	"fmt"
10 
+	"io"
11 
+	"log"
12 
+	"os"
13 
+)
14 
+
15 
+func Example() {
16 
+	// Create a buffer to write our archive to.
17 
+	buf := new(bytes.Buffer)
18 
+
19 
+	// Create a new tar archive.
20 
+	tw := tar.NewWriter(buf)
21 
+
22 
+	// Add some files to the archive.
23 
+	var files = []struct {
24 
+		Name, Body string
25 
+	}{
26 
+		{"readme.txt", "This archive contains some text files."},
27 
+		{"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
28 
+		{"todo.txt", "Get animal handling licence."},
29 
+	}
30 
+	for _, file := range files {
31 
+		hdr := &tar.Header{
32 
+			Name: file.Name,
33 
+			Mode: 0600,
34 
+			Size: int64(len(file.Body)),
35 
+		}
36 
+		if err := tw.WriteHeader(hdr); err != nil {
37 
+			log.Fatalln(err)
38 
+		}
39 
+		if _, err := tw.Write([]byte(file.Body)); err != nil {
40 
+			log.Fatalln(err)
41 
+		}
42 
+	}
43 
+	// Make sure to check the error on Close.
44 
+	if err := tw.Close(); err != nil {
45 
+		log.Fatalln(err)
46 
+	}
47 
+
48 
+	// Open the tar archive for reading.
49 
+	r := bytes.NewReader(buf.Bytes())
50 
+	tr := tar.NewReader(r)
51 
+
52 
+	// Iterate through the files in the archive.
53 
+	for {
54 
+		hdr, err := tr.Next()
55 
+		if err == io.EOF {
56 
+			// end of tar archive
57 
+			break
58 
+		}
59 
+		if err != nil {
60 
+			log.Fatalln(err)
61 
+		}
62 
+		fmt.Printf("Contents of %s:\n", hdr.Name)
63 
+		if _, err := io.Copy(os.Stdout, tr); err != nil {
64 
+			log.Fatalln(err)
65 
+		}
66 
+		fmt.Println()
67 
+	}
68 
+
69 
+	// Output:
70 
+	// Contents of readme.txt:
71 
+	// This archive contains some text files.
72 
+	// Contents of gopher.txt:
73 
+	// Gopher names:
74 
+	// George
75 
+	// Geoffrey
76 
+	// Gonzo
77 
+	// Contents of todo.txt:
78 
+	// Get animal handling licence.
79 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/reader.go
History View file @ 7fb0618
0 80 
new file mode 100644
... ... 
@@ -0,0 +1,926 @@
0 
+// Copyright 2009 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+package tar
5 
+
6 
+// TODO(dsymonds):
7 
+//   - pax extensions
8 
+
9 
+import (
10 
+	"bytes"
11 
+	"errors"
12 
+	"io"
13 
+	"io/ioutil"
14 
+	"os"
15 
+	"strconv"
16 
+	"strings"
17 
+	"time"
18 
+)
19 
+
20 
+var (
21 
+	ErrHeader = errors.New("archive/tar: invalid tar header")
22 
+)
23 
+
24 
+const maxNanoSecondIntSize = 9
25 
+
26 
+// A Reader provides sequential access to the contents of a tar archive.
27 
+// A tar archive consists of a sequence of files.
28 
+// The Next method advances to the next file in the archive (including the first),
29 
+// and then it can be treated as an io.Reader to access the file's data.
30 
+type Reader struct {
31 
+	r       io.Reader
32 
+	err     error
33 
+	pad     int64           // amount of padding (ignored) after current file entry
34 
+	curr    numBytesReader  // reader for current file entry
35 
+	hdrBuff [blockSize]byte // buffer to use in readHeader
36 
+
37 
+	RawAccounting bool          // Whether to enable the access needed to reassemble the tar from raw bytes. Some performance/memory hit for this.
38 
+	rawBytes      *bytes.Buffer // last raw bits
39 
+}
40 
+
41 
+// RawBytes accesses the raw bytes of the archive, apart from the file payload itself.
42 
+// This includes the header and padding.
43 
+//
44 
+// This call resets the current rawbytes buffer
45 
+//
46 
+// Only when RawAccounting is enabled, otherwise this returns nil
47 
+func (tr *Reader) RawBytes() []byte {
48 
+	if !tr.RawAccounting {
49 
+		return nil
50 
+	}
51 
+	if tr.rawBytes == nil {
52 
+		tr.rawBytes = bytes.NewBuffer(nil)
53 
+	}
54 
+	// if we've read them, then flush them.
55 
+	defer tr.rawBytes.Reset()
56 
+	return tr.rawBytes.Bytes()
57 
+}
58 
+
59 
+// A numBytesReader is an io.Reader with a numBytes method, returning the number
60 
+// of bytes remaining in the underlying encoded data.
61 
+type numBytesReader interface {
62 
+	io.Reader
63 
+	numBytes() int64
64 
+}
65 
+
66 
+// A regFileReader is a numBytesReader for reading file data from a tar archive.
67 
+type regFileReader struct {
68 
+	r  io.Reader // underlying reader
69 
+	nb int64     // number of unread bytes for current file entry
70 
+}
71 
+
72 
+// A sparseFileReader is a numBytesReader for reading sparse file data from a tar archive.
73 
+type sparseFileReader struct {
74 
+	rfr *regFileReader // reads the sparse-encoded file data
75 
+	sp  []sparseEntry  // the sparse map for the file
76 
+	pos int64          // keeps track of file position
77 
+	tot int64          // total size of the file
78 
+}
79 
+
80 
+// Keywords for GNU sparse files in a PAX extended header
81 
+const (
82 
+	paxGNUSparseNumBlocks = "GNU.sparse.numblocks"
83 
+	paxGNUSparseOffset    = "GNU.sparse.offset"
84 
+	paxGNUSparseNumBytes  = "GNU.sparse.numbytes"
85 
+	paxGNUSparseMap       = "GNU.sparse.map"
86 
+	paxGNUSparseName      = "GNU.sparse.name"
87 
+	paxGNUSparseMajor     = "GNU.sparse.major"
88 
+	paxGNUSparseMinor     = "GNU.sparse.minor"
89 
+	paxGNUSparseSize      = "GNU.sparse.size"
90 
+	paxGNUSparseRealSize  = "GNU.sparse.realsize"
91 
+)
92 
+
93 
+// Keywords for old GNU sparse headers
94 
+const (
95 
+	oldGNUSparseMainHeaderOffset               = 386
96 
+	oldGNUSparseMainHeaderIsExtendedOffset     = 482
97 
+	oldGNUSparseMainHeaderNumEntries           = 4
98 
+	oldGNUSparseExtendedHeaderIsExtendedOffset = 504
99 
+	oldGNUSparseExtendedHeaderNumEntries       = 21
100 
+	oldGNUSparseOffsetSize                     = 12
101 
+	oldGNUSparseNumBytesSize                   = 12
102 
+)
103 
+
104 
+// NewReader creates a new Reader reading from r.
105 
+func NewReader(r io.Reader) *Reader { return &Reader{r: r} }
106 
+
107 
+// Next advances to the next entry in the tar archive.
108 
+//
109 
+// io.EOF is returned at the end of the input.
110 
+func (tr *Reader) Next() (*Header, error) {
111 
+	var hdr *Header
112 
+	if tr.RawAccounting {
113 
+		if tr.rawBytes == nil {
114 
+			tr.rawBytes = bytes.NewBuffer(nil)
115 
+		} else {
116 
+			tr.rawBytes.Reset()
117 
+		}
118 
+	}
119 
+	if tr.err == nil {
120 
+		tr.skipUnread()
121 
+	}
122 
+	if tr.err != nil {
123 
+		return hdr, tr.err
124 
+	}
125 
+	hdr = tr.readHeader()
126 
+	if hdr == nil {
127 
+		return hdr, tr.err
128 
+	}
129 
+	// Check for PAX/GNU header.
130 
+	switch hdr.Typeflag {
131 
+	case TypeXHeader:
132 
+		//  PAX extended header
133 
+		headers, err := parsePAX(tr)
134 
+		if err != nil {
135 
+			return nil, err
136 
+		}
137 
+		// We actually read the whole file,
138 
+		// but this skips alignment padding
139 
+		tr.skipUnread()
140 
+		hdr = tr.readHeader()
141 
+		mergePAX(hdr, headers)
142 
+
143 
+		// Check for a PAX format sparse file
144 
+		sp, err := tr.checkForGNUSparsePAXHeaders(hdr, headers)
145 
+		if err != nil {
146 
+			tr.err = err
147 
+			return nil, err
148 
+		}
149 
+		if sp != nil {
150 
+			// Current file is a PAX format GNU sparse file.
151 
+			// Set the current file reader to a sparse file reader.
152 
+			tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
153 
+		}
154 
+		return hdr, nil
155 
+	case TypeGNULongName:
156 
+		// We have a GNU long name header. Its contents are the real file name.
157 
+		realname, err := ioutil.ReadAll(tr)
158 
+		if err != nil {
159 
+			return nil, err
160 
+		}
161 
+		var b []byte
162 
+		if tr.RawAccounting {
163 
+			if _, err = tr.rawBytes.Write(realname); err != nil {
164 
+				return nil, err
165 
+			}
166 
+			b = tr.RawBytes()
167 
+		}
168 
+		hdr, err := tr.Next()
169 
+		// since the above call to Next() resets the buffer, we need to throw the bytes over
170 
+		if tr.RawAccounting {
171 
+			if _, err = tr.rawBytes.Write(b); err != nil {
172 
+				return nil, err
173 
+			}
174 
+		}
175 
+		hdr.Name = cString(realname)
176 
+		return hdr, err
177 
+	case TypeGNULongLink:
178 
+		// We have a GNU long link header.
179 
+		realname, err := ioutil.ReadAll(tr)
180 
+		if err != nil {
181 
+			return nil, err
182 
+		}
183 
+		var b []byte
184 
+		if tr.RawAccounting {
185 
+			if _, err = tr.rawBytes.Write(realname); err != nil {
186 
+				return nil, err
187 
+			}
188 
+			b = tr.RawBytes()
189 
+		}
190 
+		hdr, err := tr.Next()
191 
+		// since the above call to Next() resets the buffer, we need to throw the bytes over
192 
+		if tr.RawAccounting {
193 
+			if _, err = tr.rawBytes.Write(b); err != nil {
194 
+				return nil, err
195 
+			}
196 
+		}
197 
+		hdr.Linkname = cString(realname)
198 
+		return hdr, err
199 
+	}
200 
+	return hdr, tr.err
201 
+}
202 
+
203 
+// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then
204 
+// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to
205 
+// be treated as a regular file.
206 
+func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) {
207 
+	var sparseFormat string
208 
+
209 
+	// Check for sparse format indicators
210 
+	major, majorOk := headers[paxGNUSparseMajor]
211 
+	minor, minorOk := headers[paxGNUSparseMinor]
212 
+	sparseName, sparseNameOk := headers[paxGNUSparseName]
213 
+	_, sparseMapOk := headers[paxGNUSparseMap]
214 
+	sparseSize, sparseSizeOk := headers[paxGNUSparseSize]
215 
+	sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize]
216 
+
217 
+	// Identify which, if any, sparse format applies from which PAX headers are set
218 
+	if majorOk && minorOk {
219 
+		sparseFormat = major + "." + minor
220 
+	} else if sparseNameOk && sparseMapOk {
221 
+		sparseFormat = "0.1"
222 
+	} else if sparseSizeOk {
223 
+		sparseFormat = "0.0"
224 
+	} else {
225 
+		// Not a PAX format GNU sparse file.
226 
+		return nil, nil
227 
+	}
228 
+
229 
+	// Check for unknown sparse format
230 
+	if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" {
231 
+		return nil, nil
232 
+	}
233 
+
234 
+	// Update hdr from GNU sparse PAX headers
235 
+	if sparseNameOk {
236 
+		hdr.Name = sparseName
237 
+	}
238 
+	if sparseSizeOk {
239 
+		realSize, err := strconv.ParseInt(sparseSize, 10, 0)
240 
+		if err != nil {
241 
+			return nil, ErrHeader
242 
+		}
243 
+		hdr.Size = realSize
244 
+	} else if sparseRealSizeOk {
245 
+		realSize, err := strconv.ParseInt(sparseRealSize, 10, 0)
246 
+		if err != nil {
247 
+			return nil, ErrHeader
248 
+		}
249 
+		hdr.Size = realSize
250 
+	}
251 
+
252 
+	// Set up the sparse map, according to the particular sparse format in use
253 
+	var sp []sparseEntry
254 
+	var err error
255 
+	switch sparseFormat {
256 
+	case "0.0", "0.1":
257 
+		sp, err = readGNUSparseMap0x1(headers)
258 
+	case "1.0":
259 
+		sp, err = readGNUSparseMap1x0(tr.curr)
260 
+	}
261 
+	return sp, err
262 
+}
263 
+
264 
+// mergePAX merges well known headers according to PAX standard.
265 
+// In general headers with the same name as those found
266 
+// in the header struct overwrite those found in the header
267 
+// struct with higher precision or longer values. Esp. useful
268 
+// for name and linkname fields.
269 
+func mergePAX(hdr *Header, headers map[string]string) error {
270 
+	for k, v := range headers {
271 
+		switch k {
272 
+		case paxPath:
273 
+			hdr.Name = v
274 
+		case paxLinkpath:
275 
+			hdr.Linkname = v
276 
+		case paxGname:
277 
+			hdr.Gname = v
278 
+		case paxUname:
279 
+			hdr.Uname = v
280 
+		case paxUid:
281 
+			uid, err := strconv.ParseInt(v, 10, 0)
282 
+			if err != nil {
283 
+				return err
284 
+			}
285 
+			hdr.Uid = int(uid)
286 
+		case paxGid:
287 
+			gid, err := strconv.ParseInt(v, 10, 0)
288 
+			if err != nil {
289 
+				return err
290 
+			}
291 
+			hdr.Gid = int(gid)
292 
+		case paxAtime:
293 
+			t, err := parsePAXTime(v)
294 
+			if err != nil {
295 
+				return err
296 
+			}
297 
+			hdr.AccessTime = t
298 
+		case paxMtime:
299 
+			t, err := parsePAXTime(v)
300 
+			if err != nil {
301 
+				return err
302 
+			}
303 
+			hdr.ModTime = t
304 
+		case paxCtime:
305 
+			t, err := parsePAXTime(v)
306 
+			if err != nil {
307 
+				return err
308 
+			}
309 
+			hdr.ChangeTime = t
310 
+		case paxSize:
311 
+			size, err := strconv.ParseInt(v, 10, 0)
312 
+			if err != nil {
313 
+				return err
314 
+			}
315 
+			hdr.Size = int64(size)
316 
+		default:
317 
+			if strings.HasPrefix(k, paxXattr) {
318 
+				if hdr.Xattrs == nil {
319 
+					hdr.Xattrs = make(map[string]string)
320 
+				}
321 
+				hdr.Xattrs[k[len(paxXattr):]] = v
322 
+			}
323 
+		}
324 
+	}
325 
+	return nil
326 
+}
327 
+
328 
+// parsePAXTime takes a string of the form %d.%d as described in
329 
+// the PAX specification.
330 
+func parsePAXTime(t string) (time.Time, error) {
331 
+	buf := []byte(t)
332 
+	pos := bytes.IndexByte(buf, '.')
333 
+	var seconds, nanoseconds int64
334 
+	var err error
335 
+	if pos == -1 {
336 
+		seconds, err = strconv.ParseInt(t, 10, 0)
337 
+		if err != nil {
338 
+			return time.Time{}, err
339 
+		}
340 
+	} else {
341 
+		seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0)
342 
+		if err != nil {
343 
+			return time.Time{}, err
344 
+		}
345 
+		nano_buf := string(buf[pos+1:])
346 
+		// Pad as needed before converting to a decimal.
347 
+		// For example .030 -> .030000000 -> 30000000 nanoseconds
348 
+		if len(nano_buf) < maxNanoSecondIntSize {
349 
+			// Right pad
350 
+			nano_buf += strings.Repeat("0", maxNanoSecondIntSize-len(nano_buf))
351 
+		} else if len(nano_buf) > maxNanoSecondIntSize {
352 
+			// Right truncate
353 
+			nano_buf = nano_buf[:maxNanoSecondIntSize]
354 
+		}
355 
+		nanoseconds, err = strconv.ParseInt(string(nano_buf), 10, 0)
356 
+		if err != nil {
357 
+			return time.Time{}, err
358 
+		}
359 
+	}
360 
+	ts := time.Unix(seconds, nanoseconds)
361 
+	return ts, nil
362 
+}
363 
+
364 
+// parsePAX parses PAX headers.
365 
+// If an extended header (type 'x') is invalid, ErrHeader is returned
366 
+func parsePAX(r io.Reader) (map[string]string, error) {
367 
+	buf, err := ioutil.ReadAll(r)
368 
+	if err != nil {
369 
+		return nil, err
370 
+	}
371 
+	// leaving this function for io.Reader makes it more testable
372 
+	if tr, ok := r.(*Reader); ok && tr.RawAccounting {
373 
+		if _, err = tr.rawBytes.Write(buf); err != nil {
374 
+			return nil, err
375 
+		}
376 
+	}
377 
+
378 
+	// For GNU PAX sparse format 0.0 support.
379 
+	// This function transforms the sparse format 0.0 headers into sparse format 0.1 headers.
380 
+	var sparseMap bytes.Buffer
381 
+
382 
+	headers := make(map[string]string)
383 
+	// Each record is constructed as
384 
+	//     "%d %s=%s\n", length, keyword, value
385 
+	for len(buf) > 0 {
386 
+		// or the header was empty to start with.
387 
+		var sp int
388 
+		// The size field ends at the first space.
389 
+		sp = bytes.IndexByte(buf, ' ')
390 
+		if sp == -1 {
391 
+			return nil, ErrHeader
392 
+		}
393 
+		// Parse the first token as a decimal integer.
394 
+		n, err := strconv.ParseInt(string(buf[:sp]), 10, 0)
395 
+		if err != nil {
396 
+			return nil, ErrHeader
397 
+		}
398 
+		// Extract everything between the decimal and the n -1 on the
399 
+		// beginning to eat the ' ', -1 on the end to skip the newline.
400 
+		var record []byte
401 
+		record, buf = buf[sp+1:n-1], buf[n:]
402 
+		// The first equals is guaranteed to mark the end of the key.
403 
+		// Everything else is value.
404 
+		eq := bytes.IndexByte(record, '=')
405 
+		if eq == -1 {
406 
+			return nil, ErrHeader
407 
+		}
408 
+		key, value := record[:eq], record[eq+1:]
409 
+
410 
+		keyStr := string(key)
411 
+		if keyStr == paxGNUSparseOffset || keyStr == paxGNUSparseNumBytes {
412 
+			// GNU sparse format 0.0 special key. Write to sparseMap instead of using the headers map.
413 
+			sparseMap.Write(value)
414 
+			sparseMap.Write([]byte{','})
415 
+		} else {
416 
+			// Normal key. Set the value in the headers map.
417 
+			headers[keyStr] = string(value)
418 
+		}
419 
+	}
420 
+	if sparseMap.Len() != 0 {
421 
+		// Add sparse info to headers, chopping off the extra comma
422 
+		sparseMap.Truncate(sparseMap.Len() - 1)
423 
+		headers[paxGNUSparseMap] = sparseMap.String()
424 
+	}
425 
+	return headers, nil
426 
+}
427 
+
428 
+// cString parses bytes as a NUL-terminated C-style string.
429 
+// If a NUL byte is not found then the whole slice is returned as a string.
430 
+func cString(b []byte) string {
431 
+	n := 0
432 
+	for n < len(b) && b[n] != 0 {
433 
+		n++
434 
+	}
435 
+	return string(b[0:n])
436 
+}
437 
+
438 
+func (tr *Reader) octal(b []byte) int64 {
439 
+	// Check for binary format first.
440 
+	if len(b) > 0 && b[0]&0x80 != 0 {
441 
+		var x int64
442 
+		for i, c := range b {
443 
+			if i == 0 {
444 
+				c &= 0x7f // ignore signal bit in first byte
445 
+			}
446 
+			x = x<<8 | int64(c)
447 
+		}
448 
+		return x
449 
+	}
450 
+
451 
+	// Because unused fields are filled with NULs, we need
452 
+	// to skip leading NULs. Fields may also be padded with
453 
+	// spaces or NULs.
454 
+	// So we remove leading and trailing NULs and spaces to
455 
+	// be sure.
456 
+	b = bytes.Trim(b, " \x00")
457 
+
458 
+	if len(b) == 0 {
459 
+		return 0
460 
+	}
461 
+	x, err := strconv.ParseUint(cString(b), 8, 64)
462 
+	if err != nil {
463 
+		tr.err = err
464 
+	}
465 
+	return int64(x)
466 
+}
467 
+
468 
+// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding.
469 
+func (tr *Reader) skipUnread() {
470 
+	nr := tr.numBytes() + tr.pad // number of bytes to skip
471 
+	tr.curr, tr.pad = nil, 0
472 
+	if tr.RawAccounting {
473 
+		_, tr.err = io.CopyN(tr.rawBytes, tr.r, nr)
474 
+		return
475 
+	}
476 
+	if sr, ok := tr.r.(io.Seeker); ok {
477 
+		if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil {
478 
+			return
479 
+		}
480 
+	}
481 
+	_, tr.err = io.CopyN(ioutil.Discard, tr.r, nr)
482 
+}
483 
+
484 
+func (tr *Reader) verifyChecksum(header []byte) bool {
485 
+	if tr.err != nil {
486 
+		return false
487 
+	}
488 
+
489 
+	given := tr.octal(header[148:156])
490 
+	unsigned, signed := checksum(header)
491 
+	return given == unsigned || given == signed
492 
+}
493 
+
494 
+func (tr *Reader) readHeader() *Header {
495 
+	header := tr.hdrBuff[:]
496 
+	copy(header, zeroBlock)
497 
+
498 
+	if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
499 
+		// because it could read some of the block, but reach EOF first
500 
+		if tr.err == io.EOF && tr.RawAccounting {
501 
+			if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
502 
+				return nil
503 
+			}
504 
+		}
505 
+		return nil
506 
+	}
507 
+	if tr.RawAccounting {
508 
+		if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
509 
+			return nil
510 
+		}
511 
+	}
512 
+
513 
+	// Two blocks of zero bytes marks the end of the archive.
514 
+	if bytes.Equal(header, zeroBlock[0:blockSize]) {
515 
+		if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil {
516 
+			// because it could read some of the block, but reach EOF first
517 
+			if tr.err == io.EOF && tr.RawAccounting {
518 
+				if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
519 
+					return nil
520 
+				}
521 
+			}
522 
+			return nil
523 
+		}
524 
+		if tr.RawAccounting {
525 
+			if _, tr.err = tr.rawBytes.Write(header); tr.err != nil {
526 
+				return nil
527 
+			}
528 
+		}
529 
+		if bytes.Equal(header, zeroBlock[0:blockSize]) {
530 
+			tr.err = io.EOF
531 
+		} else {
532 
+			tr.err = ErrHeader // zero block and then non-zero block
533 
+		}
534 
+		return nil
535 
+	}
536 
+
537 
+	if !tr.verifyChecksum(header) {
538 
+		tr.err = ErrHeader
539 
+		return nil
540 
+	}
541 
+
542 
+	// Unpack
543 
+	hdr := new(Header)
544 
+	s := slicer(header)
545 
+
546 
+	hdr.Name = cString(s.next(100))
547 
+	hdr.Mode = tr.octal(s.next(8))
548 
+	hdr.Uid = int(tr.octal(s.next(8)))
549 
+	hdr.Gid = int(tr.octal(s.next(8)))
550 
+	hdr.Size = tr.octal(s.next(12))
551 
+	hdr.ModTime = time.Unix(tr.octal(s.next(12)), 0)
552 
+	s.next(8) // chksum
553 
+	hdr.Typeflag = s.next(1)[0]
554 
+	hdr.Linkname = cString(s.next(100))
555 
+
556 
+	// The remainder of the header depends on the value of magic.
557 
+	// The original (v7) version of tar had no explicit magic field,
558 
+	// so its magic bytes, like the rest of the block, are NULs.
559 
+	magic := string(s.next(8)) // contains version field as well.
560 
+	var format string
561 
+	switch {
562 
+	case magic[:6] == "ustar\x00": // POSIX tar (1003.1-1988)
563 
+		if string(header[508:512]) == "tar\x00" {
564 
+			format = "star"
565 
+		} else {
566 
+			format = "posix"
567 
+		}
568 
+	case magic == "ustar  \x00": // old GNU tar
569 
+		format = "gnu"
570 
+	}
571 
+
572 
+	switch format {
573 
+	case "posix", "gnu", "star":
574 
+		hdr.Uname = cString(s.next(32))
575 
+		hdr.Gname = cString(s.next(32))
576 
+		devmajor := s.next(8)
577 
+		devminor := s.next(8)
578 
+		if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock {
579 
+			hdr.Devmajor = tr.octal(devmajor)
580 
+			hdr.Devminor = tr.octal(devminor)
581 
+		}
582 
+		var prefix string
583 
+		switch format {
584 
+		case "posix", "gnu":
585 
+			prefix = cString(s.next(155))
586 
+		case "star":
587 
+			prefix = cString(s.next(131))
588 
+			hdr.AccessTime = time.Unix(tr.octal(s.next(12)), 0)
589 
+			hdr.ChangeTime = time.Unix(tr.octal(s.next(12)), 0)
590 
+		}
591 
+		if len(prefix) > 0 {
592 
+			hdr.Name = prefix + "/" + hdr.Name
593 
+		}
594 
+	}
595 
+
596 
+	if tr.err != nil {
597 
+		tr.err = ErrHeader
598 
+		return nil
599 
+	}
600 
+
601 
+	// Maximum value of hdr.Size is 64 GB (12 octal digits),
602 
+	// so there's no risk of int64 overflowing.
603 
+	nb := int64(hdr.Size)
604 
+	tr.pad = -nb & (blockSize - 1) // blockSize is a power of two
605 
+
606 
+	// Set the current file reader.
607 
+	tr.curr = &regFileReader{r: tr.r, nb: nb}
608 
+
609 
+	// Check for old GNU sparse format entry.
610 
+	if hdr.Typeflag == TypeGNUSparse {
611 
+		// Get the real size of the file.
612 
+		hdr.Size = tr.octal(header[483:495])
613 
+
614 
+		// Read the sparse map.
615 
+		sp := tr.readOldGNUSparseMap(header)
616 
+		if tr.err != nil {
617 
+			return nil
618 
+		}
619 
+		// Current file is a GNU sparse file. Update the current file reader.
620 
+		tr.curr = &sparseFileReader{rfr: tr.curr.(*regFileReader), sp: sp, tot: hdr.Size}
621 
+	}
622 
+
623 
+	return hdr
624 
+}
625 
+
626 
+// A sparseEntry holds a single entry in a sparse file's sparse map.
627 
+// A sparse entry indicates the offset and size in a sparse file of a
628 
+// block of data.
629 
+type sparseEntry struct {
630 
+	offset   int64
631 
+	numBytes int64
632 
+}
633 
+
634 
+// readOldGNUSparseMap reads the sparse map as stored in the old GNU sparse format.
635 
+// The sparse map is stored in the tar header if it's small enough. If it's larger than four entries,
636 
+// then one or more extension headers are used to store the rest of the sparse map.
637 
+func (tr *Reader) readOldGNUSparseMap(header []byte) []sparseEntry {
638 
+	isExtended := header[oldGNUSparseMainHeaderIsExtendedOffset] != 0
639 
+	spCap := oldGNUSparseMainHeaderNumEntries
640 
+	if isExtended {
641 
+		spCap += oldGNUSparseExtendedHeaderNumEntries
642 
+	}
643 
+	sp := make([]sparseEntry, 0, spCap)
644 
+	s := slicer(header[oldGNUSparseMainHeaderOffset:])
645 
+
646 
+	// Read the four entries from the main tar header
647 
+	for i := 0; i < oldGNUSparseMainHeaderNumEntries; i++ {
648 
+		offset := tr.octal(s.next(oldGNUSparseOffsetSize))
649 
+		numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
650 
+		if tr.err != nil {
651 
+			tr.err = ErrHeader
652 
+			return nil
653 
+		}
654 
+		if offset == 0 && numBytes == 0 {
655 
+			break
656 
+		}
657 
+		sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
658 
+	}
659 
+
660 
+	for isExtended {
661 
+		// There are more entries. Read an extension header and parse its entries.
662 
+		sparseHeader := make([]byte, blockSize)
663 
+		if _, tr.err = io.ReadFull(tr.r, sparseHeader); tr.err != nil {
664 
+			return nil
665 
+		}
666 
+		if tr.RawAccounting {
667 
+			if _, tr.err = tr.rawBytes.Write(sparseHeader); tr.err != nil {
668 
+				return nil
669 
+			}
670 
+		}
671 
+
672 
+		isExtended = sparseHeader[oldGNUSparseExtendedHeaderIsExtendedOffset] != 0
673 
+		s = slicer(sparseHeader)
674 
+		for i := 0; i < oldGNUSparseExtendedHeaderNumEntries; i++ {
675 
+			offset := tr.octal(s.next(oldGNUSparseOffsetSize))
676 
+			numBytes := tr.octal(s.next(oldGNUSparseNumBytesSize))
677 
+			if tr.err != nil {
678 
+				tr.err = ErrHeader
679 
+				return nil
680 
+			}
681 
+			if offset == 0 && numBytes == 0 {
682 
+				break
683 
+			}
684 
+			sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
685 
+		}
686 
+	}
687 
+	return sp
688 
+}
689 
+
690 
+// readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format version 1.0.
691 
+// The sparse map is stored just before the file data and padded out to the nearest block boundary.
692 
+func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) {
693 
+	buf := make([]byte, 2*blockSize)
694 
+	sparseHeader := buf[:blockSize]
695 
+
696 
+	// readDecimal is a helper function to read a decimal integer from the sparse map
697 
+	// while making sure to read from the file in blocks of size blockSize
698 
+	readDecimal := func() (int64, error) {
699 
+		// Look for newline
700 
+		nl := bytes.IndexByte(sparseHeader, '\n')
701 
+		if nl == -1 {
702 
+			if len(sparseHeader) >= blockSize {
703 
+				// This is an error
704 
+				return 0, ErrHeader
705 
+			}
706 
+			oldLen := len(sparseHeader)
707 
+			newLen := oldLen + blockSize
708 
+			if cap(sparseHeader) < newLen {
709 
+				// There's more header, but we need to make room for the next block
710 
+				copy(buf, sparseHeader)
711 
+				sparseHeader = buf[:newLen]
712 
+			} else {
713 
+				// There's more header, and we can just reslice
714 
+				sparseHeader = sparseHeader[:newLen]
715 
+			}
716 
+
717 
+			// Now that sparseHeader is large enough, read next block
718 
+			if _, err := io.ReadFull(r, sparseHeader[oldLen:newLen]); err != nil {
719 
+				return 0, err
720 
+			}
721 
+			// leaving this function for io.Reader makes it more testable
722 
+			if tr, ok := r.(*Reader); ok && tr.RawAccounting {
723 
+				if _, err := tr.rawBytes.Write(sparseHeader[oldLen:newLen]); err != nil {
724 
+					return 0, err
725 
+				}
726 
+			}
727 
+
728 
+			// Look for a newline in the new data
729 
+			nl = bytes.IndexByte(sparseHeader[oldLen:newLen], '\n')
730 
+			if nl == -1 {
731 
+				// This is an error
732 
+				return 0, ErrHeader
733 
+			}
734 
+			nl += oldLen // We want the position from the beginning
735 
+		}
736 
+		// Now that we've found a newline, read a number
737 
+		n, err := strconv.ParseInt(string(sparseHeader[:nl]), 10, 0)
738 
+		if err != nil {
739 
+			return 0, ErrHeader
740 
+		}
741 
+
742 
+		// Update sparseHeader to consume this number
743 
+		sparseHeader = sparseHeader[nl+1:]
744 
+		return n, nil
745 
+	}
746 
+
747 
+	// Read the first block
748 
+	if _, err := io.ReadFull(r, sparseHeader); err != nil {
749 
+		return nil, err
750 
+	}
751 
+	// leaving this function for io.Reader makes it more testable
752 
+	if tr, ok := r.(*Reader); ok && tr.RawAccounting {
753 
+		if _, err := tr.rawBytes.Write(sparseHeader); err != nil {
754 
+			return nil, err
755 
+		}
756 
+	}
757 
+
758 
+	// The first line contains the number of entries
759 
+	numEntries, err := readDecimal()
760 
+	if err != nil {
761 
+		return nil, err
762 
+	}
763 
+
764 
+	// Read all the entries
765 
+	sp := make([]sparseEntry, 0, numEntries)
766 
+	for i := int64(0); i < numEntries; i++ {
767 
+		// Read the offset
768 
+		offset, err := readDecimal()
769 
+		if err != nil {
770 
+			return nil, err
771 
+		}
772 
+		// Read numBytes
773 
+		numBytes, err := readDecimal()
774 
+		if err != nil {
775 
+			return nil, err
776 
+		}
777 
+
778 
+		sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
779 
+	}
780 
+
781 
+	return sp, nil
782 
+}
783 
+
784 
+// readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format version 0.1.
785 
+// The sparse map is stored in the PAX headers.
786 
+func readGNUSparseMap0x1(headers map[string]string) ([]sparseEntry, error) {
787 
+	// Get number of entries
788 
+	numEntriesStr, ok := headers[paxGNUSparseNumBlocks]
789 
+	if !ok {
790 
+		return nil, ErrHeader
791 
+	}
792 
+	numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0)
793 
+	if err != nil {
794 
+		return nil, ErrHeader
795 
+	}
796 
+
797 
+	sparseMap := strings.Split(headers[paxGNUSparseMap], ",")
798 
+
799 
+	// There should be two numbers in sparseMap for each entry
800 
+	if int64(len(sparseMap)) != 2*numEntries {
801 
+		return nil, ErrHeader
802 
+	}
803 
+
804 
+	// Loop through the entries in the sparse map
805 
+	sp := make([]sparseEntry, 0, numEntries)
806 
+	for i := int64(0); i < numEntries; i++ {
807 
+		offset, err := strconv.ParseInt(sparseMap[2*i], 10, 0)
808 
+		if err != nil {
809 
+			return nil, ErrHeader
810 
+		}
811 
+		numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 0)
812 
+		if err != nil {
813 
+			return nil, ErrHeader
814 
+		}
815 
+		sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes})
816 
+	}
817 
+
818 
+	return sp, nil
819 
+}
820 
+
821 
+// numBytes returns the number of bytes left to read in the current file's entry
822 
+// in the tar archive, or 0 if there is no current file.
823 
+func (tr *Reader) numBytes() int64 {
824 
+	if tr.curr == nil {
825 
+		// No current file, so no bytes
826 
+		return 0
827 
+	}
828 
+	return tr.curr.numBytes()
829 
+}
830 
+
831 
+// Read reads from the current entry in the tar archive.
832 
+// It returns 0, io.EOF when it reaches the end of that entry,
833 
+// until Next is called to advance to the next entry.
834 
+func (tr *Reader) Read(b []byte) (n int, err error) {
835 
+	if tr.curr == nil {
836 
+		return 0, io.EOF
837 
+	}
838 
+	n, err = tr.curr.Read(b)
839 
+	if err != nil && err != io.EOF {
840 
+		tr.err = err
841 
+	}
842 
+	return
843 
+}
844 
+
845 
+func (rfr *regFileReader) Read(b []byte) (n int, err error) {
846 
+	if rfr.nb == 0 {
847 
+		// file consumed
848 
+		return 0, io.EOF
849 
+	}
850 
+	if int64(len(b)) > rfr.nb {
851 
+		b = b[0:rfr.nb]
852 
+	}
853 
+	n, err = rfr.r.Read(b)
854 
+	rfr.nb -= int64(n)
855 
+
856 
+	if err == io.EOF && rfr.nb > 0 {
857 
+		err = io.ErrUnexpectedEOF
858 
+	}
859 
+	return
860 
+}
861 
+
862 
+// numBytes returns the number of bytes left to read in the file's data in the tar archive.
863 
+func (rfr *regFileReader) numBytes() int64 {
864 
+	return rfr.nb
865 
+}
866 
+
867 
+// readHole reads a sparse file hole ending at offset toOffset
868 
+func (sfr *sparseFileReader) readHole(b []byte, toOffset int64) int {
869 
+	n64 := toOffset - sfr.pos
870 
+	if n64 > int64(len(b)) {
871 
+		n64 = int64(len(b))
872 
+	}
873 
+	n := int(n64)
874 
+	for i := 0; i < n; i++ {
875 
+		b[i] = 0
876 
+	}
877 
+	sfr.pos += n64
878 
+	return n
879 
+}
880 
+
881 
+// Read reads the sparse file data in expanded form.
882 
+func (sfr *sparseFileReader) Read(b []byte) (n int, err error) {
883 
+	if len(sfr.sp) == 0 {
884 
+		// No more data fragments to read from.
885 
+		if sfr.pos < sfr.tot {
886 
+			// We're in the last hole
887 
+			n = sfr.readHole(b, sfr.tot)
888 
+			return
889 
+		}
890 
+		// Otherwise, we're at the end of the file
891 
+		return 0, io.EOF
892 
+	}
893 
+	if sfr.pos < sfr.sp[0].offset {
894 
+		// We're in a hole
895 
+		n = sfr.readHole(b, sfr.sp[0].offset)
896 
+		return
897 
+	}
898 
+
899 
+	// We're not in a hole, so we'll read from the next data fragment
900 
+	posInFragment := sfr.pos - sfr.sp[0].offset
901 
+	bytesLeft := sfr.sp[0].numBytes - posInFragment
902 
+	if int64(len(b)) > bytesLeft {
903 
+		b = b[0:bytesLeft]
904 
+	}
905 
+
906 
+	n, err = sfr.rfr.Read(b)
907 
+	sfr.pos += int64(n)
908 
+
909 
+	if int64(n) == bytesLeft {
910 
+		// We're done with this fragment
911 
+		sfr.sp = sfr.sp[1:]
912 
+	}
913 
+
914 
+	if err == io.EOF && sfr.pos < sfr.tot {
915 
+		// We reached the end of the last fragment's data, but there's a final hole
916 
+		err = nil
917 
+	}
918 
+	return
919 
+}
920 
+
921 
+// numBytes returns the number of bytes left to read in the sparse file's
922 
+// sparse-encoded data in the tar archive.
923 
+func (sfr *sparseFileReader) numBytes() int64 {
924 
+	return sfr.rfr.nb
925 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/reader_test.go
History View file @ 7fb0618
0 926 
new file mode 100644
... ... 
@@ -0,0 +1,743 @@
0 
+// Copyright 2009 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+package tar
5 
+
6 
+import (
7 
+	"bytes"
8 
+	"crypto/md5"
9 
+	"fmt"
10 
+	"io"
11 
+	"io/ioutil"
12 
+	"os"
13 
+	"reflect"
14 
+	"strings"
15 
+	"testing"
16 
+	"time"
17 
+)
18 
+
19 
+type untarTest struct {
20 
+	file    string
21 
+	headers []*Header
22 
+	cksums  []string
23 
+}
24 
+
25 
+var gnuTarTest = &untarTest{
26 
+	file: "testdata/gnu.tar",
27 
+	headers: []*Header{
28 
+		{
29 
+			Name:     "small.txt",
30 
+			Mode:     0640,
31 
+			Uid:      73025,
32 
+			Gid:      5000,
33 
+			Size:     5,
34 
+			ModTime:  time.Unix(1244428340, 0),
35 
+			Typeflag: '0',
36 
+			Uname:    "dsymonds",
37 
+			Gname:    "eng",
38 
+		},
39 
+		{
40 
+			Name:     "small2.txt",
41 
+			Mode:     0640,
42 
+			Uid:      73025,
43 
+			Gid:      5000,
44 
+			Size:     11,
45 
+			ModTime:  time.Unix(1244436044, 0),
46 
+			Typeflag: '0',
47 
+			Uname:    "dsymonds",
48 
+			Gname:    "eng",
49 
+		},
50 
+	},
51 
+	cksums: []string{
52 
+		"e38b27eaccb4391bdec553a7f3ae6b2f",
53 
+		"c65bd2e50a56a2138bf1716f2fd56fe9",
54 
+	},
55 
+}
56 
+
57 
+var sparseTarTest = &untarTest{
58 
+	file: "testdata/sparse-formats.tar",
59 
+	headers: []*Header{
60 
+		{
61 
+			Name:     "sparse-gnu",
62 
+			Mode:     420,
63 
+			Uid:      1000,
64 
+			Gid:      1000,
65 
+			Size:     200,
66 
+			ModTime:  time.Unix(1392395740, 0),
67 
+			Typeflag: 0x53,
68 
+			Linkname: "",
69 
+			Uname:    "david",
70 
+			Gname:    "david",
71 
+			Devmajor: 0,
72 
+			Devminor: 0,
73 
+		},
74 
+		{
75 
+			Name:     "sparse-posix-0.0",
76 
+			Mode:     420,
77 
+			Uid:      1000,
78 
+			Gid:      1000,
79 
+			Size:     200,
80 
+			ModTime:  time.Unix(1392342187, 0),
81 
+			Typeflag: 0x30,
82 
+			Linkname: "",
83 
+			Uname:    "david",
84 
+			Gname:    "david",
85 
+			Devmajor: 0,
86 
+			Devminor: 0,
87 
+		},
88 
+		{
89 
+			Name:     "sparse-posix-0.1",
90 
+			Mode:     420,
91 
+			Uid:      1000,
92 
+			Gid:      1000,
93 
+			Size:     200,
94 
+			ModTime:  time.Unix(1392340456, 0),
95 
+			Typeflag: 0x30,
96 
+			Linkname: "",
97 
+			Uname:    "david",
98 
+			Gname:    "david",
99 
+			Devmajor: 0,
100 
+			Devminor: 0,
101 
+		},
102 
+		{
103 
+			Name:     "sparse-posix-1.0",
104 
+			Mode:     420,
105 
+			Uid:      1000,
106 
+			Gid:      1000,
107 
+			Size:     200,
108 
+			ModTime:  time.Unix(1392337404, 0),
109 
+			Typeflag: 0x30,
110 
+			Linkname: "",
111 
+			Uname:    "david",
112 
+			Gname:    "david",
113 
+			Devmajor: 0,
114 
+			Devminor: 0,
115 
+		},
116 
+		{
117 
+			Name:     "end",
118 
+			Mode:     420,
119 
+			Uid:      1000,
120 
+			Gid:      1000,
121 
+			Size:     4,
122 
+			ModTime:  time.Unix(1392398319, 0),
123 
+			Typeflag: 0x30,
124 
+			Linkname: "",
125 
+			Uname:    "david",
126 
+			Gname:    "david",
127 
+			Devmajor: 0,
128 
+			Devminor: 0,
129 
+		},
130 
+	},
131 
+	cksums: []string{
132 
+		"6f53234398c2449fe67c1812d993012f",
133 
+		"6f53234398c2449fe67c1812d993012f",
134 
+		"6f53234398c2449fe67c1812d993012f",
135 
+		"6f53234398c2449fe67c1812d993012f",
136 
+		"b0061974914468de549a2af8ced10316",
137 
+	},
138 
+}
139 
+
140 
+var untarTests = []*untarTest{
141 
+	gnuTarTest,
142 
+	sparseTarTest,
143 
+	{
144 
+		file: "testdata/star.tar",
145 
+		headers: []*Header{
146 
+			{
147 
+				Name:       "small.txt",
148 
+				Mode:       0640,
149 
+				Uid:        73025,
150 
+				Gid:        5000,
151 
+				Size:       5,
152 
+				ModTime:    time.Unix(1244592783, 0),
153 
+				Typeflag:   '0',
154 
+				Uname:      "dsymonds",
155 
+				Gname:      "eng",
156 
+				AccessTime: time.Unix(1244592783, 0),
157 
+				ChangeTime: time.Unix(1244592783, 0),
158 
+			},
159 
+			{
160 
+				Name:       "small2.txt",
161 
+				Mode:       0640,
162 
+				Uid:        73025,
163 
+				Gid:        5000,
164 
+				Size:       11,
165 
+				ModTime:    time.Unix(1244592783, 0),
166 
+				Typeflag:   '0',
167 
+				Uname:      "dsymonds",
168 
+				Gname:      "eng",
169 
+				AccessTime: time.Unix(1244592783, 0),
170 
+				ChangeTime: time.Unix(1244592783, 0),
171 
+			},
172 
+		},
173 
+	},
174 
+	{
175 
+		file: "testdata/v7.tar",
176 
+		headers: []*Header{
177 
+			{
178 
+				Name:     "small.txt",
179 
+				Mode:     0444,
180 
+				Uid:      73025,
181 
+				Gid:      5000,
182 
+				Size:     5,
183 
+				ModTime:  time.Unix(1244593104, 0),
184 
+				Typeflag: '\x00',
185 
+			},
186 
+			{
187 
+				Name:     "small2.txt",
188 
+				Mode:     0444,
189 
+				Uid:      73025,
190 
+				Gid:      5000,
191 
+				Size:     11,
192 
+				ModTime:  time.Unix(1244593104, 0),
193 
+				Typeflag: '\x00',
194 
+			},
195 
+		},
196 
+	},
197 
+	{
198 
+		file: "testdata/pax.tar",
199 
+		headers: []*Header{
200 
+			{
201 
+				Name:       "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100",
202 
+				Mode:       0664,
203 
+				Uid:        1000,
204 
+				Gid:        1000,
205 
+				Uname:      "shane",
206 
+				Gname:      "shane",
207 
+				Size:       7,
208 
+				ModTime:    time.Unix(1350244992, 23960108),
209 
+				ChangeTime: time.Unix(1350244992, 23960108),
210 
+				AccessTime: time.Unix(1350244992, 23960108),
211 
+				Typeflag:   TypeReg,
212 
+			},
213 
+			{
214 
+				Name:       "a/b",
215 
+				Mode:       0777,
216 
+				Uid:        1000,
217 
+				Gid:        1000,
218 
+				Uname:      "shane",
219 
+				Gname:      "shane",
220 
+				Size:       0,
221 
+				ModTime:    time.Unix(1350266320, 910238425),
222 
+				ChangeTime: time.Unix(1350266320, 910238425),
223 
+				AccessTime: time.Unix(1350266320, 910238425),
224 
+				Typeflag:   TypeSymlink,
225 
+				Linkname:   "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100",
226 
+			},
227 
+		},
228 
+	},
229 
+	{
230 
+		file: "testdata/nil-uid.tar", // golang.org/issue/5290
231 
+		headers: []*Header{
232 
+			{
233 
+				Name:     "P1050238.JPG.log",
234 
+				Mode:     0664,
235 
+				Uid:      0,
236 
+				Gid:      0,
237 
+				Size:     14,
238 
+				ModTime:  time.Unix(1365454838, 0),
239 
+				Typeflag: TypeReg,
240 
+				Linkname: "",
241 
+				Uname:    "eyefi",
242 
+				Gname:    "eyefi",
243 
+				Devmajor: 0,
244 
+				Devminor: 0,
245 
+			},
246 
+		},
247 
+	},
248 
+	{
249 
+		file: "testdata/xattrs.tar",
250 
+		headers: []*Header{
251 
+			{
252 
+				Name:       "small.txt",
253 
+				Mode:       0644,
254 
+				Uid:        1000,
255 
+				Gid:        10,
256 
+				Size:       5,
257 
+				ModTime:    time.Unix(1386065770, 448252320),
258 
+				Typeflag:   '0',
259 
+				Uname:      "alex",
260 
+				Gname:      "wheel",
261 
+				AccessTime: time.Unix(1389782991, 419875220),
262 
+				ChangeTime: time.Unix(1389782956, 794414986),
263 
+				Xattrs: map[string]string{
264 
+					"user.key":  "value",
265 
+					"user.key2": "value2",
266 
+					// Interestingly, selinux encodes the terminating null inside the xattr
267 
+					"security.selinux": "unconfined_u:object_r:default_t:s0\x00",
268 
+				},
269 
+			},
270 
+			{
271 
+				Name:       "small2.txt",
272 
+				Mode:       0644,
273 
+				Uid:        1000,
274 
+				Gid:        10,
275 
+				Size:       11,
276 
+				ModTime:    time.Unix(1386065770, 449252304),
277 
+				Typeflag:   '0',
278 
+				Uname:      "alex",
279 
+				Gname:      "wheel",
280 
+				AccessTime: time.Unix(1389782991, 419875220),
281 
+				ChangeTime: time.Unix(1386065770, 449252304),
282 
+				Xattrs: map[string]string{
283 
+					"security.selinux": "unconfined_u:object_r:default_t:s0\x00",
284 
+				},
285 
+			},
286 
+		},
287 
+	},
288 
+}
289 
+
290 
+func TestReader(t *testing.T) {
291 
+testLoop:
292 
+	for i, test := range untarTests {
293 
+		f, err := os.Open(test.file)
294 
+		if err != nil {
295 
+			t.Errorf("test %d: Unexpected error: %v", i, err)
296 
+			continue
297 
+		}
298 
+		defer f.Close()
299 
+		tr := NewReader(f)
300 
+		for j, header := range test.headers {
301 
+			hdr, err := tr.Next()
302 
+			if err != nil || hdr == nil {
303 
+				t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err)
304 
+				f.Close()
305 
+				continue testLoop
306 
+			}
307 
+			if !reflect.DeepEqual(*hdr, *header) {
308 
+				t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v",
309 
+					i, j, *hdr, *header)
310 
+			}
311 
+		}
312 
+		hdr, err := tr.Next()
313 
+		if err == io.EOF {
314 
+			continue testLoop
315 
+		}
316 
+		if hdr != nil || err != nil {
317 
+			t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, hdr, err)
318 
+		}
319 
+	}
320 
+}
321 
+
322 
+func TestPartialRead(t *testing.T) {
323 
+	f, err := os.Open("testdata/gnu.tar")
324 
+	if err != nil {
325 
+		t.Fatalf("Unexpected error: %v", err)
326 
+	}
327 
+	defer f.Close()
328 
+
329 
+	tr := NewReader(f)
330 
+
331 
+	// Read the first four bytes; Next() should skip the last byte.
332 
+	hdr, err := tr.Next()
333 
+	if err != nil || hdr == nil {
334 
+		t.Fatalf("Didn't get first file: %v", err)
335 
+	}
336 
+	buf := make([]byte, 4)
337 
+	if _, err := io.ReadFull(tr, buf); err != nil {
338 
+		t.Fatalf("Unexpected error: %v", err)
339 
+	}
340 
+	if expected := []byte("Kilt"); !bytes.Equal(buf, expected) {
341 
+		t.Errorf("Contents = %v, want %v", buf, expected)
342 
+	}
343 
+
344 
+	// Second file
345 
+	hdr, err = tr.Next()
346 
+	if err != nil || hdr == nil {
347 
+		t.Fatalf("Didn't get second file: %v", err)
348 
+	}
349 
+	buf = make([]byte, 6)
350 
+	if _, err := io.ReadFull(tr, buf); err != nil {
351 
+		t.Fatalf("Unexpected error: %v", err)
352 
+	}
353 
+	if expected := []byte("Google"); !bytes.Equal(buf, expected) {
354 
+		t.Errorf("Contents = %v, want %v", buf, expected)
355 
+	}
356 
+}
357 
+
358 
+func TestIncrementalRead(t *testing.T) {
359 
+	test := gnuTarTest
360 
+	f, err := os.Open(test.file)
361 
+	if err != nil {
362 
+		t.Fatalf("Unexpected error: %v", err)
363 
+	}
364 
+	defer f.Close()
365 
+
366 
+	tr := NewReader(f)
367 
+
368 
+	headers := test.headers
369 
+	cksums := test.cksums
370 
+	nread := 0
371 
+
372 
+	// loop over all files
373 
+	for ; ; nread++ {
374 
+		hdr, err := tr.Next()
375 
+		if hdr == nil || err == io.EOF {
376 
+			break
377 
+		}
378 
+
379 
+		// check the header
380 
+		if !reflect.DeepEqual(*hdr, *headers[nread]) {
381 
+			t.Errorf("Incorrect header:\nhave %+v\nwant %+v",
382 
+				*hdr, headers[nread])
383 
+		}
384 
+
385 
+		// read file contents in little chunks EOF,
386 
+		// checksumming all the way
387 
+		h := md5.New()
388 
+		rdbuf := make([]uint8, 8)
389 
+		for {
390 
+			nr, err := tr.Read(rdbuf)
391 
+			if err == io.EOF {
392 
+				break
393 
+			}
394 
+			if err != nil {
395 
+				t.Errorf("Read: unexpected error %v\n", err)
396 
+				break
397 
+			}
398 
+			h.Write(rdbuf[0:nr])
399 
+		}
400 
+		// verify checksum
401 
+		have := fmt.Sprintf("%x", h.Sum(nil))
402 
+		want := cksums[nread]
403 
+		if want != have {
404 
+			t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want)
405 
+		}
406 
+	}
407 
+	if nread != len(headers) {
408 
+		t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread)
409 
+	}
410 
+}
411 
+
412 
+func TestNonSeekable(t *testing.T) {
413 
+	test := gnuTarTest
414 
+	f, err := os.Open(test.file)
415 
+	if err != nil {
416 
+		t.Fatalf("Unexpected error: %v", err)
417 
+	}
418 
+	defer f.Close()
419 
+
420 
+	type readerOnly struct {
421 
+		io.Reader
422 
+	}
423 
+	tr := NewReader(readerOnly{f})
424 
+	nread := 0
425 
+
426 
+	for ; ; nread++ {
427 
+		_, err := tr.Next()
428 
+		if err == io.EOF {
429 
+			break
430 
+		}
431 
+		if err != nil {
432 
+			t.Fatalf("Unexpected error: %v", err)
433 
+		}
434 
+	}
435 
+
436 
+	if nread != len(test.headers) {
437 
+		t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(test.headers), nread)
438 
+	}
439 
+}
440 
+
441 
+func TestParsePAXHeader(t *testing.T) {
442 
+	paxTests := [][3]string{
443 
+		{"a", "a=name", "10 a=name\n"}, // Test case involving multiple acceptable lengths
444 
+		{"a", "a=name", "9 a=name\n"},  // Test case involving multiple acceptable length
445 
+		{"mtime", "mtime=1350244992.023960108", "30 mtime=1350244992.023960108\n"}}
446 
+	for _, test := range paxTests {
447 
+		key, expected, raw := test[0], test[1], test[2]
448 
+		reader := bytes.NewReader([]byte(raw))
449 
+		headers, err := parsePAX(reader)
450 
+		if err != nil {
451 
+			t.Errorf("Couldn't parse correctly formatted headers: %v", err)
452 
+			continue
453 
+		}
454 
+		if strings.EqualFold(headers[key], expected) {
455 
+			t.Errorf("mtime header incorrectly parsed: got %s, wanted %s", headers[key], expected)
456 
+			continue
457 
+		}
458 
+		trailer := make([]byte, 100)
459 
+		n, err := reader.Read(trailer)
460 
+		if err != io.EOF || n != 0 {
461 
+			t.Error("Buffer wasn't consumed")
462 
+		}
463 
+	}
464 
+	badHeader := bytes.NewReader([]byte("3 somelongkey="))
465 
+	if _, err := parsePAX(badHeader); err != ErrHeader {
466 
+		t.Fatal("Unexpected success when parsing bad header")
467 
+	}
468 
+}
469 
+
470 
+func TestParsePAXTime(t *testing.T) {
471 
+	// Some valid PAX time values
472 
+	timestamps := map[string]time.Time{
473 
+		"1350244992.023960108":  time.Unix(1350244992, 23960108), // The common case
474 
+		"1350244992.02396010":   time.Unix(1350244992, 23960100), // Lower precision value
475 
+		"1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value
476 
+		"1350244992":            time.Unix(1350244992, 0),        // Low precision value
477 
+	}
478 
+	for input, expected := range timestamps {
479 
+		ts, err := parsePAXTime(input)
480 
+		if err != nil {
481 
+			t.Fatal(err)
482 
+		}
483 
+		if !ts.Equal(expected) {
484 
+			t.Fatalf("Time parsing failure %s %s", ts, expected)
485 
+		}
486 
+	}
487 
+}
488 
+
489 
+func TestMergePAX(t *testing.T) {
490 
+	hdr := new(Header)
491 
+	// Test a string, integer, and time based value.
492 
+	headers := map[string]string{
493 
+		"path":  "a/b/c",
494 
+		"uid":   "1000",
495 
+		"mtime": "1350244992.023960108",
496 
+	}
497 
+	err := mergePAX(hdr, headers)
498 
+	if err != nil {
499 
+		t.Fatal(err)
500 
+	}
501 
+	want := &Header{
502 
+		Name:    "a/b/c",
503 
+		Uid:     1000,
504 
+		ModTime: time.Unix(1350244992, 23960108),
505 
+	}
506 
+	if !reflect.DeepEqual(hdr, want) {
507 
+		t.Errorf("incorrect merge: got %+v, want %+v", hdr, want)
508 
+	}
509 
+}
510 
+
511 
+func TestSparseEndToEnd(t *testing.T) {
512 
+	test := sparseTarTest
513 
+	f, err := os.Open(test.file)
514 
+	if err != nil {
515 
+		t.Fatalf("Unexpected error: %v", err)
516 
+	}
517 
+	defer f.Close()
518 
+
519 
+	tr := NewReader(f)
520 
+
521 
+	headers := test.headers
522 
+	cksums := test.cksums
523 
+	nread := 0
524 
+
525 
+	// loop over all files
526 
+	for ; ; nread++ {
527 
+		hdr, err := tr.Next()
528 
+		if hdr == nil || err == io.EOF {
529 
+			break
530 
+		}
531 
+
532 
+		// check the header
533 
+		if !reflect.DeepEqual(*hdr, *headers[nread]) {
534 
+			t.Errorf("Incorrect header:\nhave %+v\nwant %+v",
535 
+				*hdr, headers[nread])
536 
+		}
537 
+
538 
+		// read and checksum the file data
539 
+		h := md5.New()
540 
+		_, err = io.Copy(h, tr)
541 
+		if err != nil {
542 
+			t.Fatalf("Unexpected error: %v", err)
543 
+		}
544 
+
545 
+		// verify checksum
546 
+		have := fmt.Sprintf("%x", h.Sum(nil))
547 
+		want := cksums[nread]
548 
+		if want != have {
549 
+			t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want)
550 
+		}
551 
+	}
552 
+	if nread != len(headers) {
553 
+		t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread)
554 
+	}
555 
+}
556 
+
557 
+type sparseFileReadTest struct {
558 
+	sparseData []byte
559 
+	sparseMap  []sparseEntry
560 
+	realSize   int64
561 
+	expected   []byte
562 
+}
563 
+
564 
+var sparseFileReadTests = []sparseFileReadTest{
565 
+	{
566 
+		sparseData: []byte("abcde"),
567 
+		sparseMap: []sparseEntry{
568 
+			{offset: 0, numBytes: 2},
569 
+			{offset: 5, numBytes: 3},
570 
+		},
571 
+		realSize: 8,
572 
+		expected: []byte("ab\x00\x00\x00cde"),
573 
+	},
574 
+	{
575 
+		sparseData: []byte("abcde"),
576 
+		sparseMap: []sparseEntry{
577 
+			{offset: 0, numBytes: 2},
578 
+			{offset: 5, numBytes: 3},
579 
+		},
580 
+		realSize: 10,
581 
+		expected: []byte("ab\x00\x00\x00cde\x00\x00"),
582 
+	},
583 
+	{
584 
+		sparseData: []byte("abcde"),
585 
+		sparseMap: []sparseEntry{
586 
+			{offset: 1, numBytes: 3},
587 
+			{offset: 6, numBytes: 2},
588 
+		},
589 
+		realSize: 8,
590 
+		expected: []byte("\x00abc\x00\x00de"),
591 
+	},
592 
+	{
593 
+		sparseData: []byte("abcde"),
594 
+		sparseMap: []sparseEntry{
595 
+			{offset: 1, numBytes: 3},
596 
+			{offset: 6, numBytes: 2},
597 
+		},
598 
+		realSize: 10,
599 
+		expected: []byte("\x00abc\x00\x00de\x00\x00"),
600 
+	},
601 
+	{
602 
+		sparseData: []byte(""),
603 
+		sparseMap:  nil,
604 
+		realSize:   2,
605 
+		expected:   []byte("\x00\x00"),
606 
+	},
607 
+}
608 
+
609 
+func TestSparseFileReader(t *testing.T) {
610 
+	for i, test := range sparseFileReadTests {
611 
+		r := bytes.NewReader(test.sparseData)
612 
+		nb := int64(r.Len())
613 
+		sfr := &sparseFileReader{
614 
+			rfr: &regFileReader{r: r, nb: nb},
615 
+			sp:  test.sparseMap,
616 
+			pos: 0,
617 
+			tot: test.realSize,
618 
+		}
619 
+		if sfr.numBytes() != nb {
620 
+			t.Errorf("test %d: Before reading, sfr.numBytes() = %d, want %d", i, sfr.numBytes(), nb)
621 
+		}
622 
+		buf, err := ioutil.ReadAll(sfr)
623 
+		if err != nil {
624 
+			t.Errorf("test %d: Unexpected error: %v", i, err)
625 
+		}
626 
+		if e := test.expected; !bytes.Equal(buf, e) {
627 
+			t.Errorf("test %d: Contents = %v, want %v", i, buf, e)
628 
+		}
629 
+		if sfr.numBytes() != 0 {
630 
+			t.Errorf("test %d: After draining the reader, numBytes() was nonzero", i)
631 
+		}
632 
+	}
633 
+}
634 
+
635 
+func TestSparseIncrementalRead(t *testing.T) {
636 
+	sparseMap := []sparseEntry{{10, 2}}
637 
+	sparseData := []byte("Go")
638 
+	expected := "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Go\x00\x00\x00\x00\x00\x00\x00\x00"
639 
+
640 
+	r := bytes.NewReader(sparseData)
641 
+	nb := int64(r.Len())
642 
+	sfr := &sparseFileReader{
643 
+		rfr: &regFileReader{r: r, nb: nb},
644 
+		sp:  sparseMap,
645 
+		pos: 0,
646 
+		tot: int64(len(expected)),
647 
+	}
648 
+
649 
+	// We'll read the data 6 bytes at a time, with a hole of size 10 at
650 
+	// the beginning and one of size 8 at the end.
651 
+	var outputBuf bytes.Buffer
652 
+	buf := make([]byte, 6)
653 
+	for {
654 
+		n, err := sfr.Read(buf)
655 
+		if err == io.EOF {
656 
+			break
657 
+		}
658 
+		if err != nil {
659 
+			t.Errorf("Read: unexpected error %v\n", err)
660 
+		}
661 
+		if n > 0 {
662 
+			_, err := outputBuf.Write(buf[:n])
663 
+			if err != nil {
664 
+				t.Errorf("Write: unexpected error %v\n", err)
665 
+			}
666 
+		}
667 
+	}
668 
+	got := outputBuf.String()
669 
+	if got != expected {
670 
+		t.Errorf("Contents = %v, want %v", got, expected)
671 
+	}
672 
+}
673 
+
674 
+func TestReadGNUSparseMap0x1(t *testing.T) {
675 
+	headers := map[string]string{
676 
+		paxGNUSparseNumBlocks: "4",
677 
+		paxGNUSparseMap:       "0,5,10,5,20,5,30,5",
678 
+	}
679 
+	expected := []sparseEntry{
680 
+		{offset: 0, numBytes: 5},
681 
+		{offset: 10, numBytes: 5},
682 
+		{offset: 20, numBytes: 5},
683 
+		{offset: 30, numBytes: 5},
684 
+	}
685 
+
686 
+	sp, err := readGNUSparseMap0x1(headers)
687 
+	if err != nil {
688 
+		t.Errorf("Unexpected error: %v", err)
689 
+	}
690 
+	if !reflect.DeepEqual(sp, expected) {
691 
+		t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected)
692 
+	}
693 
+}
694 
+
695 
+func TestReadGNUSparseMap1x0(t *testing.T) {
696 
+	// This test uses lots of holes so the sparse header takes up more than two blocks
697 
+	numEntries := 100
698 
+	expected := make([]sparseEntry, 0, numEntries)
699 
+	sparseMap := new(bytes.Buffer)
700 
+
701 
+	fmt.Fprintf(sparseMap, "%d\n", numEntries)
702 
+	for i := 0; i < numEntries; i++ {
703 
+		offset := int64(2048 * i)
704 
+		numBytes := int64(1024)
705 
+		expected = append(expected, sparseEntry{offset: offset, numBytes: numBytes})
706 
+		fmt.Fprintf(sparseMap, "%d\n%d\n", offset, numBytes)
707 
+	}
708 
+
709 
+	// Make the header the smallest multiple of blockSize that fits the sparseMap
710 
+	headerBlocks := (sparseMap.Len() + blockSize - 1) / blockSize
711 
+	bufLen := blockSize * headerBlocks
712 
+	buf := make([]byte, bufLen)
713 
+	copy(buf, sparseMap.Bytes())
714 
+
715 
+	// Get an reader to read the sparse map
716 
+	r := bytes.NewReader(buf)
717 
+
718 
+	// Read the sparse map
719 
+	sp, err := readGNUSparseMap1x0(r)
720 
+	if err != nil {
721 
+		t.Errorf("Unexpected error: %v", err)
722 
+	}
723 
+	if !reflect.DeepEqual(sp, expected) {
724 
+		t.Errorf("Incorrect sparse map: got %v, wanted %v", sp, expected)
725 
+	}
726 
+}
727 
+
728 
+func TestUninitializedRead(t *testing.T) {
729 
+	test := gnuTarTest
730 
+	f, err := os.Open(test.file)
731 
+	if err != nil {
732 
+		t.Fatalf("Unexpected error: %v", err)
733 
+	}
734 
+	defer f.Close()
735 
+
736 
+	tr := NewReader(f)
737 
+	_, err = tr.Read([]byte{})
738 
+	if err == nil || err != io.EOF {
739 
+		t.Errorf("Unexpected error: %v, wanted %v", err, io.EOF)
740 
+	}
741 
+
742 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/stat_atim.go
History View file @ 7fb0618
0 743 
new file mode 100644
... ... 
@@ -0,0 +1,20 @@
0 
+// Copyright 2012 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+// +build linux dragonfly openbsd solaris
5 
+
6 
+package tar
7 
+
8 
+import (
9 
+	"syscall"
10 
+	"time"
11 
+)
12 
+
13 
+func statAtime(st *syscall.Stat_t) time.Time {
14 
+	return time.Unix(st.Atim.Unix())
15 
+}
16 
+
17 
+func statCtime(st *syscall.Stat_t) time.Time {
18 
+	return time.Unix(st.Ctim.Unix())
19 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/stat_atimespec.go
History View file @ 7fb0618
0 20 
new file mode 100644
... ... 
@@ -0,0 +1,20 @@
0 
+// Copyright 2012 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+// +build darwin freebsd netbsd
5 
+
6 
+package tar
7 
+
8 
+import (
9 
+	"syscall"
10 
+	"time"
11 
+)
12 
+
13 
+func statAtime(st *syscall.Stat_t) time.Time {
14 
+	return time.Unix(st.Atimespec.Unix())
15 
+}
16 
+
17 
+func statCtime(st *syscall.Stat_t) time.Time {
18 
+	return time.Unix(st.Ctimespec.Unix())
19 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/stat_unix.go
History View file @ 7fb0618
0 20 
new file mode 100644
... ... 
@@ -0,0 +1,32 @@
0 
+// Copyright 2012 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+// +build linux darwin dragonfly freebsd openbsd netbsd solaris
5 
+
6 
+package tar
7 
+
8 
+import (
9 
+	"os"
10 
+	"syscall"
11 
+)
12 
+
13 
+func init() {
14 
+	sysStat = statUnix
15 
+}
16 
+
17 
+func statUnix(fi os.FileInfo, h *Header) error {
18 
+	sys, ok := fi.Sys().(*syscall.Stat_t)
19 
+	if !ok {
20 
+		return nil
21 
+	}
22 
+	h.Uid = int(sys.Uid)
23 
+	h.Gid = int(sys.Gid)
24 
+	// TODO(bradfitz): populate username & group.  os/user
25 
+	// doesn't cache LookupId lookups, and lacks group
26 
+	// lookup functions.
27 
+	h.AccessTime = statAtime(sys)
28 
+	h.ChangeTime = statCtime(sys)
29 
+	// TODO(bradfitz): major/minor device numbers?
30 
+	return nil
31 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/tar_test.go
History View file @ 7fb0618
0 32 
new file mode 100644
... ... 
@@ -0,0 +1,284 @@
0 
+// Copyright 2012 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+package tar
5 
+
6 
+import (
7 
+	"bytes"
8 
+	"io/ioutil"
9 
+	"os"
10 
+	"path"
11 
+	"reflect"
12 
+	"strings"
13 
+	"testing"
14 
+	"time"
15 
+)
16 
+
17 
+func TestFileInfoHeader(t *testing.T) {
18 
+	fi, err := os.Stat("testdata/small.txt")
19 
+	if err != nil {
20 
+		t.Fatal(err)
21 
+	}
22 
+	h, err := FileInfoHeader(fi, "")
23 
+	if err != nil {
24 
+		t.Fatalf("FileInfoHeader: %v", err)
25 
+	}
26 
+	if g, e := h.Name, "small.txt"; g != e {
27 
+		t.Errorf("Name = %q; want %q", g, e)
28 
+	}
29 
+	if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e {
30 
+		t.Errorf("Mode = %#o; want %#o", g, e)
31 
+	}
32 
+	if g, e := h.Size, int64(5); g != e {
33 
+		t.Errorf("Size = %v; want %v", g, e)
34 
+	}
35 
+	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
36 
+		t.Errorf("ModTime = %v; want %v", g, e)
37 
+	}
38 
+	// FileInfoHeader should error when passing nil FileInfo
39 
+	if _, err := FileInfoHeader(nil, ""); err == nil {
40 
+		t.Fatalf("Expected error when passing nil to FileInfoHeader")
41 
+	}
42 
+}
43 
+
44 
+func TestFileInfoHeaderDir(t *testing.T) {
45 
+	fi, err := os.Stat("testdata")
46 
+	if err != nil {
47 
+		t.Fatal(err)
48 
+	}
49 
+	h, err := FileInfoHeader(fi, "")
50 
+	if err != nil {
51 
+		t.Fatalf("FileInfoHeader: %v", err)
52 
+	}
53 
+	if g, e := h.Name, "testdata/"; g != e {
54 
+		t.Errorf("Name = %q; want %q", g, e)
55 
+	}
56 
+	// Ignoring c_ISGID for golang.org/issue/4867
57 
+	if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e {
58 
+		t.Errorf("Mode = %#o; want %#o", g, e)
59 
+	}
60 
+	if g, e := h.Size, int64(0); g != e {
61 
+		t.Errorf("Size = %v; want %v", g, e)
62 
+	}
63 
+	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
64 
+		t.Errorf("ModTime = %v; want %v", g, e)
65 
+	}
66 
+}
67 
+
68 
+func TestFileInfoHeaderSymlink(t *testing.T) {
69 
+	h, err := FileInfoHeader(symlink{}, "some-target")
70 
+	if err != nil {
71 
+		t.Fatal(err)
72 
+	}
73 
+	if g, e := h.Name, "some-symlink"; g != e {
74 
+		t.Errorf("Name = %q; want %q", g, e)
75 
+	}
76 
+	if g, e := h.Linkname, "some-target"; g != e {
77 
+		t.Errorf("Linkname = %q; want %q", g, e)
78 
+	}
79 
+}
80 
+
81 
+type symlink struct{}
82 
+
83 
+func (symlink) Name() string       { return "some-symlink" }
84 
+func (symlink) Size() int64        { return 0 }
85 
+func (symlink) Mode() os.FileMode  { return os.ModeSymlink }
86 
+func (symlink) ModTime() time.Time { return time.Time{} }
87 
+func (symlink) IsDir() bool        { return false }
88 
+func (symlink) Sys() interface{}   { return nil }
89 
+
90 
+func TestRoundTrip(t *testing.T) {
91 
+	data := []byte("some file contents")
92 
+
93 
+	var b bytes.Buffer
94 
+	tw := NewWriter(&b)
95 
+	hdr := &Header{
96 
+		Name:    "file.txt",
97 
+		Uid:     1 << 21, // too big for 8 octal digits
98 
+		Size:    int64(len(data)),
99 
+		ModTime: time.Now(),
100 
+	}
101 
+	// tar only supports second precision.
102 
+	hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
103 
+	if err := tw.WriteHeader(hdr); err != nil {
104 
+		t.Fatalf("tw.WriteHeader: %v", err)
105 
+	}
106 
+	if _, err := tw.Write(data); err != nil {
107 
+		t.Fatalf("tw.Write: %v", err)
108 
+	}
109 
+	if err := tw.Close(); err != nil {
110 
+		t.Fatalf("tw.Close: %v", err)
111 
+	}
112 
+
113 
+	// Read it back.
114 
+	tr := NewReader(&b)
115 
+	rHdr, err := tr.Next()
116 
+	if err != nil {
117 
+		t.Fatalf("tr.Next: %v", err)
118 
+	}
119 
+	if !reflect.DeepEqual(rHdr, hdr) {
120 
+		t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr)
121 
+	}
122 
+	rData, err := ioutil.ReadAll(tr)
123 
+	if err != nil {
124 
+		t.Fatalf("Read: %v", err)
125 
+	}
126 
+	if !bytes.Equal(rData, data) {
127 
+		t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data)
128 
+	}
129 
+}
130 
+
131 
+type headerRoundTripTest struct {
132 
+	h  *Header
133 
+	fm os.FileMode
134 
+}
135 
+
136 
+func TestHeaderRoundTrip(t *testing.T) {
137 
+	golden := []headerRoundTripTest{
138 
+		// regular file.
139 
+		{
140 
+			h: &Header{
141 
+				Name:     "test.txt",
142 
+				Mode:     0644 | c_ISREG,
143 
+				Size:     12,
144 
+				ModTime:  time.Unix(1360600916, 0),
145 
+				Typeflag: TypeReg,
146 
+			},
147 
+			fm: 0644,
148 
+		},
149 
+		// hard link.
150 
+		{
151 
+			h: &Header{
152 
+				Name:     "hard.txt",
153 
+				Mode:     0644 | c_ISLNK,
154 
+				Size:     0,
155 
+				ModTime:  time.Unix(1360600916, 0),
156 
+				Typeflag: TypeLink,
157 
+			},
158 
+			fm: 0644 | os.ModeSymlink,
159 
+		},
160 
+		// symbolic link.
161 
+		{
162 
+			h: &Header{
163 
+				Name:     "link.txt",
164 
+				Mode:     0777 | c_ISLNK,
165 
+				Size:     0,
166 
+				ModTime:  time.Unix(1360600852, 0),
167 
+				Typeflag: TypeSymlink,
168 
+			},
169 
+			fm: 0777 | os.ModeSymlink,
170 
+		},
171 
+		// character device node.
172 
+		{
173 
+			h: &Header{
174 
+				Name:     "dev/null",
175 
+				Mode:     0666 | c_ISCHR,
176 
+				Size:     0,
177 
+				ModTime:  time.Unix(1360578951, 0),
178 
+				Typeflag: TypeChar,
179 
+			},
180 
+			fm: 0666 | os.ModeDevice | os.ModeCharDevice,
181 
+		},
182 
+		// block device node.
183 
+		{
184 
+			h: &Header{
185 
+				Name:     "dev/sda",
186 
+				Mode:     0660 | c_ISBLK,
187 
+				Size:     0,
188 
+				ModTime:  time.Unix(1360578954, 0),
189 
+				Typeflag: TypeBlock,
190 
+			},
191 
+			fm: 0660 | os.ModeDevice,
192 
+		},
193 
+		// directory.
194 
+		{
195 
+			h: &Header{
196 
+				Name:     "dir/",
197 
+				Mode:     0755 | c_ISDIR,
198 
+				Size:     0,
199 
+				ModTime:  time.Unix(1360601116, 0),
200 
+				Typeflag: TypeDir,
201 
+			},
202 
+			fm: 0755 | os.ModeDir,
203 
+		},
204 
+		// fifo node.
205 
+		{
206 
+			h: &Header{
207 
+				Name:     "dev/initctl",
208 
+				Mode:     0600 | c_ISFIFO,
209 
+				Size:     0,
210 
+				ModTime:  time.Unix(1360578949, 0),
211 
+				Typeflag: TypeFifo,
212 
+			},
213 
+			fm: 0600 | os.ModeNamedPipe,
214 
+		},
215 
+		// setuid.
216 
+		{
217 
+			h: &Header{
218 
+				Name:     "bin/su",
219 
+				Mode:     0755 | c_ISREG | c_ISUID,
220 
+				Size:     23232,
221 
+				ModTime:  time.Unix(1355405093, 0),
222 
+				Typeflag: TypeReg,
223 
+			},
224 
+			fm: 0755 | os.ModeSetuid,
225 
+		},
226 
+		// setguid.
227 
+		{
228 
+			h: &Header{
229 
+				Name:     "group.txt",
230 
+				Mode:     0750 | c_ISREG | c_ISGID,
231 
+				Size:     0,
232 
+				ModTime:  time.Unix(1360602346, 0),
233 
+				Typeflag: TypeReg,
234 
+			},
235 
+			fm: 0750 | os.ModeSetgid,
236 
+		},
237 
+		// sticky.
238 
+		{
239 
+			h: &Header{
240 
+				Name:     "sticky.txt",
241 
+				Mode:     0600 | c_ISREG | c_ISVTX,
242 
+				Size:     7,
243 
+				ModTime:  time.Unix(1360602540, 0),
244 
+				Typeflag: TypeReg,
245 
+			},
246 
+			fm: 0600 | os.ModeSticky,
247 
+		},
248 
+	}
249 
+
250 
+	for i, g := range golden {
251 
+		fi := g.h.FileInfo()
252 
+		h2, err := FileInfoHeader(fi, "")
253 
+		if err != nil {
254 
+			t.Error(err)
255 
+			continue
256 
+		}
257 
+		if strings.Contains(fi.Name(), "/") {
258 
+			t.Errorf("FileInfo of %q contains slash: %q", g.h.Name, fi.Name())
259 
+		}
260 
+		name := path.Base(g.h.Name)
261 
+		if fi.IsDir() {
262 
+			name += "/"
263 
+		}
264 
+		if got, want := h2.Name, name; got != want {
265 
+			t.Errorf("i=%d: Name: got %v, want %v", i, got, want)
266 
+		}
267 
+		if got, want := h2.Size, g.h.Size; got != want {
268 
+			t.Errorf("i=%d: Size: got %v, want %v", i, got, want)
269 
+		}
270 
+		if got, want := h2.Mode, g.h.Mode; got != want {
271 
+			t.Errorf("i=%d: Mode: got %o, want %o", i, got, want)
272 
+		}
273 
+		if got, want := fi.Mode(), g.fm; got != want {
274 
+			t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want)
275 
+		}
276 
+		if got, want := h2.ModTime, g.h.ModTime; got != want {
277 
+			t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want)
278 
+		}
279 
+		if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h {
280 
+			t.Errorf("i=%d: Sys didn't return original *Header", i)
281 
+		}
282 
+	}
283 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/gnu.tar
History View file @ 7fb0618
0 284 
new file mode 100644
1 285 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/gnu.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/nil-uid.tar
History View file @ 7fb0618
2 286 
new file mode 100644
3 287 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/nil-uid.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/pax.tar
History View file @ 7fb0618
4 288 
new file mode 100644
5 289 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/pax.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/small.txt
History View file @ 7fb0618
6 290 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+Kilts
0 1 
\ No newline at end of file
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/small2.txt
History View file @ 7fb0618
1 2 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+Google.com
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/sparse-formats.tar
History View file @ 7fb0618
0 1 
new file mode 100644
1 2 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/sparse-formats.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/star.tar
History View file @ 7fb0618
2 3 
new file mode 100644
3 4 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/star.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/ustar.tar
History View file @ 7fb0618
4 5 
new file mode 100644
5 6 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/ustar.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/v7.tar
History View file @ 7fb0618
6 7 
new file mode 100644
7 8 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/v7.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/writer-big-long.tar
History View file @ 7fb0618
8 9 
new file mode 100644
9 10 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/writer-big-long.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/writer-big.tar
History View file @ 7fb0618
10 11 
new file mode 100644
11 12 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/writer-big.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/writer.tar
History View file @ 7fb0618
12 13 
new file mode 100644
13 14 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/writer.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/xattrs.tar
History View file @ 7fb0618
14 15 
new file mode 100644
15 16 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/archive/tar/testdata/xattrs.tar differ
vendor/src/github.com/vbatts/tar-split/archive/tar/writer.go
History View file @ 7fb0618
16 17 
new file mode 100644
... ... 
@@ -0,0 +1,396 @@
0 
+// Copyright 2009 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+package tar
5 
+
6 
+// TODO(dsymonds):
7 
+// - catch more errors (no first header, etc.)
8 
+
9 
+import (
10 
+	"bytes"
11 
+	"errors"
12 
+	"fmt"
13 
+	"io"
14 
+	"os"
15 
+	"path"
16 
+	"strconv"
17 
+	"strings"
18 
+	"time"
19 
+)
20 
+
21 
+var (
22 
+	ErrWriteTooLong    = errors.New("archive/tar: write too long")
23 
+	ErrFieldTooLong    = errors.New("archive/tar: header field too long")
24 
+	ErrWriteAfterClose = errors.New("archive/tar: write after close")
25 
+	errNameTooLong     = errors.New("archive/tar: name too long")
26 
+	errInvalidHeader   = errors.New("archive/tar: header field too long or contains invalid values")
27 
+)
28 
+
29 
+// A Writer provides sequential writing of a tar archive in POSIX.1 format.
30 
+// A tar archive consists of a sequence of files.
31 
+// Call WriteHeader to begin a new file, and then call Write to supply that file's data,
32 
+// writing at most hdr.Size bytes in total.
33 
+type Writer struct {
34 
+	w          io.Writer
35 
+	err        error
36 
+	nb         int64 // number of unwritten bytes for current file entry
37 
+	pad        int64 // amount of padding to write after current file entry
38 
+	closed     bool
39 
+	usedBinary bool            // whether the binary numeric field extension was used
40 
+	preferPax  bool            // use pax header instead of binary numeric header
41 
+	hdrBuff    [blockSize]byte // buffer to use in writeHeader when writing a regular header
42 
+	paxHdrBuff [blockSize]byte // buffer to use in writeHeader when writing a pax header
43 
+}
44 
+
45 
+// NewWriter creates a new Writer writing to w.
46 
+func NewWriter(w io.Writer) *Writer { return &Writer{w: w} }
47 
+
48 
+// Flush finishes writing the current file (optional).
49 
+func (tw *Writer) Flush() error {
50 
+	if tw.nb > 0 {
51 
+		tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb)
52 
+		return tw.err
53 
+	}
54 
+
55 
+	n := tw.nb + tw.pad
56 
+	for n > 0 && tw.err == nil {
57 
+		nr := n
58 
+		if nr > blockSize {
59 
+			nr = blockSize
60 
+		}
61 
+		var nw int
62 
+		nw, tw.err = tw.w.Write(zeroBlock[0:nr])
63 
+		n -= int64(nw)
64 
+	}
65 
+	tw.nb = 0
66 
+	tw.pad = 0
67 
+	return tw.err
68 
+}
69 
+
70 
+// Write s into b, terminating it with a NUL if there is room.
71 
+// If the value is too long for the field and allowPax is true add a paxheader record instead
72 
+func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string, paxHeaders map[string]string) {
73 
+	needsPaxHeader := allowPax && len(s) > len(b) || !isASCII(s)
74 
+	if needsPaxHeader {
75 
+		paxHeaders[paxKeyword] = s
76 
+		return
77 
+	}
78 
+	if len(s) > len(b) {
79 
+		if tw.err == nil {
80 
+			tw.err = ErrFieldTooLong
81 
+		}
82 
+		return
83 
+	}
84 
+	ascii := toASCII(s)
85 
+	copy(b, ascii)
86 
+	if len(ascii) < len(b) {
87 
+		b[len(ascii)] = 0
88 
+	}
89 
+}
90 
+
91 
+// Encode x as an octal ASCII string and write it into b with leading zeros.
92 
+func (tw *Writer) octal(b []byte, x int64) {
93 
+	s := strconv.FormatInt(x, 8)
94 
+	// leading zeros, but leave room for a NUL.
95 
+	for len(s)+1 < len(b) {
96 
+		s = "0" + s
97 
+	}
98 
+	tw.cString(b, s, false, paxNone, nil)
99 
+}
100 
+
101 
+// Write x into b, either as octal or as binary (GNUtar/star extension).
102 
+// If the value is too long for the field and writingPax is enabled both for the field and the add a paxheader record instead
103 
+func (tw *Writer) numeric(b []byte, x int64, allowPax bool, paxKeyword string, paxHeaders map[string]string) {
104 
+	// Try octal first.
105 
+	s := strconv.FormatInt(x, 8)
106 
+	if len(s) < len(b) {
107 
+		tw.octal(b, x)
108 
+		return
109 
+	}
110 
+
111 
+	// If it is too long for octal, and pax is preferred, use a pax header
112 
+	if allowPax && tw.preferPax {
113 
+		tw.octal(b, 0)
114 
+		s := strconv.FormatInt(x, 10)
115 
+		paxHeaders[paxKeyword] = s
116 
+		return
117 
+	}
118 
+
119 
+	// Too big: use binary (big-endian).
120 
+	tw.usedBinary = true
121 
+	for i := len(b) - 1; x > 0 && i >= 0; i-- {
122 
+		b[i] = byte(x)
123 
+		x >>= 8
124 
+	}
125 
+	b[0] |= 0x80 // highest bit indicates binary format
126 
+}
127 
+
128 
+var (
129 
+	minTime = time.Unix(0, 0)
130 
+	// There is room for 11 octal digits (33 bits) of mtime.
131 
+	maxTime = minTime.Add((1<<33 - 1) * time.Second)
132 
+)
133 
+
134 
+// WriteHeader writes hdr and prepares to accept the file's contents.
135 
+// WriteHeader calls Flush if it is not the first header.
136 
+// Calling after a Close will return ErrWriteAfterClose.
137 
+func (tw *Writer) WriteHeader(hdr *Header) error {
138 
+	return tw.writeHeader(hdr, true)
139 
+}
140 
+
141 
+// WriteHeader writes hdr and prepares to accept the file's contents.
142 
+// WriteHeader calls Flush if it is not the first header.
143 
+// Calling after a Close will return ErrWriteAfterClose.
144 
+// As this method is called internally by writePax header to allow it to
145 
+// suppress writing the pax header.
146 
+func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
147 
+	if tw.closed {
148 
+		return ErrWriteAfterClose
149 
+	}
150 
+	if tw.err == nil {
151 
+		tw.Flush()
152 
+	}
153 
+	if tw.err != nil {
154 
+		return tw.err
155 
+	}
156 
+
157 
+	// a map to hold pax header records, if any are needed
158 
+	paxHeaders := make(map[string]string)
159 
+
160 
+	// TODO(shanemhansen): we might want to use PAX headers for
161 
+	// subsecond time resolution, but for now let's just capture
162 
+	// too long fields or non ascii characters
163 
+
164 
+	var header []byte
165 
+
166 
+	// We need to select which scratch buffer to use carefully,
167 
+	// since this method is called recursively to write PAX headers.
168 
+	// If allowPax is true, this is the non-recursive call, and we will use hdrBuff.
169 
+	// If allowPax is false, we are being called by writePAXHeader, and hdrBuff is
170 
+	// already being used by the non-recursive call, so we must use paxHdrBuff.
171 
+	header = tw.hdrBuff[:]
172 
+	if !allowPax {
173 
+		header = tw.paxHdrBuff[:]
174 
+	}
175 
+	copy(header, zeroBlock)
176 
+	s := slicer(header)
177 
+
178 
+	// keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
179 
+	pathHeaderBytes := s.next(fileNameSize)
180 
+
181 
+	tw.cString(pathHeaderBytes, hdr.Name, true, paxPath, paxHeaders)
182 
+
183 
+	// Handle out of range ModTime carefully.
184 
+	var modTime int64
185 
+	if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) {
186 
+		modTime = hdr.ModTime.Unix()
187 
+	}
188 
+
189 
+	tw.octal(s.next(8), hdr.Mode)                                   // 100:108
190 
+	tw.numeric(s.next(8), int64(hdr.Uid), true, paxUid, paxHeaders) // 108:116
191 
+	tw.numeric(s.next(8), int64(hdr.Gid), true, paxGid, paxHeaders) // 116:124
192 
+	tw.numeric(s.next(12), hdr.Size, true, paxSize, paxHeaders)     // 124:136
193 
+	tw.numeric(s.next(12), modTime, false, paxNone, nil)            // 136:148 --- consider using pax for finer granularity
194 
+	s.next(8)                                                       // chksum (148:156)
195 
+	s.next(1)[0] = hdr.Typeflag                                     // 156:157
196 
+
197 
+	tw.cString(s.next(100), hdr.Linkname, true, paxLinkpath, paxHeaders)
198 
+
199 
+	copy(s.next(8), []byte("ustar\x0000"))                        // 257:265
200 
+	tw.cString(s.next(32), hdr.Uname, true, paxUname, paxHeaders) // 265:297
201 
+	tw.cString(s.next(32), hdr.Gname, true, paxGname, paxHeaders) // 297:329
202 
+	tw.numeric(s.next(8), hdr.Devmajor, false, paxNone, nil)      // 329:337
203 
+	tw.numeric(s.next(8), hdr.Devminor, false, paxNone, nil)      // 337:345
204 
+
205 
+	// keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
206 
+	prefixHeaderBytes := s.next(155)
207 
+	tw.cString(prefixHeaderBytes, "", false, paxNone, nil) // 345:500  prefix
208 
+
209 
+	// Use the GNU magic instead of POSIX magic if we used any GNU extensions.
210 
+	if tw.usedBinary {
211 
+		copy(header[257:265], []byte("ustar  \x00"))
212 
+	}
213 
+
214 
+	_, paxPathUsed := paxHeaders[paxPath]
215 
+	// try to use a ustar header when only the name is too long
216 
+	if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
217 
+		suffix := hdr.Name
218 
+		prefix := ""
219 
+		if len(hdr.Name) > fileNameSize && isASCII(hdr.Name) {
220 
+			var err error
221 
+			prefix, suffix, err = tw.splitUSTARLongName(hdr.Name)
222 
+			if err == nil {
223 
+				// ok we can use a ustar long name instead of pax, now correct the fields
224 
+
225 
+				// remove the path field from the pax header. this will suppress the pax header
226 
+				delete(paxHeaders, paxPath)
227 
+
228 
+				// update the path fields
229 
+				tw.cString(pathHeaderBytes, suffix, false, paxNone, nil)
230 
+				tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil)
231 
+
232 
+				// Use the ustar magic if we used ustar long names.
233 
+				if len(prefix) > 0 && !tw.usedBinary {
234 
+					copy(header[257:265], []byte("ustar\x00"))
235 
+				}
236 
+			}
237 
+		}
238 
+	}
239 
+
240 
+	// The chksum field is terminated by a NUL and a space.
241 
+	// This is different from the other octal fields.
242 
+	chksum, _ := checksum(header)
243 
+	tw.octal(header[148:155], chksum)
244 
+	header[155] = ' '
245 
+
246 
+	if tw.err != nil {
247 
+		// problem with header; probably integer too big for a field.
248 
+		return tw.err
249 
+	}
250 
+
251 
+	if allowPax {
252 
+		for k, v := range hdr.Xattrs {
253 
+			paxHeaders[paxXattr+k] = v
254 
+		}
255 
+	}
256 
+
257 
+	if len(paxHeaders) > 0 {
258 
+		if !allowPax {
259 
+			return errInvalidHeader
260 
+		}
261 
+		if err := tw.writePAXHeader(hdr, paxHeaders); err != nil {
262 
+			return err
263 
+		}
264 
+	}
265 
+	tw.nb = int64(hdr.Size)
266 
+	tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
267 
+
268 
+	_, tw.err = tw.w.Write(header)
269 
+	return tw.err
270 
+}
271 
+
272 
+// writeUSTARLongName splits a USTAR long name hdr.Name.
273 
+// name must be < 256 characters. errNameTooLong is returned
274 
+// if hdr.Name can't be split. The splitting heuristic
275 
+// is compatible with gnu tar.
276 
+func (tw *Writer) splitUSTARLongName(name string) (prefix, suffix string, err error) {
277 
+	length := len(name)
278 
+	if length > fileNamePrefixSize+1 {
279 
+		length = fileNamePrefixSize + 1
280 
+	} else if name[length-1] == '/' {
281 
+		length--
282 
+	}
283 
+	i := strings.LastIndex(name[:length], "/")
284 
+	// nlen contains the resulting length in the name field.
285 
+	// plen contains the resulting length in the prefix field.
286 
+	nlen := len(name) - i - 1
287 
+	plen := i
288 
+	if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize {
289 
+		err = errNameTooLong
290 
+		return
291 
+	}
292 
+	prefix, suffix = name[:i], name[i+1:]
293 
+	return
294 
+}
295 
+
296 
+// writePaxHeader writes an extended pax header to the
297 
+// archive.
298 
+func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error {
299 
+	// Prepare extended header
300 
+	ext := new(Header)
301 
+	ext.Typeflag = TypeXHeader
302 
+	// Setting ModTime is required for reader parsing to
303 
+	// succeed, and seems harmless enough.
304 
+	ext.ModTime = hdr.ModTime
305 
+	// The spec asks that we namespace our pseudo files
306 
+	// with the current pid.
307 
+	pid := os.Getpid()
308 
+	dir, file := path.Split(hdr.Name)
309 
+	fullName := path.Join(dir,
310 
+		fmt.Sprintf("PaxHeaders.%d", pid), file)
311 
+
312 
+	ascii := toASCII(fullName)
313 
+	if len(ascii) > 100 {
314 
+		ascii = ascii[:100]
315 
+	}
316 
+	ext.Name = ascii
317 
+	// Construct the body
318 
+	var buf bytes.Buffer
319 
+
320 
+	for k, v := range paxHeaders {
321 
+		fmt.Fprint(&buf, paxHeader(k+"="+v))
322 
+	}
323 
+
324 
+	ext.Size = int64(len(buf.Bytes()))
325 
+	if err := tw.writeHeader(ext, false); err != nil {
326 
+		return err
327 
+	}
328 
+	if _, err := tw.Write(buf.Bytes()); err != nil {
329 
+		return err
330 
+	}
331 
+	if err := tw.Flush(); err != nil {
332 
+		return err
333 
+	}
334 
+	return nil
335 
+}
336 
+
337 
+// paxHeader formats a single pax record, prefixing it with the appropriate length
338 
+func paxHeader(msg string) string {
339 
+	const padding = 2 // Extra padding for space and newline
340 
+	size := len(msg) + padding
341 
+	size += len(strconv.Itoa(size))
342 
+	record := fmt.Sprintf("%d %s\n", size, msg)
343 
+	if len(record) != size {
344 
+		// Final adjustment if adding size increased
345 
+		// the number of digits in size
346 
+		size = len(record)
347 
+		record = fmt.Sprintf("%d %s\n", size, msg)
348 
+	}
349 
+	return record
350 
+}
351 
+
352 
+// Write writes to the current entry in the tar archive.
353 
+// Write returns the error ErrWriteTooLong if more than
354 
+// hdr.Size bytes are written after WriteHeader.
355 
+func (tw *Writer) Write(b []byte) (n int, err error) {
356 
+	if tw.closed {
357 
+		err = ErrWriteTooLong
358 
+		return
359 
+	}
360 
+	overwrite := false
361 
+	if int64(len(b)) > tw.nb {
362 
+		b = b[0:tw.nb]
363 
+		overwrite = true
364 
+	}
365 
+	n, err = tw.w.Write(b)
366 
+	tw.nb -= int64(n)
367 
+	if err == nil && overwrite {
368 
+		err = ErrWriteTooLong
369 
+		return
370 
+	}
371 
+	tw.err = err
372 
+	return
373 
+}
374 
+
375 
+// Close closes the tar archive, flushing any unwritten
376 
+// data to the underlying writer.
377 
+func (tw *Writer) Close() error {
378 
+	if tw.err != nil || tw.closed {
379 
+		return tw.err
380 
+	}
381 
+	tw.Flush()
382 
+	tw.closed = true
383 
+	if tw.err != nil {
384 
+		return tw.err
385 
+	}
386 
+
387 
+	// trailer: two zero blocks
388 
+	for i := 0; i < 2; i++ {
389 
+		_, tw.err = tw.w.Write(zeroBlock)
390 
+		if tw.err != nil {
391 
+			break
392 
+		}
393 
+	}
394 
+	return tw.err
395 
+}
vendor/src/github.com/vbatts/tar-split/archive/tar/writer_test.go
History View file @ 7fb0618
0 396 
new file mode 100644
... ... 
@@ -0,0 +1,491 @@
0 
+// Copyright 2009 The Go Authors. All rights reserved.
1 
+// Use of this source code is governed by a BSD-style
2 
+// license that can be found in the LICENSE file.
3 
+
4 
+package tar
5 
+
6 
+import (
7 
+	"bytes"
8 
+	"fmt"
9 
+	"io"
10 
+	"io/ioutil"
11 
+	"os"
12 
+	"reflect"
13 
+	"strings"
14 
+	"testing"
15 
+	"testing/iotest"
16 
+	"time"
17 
+)
18 
+
19 
+type writerTestEntry struct {
20 
+	header   *Header
21 
+	contents string
22 
+}
23 
+
24 
+type writerTest struct {
25 
+	file    string // filename of expected output
26 
+	entries []*writerTestEntry
27 
+}
28 
+
29 
+var writerTests = []*writerTest{
30 
+	// The writer test file was produced with this command:
31 
+	// tar (GNU tar) 1.26
32 
+	//   ln -s small.txt link.txt
33 
+	//   tar -b 1 --format=ustar -c -f writer.tar small.txt small2.txt link.txt
34 
+	{
35 
+		file: "testdata/writer.tar",
36 
+		entries: []*writerTestEntry{
37 
+			{
38 
+				header: &Header{
39 
+					Name:     "small.txt",
40 
+					Mode:     0640,
41 
+					Uid:      73025,
42 
+					Gid:      5000,
43 
+					Size:     5,
44 
+					ModTime:  time.Unix(1246508266, 0),
45 
+					Typeflag: '0',
46 
+					Uname:    "dsymonds",
47 
+					Gname:    "eng",
48 
+				},
49 
+				contents: "Kilts",
50 
+			},
51 
+			{
52 
+				header: &Header{
53 
+					Name:     "small2.txt",
54 
+					Mode:     0640,
55 
+					Uid:      73025,
56 
+					Gid:      5000,
57 
+					Size:     11,
58 
+					ModTime:  time.Unix(1245217492, 0),
59 
+					Typeflag: '0',
60 
+					Uname:    "dsymonds",
61 
+					Gname:    "eng",
62 
+				},
63 
+				contents: "Google.com\n",
64 
+			},
65 
+			{
66 
+				header: &Header{
67 
+					Name:     "link.txt",
68 
+					Mode:     0777,
69 
+					Uid:      1000,
70 
+					Gid:      1000,
71 
+					Size:     0,
72 
+					ModTime:  time.Unix(1314603082, 0),
73 
+					Typeflag: '2',
74 
+					Linkname: "small.txt",
75 
+					Uname:    "strings",
76 
+					Gname:    "strings",
77 
+				},
78 
+				// no contents
79 
+			},
80 
+		},
81 
+	},
82 
+	// The truncated test file was produced using these commands:
83 
+	//   dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt
84 
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
85 
+	{
86 
+		file: "testdata/writer-big.tar",
87 
+		entries: []*writerTestEntry{
88 
+			{
89 
+				header: &Header{
90 
+					Name:     "tmp/16gig.txt",
91 
+					Mode:     0640,
92 
+					Uid:      73025,
93 
+					Gid:      5000,
94 
+					Size:     16 << 30,
95 
+					ModTime:  time.Unix(1254699560, 0),
96 
+					Typeflag: '0',
97 
+					Uname:    "dsymonds",
98 
+					Gname:    "eng",
99 
+				},
100 
+				// fake contents
101 
+				contents: strings.Repeat("\x00", 4<<10),
102 
+			},
103 
+		},
104 
+	},
105 
+	// The truncated test file was produced using these commands:
106 
+	//   dd if=/dev/zero bs=1048576 count=16384 > (longname/)*15 /16gig.txt
107 
+	//   tar -b 1 -c -f- (longname/)*15 /16gig.txt | dd bs=512 count=8 > writer-big-long.tar
108 
+	{
109 
+		file: "testdata/writer-big-long.tar",
110 
+		entries: []*writerTestEntry{
111 
+			{
112 
+				header: &Header{
113 
+					Name:     strings.Repeat("longname/", 15) + "16gig.txt",
114 
+					Mode:     0644,
115 
+					Uid:      1000,
116 
+					Gid:      1000,
117 
+					Size:     16 << 30,
118 
+					ModTime:  time.Unix(1399583047, 0),
119 
+					Typeflag: '0',
120 
+					Uname:    "guillaume",
121 
+					Gname:    "guillaume",
122 
+				},
123 
+				// fake contents
124 
+				contents: strings.Repeat("\x00", 4<<10),
125 
+			},
126 
+		},
127 
+	},
128 
+	// This file was produced using gnu tar 1.17
129 
+	// gnutar  -b 4 --format=ustar (longname/)*15 + file.txt
130 
+	{
131 
+		file: "testdata/ustar.tar",
132 
+		entries: []*writerTestEntry{
133 
+			{
134 
+				header: &Header{
135 
+					Name:     strings.Repeat("longname/", 15) + "file.txt",
136 
+					Mode:     0644,
137 
+					Uid:      0765,
138 
+					Gid:      024,
139 
+					Size:     06,
140 
+					ModTime:  time.Unix(1360135598, 0),
141 
+					Typeflag: '0',
142 
+					Uname:    "shane",
143 
+					Gname:    "staff",
144 
+				},
145 
+				contents: "hello\n",
146 
+			},
147 
+		},
148 
+	},
149 
+}
150 
+
151 
+// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection.
152 
+func bytestr(offset int, b []byte) string {
153 
+	const rowLen = 32
154 
+	s := fmt.Sprintf("%04x ", offset)
155 
+	for _, ch := range b {
156 
+		switch {
157 
+		case '0' <= ch && ch <= '9', 'A' <= ch && ch <= 'Z', 'a' <= ch && ch <= 'z':
158 
+			s += fmt.Sprintf("  %c", ch)
159 
+		default:
160 
+			s += fmt.Sprintf(" %02x", ch)
161 
+		}
162 
+	}
163 
+	return s
164 
+}
165 
+
166 
+// Render a pseudo-diff between two blocks of bytes.
167 
+func bytediff(a []byte, b []byte) string {
168 
+	const rowLen = 32
169 
+	s := fmt.Sprintf("(%d bytes vs. %d bytes)\n", len(a), len(b))
170 
+	for offset := 0; len(a)+len(b) > 0; offset += rowLen {
171 
+		na, nb := rowLen, rowLen
172 
+		if na > len(a) {
173 
+			na = len(a)
174 
+		}
175 
+		if nb > len(b) {
176 
+			nb = len(b)
177 
+		}
178 
+		sa := bytestr(offset, a[0:na])
179 
+		sb := bytestr(offset, b[0:nb])
180 
+		if sa != sb {
181 
+			s += fmt.Sprintf("-%v\n+%v\n", sa, sb)
182 
+		}
183 
+		a = a[na:]
184 
+		b = b[nb:]
185 
+	}
186 
+	return s
187 
+}
188 
+
189 
+func TestWriter(t *testing.T) {
190 
+testLoop:
191 
+	for i, test := range writerTests {
192 
+		expected, err := ioutil.ReadFile(test.file)
193 
+		if err != nil {
194 
+			t.Errorf("test %d: Unexpected error: %v", i, err)
195 
+			continue
196 
+		}
197 
+
198 
+		buf := new(bytes.Buffer)
199 
+		tw := NewWriter(iotest.TruncateWriter(buf, 4<<10)) // only catch the first 4 KB
200 
+		big := false
201 
+		for j, entry := range test.entries {
202 
+			big = big || entry.header.Size > 1<<10
203 
+			if err := tw.WriteHeader(entry.header); err != nil {
204 
+				t.Errorf("test %d, entry %d: Failed writing header: %v", i, j, err)
205 
+				continue testLoop
206 
+			}
207 
+			if _, err := io.WriteString(tw, entry.contents); err != nil {
208 
+				t.Errorf("test %d, entry %d: Failed writing contents: %v", i, j, err)
209 
+				continue testLoop
210 
+			}
211 
+		}
212 
+		// Only interested in Close failures for the small tests.
213 
+		if err := tw.Close(); err != nil && !big {
214 
+			t.Errorf("test %d: Failed closing archive: %v", i, err)
215 
+			continue testLoop
216 
+		}
217 
+
218 
+		actual := buf.Bytes()
219 
+		if !bytes.Equal(expected, actual) {
220 
+			t.Errorf("test %d: Incorrect result: (-=expected, +=actual)\n%v",
221 
+				i, bytediff(expected, actual))
222 
+		}
223 
+		if testing.Short() { // The second test is expensive.
224 
+			break
225 
+		}
226 
+	}
227 
+}
228 
+
229 
+func TestPax(t *testing.T) {
230 
+	// Create an archive with a large name
231 
+	fileinfo, err := os.Stat("testdata/small.txt")
232 
+	if err != nil {
233 
+		t.Fatal(err)
234 
+	}
235 
+	hdr, err := FileInfoHeader(fileinfo, "")
236 
+	if err != nil {
237 
+		t.Fatalf("os.Stat: %v", err)
238 
+	}
239 
+	// Force a PAX long name to be written
240 
+	longName := strings.Repeat("ab", 100)
241 
+	contents := strings.Repeat(" ", int(hdr.Size))
242 
+	hdr.Name = longName
243 
+	var buf bytes.Buffer
244 
+	writer := NewWriter(&buf)
245 
+	if err := writer.WriteHeader(hdr); err != nil {
246 
+		t.Fatal(err)
247 
+	}
248 
+	if _, err = writer.Write([]byte(contents)); err != nil {
249 
+		t.Fatal(err)
250 
+	}
251 
+	if err := writer.Close(); err != nil {
252 
+		t.Fatal(err)
253 
+	}
254 
+	// Simple test to make sure PAX extensions are in effect
255 
+	if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
256 
+		t.Fatal("Expected at least one PAX header to be written.")
257 
+	}
258 
+	// Test that we can get a long name back out of the archive.
259 
+	reader := NewReader(&buf)
260 
+	hdr, err = reader.Next()
261 
+	if err != nil {
262 
+		t.Fatal(err)
263 
+	}
264 
+	if hdr.Name != longName {
265 
+		t.Fatal("Couldn't recover long file name")
266 
+	}
267 
+}
268 
+
269 
+func TestPaxSymlink(t *testing.T) {
270 
+	// Create an archive with a large linkname
271 
+	fileinfo, err := os.Stat("testdata/small.txt")
272 
+	if err != nil {
273 
+		t.Fatal(err)
274 
+	}
275 
+	hdr, err := FileInfoHeader(fileinfo, "")
276 
+	hdr.Typeflag = TypeSymlink
277 
+	if err != nil {
278 
+		t.Fatalf("os.Stat:1 %v", err)
279 
+	}
280 
+	// Force a PAX long linkname to be written
281 
+	longLinkname := strings.Repeat("1234567890/1234567890", 10)
282 
+	hdr.Linkname = longLinkname
283 
+
284 
+	hdr.Size = 0
285 
+	var buf bytes.Buffer
286 
+	writer := NewWriter(&buf)
287 
+	if err := writer.WriteHeader(hdr); err != nil {
288 
+		t.Fatal(err)
289 
+	}
290 
+	if err := writer.Close(); err != nil {
291 
+		t.Fatal(err)
292 
+	}
293 
+	// Simple test to make sure PAX extensions are in effect
294 
+	if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
295 
+		t.Fatal("Expected at least one PAX header to be written.")
296 
+	}
297 
+	// Test that we can get a long name back out of the archive.
298 
+	reader := NewReader(&buf)
299 
+	hdr, err = reader.Next()
300 
+	if err != nil {
301 
+		t.Fatal(err)
302 
+	}
303 
+	if hdr.Linkname != longLinkname {
304 
+		t.Fatal("Couldn't recover long link name")
305 
+	}
306 
+}
307 
+
308 
+func TestPaxNonAscii(t *testing.T) {
309 
+	// Create an archive with non ascii. These should trigger a pax header
310 
+	// because pax headers have a defined utf-8 encoding.
311 
+	fileinfo, err := os.Stat("testdata/small.txt")
312 
+	if err != nil {
313 
+		t.Fatal(err)
314 
+	}
315 
+
316 
+	hdr, err := FileInfoHeader(fileinfo, "")
317 
+	if err != nil {
318 
+		t.Fatalf("os.Stat:1 %v", err)
319 
+	}
320 
+
321 
+	// some sample data
322 
+	chineseFilename := "文件名"
323 
+	chineseGroupname := "組"
324 
+	chineseUsername := "用戶名"
325 
+
326 
+	hdr.Name = chineseFilename
327 
+	hdr.Gname = chineseGroupname
328 
+	hdr.Uname = chineseUsername
329 
+
330 
+	contents := strings.Repeat(" ", int(hdr.Size))
331 
+
332 
+	var buf bytes.Buffer
333 
+	writer := NewWriter(&buf)
334 
+	if err := writer.WriteHeader(hdr); err != nil {
335 
+		t.Fatal(err)
336 
+	}
337 
+	if _, err = writer.Write([]byte(contents)); err != nil {
338 
+		t.Fatal(err)
339 
+	}
340 
+	if err := writer.Close(); err != nil {
341 
+		t.Fatal(err)
342 
+	}
343 
+	// Simple test to make sure PAX extensions are in effect
344 
+	if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
345 
+		t.Fatal("Expected at least one PAX header to be written.")
346 
+	}
347 
+	// Test that we can get a long name back out of the archive.
348 
+	reader := NewReader(&buf)
349 
+	hdr, err = reader.Next()
350 
+	if err != nil {
351 
+		t.Fatal(err)
352 
+	}
353 
+	if hdr.Name != chineseFilename {
354 
+		t.Fatal("Couldn't recover unicode name")
355 
+	}
356 
+	if hdr.Gname != chineseGroupname {
357 
+		t.Fatal("Couldn't recover unicode group")
358 
+	}
359 
+	if hdr.Uname != chineseUsername {
360 
+		t.Fatal("Couldn't recover unicode user")
361 
+	}
362 
+}
363 
+
364 
+func TestPaxXattrs(t *testing.T) {
365 
+	xattrs := map[string]string{
366 
+		"user.key": "value",
367 
+	}
368 
+
369 
+	// Create an archive with an xattr
370 
+	fileinfo, err := os.Stat("testdata/small.txt")
371 
+	if err != nil {
372 
+		t.Fatal(err)
373 
+	}
374 
+	hdr, err := FileInfoHeader(fileinfo, "")
375 
+	if err != nil {
376 
+		t.Fatalf("os.Stat: %v", err)
377 
+	}
378 
+	contents := "Kilts"
379 
+	hdr.Xattrs = xattrs
380 
+	var buf bytes.Buffer
381 
+	writer := NewWriter(&buf)
382 
+	if err := writer.WriteHeader(hdr); err != nil {
383 
+		t.Fatal(err)
384 
+	}
385 
+	if _, err = writer.Write([]byte(contents)); err != nil {
386 
+		t.Fatal(err)
387 
+	}
388 
+	if err := writer.Close(); err != nil {
389 
+		t.Fatal(err)
390 
+	}
391 
+	// Test that we can get the xattrs back out of the archive.
392 
+	reader := NewReader(&buf)
393 
+	hdr, err = reader.Next()
394 
+	if err != nil {
395 
+		t.Fatal(err)
396 
+	}
397 
+	if !reflect.DeepEqual(hdr.Xattrs, xattrs) {
398 
+		t.Fatalf("xattrs did not survive round trip: got %+v, want %+v",
399 
+			hdr.Xattrs, xattrs)
400 
+	}
401 
+}
402 
+
403 
+func TestPAXHeader(t *testing.T) {
404 
+	medName := strings.Repeat("CD", 50)
405 
+	longName := strings.Repeat("AB", 100)
406 
+	paxTests := [][2]string{
407 
+		{paxPath + "=/etc/hosts", "19 path=/etc/hosts\n"},
408 
+		{"a=b", "6 a=b\n"},          // Single digit length
409 
+		{"a=names", "11 a=names\n"}, // Test case involving carries
410 
+		{paxPath + "=" + longName, fmt.Sprintf("210 path=%s\n", longName)},
411 
+		{paxPath + "=" + medName, fmt.Sprintf("110 path=%s\n", medName)}}
412 
+
413 
+	for _, test := range paxTests {
414 
+		key, expected := test[0], test[1]
415 
+		if result := paxHeader(key); result != expected {
416 
+			t.Fatalf("paxHeader: got %s, expected %s", result, expected)
417 
+		}
418 
+	}
419 
+}
420 
+
421 
+func TestUSTARLongName(t *testing.T) {
422 
+	// Create an archive with a path that failed to split with USTAR extension in previous versions.
423 
+	fileinfo, err := os.Stat("testdata/small.txt")
424 
+	if err != nil {
425 
+		t.Fatal(err)
426 
+	}
427 
+	hdr, err := FileInfoHeader(fileinfo, "")
428 
+	hdr.Typeflag = TypeDir
429 
+	if err != nil {
430 
+		t.Fatalf("os.Stat:1 %v", err)
431 
+	}
432 
+	// Force a PAX long name to be written. The name was taken from a practical example
433 
+	// that fails and replaced ever char through numbers to anonymize the sample.
434 
+	longName := "/0000_0000000/00000-000000000/0000_0000000/00000-0000000000000/0000_0000000/00000-0000000-00000000/0000_0000000/00000000/0000_0000000/000/0000_0000000/00000000v00/0000_0000000/000000/0000_0000000/0000000/0000_0000000/00000y-00/0000/0000/00000000/0x000000/"
435 
+	hdr.Name = longName
436 
+
437 
+	hdr.Size = 0
438 
+	var buf bytes.Buffer
439 
+	writer := NewWriter(&buf)
440 
+	if err := writer.WriteHeader(hdr); err != nil {
441 
+		t.Fatal(err)
442 
+	}
443 
+	if err := writer.Close(); err != nil {
444 
+		t.Fatal(err)
445 
+	}
446 
+	// Test that we can get a long name back out of the archive.
447 
+	reader := NewReader(&buf)
448 
+	hdr, err = reader.Next()
449 
+	if err != nil {
450 
+		t.Fatal(err)
451 
+	}
452 
+	if hdr.Name != longName {
453 
+		t.Fatal("Couldn't recover long name")
454 
+	}
455 
+}
456 
+
457 
+func TestValidTypeflagWithPAXHeader(t *testing.T) {
458 
+	var buffer bytes.Buffer
459 
+	tw := NewWriter(&buffer)
460 
+
461 
+	fileName := strings.Repeat("ab", 100)
462 
+
463 
+	hdr := &Header{
464 
+		Name:     fileName,
465 
+		Size:     4,
466 
+		Typeflag: 0,
467 
+	}
468 
+	if err := tw.WriteHeader(hdr); err != nil {
469 
+		t.Fatalf("Failed to write header: %s", err)
470 
+	}
471 
+	if _, err := tw.Write([]byte("fooo")); err != nil {
472 
+		t.Fatalf("Failed to write the file's data: %s", err)
473 
+	}
474 
+	tw.Close()
475 
+
476 
+	tr := NewReader(&buffer)
477 
+
478 
+	for {
479 
+		header, err := tr.Next()
480 
+		if err == io.EOF {
481 
+			break
482 
+		}
483 
+		if err != nil {
484 
+			t.Fatalf("Failed to read header: %s", err)
485 
+		}
486 
+		if header.Typeflag != 0 {
487 
+			t.Fatalf("Typeflag should've been 0, found %d", header.Typeflag)
488 
+		}
489 
+	}
490 
+}
vendor/src/github.com/vbatts/tar-split/checksize.go
History View file @ 7fb0618
0 491 
new file mode 100644
... ... 
@@ -0,0 +1,111 @@
0 
+// +build ignore
1 
+
2 
+package main
3 
+
4 
+import (
5 
+	"archive/tar"
6 
+	"compress/gzip"
7 
+	"flag"
8 
+	"fmt"
9 
+	"io"
10 
+	"io/ioutil"
11 
+	"log"
12 
+	"os"
13 
+
14 
+	"github.com/vbatts/tar-split/tar/asm"
15 
+	"github.com/vbatts/tar-split/tar/storage"
16 
+)
17 
+
18 
+var (
19 
+	flCleanup = flag.Bool("cleanup", true, "cleanup tempfiles")
20 
+)
21 
+
22 
+func main() {
23 
+	flag.Parse()
24 
+
25 
+	for _, arg := range flag.Args() {
26 
+		fh, err := os.Open(arg)
27 
+		if err != nil {
28 
+			log.Fatal(err)
29 
+		}
30 
+		defer fh.Close()
31 
+		fi, err := fh.Stat()
32 
+		if err != nil {
33 
+			log.Fatal(err)
34 
+		}
35 
+		fmt.Printf("inspecting %q (size %dk)\n", fh.Name(), fi.Size()/1024)
36 
+
37 
+		packFh, err := ioutil.TempFile("", "packed.")
38 
+		if err != nil {
39 
+			log.Fatal(err)
40 
+		}
41 
+		defer packFh.Close()
42 
+		if *flCleanup {
43 
+			defer os.Remove(packFh.Name())
44 
+		}
45 
+
46 
+		sp := storage.NewJSONPacker(packFh)
47 
+		fp := storage.NewDiscardFilePutter()
48 
+		dissam, err := asm.NewInputTarStream(fh, sp, fp)
49 
+		if err != nil {
50 
+			log.Fatal(err)
51 
+		}
52 
+
53 
+		var num int
54 
+		tr := tar.NewReader(dissam)
55 
+		for {
56 
+			_, err = tr.Next()
57 
+			if err != nil {
58 
+				if err == io.EOF {
59 
+					break
60 
+				}
61 
+				log.Fatal(err)
62 
+			}
63 
+			num++
64 
+			if _, err := io.Copy(ioutil.Discard, tr); err != nil {
65 
+				log.Fatal(err)
66 
+			}
67 
+		}
68 
+		fmt.Printf(" -- number of files: %d\n", num)
69 
+
70 
+		if err := packFh.Sync(); err != nil {
71 
+			log.Fatal(err)
72 
+		}
73 
+
74 
+		fi, err = packFh.Stat()
75 
+		if err != nil {
76 
+			log.Fatal(err)
77 
+		}
78 
+		fmt.Printf(" -- size of metadata uncompressed: %dk\n", fi.Size()/1024)
79 
+
80 
+		gzPackFh, err := ioutil.TempFile("", "packed.gz.")
81 
+		if err != nil {
82 
+			log.Fatal(err)
83 
+		}
84 
+		defer gzPackFh.Close()
85 
+		if *flCleanup {
86 
+			defer os.Remove(gzPackFh.Name())
87 
+		}
88 
+
89 
+		gzWrtr := gzip.NewWriter(gzPackFh)
90 
+
91 
+		if _, err := packFh.Seek(0, 0); err != nil {
92 
+			log.Fatal(err)
93 
+		}
94 
+
95 
+		if _, err := io.Copy(gzWrtr, packFh); err != nil {
96 
+			log.Fatal(err)
97 
+		}
98 
+		gzWrtr.Close()
99 
+
100 
+		if err := gzPackFh.Sync(); err != nil {
101 
+			log.Fatal(err)
102 
+		}
103 
+
104 
+		fi, err = gzPackFh.Stat()
105 
+		if err != nil {
106 
+			log.Fatal(err)
107 
+		}
108 
+		fmt.Printf(" -- size of gzip compressed metadata: %dk\n", fi.Size()/1024)
109 
+	}
110 
+}
vendor/src/github.com/vbatts/tar-split/cmd/tar-split/README.md
History View file @ 7fb0618
0 111 
new file mode 100644
... ... 
@@ -0,0 +1,25 @@
0 
+## tar-split utility
1 
+
2 
+
3 
+## Usage
4 
+
5 
+### Disassembly
6 
+
7 
+```bash
8 
+$ sha256sum archive.tar
9 
+d734a748db93ec873392470510b8a1c88929abd8fae2540dc43d5b26f7537868  archive.tar
10 
+$ mkdir ./x
11 
+$ tar-split d --output tar-data.json.gz ./archive.tar | tar -C ./x -x
12 
+time="2015-07-20T15:45:04-04:00" level=info msg="created tar-data.json.gz from ./archive.tar (read 204800 bytes)"
13 
+```
14 
+
15 
+### Assembly
16 
+
17 
+```bash
18 
+$ tar-split a --output new.tar --input ./tar-data.json.gz  --path ./x/
19 
+INFO[0000] created new.tar from ./x/ and ./tar-data.json.gz (wrote 204800 bytes)
20 
+$ sha256sum new.tar
21 
+d734a748db93ec873392470510b8a1c88929abd8fae2540dc43d5b26f7537868  new.tar
22 
+```
23 
+
24 
+
vendor/src/github.com/vbatts/tar-split/cmd/tar-split/main.go
History View file @ 7fb0618
0 25 
new file mode 100644
... ... 
@@ -0,0 +1,175 @@
0 
+// go:generate git tag | tail -1
1 
+package main
2 
+
3 
+import (
4 
+	"compress/gzip"
5 
+	"io"
6 
+	"os"
7 
+
8 
+	"github.com/Sirupsen/logrus"
9 
+	"github.com/codegangsta/cli"
10 
+	"github.com/vbatts/tar-split/tar/asm"
11 
+	"github.com/vbatts/tar-split/tar/storage"
12 
+)
13 
+
14 
+func main() {
15 
+	app := cli.NewApp()
16 
+	app.Name = "tar-split"
17 
+	app.Usage = "tar assembly and disassembly utility"
18 
+	app.Version = "0.9.2"
19 
+	app.Author = "Vincent Batts"
20 
+	app.Email = "vbatts@hashbangbash.com"
21 
+	app.Action = cli.ShowAppHelp
22 
+	app.Before = func(c *cli.Context) error {
23 
+		logrus.SetOutput(os.Stderr)
24 
+		if c.Bool("debug") {
25 
+			logrus.SetLevel(logrus.DebugLevel)
26 
+		}
27 
+		return nil
28 
+	}
29 
+	app.Flags = []cli.Flag{
30 
+		cli.BoolFlag{
31 
+			Name:  "debug, D",
32 
+			Usage: "debug output",
33 
+			// defaults to false
34 
+		},
35 
+	}
36 
+	app.Commands = []cli.Command{
37 
+		{
38 
+			Name:    "disasm",
39 
+			Aliases: []string{"d"},
40 
+			Usage:   "disassemble the input tar stream",
41 
+			Action:  CommandDisasm,
42 
+			Flags: []cli.Flag{
43 
+				cli.StringFlag{
44 
+					Name:  "output",
45 
+					Value: "tar-data.json.gz",
46 
+					Usage: "output of disassembled tar stream",
47 
+				},
48 
+			},
49 
+		},
50 
+		{
51 
+			Name:    "asm",
52 
+			Aliases: []string{"a"},
53 
+			Usage:   "assemble tar stream",
54 
+			Action:  CommandAsm,
55 
+			Flags: []cli.Flag{
56 
+				cli.StringFlag{
57 
+					Name:  "input",
58 
+					Value: "tar-data.json.gz",
59 
+					Usage: "input of disassembled tar stream",
60 
+				},
61 
+				cli.StringFlag{
62 
+					Name:  "output",
63 
+					Value: "-",
64 
+					Usage: "reassembled tar archive",
65 
+				},
66 
+				cli.StringFlag{
67 
+					Name:  "path",
68 
+					Value: "",
69 
+					Usage: "relative path of extracted tar",
70 
+				},
71 
+			},
72 
+		},
73 
+	}
74 
+
75 
+	if err := app.Run(os.Args); err != nil {
76 
+		logrus.Fatal(err)
77 
+	}
78 
+}
79 
+
80 
+func CommandDisasm(c *cli.Context) {
81 
+	if len(c.Args()) != 1 {
82 
+		logrus.Fatalf("please specify tar to be disabled <NAME|->")
83 
+	}
84 
+	if len(c.String("output")) == 0 {
85 
+		logrus.Fatalf("--output filename must be set")
86 
+	}
87 
+
88 
+	// Set up the tar input stream
89 
+	var inputStream io.Reader
90 
+	if c.Args()[0] == "-" {
91 
+		inputStream = os.Stdin
92 
+	} else {
93 
+		fh, err := os.Open(c.Args()[0])
94 
+		if err != nil {
95 
+			logrus.Fatal(err)
96 
+		}
97 
+		defer fh.Close()
98 
+		inputStream = fh
99 
+	}
100 
+
101 
+	// Set up the metadata storage
102 
+	mf, err := os.OpenFile(c.String("output"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
103 
+	if err != nil {
104 
+		logrus.Fatal(err)
105 
+	}
106 
+	defer mf.Close()
107 
+	mfz := gzip.NewWriter(mf)
108 
+	defer mfz.Close()
109 
+	metaPacker := storage.NewJSONPacker(mfz)
110 
+
111 
+	// we're passing nil here for the file putter, because the ApplyDiff will
112 
+	// handle the extraction of the archive
113 
+	its, err := asm.NewInputTarStream(inputStream, metaPacker, nil)
114 
+	if err != nil {
115 
+		logrus.Fatal(err)
116 
+	}
117 
+	i, err := io.Copy(os.Stdout, its)
118 
+	if err != nil {
119 
+		logrus.Fatal(err)
120 
+	}
121 
+	logrus.Infof("created %s from %s (read %d bytes)", c.String("output"), c.Args()[0], i)
122 
+}
123 
+
124 
+func CommandAsm(c *cli.Context) {
125 
+	if len(c.Args()) > 0 {
126 
+		logrus.Warnf("%d additional arguments passed are ignored", len(c.Args()))
127 
+	}
128 
+	if len(c.String("input")) == 0 {
129 
+		logrus.Fatalf("--input filename must be set")
130 
+	}
131 
+	if len(c.String("output")) == 0 {
132 
+		logrus.Fatalf("--output filename must be set ([FILENAME|-])")
133 
+	}
134 
+	if len(c.String("path")) == 0 {
135 
+		logrus.Fatalf("--path must be set")
136 
+	}
137 
+
138 
+	var outputStream io.Writer
139 
+	if c.String("output") == "-" {
140 
+		outputStream = os.Stdout
141 
+	} else {
142 
+		fh, err := os.Create(c.String("output"))
143 
+		if err != nil {
144 
+			logrus.Fatal(err)
145 
+		}
146 
+		defer fh.Close()
147 
+		outputStream = fh
148 
+	}
149 
+
150 
+	// Get the tar metadata reader
151 
+	mf, err := os.Open(c.String("input"))
152 
+	if err != nil {
153 
+		logrus.Fatal(err)
154 
+	}
155 
+	defer mf.Close()
156 
+	mfz, err := gzip.NewReader(mf)
157 
+	if err != nil {
158 
+		logrus.Fatal(err)
159 
+	}
160 
+	defer mfz.Close()
161 
+
162 
+	metaUnpacker := storage.NewJSONUnpacker(mfz)
163 
+	// XXX maybe get the absolute path here
164 
+	fileGetter := storage.NewPathFileGetter(c.String("path"))
165 
+
166 
+	ots := asm.NewOutputTarStream(fileGetter, metaUnpacker)
167 
+	defer ots.Close()
168 
+	i, err := io.Copy(outputStream, ots)
169 
+	if err != nil {
170 
+		logrus.Fatal(err)
171 
+	}
172 
+
173 
+	logrus.Infof("created %s from %s and %s (wrote %d bytes)", c.String("output"), c.String("path"), c.String("input"), i)
174 
+}
vendor/src/github.com/vbatts/tar-split/concept/main.go
History View file @ 7fb0618
0 175 
new file mode 100644
... ... 
@@ -0,0 +1,91 @@
0 
+// +build ignore
1 
+
2 
+package main
3 
+
4 
+import (
5 
+	"flag"
6 
+	"fmt"
7 
+	"io"
8 
+	"io/ioutil"
9 
+	"log"
10 
+	"os"
11 
+
12 
+	"github.com/vbatts/tar-split/archive/tar"
13 
+)
14 
+
15 
+func main() {
16 
+	flag.Parse()
17 
+	log.SetOutput(os.Stderr)
18 
+	for _, arg := range flag.Args() {
19 
+		func() {
20 
+			// Open the tar archive
21 
+			fh, err := os.Open(arg)
22 
+			if err != nil {
23 
+				log.Fatal(err, arg)
24 
+			}
25 
+			defer fh.Close()
26 
+
27 
+			output, err := os.Create(fmt.Sprintf("%s.out", arg))
28 
+			if err != nil {
29 
+				log.Fatal(err)
30 
+			}
31 
+			defer output.Close()
32 
+			log.Printf("writing %q to %q", fh.Name(), output.Name())
33 
+
34 
+			fi, err := fh.Stat()
35 
+			if err != nil {
36 
+				log.Fatal(err, fh.Name())
37 
+			}
38 
+			size := fi.Size()
39 
+			var sum int64
40 
+			tr := tar.NewReader(fh)
41 
+			tr.RawAccounting = true
42 
+			for {
43 
+				hdr, err := tr.Next()
44 
+				if err != nil {
45 
+					if err != io.EOF {
46 
+						log.Println(err)
47 
+					}
48 
+					// even when an EOF is reached, there is often 1024 null bytes on
49 
+					// the end of an archive. Collect them too.
50 
+					post := tr.RawBytes()
51 
+					output.Write(post)
52 
+					sum += int64(len(post))
53 
+
54 
+					fmt.Printf("EOF padding: %d\n", len(post))
55 
+					break
56 
+				}
57 
+
58 
+				pre := tr.RawBytes()
59 
+				output.Write(pre)
60 
+				sum += int64(len(pre))
61 
+
62 
+				var i int64
63 
+				if i, err = io.Copy(output, tr); err != nil {
64 
+					log.Println(err)
65 
+					break
66 
+				}
67 
+				sum += i
68 
+
69 
+				fmt.Println(hdr.Name, "pre:", len(pre), "read:", i)
70 
+			}
71 
+
72 
+			// it is allowable, and not uncommon that there is further padding on the
73 
+			// end of an archive, apart from the expected 1024 null bytes
74 
+			remainder, err := ioutil.ReadAll(fh)
75 
+			if err != nil && err != io.EOF {
76 
+				log.Fatal(err, fh.Name())
77 
+			}
78 
+			output.Write(remainder)
79 
+			sum += int64(len(remainder))
80 
+			fmt.Printf("Remainder: %d\n", len(remainder))
81 
+
82 
+			if size != sum {
83 
+				fmt.Printf("Size: %d; Sum: %d; Diff: %d\n", size, sum, size-sum)
84 
+				fmt.Printf("Compare like `cmp -bl %s %s | less`\n", fh.Name(), output.Name())
85 
+			} else {
86 
+				fmt.Printf("Size: %d; Sum: %d\n", size, sum)
87 
+			}
88 
+		}()
89 
+	}
90 
+}
vendor/src/github.com/vbatts/tar-split/tar/asm/README.md
History View file @ 7fb0618
0 91 
new file mode 100644
... ... 
@@ -0,0 +1,44 @@
0 
+asm
1 
+===
2 
+
3 
+This library for assembly and disassembly of tar archives, facilitated by
4 
+`github.com/vbatts/tar-split/tar/storage`.
5 
+
6 
+
7 
+Concerns
8 
+--------
9 
+
10 
+For completely safe assembly/disassembly, there will need to be a Content
11 
+Addressable Storage (CAS) directory, that maps to a checksum in the
12 
+`storage.Entity` of `storage.FileType`.
13 
+
14 
+This is due to the fact that tar archives _can_ allow multiple records for the
15 
+same path, but the last one effectively wins. Even if the prior records had a
16 
+different payload.
17 
+
18 
+In this way, when assembling an archive from relative paths, if the archive has
19 
+multiple entries for the same path, then all payloads read in from a relative
20 
+path would be identical.
21 
+
22 
+
23 
+Thoughts
24 
+--------
25 
+
26 
+Have a look-aside directory or storage. This way when a clobbering record is
27 
+encountered from the tar stream, then the payload of the prior/existing file is
28 
+stored to the CAS. This way the clobbering record's file payload can be
29 
+extracted, but we'll have preserved the payload needed to reassemble a precise
30 
+tar archive.
31 
+
32 
+clobbered/path/to/file.[0-N]
33 
+
34 
+*alternatively*
35 
+
36 
+We could just _not_ support tar streams that have clobbering file paths.
37 
+Appending records to the archive is not incredibly common, and doesn't happen
38 
+by default for most implementations.  Not supporting them wouldn't be a
39 
+security concern either, as if it did occur, we would reassemble an archive
40 
+that doesn't validate signature/checksum, so it shouldn't be trusted anyway.
41 
+
42 
+Otherwise, this will allow us to defer support for appended files as a FUTURE FEATURE.
43 
+
vendor/src/github.com/vbatts/tar-split/tar/asm/assemble.go
History View file @ 7fb0618
0 44 
new file mode 100644
... ... 
@@ -0,0 +1,68 @@
0 
+package asm
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"fmt"
5 
+	"hash/crc64"
6 
+	"io"
7 
+
8 
+	"github.com/vbatts/tar-split/tar/storage"
9 
+)
10 
+
11 
+// NewOutputTarStream returns an io.ReadCloser that is an assemble tar archive
12 
+// stream.
13 
+//
14 
+// It takes a storage.FileGetter, for mapping the file payloads that are to be read in,
15 
+// and a storage.Unpacker, which has access to the rawbytes and file order
16 
+// metadata. With the combination of these two items, a precise assembled Tar
17 
+// archive is possible.
18 
+func NewOutputTarStream(fg storage.FileGetter, up storage.Unpacker) io.ReadCloser {
19 
+	// ... Since these are interfaces, this is possible, so let's not have a nil pointer
20 
+	if fg == nil || up == nil {
21 
+		return nil
22 
+	}
23 
+	pr, pw := io.Pipe()
24 
+	go func() {
25 
+		for {
26 
+			entry, err := up.Next()
27 
+			if err != nil {
28 
+				pw.CloseWithError(err)
29 
+				return
30 
+			}
31 
+			switch entry.Type {
32 
+			case storage.SegmentType:
33 
+				if _, err := pw.Write(entry.Payload); err != nil {
34 
+					pw.CloseWithError(err)
35 
+					return
36 
+				}
37 
+			case storage.FileType:
38 
+				if entry.Size == 0 {
39 
+					continue
40 
+				}
41 
+				fh, err := fg.Get(entry.Name)
42 
+				if err != nil {
43 
+					pw.CloseWithError(err)
44 
+					return
45 
+				}
46 
+				c := crc64.New(storage.CRCTable)
47 
+				tRdr := io.TeeReader(fh, c)
48 
+				if _, err := io.Copy(pw, tRdr); err != nil {
49 
+					fh.Close()
50 
+					pw.CloseWithError(err)
51 
+					return
52 
+				}
53 
+				if !bytes.Equal(c.Sum(nil), entry.Payload) {
54 
+					// I would rather this be a comparable ErrInvalidChecksum or such,
55 
+					// but since it's coming through the PipeReader, the context of
56 
+					// _which_ file would be lost...
57 
+					fh.Close()
58 
+					pw.CloseWithError(fmt.Errorf("file integrity checksum failed for %q", entry.Name))
59 
+					return
60 
+				}
61 
+				fh.Close()
62 
+			}
63 
+		}
64 
+		pw.Close()
65 
+	}()
66 
+	return pr
67 
+}
vendor/src/github.com/vbatts/tar-split/tar/asm/assemble_test.go
History View file @ 7fb0618
0 68 
new file mode 100644
... ... 
@@ -0,0 +1,181 @@
0 
+package asm
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"compress/gzip"
5 
+	"crypto/sha1"
6 
+	"fmt"
7 
+	"hash/crc64"
8 
+	"io"
9 
+	"io/ioutil"
10 
+	"os"
11 
+	"testing"
12 
+
13 
+	"github.com/vbatts/tar-split/tar/storage"
14 
+)
15 
+
16 
+var entries = []struct {
17 
+	Entry storage.Entry
18 
+	Body  []byte
19 
+}{
20 
+	{
21 
+		Entry: storage.Entry{
22 
+			Type:    storage.FileType,
23 
+			Name:    "./hurr.txt",
24 
+			Payload: []byte{2, 116, 164, 177, 171, 236, 107, 78},
25 
+			Size:    20,
26 
+		},
27 
+		Body: []byte("imma hurr til I derp"),
28 
+	},
29 
+	{
30 
+		Entry: storage.Entry{
31 
+			Type:    storage.FileType,
32 
+			Name:    "./ermahgerd.txt",
33 
+			Payload: []byte{126, 72, 89, 239, 230, 252, 160, 187},
34 
+			Size:    26,
35 
+		},
36 
+		Body: []byte("café con leche, por favor"),
37 
+	},
38 
+}
39 
+var entriesMangled = []struct {
40 
+	Entry storage.Entry
41 
+	Body  []byte
42 
+}{
43 
+	{
44 
+		Entry: storage.Entry{
45 
+			Type:    storage.FileType,
46 
+			Name:    "./hurr.txt",
47 
+			Payload: []byte{3, 116, 164, 177, 171, 236, 107, 78},
48 
+			Size:    20,
49 
+		},
50 
+		// switch
51 
+		Body: []byte("imma derp til I hurr"),
52 
+	},
53 
+	{
54 
+		Entry: storage.Entry{
55 
+			Type:    storage.FileType,
56 
+			Name:    "./ermahgerd.txt",
57 
+			Payload: []byte{127, 72, 89, 239, 230, 252, 160, 187},
58 
+			Size:    26,
59 
+		},
60 
+		// san not con
61 
+		Body: []byte("café sans leche, por favor"),
62 
+	},
63 
+}
64 
+
65 
+func TestTarStreamMangledGetterPutter(t *testing.T) {
66 
+	fgp := storage.NewBufferFileGetPutter()
67 
+
68 
+	// first lets prep a GetPutter and Packer
69 
+	for i := range entries {
70 
+		if entries[i].Entry.Type == storage.FileType {
71 
+			j, csum, err := fgp.Put(entries[i].Entry.Name, bytes.NewBuffer(entries[i].Body))
72 
+			if err != nil {
73 
+				t.Error(err)
74 
+			}
75 
+			if j != entries[i].Entry.Size {
76 
+				t.Errorf("size %q: expected %d; got %d",
77 
+					entries[i].Entry.Name,
78 
+					entries[i].Entry.Size,
79 
+					j)
80 
+			}
81 
+			if !bytes.Equal(csum, entries[i].Entry.Payload) {
82 
+				t.Errorf("checksum %q: expected %v; got %v",
83 
+					entries[i].Entry.Name,
84 
+					entries[i].Entry.Payload,
85 
+					csum)
86 
+			}
87 
+		}
88 
+	}
89 
+
90 
+	for _, e := range entriesMangled {
91 
+		if e.Entry.Type == storage.FileType {
92 
+			rdr, err := fgp.Get(e.Entry.Name)
93 
+			if err != nil {
94 
+				t.Error(err)
95 
+			}
96 
+			c := crc64.New(storage.CRCTable)
97 
+			i, err := io.Copy(c, rdr)
98 
+			if err != nil {
99 
+				t.Fatal(err)
100 
+			}
101 
+			rdr.Close()
102 
+
103 
+			csum := c.Sum(nil)
104 
+			if bytes.Equal(csum, e.Entry.Payload) {
105 
+				t.Errorf("wrote %d bytes. checksum for %q should not have matched! %v",
106 
+					i,
107 
+					e.Entry.Name,
108 
+					csum)
109 
+			}
110 
+		}
111 
+	}
112 
+}
113 
+
114 
+func TestTarStream(t *testing.T) {
115 
+	var (
116 
+		expectedSum        = "1eb237ff69bca6e22789ecb05b45d35ca307adbd"
117 
+		expectedSize int64 = 10240
118 
+	)
119 
+
120 
+	fh, err := os.Open("./testdata/t.tar.gz")
121 
+	if err != nil {
122 
+		t.Fatal(err)
123 
+	}
124 
+	defer fh.Close()
125 
+	gzRdr, err := gzip.NewReader(fh)
126 
+	if err != nil {
127 
+		t.Fatal(err)
128 
+	}
129 
+	defer gzRdr.Close()
130 
+
131 
+	// Setup where we'll store the metadata
132 
+	w := bytes.NewBuffer([]byte{})
133 
+	sp := storage.NewJSONPacker(w)
134 
+	fgp := storage.NewBufferFileGetPutter()
135 
+
136 
+	// wrap the disassembly stream
137 
+	tarStream, err := NewInputTarStream(gzRdr, sp, fgp)
138 
+	if err != nil {
139 
+		t.Fatal(err)
140 
+	}
141 
+
142 
+	// get a sum of the stream after it has passed through to ensure it's the same.
143 
+	h0 := sha1.New()
144 
+	tRdr0 := io.TeeReader(tarStream, h0)
145 
+
146 
+	// read it all to the bit bucket
147 
+	i, err := io.Copy(ioutil.Discard, tRdr0)
148 
+	if err != nil {
149 
+		t.Fatal(err)
150 
+	}
151 
+
152 
+	if i != expectedSize {
153 
+		t.Errorf("size of tar: expected %d; got %d", expectedSize, i)
154 
+	}
155 
+	if fmt.Sprintf("%x", h0.Sum(nil)) != expectedSum {
156 
+		t.Fatalf("checksum of tar: expected %s; got %x", expectedSum, h0.Sum(nil))
157 
+	}
158 
+
159 
+	t.Logf("%s", w.String()) // if we fail, then show the packed info
160 
+
161 
+	// If we've made it this far, then we'll turn it around and create a tar
162 
+	// stream from the packed metadata and buffered file contents.
163 
+	r := bytes.NewBuffer(w.Bytes())
164 
+	sup := storage.NewJSONUnpacker(r)
165 
+	// and reuse the fgp that we Put the payloads to.
166 
+
167 
+	rc := NewOutputTarStream(fgp, sup)
168 
+	h1 := sha1.New()
169 
+	i, err = io.Copy(h1, rc)
170 
+	if err != nil {
171 
+		t.Fatal(err)
172 
+	}
173 
+
174 
+	if i != expectedSize {
175 
+		t.Errorf("size of output tar: expected %d; got %d", expectedSize, i)
176 
+	}
177 
+	if fmt.Sprintf("%x", h1.Sum(nil)) != expectedSum {
178 
+		t.Fatalf("checksum of output tar: expected %s; got %x", expectedSum, h1.Sum(nil))
179 
+	}
180 
+}
vendor/src/github.com/vbatts/tar-split/tar/asm/disassemble.go
History View file @ 7fb0618
0 181 
new file mode 100644
... ... 
@@ -0,0 +1,133 @@
0 
+package asm
1 
+
2 
+import (
3 
+	"io"
4 
+	"io/ioutil"
5 
+
6 
+	"github.com/vbatts/tar-split/archive/tar"
7 
+	"github.com/vbatts/tar-split/tar/storage"
8 
+)
9 
+
10 
+// NewInputTarStream wraps the Reader stream of a tar archive and provides a
11 
+// Reader stream of the same.
12 
+//
13 
+// In the middle it will pack the segments and file metadata to storage.Packer
14 
+// `p`.
15 
+//
16 
+// The the storage.FilePutter is where payload of files in the stream are
17 
+// stashed. If this stashing is not needed, you can provide a nil
18 
+// storage.FilePutter. Since the checksumming is still needed, then a default
19 
+// of NewDiscardFilePutter will be used internally
20 
+func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io.Reader, error) {
21 
+	// What to do here... folks will want their own access to the Reader that is
22 
+	// their tar archive stream, but we'll need that same stream to use our
23 
+	// forked 'archive/tar'.
24 
+	// Perhaps do an io.TeeReader that hand back an io.Reader for them to read
25 
+	// from, and we'll mitm the stream to store metadata.
26 
+	// We'll need a storage.FilePutter too ...
27 
+
28 
+	// Another concern, whether to do any storage.FilePutter operations, such that we
29 
+	// don't extract any amount of the archive. But then again, we're not making
30 
+	// files/directories, hardlinks, etc. Just writing the io to the storage.FilePutter.
31 
+	// Perhaps we have a DiscardFilePutter that is a bit bucket.
32 
+
33 
+	// we'll return the pipe reader, since TeeReader does not buffer and will
34 
+	// only read what the outputRdr Read's. Since Tar archive's have padding on
35 
+	// the end, we want to be the one reading the padding, even if the user's
36 
+	// `archive/tar` doesn't care.
37 
+	pR, pW := io.Pipe()
38 
+	outputRdr := io.TeeReader(r, pW)
39 
+
40 
+	// we need a putter that will generate the crc64 sums of file payloads
41 
+	if fp == nil {
42 
+		fp = storage.NewDiscardFilePutter()
43 
+	}
44 
+
45 
+	go func() {
46 
+		tr := tar.NewReader(outputRdr)
47 
+		tr.RawAccounting = true
48 
+		for {
49 
+			hdr, err := tr.Next()
50 
+			if err != nil {
51 
+				if err != io.EOF {
52 
+					pW.CloseWithError(err)
53 
+					return
54 
+				}
55 
+				// even when an EOF is reached, there is often 1024 null bytes on
56 
+				// the end of an archive. Collect them too.
57 
+				_, err := p.AddEntry(storage.Entry{
58 
+					Type:    storage.SegmentType,
59 
+					Payload: tr.RawBytes(),
60 
+				})
61 
+				if err != nil {
62 
+					pW.CloseWithError(err)
63 
+					return
64 
+				}
65 
+				break // not return. We need the end of the reader.
66 
+			}
67 
+			if hdr == nil {
68 
+				break // not return. We need the end of the reader.
69 
+			}
70 
+
71 
+			if _, err := p.AddEntry(storage.Entry{
72 
+				Type:    storage.SegmentType,
73 
+				Payload: tr.RawBytes(),
74 
+			}); err != nil {
75 
+				pW.CloseWithError(err)
76 
+				return
77 
+			}
78 
+
79 
+			var csum []byte
80 
+			if hdr.Size > 0 {
81 
+				var err error
82 
+				_, csum, err = fp.Put(hdr.Name, tr)
83 
+				if err != nil {
84 
+					pW.CloseWithError(err)
85 
+					return
86 
+				}
87 
+			}
88 
+
89 
+			// File entries added, regardless of size
90 
+			_, err = p.AddEntry(storage.Entry{
91 
+				Type:    storage.FileType,
92 
+				Name:    hdr.Name,
93 
+				Size:    hdr.Size,
94 
+				Payload: csum,
95 
+			})
96 
+			if err != nil {
97 
+				pW.CloseWithError(err)
98 
+				return
99 
+			}
100 
+
101 
+			if b := tr.RawBytes(); len(b) > 0 {
102 
+				_, err = p.AddEntry(storage.Entry{
103 
+					Type:    storage.SegmentType,
104 
+					Payload: b,
105 
+				})
106 
+				if err != nil {
107 
+					pW.CloseWithError(err)
108 
+					return
109 
+				}
110 
+			}
111 
+		}
112 
+
113 
+		// it is allowable, and not uncommon that there is further padding on the
114 
+		// end of an archive, apart from the expected 1024 null bytes.
115 
+		remainder, err := ioutil.ReadAll(outputRdr)
116 
+		if err != nil && err != io.EOF {
117 
+			pW.CloseWithError(err)
118 
+			return
119 
+		}
120 
+		_, err = p.AddEntry(storage.Entry{
121 
+			Type:    storage.SegmentType,
122 
+			Payload: remainder,
123 
+		})
124 
+		if err != nil {
125 
+			pW.CloseWithError(err)
126 
+			return
127 
+		}
128 
+		pW.Close()
129 
+	}()
130 
+
131 
+	return pR, nil
132 
+}
vendor/src/github.com/vbatts/tar-split/tar/asm/doc.go
History View file @ 7fb0618
0 133 
new file mode 100644
... ... 
@@ -0,0 +1,9 @@
0 
+/*
1 
+Package asm provides the API for streaming assembly and disassembly of tar
2 
+archives.
3 
+
4 
+Using the `github.com/vbatts/tar-split/tar/storage` for Packing/Unpacking the
5 
+metadata for a stream, as well as an implementation of Getting/Putting the file
6 
+entries' payload.
7 
+*/
8 
+package asm
vendor/src/github.com/vbatts/tar-split/tar/asm/testdata/t.tar.gz
History View file @ 7fb0618
0 9 
new file mode 100644
1 10 
Binary files /dev/null and b/vendor/src/github.com/vbatts/tar-split/tar/asm/testdata/t.tar.gz differ
vendor/src/github.com/vbatts/tar-split/tar/storage/doc.go
History View file @ 7fb0618
2 11 
new file mode 100644
... ... 
@@ -0,0 +1,12 @@
0 
+/*
1 
+Package storage is for metadata of a tar archive.
2 
+
3 
+Packing and unpacking the Entries of the stream. The types of streams are
4 
+either segments of raw bytes (for the raw headers and various padding) and for
5 
+an entry marking a file payload.
6 
+
7 
+The raw bytes are stored precisely in the packed (marshalled) Entry. Where as
8 
+the file payload marker include the name of the file, size, and crc64 checksum
9 
+(for basic file integrity).
10 
+*/
11 
+package storage
vendor/src/github.com/vbatts/tar-split/tar/storage/entry.go
History View file @ 7fb0618
0 12 
new file mode 100644
... ... 
@@ -0,0 +1,39 @@
0 
+package storage
1 
+
2 
+// Entries is for sorting by Position
3 
+type Entries []Entry
4 
+
5 
+func (e Entries) Len() int           { return len(e) }
6 
+func (e Entries) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
7 
+func (e Entries) Less(i, j int) bool { return e[i].Position < e[j].Position }
8 
+
9 
+// Type of Entry
10 
+type Type int
11 
+
12 
+const (
13 
+	// FileType represents a file payload from the tar stream.
14 
+	//
15 
+	// This will be used to map to relative paths on disk. Only Size > 0 will get
16 
+	// read into a resulting output stream (due to hardlinks).
17 
+	FileType Type = 1 + iota
18 
+	// SegmentType represents a raw bytes segment from the archive stream. These raw
19 
+	// byte segments consist of the raw headers and various padding.
20 
+	//
21 
+	// It's payload is to be marshalled base64 encoded.
22 
+	SegmentType
23 
+)
24 
+
25 
+// Entry is a the structure for packing and unpacking the information read from
26 
+// the Tar archive.
27 
+//
28 
+// FileType Payload checksum is using `hash/crc64` for basic file integrity,
29 
+// _not_ for cryptography.
30 
+// From http://www.backplane.com/matt/crc64.html, CRC32 has almost 40,000
31 
+// collisions in a sample of 18.2 million, CRC64 had none.
32 
+type Entry struct {
33 
+	Type     Type   `json:"type"`
34 
+	Name     string `json:"name",omitempty`
35 
+	Size     int64  `json:"size",omitempty`
36 
+	Payload  []byte `json:"payload"` // SegmentType store payload here; FileType store crc64 checksum here;
37 
+	Position int    `json:"position"`
38 
+}
vendor/src/github.com/vbatts/tar-split/tar/storage/entry_test.go
History View file @ 7fb0618
0 39 
new file mode 100644
... ... 
@@ -0,0 +1,66 @@
0 
+package storage
1 
+
2 
+import (
3 
+	"encoding/json"
4 
+	"sort"
5 
+	"testing"
6 
+)
7 
+
8 
+func TestEntries(t *testing.T) {
9 
+	e := Entries{
10 
+		Entry{
11 
+			Type:     SegmentType,
12 
+			Payload:  []byte("y'all"),
13 
+			Position: 1,
14 
+		},
15 
+		Entry{
16 
+			Type:     SegmentType,
17 
+			Payload:  []byte("doin"),
18 
+			Position: 3,
19 
+		},
20 
+		Entry{
21 
+			Type:     FileType,
22 
+			Name:     "./hurr.txt",
23 
+			Payload:  []byte("deadbeef"),
24 
+			Position: 2,
25 
+		},
26 
+		Entry{
27 
+			Type:     SegmentType,
28 
+			Payload:  []byte("how"),
29 
+			Position: 0,
30 
+		},
31 
+	}
32 
+	sort.Sort(e)
33 
+	if e[0].Position != 0 {
34 
+		t.Errorf("expected Position 0, but got %d", e[0].Position)
35 
+	}
36 
+}
37 
+
38 
+func TestFile(t *testing.T) {
39 
+	f := Entry{
40 
+		Type:     FileType,
41 
+		Name:     "./hello.txt",
42 
+		Size:     100,
43 
+		Position: 2,
44 
+	}
45 
+
46 
+	buf, err := json.Marshal(f)
47 
+	if err != nil {
48 
+		t.Fatal(err)
49 
+	}
50 
+
51 
+	f1 := Entry{}
52 
+	if err = json.Unmarshal(buf, &f1); err != nil {
53 
+		t.Fatal(err)
54 
+	}
55 
+
56 
+	if f.Name != f1.Name {
57 
+		t.Errorf("expected Name %q, got %q", f.Name, f1.Name)
58 
+	}
59 
+	if f.Size != f1.Size {
60 
+		t.Errorf("expected Size %q, got %q", f.Size, f1.Size)
61 
+	}
62 
+	if f.Position != f1.Position {
63 
+		t.Errorf("expected Position %q, got %q", f.Position, f1.Position)
64 
+	}
65 
+}
vendor/src/github.com/vbatts/tar-split/tar/storage/getter.go
History View file @ 7fb0618
0 66 
new file mode 100644
... ... 
@@ -0,0 +1,106 @@
0 
+package storage
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"errors"
5 
+	"hash/crc64"
6 
+	"io"
7 
+	"io/ioutil"
8 
+	"os"
9 
+	"path/filepath"
10 
+)
11 
+
12 
+// FileGetter is the interface for getting a stream of a file payload, address
13 
+// by name/filename. Presumably, the names will be scoped to relative file
14 
+// paths.
15 
+type FileGetter interface {
16 
+	// Get returns a stream for the provided file path
17 
+	Get(filename string) (output io.ReadCloser, err error)
18 
+}
19 
+
20 
+// FilePutter is the interface for storing a stream of a file payload,
21 
+// addressed by name/filename.
22 
+type FilePutter interface {
23 
+	// Put returns the size of the stream received, and the crc64 checksum for
24 
+	// the provided stream
25 
+	Put(filename string, input io.Reader) (size int64, checksum []byte, err error)
26 
+}
27 
+
28 
+// FileGetPutter is the interface that groups both Getting and Putting file
29 
+// payloads.
30 
+type FileGetPutter interface {
31 
+	FileGetter
32 
+	FilePutter
33 
+}
34 
+
35 
+// NewPathFileGetter returns a FileGetter that is for files relative to path
36 
+// relpath.
37 
+func NewPathFileGetter(relpath string) FileGetter {
38 
+	return &pathFileGetter{root: relpath}
39 
+}
40 
+
41 
+type pathFileGetter struct {
42 
+	root string
43 
+}
44 
+
45 
+func (pfg pathFileGetter) Get(filename string) (io.ReadCloser, error) {
46 
+	return os.Open(filepath.Join(pfg.root, filename))
47 
+}
48 
+
49 
+type bufferFileGetPutter struct {
50 
+	files map[string][]byte
51 
+}
52 
+
53 
+func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) {
54 
+	if _, ok := bfgp.files[name]; !ok {
55 
+		return nil, errors.New("no such file")
56 
+	}
57 
+	b := bytes.NewBuffer(bfgp.files[name])
58 
+	return &readCloserWrapper{b}, nil
59 
+}
60 
+
61 
+func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) (int64, []byte, error) {
62 
+	c := crc64.New(CRCTable)
63 
+	tRdr := io.TeeReader(r, c)
64 
+	b := bytes.NewBuffer([]byte{})
65 
+	i, err := io.Copy(b, tRdr)
66 
+	if err != nil {
67 
+		return 0, nil, err
68 
+	}
69 
+	bfgp.files[name] = b.Bytes()
70 
+	return i, c.Sum(nil), nil
71 
+}
72 
+
73 
+type readCloserWrapper struct {
74 
+	io.Reader
75 
+}
76 
+
77 
+func (w *readCloserWrapper) Close() error { return nil }
78 
+
79 
+// NewBufferFileGetPutter is simple in memory FileGetPutter
80 
+//
81 
+// Implication is this is memory intensive...
82 
+// Probably best for testing or light weight cases.
83 
+func NewBufferFileGetPutter() FileGetPutter {
84 
+	return &bufferFileGetPutter{
85 
+		files: map[string][]byte{},
86 
+	}
87 
+}
88 
+
89 
+// NewDiscardFilePutter is a bit bucket FilePutter
90 
+func NewDiscardFilePutter() FilePutter {
91 
+	return &bitBucketFilePutter{}
92 
+}
93 
+
94 
+type bitBucketFilePutter struct {
95 
+}
96 
+
97 
+func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) (int64, []byte, error) {
98 
+	c := crc64.New(CRCTable)
99 
+	tRdr := io.TeeReader(r, c)
100 
+	i, err := io.Copy(ioutil.Discard, tRdr)
101 
+	return i, c.Sum(nil), err
102 
+}
103 
+
104 
+// CRCTable is the default table used for crc64 sum calculations
105 
+var CRCTable = crc64.MakeTable(crc64.ISO)
vendor/src/github.com/vbatts/tar-split/tar/storage/getter_test.go
History View file @ 7fb0618
0 106 
new file mode 100644
... ... 
@@ -0,0 +1,62 @@
0 
+package storage
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"io/ioutil"
5 
+	"testing"
6 
+)
7 
+
8 
+func TestGetter(t *testing.T) {
9 
+	fgp := NewBufferFileGetPutter()
10 
+	files := map[string]map[string][]byte{
11 
+		"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
12 
+		"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
13 
+	}
14 
+	for n, b := range files {
15 
+		for body, sum := range b {
16 
+			_, csum, err := fgp.Put(n, bytes.NewBufferString(body))
17 
+			if err != nil {
18 
+				t.Error(err)
19 
+			}
20 
+			if !bytes.Equal(csum, sum) {
21 
+				t.Errorf("checksum: expected 0x%x; got 0x%x", sum, csum)
22 
+			}
23 
+		}
24 
+	}
25 
+	for n, b := range files {
26 
+		for body := range b {
27 
+			r, err := fgp.Get(n)
28 
+			if err != nil {
29 
+				t.Error(err)
30 
+			}
31 
+			buf, err := ioutil.ReadAll(r)
32 
+			if err != nil {
33 
+				t.Error(err)
34 
+			}
35 
+			if body != string(buf) {
36 
+				t.Errorf("expected %q, got %q", body, string(buf))
37 
+			}
38 
+		}
39 
+	}
40 
+}
41 
+func TestPutter(t *testing.T) {
42 
+	fp := NewDiscardFilePutter()
43 
+	// map[filename]map[body]crc64sum
44 
+	files := map[string]map[string][]byte{
45 
+		"file1.txt": {"foo": []byte{60, 60, 48, 48, 0, 0, 0, 0}},
46 
+		"file2.txt": {"bar": []byte{45, 196, 22, 240, 0, 0, 0, 0}},
47 
+		"file3.txt": {"baz": []byte{32, 68, 22, 240, 0, 0, 0, 0}},
48 
+		"file4.txt": {"bif": []byte{48, 9, 150, 240, 0, 0, 0, 0}},
49 
+	}
50 
+	for n, b := range files {
51 
+		for body, sum := range b {
52 
+			_, csum, err := fp.Put(n, bytes.NewBufferString(body))
53 
+			if err != nil {
54 
+				t.Error(err)
55 
+			}
56 
+			if !bytes.Equal(csum, sum) {
57 
+				t.Errorf("checksum on %q: expected %v; got %v", n, sum, csum)
58 
+			}
59 
+		}
60 
+	}
61 
+}
vendor/src/github.com/vbatts/tar-split/tar/storage/packer.go
History View file @ 7fb0618
0 62 
new file mode 100644
... ... 
@@ -0,0 +1,140 @@
0 
+package storage
1 
+
2 
+import (
3 
+	"bufio"
4 
+	"encoding/json"
5 
+	"errors"
6 
+	"io"
7 
+	"path/filepath"
8 
+)
9 
+
10 
+// ErrDuplicatePath is occured when a tar archive has more than one entry for
11 
+// the same file path
12 
+var ErrDuplicatePath = errors.New("duplicates of file paths not supported")
13 
+
14 
+// Packer describes the methods to pack Entries to a storage destination
15 
+type Packer interface {
16 
+	// AddEntry packs the Entry and returns its position
17 
+	AddEntry(e Entry) (int, error)
18 
+}
19 
+
20 
+// Unpacker describes the methods to read Entries from a source
21 
+type Unpacker interface {
22 
+	// Next returns the next Entry being unpacked, or error, until io.EOF
23 
+	Next() (*Entry, error)
24 
+}
25 
+
26 
+/* TODO(vbatts) figure out a good model for this
27 
+type PackUnpacker interface {
28 
+	Packer
29 
+	Unpacker
30 
+}
31 
+*/
32 
+
33 
+type jsonUnpacker struct {
34 
+	r     io.Reader
35 
+	b     *bufio.Reader
36 
+	isEOF bool
37 
+	seen  seenNames
38 
+}
39 
+
40 
+func (jup *jsonUnpacker) Next() (*Entry, error) {
41 
+	var e Entry
42 
+	if jup.isEOF {
43 
+		// since ReadBytes() will return read bytes AND an EOF, we handle it this
44 
+		// round-a-bout way so we can Unmarshal the tail with relevant errors, but
45 
+		// still get an io.EOF when the stream is ended.
46 
+		return nil, io.EOF
47 
+	}
48 
+	line, err := jup.b.ReadBytes('\n')
49 
+	if err != nil && err != io.EOF {
50 
+		return nil, err
51 
+	} else if err == io.EOF {
52 
+		jup.isEOF = true
53 
+	}
54 
+
55 
+	err = json.Unmarshal(line, &e)
56 
+	if err != nil && jup.isEOF {
57 
+		// if the remainder actually _wasn't_ a remaining json structure, then just EOF
58 
+		return nil, io.EOF
59 
+	}
60 
+
61 
+	// check for dup name
62 
+	if e.Type == FileType {
63 
+		cName := filepath.Clean(e.Name)
64 
+		if _, ok := jup.seen[cName]; ok {
65 
+			return nil, ErrDuplicatePath
66 
+		}
67 
+		jup.seen[cName] = emptyByte
68 
+	}
69 
+
70 
+	return &e, err
71 
+}
72 
+
73 
+// NewJSONUnpacker provides an Unpacker that reads Entries (SegmentType and
74 
+// FileType) as a json document.
75 
+//
76 
+// Each Entry read are expected to be delimited by new line.
77 
+func NewJSONUnpacker(r io.Reader) Unpacker {
78 
+	return &jsonUnpacker{
79 
+		r:    r,
80 
+		b:    bufio.NewReader(r),
81 
+		seen: seenNames{},
82 
+	}
83 
+}
84 
+
85 
+type jsonPacker struct {
86 
+	w    io.Writer
87 
+	e    *json.Encoder
88 
+	pos  int
89 
+	seen seenNames
90 
+}
91 
+
92 
+type seenNames map[string]byte
93 
+
94 
+// used in the seenNames map. byte is a uint8, and we'll re-use the same one
95 
+// for minimalism.
96 
+const emptyByte byte = 0
97 
+
98 
+func (jp *jsonPacker) AddEntry(e Entry) (int, error) {
99 
+	// check early for dup name
100 
+	if e.Type == FileType {
101 
+		cName := filepath.Clean(e.Name)
102 
+		if _, ok := jp.seen[cName]; ok {
103 
+			return -1, ErrDuplicatePath
104 
+		}
105 
+		jp.seen[cName] = emptyByte
106 
+	}
107 
+
108 
+	e.Position = jp.pos
109 
+	err := jp.e.Encode(e)
110 
+	if err != nil {
111 
+		return -1, err
112 
+	}
113 
+
114 
+	// made it this far, increment now
115 
+	jp.pos++
116 
+	return e.Position, nil
117 
+}
118 
+
119 
+// NewJSONPacker provides an Packer that writes each Entry (SegmentType and
120 
+// FileType) as a json document.
121 
+//
122 
+// The Entries are delimited by new line.
123 
+func NewJSONPacker(w io.Writer) Packer {
124 
+	return &jsonPacker{
125 
+		w:    w,
126 
+		e:    json.NewEncoder(w),
127 
+		seen: seenNames{},
128 
+	}
129 
+}
130 
+
131 
+/*
132 
+TODO(vbatts) perhaps have a more compact packer/unpacker, maybe using msgapck
133 
+(https://github.com/ugorji/go)
134 
+
135 
+
136 
+Even though, since our jsonUnpacker and jsonPacker just take
137 
+io.Reader/io.Writer, then we can get away with passing them a
138 
+gzip.Reader/gzip.Writer
139 
+*/
vendor/src/github.com/vbatts/tar-split/tar/storage/packer_test.go
History View file @ 7fb0618
0 140 
new file mode 100644
... ... 
@@ -0,0 +1,163 @@
0 
+package storage
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"compress/gzip"
5 
+	"io"
6 
+	"testing"
7 
+)
8 
+
9 
+func TestDuplicateFail(t *testing.T) {
10 
+	e := []Entry{
11 
+		Entry{
12 
+			Type:    FileType,
13 
+			Name:    "./hurr.txt",
14 
+			Payload: []byte("abcde"),
15 
+		},
16 
+		Entry{
17 
+			Type:    FileType,
18 
+			Name:    "./hurr.txt",
19 
+			Payload: []byte("deadbeef"),
20 
+		},
21 
+		Entry{
22 
+			Type:    FileType,
23 
+			Name:    "hurr.txt", // slightly different path, same file though
24 
+			Payload: []byte("deadbeef"),
25 
+		},
26 
+	}
27 
+	buf := []byte{}
28 
+	b := bytes.NewBuffer(buf)
29 
+
30 
+	jp := NewJSONPacker(b)
31 
+	if _, err := jp.AddEntry(e[0]); err != nil {
32 
+		t.Error(err)
33 
+	}
34 
+	if _, err := jp.AddEntry(e[1]); err != ErrDuplicatePath {
35 
+		t.Errorf("expected failure on duplicate path")
36 
+	}
37 
+	if _, err := jp.AddEntry(e[2]); err != ErrDuplicatePath {
38 
+		t.Errorf("expected failure on duplicate path")
39 
+	}
40 
+}
41 
+
42 
+func TestJSONPackerUnpacker(t *testing.T) {
43 
+	e := []Entry{
44 
+		Entry{
45 
+			Type:    SegmentType,
46 
+			Payload: []byte("how"),
47 
+		},
48 
+		Entry{
49 
+			Type:    SegmentType,
50 
+			Payload: []byte("y'all"),
51 
+		},
52 
+		Entry{
53 
+			Type:    FileType,
54 
+			Name:    "./hurr.txt",
55 
+			Payload: []byte("deadbeef"),
56 
+		},
57 
+		Entry{
58 
+			Type:    SegmentType,
59 
+			Payload: []byte("doin"),
60 
+		},
61 
+	}
62 
+
63 
+	buf := []byte{}
64 
+	b := bytes.NewBuffer(buf)
65 
+
66 
+	func() {
67 
+		jp := NewJSONPacker(b)
68 
+		for i := range e {
69 
+			if _, err := jp.AddEntry(e[i]); err != nil {
70 
+				t.Error(err)
71 
+			}
72 
+		}
73 
+	}()
74 
+
75 
+	// >> packer_test.go:43: uncompressed: 266
76 
+	//t.Errorf("uncompressed: %d", len(b.Bytes()))
77 
+
78 
+	b = bytes.NewBuffer(b.Bytes())
79 
+	entries := Entries{}
80 
+	func() {
81 
+		jup := NewJSONUnpacker(b)
82 
+		for {
83 
+			entry, err := jup.Next()
84 
+			if err != nil {
85 
+				if err == io.EOF {
86 
+					break
87 
+				}
88 
+				t.Error(err)
89 
+			}
90 
+			entries = append(entries, *entry)
91 
+			t.Logf("got %#v", entry)
92 
+		}
93 
+	}()
94 
+	if len(entries) != len(e) {
95 
+		t.Errorf("expected %d entries, got %d", len(e), len(entries))
96 
+	}
97 
+}
98 
+
99 
+// you can use a compress Reader/Writer and make nice savings.
100 
+//
101 
+// For these two tests that are using the same set, it the difference of 266
102 
+// bytes uncompressed vs 138 bytes compressed.
103 
+func TestGzip(t *testing.T) {
104 
+	e := []Entry{
105 
+		Entry{
106 
+			Type:    SegmentType,
107 
+			Payload: []byte("how"),
108 
+		},
109 
+		Entry{
110 
+			Type:    SegmentType,
111 
+			Payload: []byte("y'all"),
112 
+		},
113 
+		Entry{
114 
+			Type:    FileType,
115 
+			Name:    "./hurr.txt",
116 
+			Payload: []byte("deadbeef"),
117 
+		},
118 
+		Entry{
119 
+			Type:    SegmentType,
120 
+			Payload: []byte("doin"),
121 
+		},
122 
+	}
123 
+
124 
+	buf := []byte{}
125 
+	b := bytes.NewBuffer(buf)
126 
+	gzW := gzip.NewWriter(b)
127 
+	jp := NewJSONPacker(gzW)
128 
+	for i := range e {
129 
+		if _, err := jp.AddEntry(e[i]); err != nil {
130 
+			t.Error(err)
131 
+		}
132 
+	}
133 
+	gzW.Close()
134 
+
135 
+	// >> packer_test.go:99: compressed: 138
136 
+	//t.Errorf("compressed: %d", len(b.Bytes()))
137 
+
138 
+	b = bytes.NewBuffer(b.Bytes())
139 
+	gzR, err := gzip.NewReader(b)
140 
+	if err != nil {
141 
+		t.Fatal(err)
142 
+	}
143 
+	entries := Entries{}
144 
+	func() {
145 
+		jup := NewJSONUnpacker(gzR)
146 
+		for {
147 
+			entry, err := jup.Next()
148 
+			if err != nil {
149 
+				if err == io.EOF {
150 
+					break
151 
+				}
152 
+				t.Error(err)
153 
+			}
154 
+			entries = append(entries, *entry)
155 
+			t.Logf("got %#v", entry)
156 
+		}
157 
+	}()
158 
+	if len(entries) != len(e) {
159 
+		t.Errorf("expected %d entries, got %d", len(e), len(entries))
160 
+	}
161 
+
162 
+}