Browse code

move some io related utils to pkg/ioutils

Docker-DCO-1.1-Signed-off-by: Cristian Staretu <cristian.staretu@gmail.com> (github: unclejack)

unclejack authored on 2014/08/13 01:10:43
Showing 12 changed files
... ...
@@ -18,6 +18,7 @@ import (
18 18
 
19 19
 	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
20 20
 
21
+	"github.com/docker/docker/pkg/ioutils"
21 22
 	"github.com/docker/docker/pkg/log"
22 23
 	"github.com/docker/docker/pkg/system"
23 24
 	"github.com/docker/docker/utils"
... ...
@@ -107,7 +108,7 @@ func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteClose
107 107
 
108 108
 	switch compression {
109 109
 	case Uncompressed:
110
-		return utils.NopWriteCloser(dest), nil
110
+		return ioutils.NopWriteCloser(dest), nil
111 111
 	case Gzip:
112 112
 		return gzip.NewWriter(dest), nil
113 113
 	case Bzip2, Xz:
... ...
@@ -8,6 +8,7 @@ import (
8 8
 	"time"
9 9
 
10 10
 	"github.com/docker/docker/engine"
11
+	"github.com/docker/docker/pkg/ioutils"
11 12
 	"github.com/docker/docker/pkg/jsonlog"
12 13
 	"github.com/docker/docker/pkg/log"
13 14
 	"github.com/docker/docker/utils"
... ...
@@ -195,7 +196,7 @@ func (daemon *Daemon) Attach(container *Container, stdin io.ReadCloser, stdinClo
195 195
 			if cStdout, err := container.StdoutPipe(); err != nil {
196 196
 				log.Errorf("attach: stdout pipe: %s", err)
197 197
 			} else {
198
-				io.Copy(&utils.NopWriter{}, cStdout)
198
+				io.Copy(&ioutils.NopWriter{}, cStdout)
199 199
 			}
200 200
 		}()
201 201
 	}
... ...
@@ -234,7 +235,7 @@ func (daemon *Daemon) Attach(container *Container, stdin io.ReadCloser, stdinClo
234 234
 			if cStderr, err := container.StderrPipe(); err != nil {
235 235
 				log.Errorf("attach: stdout pipe: %s", err)
236 236
 			} else {
237
-				io.Copy(&utils.NopWriter{}, cStderr)
237
+				io.Copy(&ioutils.NopWriter{}, cStderr)
238 238
 			}
239 239
 		}()
240 240
 	}
... ...
@@ -24,6 +24,7 @@ import (
24 24
 	"github.com/docker/docker/links"
25 25
 	"github.com/docker/docker/nat"
26 26
 	"github.com/docker/docker/pkg/broadcastwriter"
27
+	"github.com/docker/docker/pkg/ioutils"
27 28
 	"github.com/docker/docker/pkg/log"
28 29
 	"github.com/docker/docker/pkg/networkfs/etchosts"
29 30
 	"github.com/docker/docker/pkg/networkfs/resolvconf"
... ...
@@ -366,25 +367,25 @@ func (streamConfig *StreamConfig) StdinPipe() (io.WriteCloser, error) {
366 366
 func (streamConfig *StreamConfig) StdoutPipe() (io.ReadCloser, error) {
367 367
 	reader, writer := io.Pipe()
368 368
 	streamConfig.stdout.AddWriter(writer, "")
369
-	return utils.NewBufReader(reader), nil
369
+	return ioutils.NewBufReader(reader), nil
370 370
 }
371 371
 
372 372
 func (streamConfig *StreamConfig) StderrPipe() (io.ReadCloser, error) {
373 373
 	reader, writer := io.Pipe()
374 374
 	streamConfig.stderr.AddWriter(writer, "")
375
-	return utils.NewBufReader(reader), nil
375
+	return ioutils.NewBufReader(reader), nil
376 376
 }
377 377
 
378 378
 func (streamConfig *StreamConfig) StdoutLogPipe() io.ReadCloser {
379 379
 	reader, writer := io.Pipe()
380 380
 	streamConfig.stdout.AddWriter(writer, "stdout")
381
-	return utils.NewBufReader(reader)
381
+	return ioutils.NewBufReader(reader)
382 382
 }
383 383
 
384 384
 func (streamConfig *StreamConfig) StderrLogPipe() io.ReadCloser {
385 385
 	reader, writer := io.Pipe()
386 386
 	streamConfig.stderr.AddWriter(writer, "stderr")
387
-	return utils.NewBufReader(reader)
387
+	return ioutils.NewBufReader(reader)
388 388
 }
389 389
 
390 390
 func (container *Container) buildHostnameFile() error {
... ...
@@ -655,7 +656,7 @@ func (container *Container) ExportRw() (archive.Archive, error) {
655 655
 		container.Unmount()
656 656
 		return nil, err
657 657
 	}
658
-	return utils.NewReadCloserWrapper(archive, func() error {
658
+	return ioutils.NewReadCloserWrapper(archive, func() error {
659 659
 			err := archive.Close()
660 660
 			container.Unmount()
661 661
 			return err
... ...
@@ -673,7 +674,7 @@ func (container *Container) Export() (archive.Archive, error) {
673 673
 		container.Unmount()
674 674
 		return nil, err
675 675
 	}
676
-	return utils.NewReadCloserWrapper(archive, func() error {
676
+	return ioutils.NewReadCloserWrapper(archive, func() error {
677 677
 			err := archive.Close()
678 678
 			container.Unmount()
679 679
 			return err
... ...
@@ -809,7 +810,7 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) {
809 809
 		container.Unmount()
810 810
 		return nil, err
811 811
 	}
812
-	return utils.NewReadCloserWrapper(archive, func() error {
812
+	return ioutils.NewReadCloserWrapper(archive, func() error {
813 813
 			err := archive.Close()
814 814
 			container.Unmount()
815 815
 			return err
... ...
@@ -28,6 +28,7 @@ import (
28 28
 	"github.com/docker/docker/image"
29 29
 	"github.com/docker/docker/pkg/broadcastwriter"
30 30
 	"github.com/docker/docker/pkg/graphdb"
31
+	"github.com/docker/docker/pkg/ioutils"
31 32
 	"github.com/docker/docker/pkg/log"
32 33
 	"github.com/docker/docker/pkg/namesgenerator"
33 34
 	"github.com/docker/docker/pkg/networkfs/resolvconf"
... ...
@@ -201,7 +202,7 @@ func (daemon *Daemon) register(container *Container, updateSuffixarray bool) err
201 201
 	if container.Config.OpenStdin {
202 202
 		container.stdin, container.stdinPipe = io.Pipe()
203 203
 	} else {
204
-		container.stdinPipe = utils.NopWriteCloser(ioutil.Discard) // Silently drop stdin
204
+		container.stdinPipe = ioutils.NopWriteCloser(ioutil.Discard) // Silently drop stdin
205 205
 	}
206 206
 	// done
207 207
 	daemon.containers.Add(container.ID, container)
... ...
@@ -965,7 +966,7 @@ func (daemon *Daemon) Diff(container *Container) (archive.Archive, error) {
965 965
 	if err != nil {
966 966
 		return nil, err
967 967
 	}
968
-	return utils.NewReadCloserWrapper(archive, func() error {
968
+	return ioutils.NewReadCloserWrapper(archive, func() error {
969 969
 		err := archive.Close()
970 970
 		daemon.driver.Put(container.ID)
971 971
 		return err
... ...
@@ -10,6 +10,7 @@ import (
10 10
 	"sync"
11 11
 	"time"
12 12
 
13
+	"github.com/docker/docker/pkg/ioutils"
13 14
 	"github.com/docker/docker/utils"
14 15
 )
15 16
 
... ...
@@ -123,7 +124,7 @@ func (eng *Engine) Job(name string, args ...string) *Job {
123 123
 		env:    &Env{},
124 124
 	}
125 125
 	if eng.Logging {
126
-		job.Stderr.Add(utils.NopWriteCloser(eng.Stderr))
126
+		job.Stderr.Add(ioutils.NopWriteCloser(eng.Stderr))
127 127
 	}
128 128
 
129 129
 	// Catchall is shadowed by specific Register.
... ...
@@ -11,6 +11,7 @@ import (
11 11
 
12 12
 	"github.com/docker/docker/archive"
13 13
 	"github.com/docker/docker/daemon/graphdriver"
14
+	"github.com/docker/docker/pkg/ioutils"
14 15
 	"github.com/docker/docker/pkg/log"
15 16
 	"github.com/docker/docker/runconfig"
16 17
 	"github.com/docker/docker/utils"
... ...
@@ -198,7 +199,7 @@ func (img *Image) TarLayer() (arch archive.Archive, err error) {
198 198
 		if err != nil {
199 199
 			return nil, err
200 200
 		}
201
-		return utils.NewReadCloserWrapper(archive, func() error {
201
+		return ioutils.NewReadCloserWrapper(archive, func() error {
202 202
 			err := archive.Close()
203 203
 			driver.Put(img.ID)
204 204
 			return err
... ...
@@ -218,7 +219,7 @@ func (img *Image) TarLayer() (arch archive.Archive, err error) {
218 218
 	if err != nil {
219 219
 		return nil, err
220 220
 	}
221
-	return utils.NewReadCloserWrapper(archive, func() error {
221
+	return ioutils.NewReadCloserWrapper(archive, func() error {
222 222
 		err := archive.Close()
223 223
 		driver.Put(img.ID)
224 224
 		return err
... ...
@@ -20,6 +20,7 @@ import (
20 20
 	"github.com/docker/docker/engine"
21 21
 	"github.com/docker/docker/image"
22 22
 	"github.com/docker/docker/nat"
23
+	"github.com/docker/docker/pkg/ioutils"
23 24
 	"github.com/docker/docker/pkg/log"
24 25
 	"github.com/docker/docker/reexec"
25 26
 	"github.com/docker/docker/runconfig"
... ...
@@ -141,7 +142,7 @@ func setupBaseImage() {
141 141
 	if err := job.Run(); err != nil || img.Get("Id") != unitTestImageID {
142 142
 		// Retrieve the Image
143 143
 		job = eng.Job("pull", unitTestImageName)
144
-		job.Stdout.Add(utils.NopWriteCloser(os.Stdout))
144
+		job.Stdout.Add(ioutils.NopWriteCloser(os.Stdout))
145 145
 		if err := job.Run(); err != nil {
146 146
 			log.Fatalf("Unable to pull the test image: %s", err)
147 147
 		}
148 148
new file mode 100644
... ...
@@ -0,0 +1,82 @@
0
+package ioutils
1
+
2
+import (
3
+	"bytes"
4
+	"io"
5
+	"sync"
6
+)
7
+
8
+type readCloserWrapper struct {
9
+	io.Reader
10
+	closer func() error
11
+}
12
+
13
+func (r *readCloserWrapper) Close() error {
14
+	return r.closer()
15
+}
16
+
17
+func NewReadCloserWrapper(r io.Reader, closer func() error) io.ReadCloser {
18
+	return &readCloserWrapper{
19
+		Reader: r,
20
+		closer: closer,
21
+	}
22
+}
23
+
24
+type bufReader struct {
25
+	sync.Mutex
26
+	buf    *bytes.Buffer
27
+	reader io.Reader
28
+	err    error
29
+	wait   sync.Cond
30
+}
31
+
32
+func NewBufReader(r io.Reader) *bufReader {
33
+	reader := &bufReader{
34
+		buf:    &bytes.Buffer{},
35
+		reader: r,
36
+	}
37
+	reader.wait.L = &reader.Mutex
38
+	go reader.drain()
39
+	return reader
40
+}
41
+
42
+func (r *bufReader) drain() {
43
+	buf := make([]byte, 1024)
44
+	for {
45
+		n, err := r.reader.Read(buf)
46
+		r.Lock()
47
+		if err != nil {
48
+			r.err = err
49
+		} else {
50
+			r.buf.Write(buf[0:n])
51
+		}
52
+		r.wait.Signal()
53
+		r.Unlock()
54
+		if err != nil {
55
+			break
56
+		}
57
+	}
58
+}
59
+
60
+func (r *bufReader) Read(p []byte) (n int, err error) {
61
+	r.Lock()
62
+	defer r.Unlock()
63
+	for {
64
+		n, err = r.buf.Read(p)
65
+		if n > 0 {
66
+			return n, err
67
+		}
68
+		if r.err != nil {
69
+			return 0, r.err
70
+		}
71
+		r.wait.Wait()
72
+	}
73
+}
74
+
75
+func (r *bufReader) Close() error {
76
+	closer, ok := r.reader.(io.ReadCloser)
77
+	if !ok {
78
+		return nil
79
+	}
80
+	return closer.Close()
81
+}
0 82
new file mode 100644
... ...
@@ -0,0 +1,34 @@
0
+package ioutils
1
+
2
+import (
3
+	"bytes"
4
+	"io"
5
+	"io/ioutil"
6
+	"testing"
7
+)
8
+
9
+func TestBufReader(t *testing.T) {
10
+	reader, writer := io.Pipe()
11
+	bufreader := NewBufReader(reader)
12
+
13
+	// Write everything down to a Pipe
14
+	// Usually, a pipe should block but because of the buffered reader,
15
+	// the writes will go through
16
+	done := make(chan bool)
17
+	go func() {
18
+		writer.Write([]byte("hello world"))
19
+		writer.Close()
20
+		done <- true
21
+	}()
22
+
23
+	// Drain the reader *after* everything has been written, just to verify
24
+	// it is indeed buffering
25
+	<-done
26
+	output, err := ioutil.ReadAll(bufreader)
27
+	if err != nil {
28
+		t.Fatal(err)
29
+	}
30
+	if !bytes.Equal(output, []byte("hello world")) {
31
+		t.Error(string(output))
32
+	}
33
+}
0 34
new file mode 100644
... ...
@@ -0,0 +1,23 @@
0
+package ioutils
1
+
2
+import "io"
3
+
4
+type NopWriter struct{}
5
+
6
+func (*NopWriter) Write(buf []byte) (int, error) {
7
+	return len(buf), nil
8
+}
9
+
10
+type nopWriteCloser struct {
11
+	io.Writer
12
+}
13
+
14
+func (w *nopWriteCloser) Close() error { return nil }
15
+
16
+func NopWriteCloser(w io.Writer) io.WriteCloser {
17
+	return &nopWriteCloser{w}
18
+}
19
+
20
+type NopFlusher struct{}
21
+
22
+func (f *NopFlusher) Flush() {}
... ...
@@ -20,6 +20,7 @@ import (
20 20
 	"syscall"
21 21
 
22 22
 	"github.com/docker/docker/dockerversion"
23
+	"github.com/docker/docker/pkg/ioutils"
23 24
 	"github.com/docker/docker/pkg/log"
24 25
 )
25 26
 
... ...
@@ -157,81 +158,6 @@ func DockerInitPath(localCopy string) string {
157 157
 	return ""
158 158
 }
159 159
 
160
-type NopWriter struct{}
161
-
162
-func (*NopWriter) Write(buf []byte) (int, error) {
163
-	return len(buf), nil
164
-}
165
-
166
-type nopWriteCloser struct {
167
-	io.Writer
168
-}
169
-
170
-func (w *nopWriteCloser) Close() error { return nil }
171
-
172
-func NopWriteCloser(w io.Writer) io.WriteCloser {
173
-	return &nopWriteCloser{w}
174
-}
175
-
176
-type bufReader struct {
177
-	sync.Mutex
178
-	buf    *bytes.Buffer
179
-	reader io.Reader
180
-	err    error
181
-	wait   sync.Cond
182
-}
183
-
184
-func NewBufReader(r io.Reader) *bufReader {
185
-	reader := &bufReader{
186
-		buf:    &bytes.Buffer{},
187
-		reader: r,
188
-	}
189
-	reader.wait.L = &reader.Mutex
190
-	go reader.drain()
191
-	return reader
192
-}
193
-
194
-func (r *bufReader) drain() {
195
-	buf := make([]byte, 1024)
196
-	for {
197
-		n, err := r.reader.Read(buf)
198
-		r.Lock()
199
-		if err != nil {
200
-			r.err = err
201
-		} else {
202
-			r.buf.Write(buf[0:n])
203
-		}
204
-		r.wait.Signal()
205
-		r.Unlock()
206
-		if err != nil {
207
-			break
208
-		}
209
-	}
210
-}
211
-
212
-func (r *bufReader) Read(p []byte) (n int, err error) {
213
-	r.Lock()
214
-	defer r.Unlock()
215
-	for {
216
-		n, err = r.buf.Read(p)
217
-		if n > 0 {
218
-			return n, err
219
-		}
220
-		if r.err != nil {
221
-			return 0, r.err
222
-		}
223
-		r.wait.Wait()
224
-	}
225
-}
226
-
227
-func (r *bufReader) Close() error {
228
-	closer, ok := r.reader.(io.ReadCloser)
229
-	if !ok {
230
-		return nil
231
-	}
232
-	return closer.Close()
233
-}
234
-
235 160
 func GetTotalUsedFds() int {
236 161
 	if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
237 162
 		log.Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
... ...
@@ -340,10 +266,6 @@ func CopyDirectory(source, dest string) error {
340 340
 	return nil
341 341
 }
342 342
 
343
-type NopFlusher struct{}
344
-
345
-func (f *NopFlusher) Flush() {}
346
-
347 343
 type WriteFlusher struct {
348 344
 	sync.Mutex
349 345
 	w       io.Writer
... ...
@@ -370,7 +292,7 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher {
370 370
 	if f, ok := w.(http.Flusher); ok {
371 371
 		flusher = f
372 372
 	} else {
373
-		flusher = &NopFlusher{}
373
+		flusher = &ioutils.NopFlusher{}
374 374
 	}
375 375
 	return &WriteFlusher{w: w, flusher: flusher}
376 376
 }
... ...
@@ -527,22 +449,6 @@ func CopyFile(src, dst string) (int64, error) {
527 527
 	return io.Copy(df, sf)
528 528
 }
529 529
 
530
-type readCloserWrapper struct {
531
-	io.Reader
532
-	closer func() error
533
-}
534
-
535
-func (r *readCloserWrapper) Close() error {
536
-	return r.closer()
537
-}
538
-
539
-func NewReadCloserWrapper(r io.Reader, closer func() error) io.ReadCloser {
540
-	return &readCloserWrapper{
541
-		Reader: r,
542
-		closer: closer,
543
-	}
544
-}
545
-
546 530
 // ReplaceOrAppendValues returns the defaults with the overrides either
547 531
 // replaced by env key or appended to the list
548 532
 func ReplaceOrAppendEnvValues(defaults, overrides []string) []string {
... ...
@@ -1,39 +1,10 @@
1 1
 package utils
2 2
 
3 3
 import (
4
-	"bytes"
5
-	"io"
6
-	"io/ioutil"
7 4
 	"os"
8 5
 	"testing"
9 6
 )
10 7
 
11
-func TestBufReader(t *testing.T) {
12
-	reader, writer := io.Pipe()
13
-	bufreader := NewBufReader(reader)
14
-
15
-	// Write everything down to a Pipe
16
-	// Usually, a pipe should block but because of the buffered reader,
17
-	// the writes will go through
18
-	done := make(chan bool)
19
-	go func() {
20
-		writer.Write([]byte("hello world"))
21
-		writer.Close()
22
-		done <- true
23
-	}()
24
-
25
-	// Drain the reader *after* everything has been written, just to verify
26
-	// it is indeed buffering
27
-	<-done
28
-	output, err := ioutil.ReadAll(bufreader)
29
-	if err != nil {
30
-		t.Fatal(err)
31
-	}
32
-	if !bytes.Equal(output, []byte("hello world")) {
33
-		t.Error(string(output))
34
-	}
35
-}
36
-
37 8
 func TestCheckLocalDns(t *testing.T) {
38 9
 	for resolv, result := range map[string]bool{`# Dynamic
39 10
 nameserver 10.0.2.3