Browse code

Implement tail for docker logs

Fixes #4330
Docker-DCO-1.1-Signed-off-by: Alexandr Morozov <lk4d4math@gmail.com> (github: LK4D4)

Alexandr Morozov authored on 2014/06/03 20:09:33
Showing 8 changed files
... ...
@@ -1693,6 +1693,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error {
1693 1693
 		cmd    = cli.Subcmd("logs", "CONTAINER", "Fetch the logs of a container")
1694 1694
 		follow = cmd.Bool([]string{"f", "-follow"}, false, "Follow log output")
1695 1695
 		times  = cmd.Bool([]string{"t", "-timestamps"}, false, "Show timestamps")
1696
+		tail   = cmd.String([]string{"-tail"}, "all", "Output the specified number of lines at the end of logs(all logs by default)")
1696 1697
 	)
1697 1698
 
1698 1699
 	if err := cmd.Parse(args); err != nil {
... ...
@@ -1726,6 +1727,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error {
1726 1726
 	if *follow {
1727 1727
 		v.Set("follow", "1")
1728 1728
 	}
1729
+	v.Set("tail", *tail)
1729 1730
 
1730 1731
 	return cli.streamHelper("GET", "/containers/"+name+"/logs?"+v.Encode(), env.GetSubEnv("Config").GetBool("Tty"), nil, cli.out, cli.err, nil)
1731 1732
 }
... ...
@@ -378,6 +378,7 @@ func getContainersLogs(eng *engine.Engine, version version.Version, w http.Respo
378 378
 		return err
379 379
 	}
380 380
 	logsJob.Setenv("follow", r.Form.Get("follow"))
381
+	logsJob.Setenv("tail", r.Form.Get("tail"))
381 382
 	logsJob.Setenv("stdout", r.Form.Get("stdout"))
382 383
 	logsJob.Setenv("stderr", r.Form.Get("stderr"))
383 384
 	logsJob.Setenv("timestamps", r.Form.Get("timestamps"))
... ...
@@ -306,7 +306,7 @@ Get stdout and stderr logs from the container ``id``
306 306
 
307 307
     **Example request**:
308 308
 
309
-       GET /containers/4fa6e0f0c678/logs?stderr=1&stdout=1&timestamps=1&follow=1 HTTP/1.1
309
+       GET /containers/4fa6e0f0c678/logs?stderr=1&stdout=1&timestamps=1&follow=1&tail=10 HTTP/1.1
310 310
 
311 311
     **Example response**:
312 312
 
... ...
@@ -319,14 +319,12 @@ Get stdout and stderr logs from the container ``id``
319 319
 
320 320
      
321 321
 
322
-    -   **follow** – 1/True/true or 0/False/false, return stream.
323
-        Default false
324
-    -   **stdout** – 1/True/true or 0/False/false, if logs=true, return
325
-        stdout log. Default false
326
-    -   **stderr** – 1/True/true or 0/False/false, if logs=true, return
327
-        stderr log. Default false
328
-    -   **timestamps** – 1/True/true or 0/False/false, if logs=true, print
329
-        timestamps for every log line. Default false
322
+    -   **follow** – 1/True/true or 0/False/false, return stream. Default false
323
+    -   **stdout** – 1/True/true or 0/False/false, show stdout log. Default false
324
+    -   **stderr** – 1/True/true or 0/False/false, show stderr log. Default false
325
+    -   **timestamps** – 1/True/true or 0/False/false, print timestamps for
326
+        every log line. Default false
327
+    -   **tail** – Output specified number of lines at the end of logs: `all` or `<number>`. Default all
330 328
 
331 329
     Status Codes:
332 330
 
... ...
@@ -738,13 +738,15 @@ specify this by adding the server name.
738 738
 
739 739
       -f, --follow=false        Follow log output
740 740
       -t, --timestamps=false    Show timestamps
741
+      --tail="all"              Output the specified number of lines at the end of logs (all logs by default)
741 742
 
742
-The `docker logs` command batch-retrieves all logs
743
-present at the time of execution.
743
+The `docker logs` command batch-retrieves logs present at the time of execution.
744 744
 
745
-The ``docker logs --follow`` command will first return all logs from the
746
-beginning and then continue streaming new output from the container's `STDOUT`
747
-and `STDERR`.
745
+The `docker logs --follow` command will continue streaming the new output from
746
+the container's `STDOUT` and `STDERR`.
747
+
748
+Passing a negative number or a non-integer to --tail is invalid and the
749
+value is set to all in that case. This behavior may change in the future.
748 750
 
749 751
 ## port
750 752
 
... ...
@@ -169,3 +169,47 @@ func TestLogsStderrInStdout(t *testing.T) {
169 169
 
170 170
 	logDone("logs - stderr in stdout (with pseudo-tty)")
171 171
 }
172
+
173
+func TestLogsTail(t *testing.T) {
174
+	testLen := 100
175
+	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", fmt.Sprintf("for i in $(seq 1 %d); do echo =; done;", testLen))
176
+
177
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
178
+	errorOut(err, t, fmt.Sprintf("run failed with errors: %v", err))
179
+
180
+	cleanedContainerID := stripTrailingCharacters(out)
181
+	exec.Command(dockerBinary, "wait", cleanedContainerID).Run()
182
+
183
+	logsCmd := exec.Command(dockerBinary, "logs", "--tail", "5", cleanedContainerID)
184
+	out, _, _, err = runCommandWithStdoutStderr(logsCmd)
185
+	errorOut(err, t, fmt.Sprintf("failed to log container: %v %v", out, err))
186
+
187
+	lines := strings.Split(out, "\n")
188
+
189
+	if len(lines) != 6 {
190
+		t.Fatalf("Expected log %d lines, received %d\n", 6, len(lines))
191
+	}
192
+
193
+	logsCmd = exec.Command(dockerBinary, "logs", "--tail", "all", cleanedContainerID)
194
+	out, _, _, err = runCommandWithStdoutStderr(logsCmd)
195
+	errorOut(err, t, fmt.Sprintf("failed to log container: %v %v", out, err))
196
+
197
+	lines = strings.Split(out, "\n")
198
+
199
+	if len(lines) != testLen+1 {
200
+		t.Fatalf("Expected log %d lines, received %d\n", testLen+1, len(lines))
201
+	}
202
+
203
+	logsCmd = exec.Command(dockerBinary, "logs", "--tail", "random", cleanedContainerID)
204
+	out, _, _, err = runCommandWithStdoutStderr(logsCmd)
205
+	errorOut(err, t, fmt.Sprintf("failed to log container: %v %v", out, err))
206
+
207
+	lines = strings.Split(out, "\n")
208
+
209
+	if len(lines) != testLen+1 {
210
+		t.Fatalf("Expected log %d lines, received %d\n", testLen+1, len(lines))
211
+	}
212
+
213
+	deleteContainer(cleanedContainerID)
214
+	logDone("logs - logs tail")
215
+}
172 216
new file mode 100644
... ...
@@ -0,0 +1,61 @@
0
+package tailfile
1
+
2
+import (
3
+	"bytes"
4
+	"errors"
5
+	"os"
6
+)
7
+
8
+const blockSize = 1024
9
+
10
+var eol = []byte("\n")
11
+var ErrNonPositiveLinesNumber = errors.New("Lines number must be positive")
12
+
13
+//TailFile returns last n lines of file f
14
+func TailFile(f *os.File, n int) ([][]byte, error) {
15
+	if n <= 0 {
16
+		return nil, ErrNonPositiveLinesNumber
17
+	}
18
+	size, err := f.Seek(0, os.SEEK_END)
19
+	if err != nil {
20
+		return nil, err
21
+	}
22
+	block := -1
23
+	var data []byte
24
+	var cnt int
25
+	for {
26
+		var b []byte
27
+		step := int64(block * blockSize)
28
+		left := size + step // how many bytes to beginning
29
+		if left < 0 {
30
+			if _, err := f.Seek(0, os.SEEK_SET); err != nil {
31
+				return nil, err
32
+			}
33
+			b = make([]byte, blockSize+left)
34
+			if _, err := f.Read(b); err != nil {
35
+				return nil, err
36
+			}
37
+			data = append(b, data...)
38
+			break
39
+		} else {
40
+			b = make([]byte, blockSize)
41
+			if _, err := f.Seek(step, os.SEEK_END); err != nil {
42
+				return nil, err
43
+			}
44
+			if _, err := f.Read(b); err != nil {
45
+				return nil, err
46
+			}
47
+			data = append(b, data...)
48
+		}
49
+		cnt += bytes.Count(b, eol)
50
+		if cnt > n {
51
+			break
52
+		}
53
+		block--
54
+	}
55
+	lines := bytes.Split(data, eol)
56
+	if n < len(lines) {
57
+		return lines[len(lines)-n-1 : len(lines)-1], nil
58
+	}
59
+	return lines[:len(lines)-1], nil
60
+}
0 61
new file mode 100644
... ...
@@ -0,0 +1,148 @@
0
+package tailfile
1
+
2
+import (
3
+	"io/ioutil"
4
+	"os"
5
+	"testing"
6
+)
7
+
8
+func TestTailFile(t *testing.T) {
9
+	f, err := ioutil.TempFile("", "tail-test")
10
+	if err != nil {
11
+		t.Fatal(err)
12
+	}
13
+	defer f.Close()
14
+	defer os.RemoveAll(f.Name())
15
+	testFile := []byte(`first line
16
+second line
17
+third line
18
+fourth line
19
+fifth line
20
+next first line
21
+next second line
22
+next third line
23
+next fourth line
24
+next fifth line
25
+last first line
26
+next first line
27
+next second line
28
+next third line
29
+next fourth line
30
+next fifth line
31
+next first line
32
+next second line
33
+next third line
34
+next fourth line
35
+next fifth line
36
+last second line
37
+last third line
38
+last fourth line
39
+last fifth line
40
+truncated line`)
41
+	if _, err := f.Write(testFile); err != nil {
42
+		t.Fatal(err)
43
+	}
44
+	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
45
+		t.Fatal(err)
46
+	}
47
+	expected := []string{"last fourth line", "last fifth line"}
48
+	res, err := TailFile(f, 2)
49
+	if err != nil {
50
+		t.Fatal(err)
51
+	}
52
+	for i, l := range res {
53
+		t.Logf("%s", l)
54
+		if expected[i] != string(l) {
55
+			t.Fatalf("Expected line %s, got %s", expected[i], l)
56
+		}
57
+	}
58
+}
59
+
60
+func TestTailFileManyLines(t *testing.T) {
61
+	f, err := ioutil.TempFile("", "tail-test")
62
+	if err != nil {
63
+		t.Fatal(err)
64
+	}
65
+	defer f.Close()
66
+	defer os.RemoveAll(f.Name())
67
+	testFile := []byte(`first line
68
+second line
69
+truncated line`)
70
+	if _, err := f.Write(testFile); err != nil {
71
+		t.Fatal(err)
72
+	}
73
+	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
74
+		t.Fatal(err)
75
+	}
76
+	expected := []string{"first line", "second line"}
77
+	res, err := TailFile(f, 10000)
78
+	if err != nil {
79
+		t.Fatal(err)
80
+	}
81
+	for i, l := range res {
82
+		t.Logf("%s", l)
83
+		if expected[i] != string(l) {
84
+			t.Fatalf("Expected line %s, got %s", expected[i], l)
85
+		}
86
+	}
87
+}
88
+
89
+func TestTailEmptyFile(t *testing.T) {
90
+	f, err := ioutil.TempFile("", "tail-test")
91
+	if err != nil {
92
+		t.Fatal(err)
93
+	}
94
+	defer f.Close()
95
+	defer os.RemoveAll(f.Name())
96
+	res, err := TailFile(f, 10000)
97
+	if err != nil {
98
+		t.Fatal(err)
99
+	}
100
+	if len(res) != 0 {
101
+		t.Fatal("Must be empty slice from empty file")
102
+	}
103
+}
104
+
105
+func TestTailNegativeN(t *testing.T) {
106
+	f, err := ioutil.TempFile("", "tail-test")
107
+	if err != nil {
108
+		t.Fatal(err)
109
+	}
110
+	defer f.Close()
111
+	defer os.RemoveAll(f.Name())
112
+	testFile := []byte(`first line
113
+second line
114
+truncated line`)
115
+	if _, err := f.Write(testFile); err != nil {
116
+		t.Fatal(err)
117
+	}
118
+	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
119
+		t.Fatal(err)
120
+	}
121
+	if _, err := TailFile(f, -1); err != ErrNonPositiveLinesNumber {
122
+		t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err)
123
+	}
124
+	if _, err := TailFile(f, 0); err != ErrNonPositiveLinesNumber {
125
+		t.Fatalf("Expected ErrNonPositiveLinesNumber, got %s", err)
126
+	}
127
+}
128
+
129
+func BenchmarkTail(b *testing.B) {
130
+	f, err := ioutil.TempFile("", "tail-test")
131
+	if err != nil {
132
+		b.Fatal(err)
133
+	}
134
+	defer f.Close()
135
+	defer os.RemoveAll(f.Name())
136
+	for i := 0; i < 10000; i++ {
137
+		if _, err := f.Write([]byte("tailfile pretty interesting line\n")); err != nil {
138
+			b.Fatal(err)
139
+		}
140
+	}
141
+	b.ResetTimer()
142
+	for i := 0; i < b.N; i++ {
143
+		if _, err := TailFile(f, 1000); err != nil {
144
+			b.Fatal(err)
145
+		}
146
+	}
147
+}
... ...
@@ -22,6 +22,7 @@
22 22
 package server
23 23
 
24 24
 import (
25
+	"bytes"
25 26
 	"encoding/json"
26 27
 	"fmt"
27 28
 	"io"
... ...
@@ -52,6 +53,7 @@ import (
52 52
 	"github.com/dotcloud/docker/image"
53 53
 	"github.com/dotcloud/docker/pkg/graphdb"
54 54
 	"github.com/dotcloud/docker/pkg/signal"
55
+	"github.com/dotcloud/docker/pkg/tailfile"
55 56
 	"github.com/dotcloud/docker/registry"
56 57
 	"github.com/dotcloud/docker/runconfig"
57 58
 	"github.com/dotcloud/docker/utils"
... ...
@@ -2153,8 +2155,10 @@ func (srv *Server) ContainerLogs(job *engine.Job) engine.Status {
2153 2153
 		name   = job.Args[0]
2154 2154
 		stdout = job.GetenvBool("stdout")
2155 2155
 		stderr = job.GetenvBool("stderr")
2156
+		tail   = job.Getenv("tail")
2156 2157
 		follow = job.GetenvBool("follow")
2157 2158
 		times  = job.GetenvBool("timestamps")
2159
+		lines  = -1
2158 2160
 		format string
2159 2161
 	)
2160 2162
 	if !(stdout || stderr) {
... ...
@@ -2163,6 +2167,9 @@ func (srv *Server) ContainerLogs(job *engine.Job) engine.Status {
2163 2163
 	if times {
2164 2164
 		format = time.StampMilli
2165 2165
 	}
2166
+	if tail == "" {
2167
+		tail = "all"
2168
+	}
2166 2169
 	container := srv.daemon.Get(name)
2167 2170
 	if container == nil {
2168 2171
 		return job.Errorf("No such container: %s", name)
... ...
@@ -2190,25 +2197,47 @@ func (srv *Server) ContainerLogs(job *engine.Job) engine.Status {
2190 2190
 	} else if err != nil {
2191 2191
 		utils.Errorf("Error reading logs (json): %s", err)
2192 2192
 	} else {
2193
-		dec := json.NewDecoder(cLog)
2194
-		for {
2195
-			l := &utils.JSONLog{}
2196
-
2197
-			if err := dec.Decode(l); err == io.EOF {
2198
-				break
2199
-			} else if err != nil {
2200
-				utils.Errorf("Error streaming logs: %s", err)
2201
-				break
2202
-			}
2203
-			logLine := l.Log
2204
-			if times {
2205
-				logLine = fmt.Sprintf("[%s] %s", l.Created.Format(format), logLine)
2193
+		if tail != "all" {
2194
+			var err error
2195
+			lines, err = strconv.Atoi(tail)
2196
+			if err != nil {
2197
+				utils.Errorf("Failed to parse tail %s, error: %v, show all logs", err)
2198
+				lines = -1
2206 2199
 			}
2207
-			if l.Stream == "stdout" && stdout {
2208
-				fmt.Fprintf(job.Stdout, "%s", logLine)
2200
+		}
2201
+		if lines != 0 {
2202
+			if lines > 0 {
2203
+				f := cLog.(*os.File)
2204
+				ls, err := tailfile.TailFile(f, lines)
2205
+				if err != nil {
2206
+					return job.Error(err)
2207
+				}
2208
+				tmp := bytes.NewBuffer([]byte{})
2209
+				for _, l := range ls {
2210
+					fmt.Fprintf(tmp, "%s\n", l)
2211
+				}
2212
+				cLog = tmp
2209 2213
 			}
2210
-			if l.Stream == "stderr" && stderr {
2211
-				fmt.Fprintf(job.Stderr, "%s", logLine)
2214
+			dec := json.NewDecoder(cLog)
2215
+			for {
2216
+				l := &utils.JSONLog{}
2217
+
2218
+				if err := dec.Decode(l); err == io.EOF {
2219
+					break
2220
+				} else if err != nil {
2221
+					utils.Errorf("Error streaming logs: %s", err)
2222
+					break
2223
+				}
2224
+				logLine := l.Log
2225
+				if times {
2226
+					logLine = fmt.Sprintf("[%s] %s", l.Created.Format(format), logLine)
2227
+				}
2228
+				if l.Stream == "stdout" && stdout {
2229
+					fmt.Fprintf(job.Stdout, "%s", logLine)
2230
+				}
2231
+				if l.Stream == "stderr" && stderr {
2232
+					fmt.Fprintf(job.Stderr, "%s", logLine)
2233
+				}
2212 2234
 			}
2213 2235
 		}
2214 2236
 	}