Browse code

Refactor log file writer

Make the `*RotateFileWriter` specifically about writing
`logger.Message`'s, which is what it's used for.

This allows for future changes where the log writer can cache details
about log entries such as (e.g.) the timestamps included in a particular
log file, which can be used to optimize reads.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Brian Goff authored on 2017/07/18 05:29:11
Showing 3 changed files
... ...
@@ -7,7 +7,6 @@ import (
7 7
 	"bytes"
8 8
 	"encoding/json"
9 9
 	"fmt"
10
-	"io"
11 10
 	"strconv"
12 11
 	"sync"
13 12
 
... ...
@@ -24,10 +23,7 @@ const Name = "json-file"
24 24
 
25 25
 // JSONFileLogger is Logger implementation for default Docker logging.
26 26
 type JSONFileLogger struct {
27
-	extra []byte // json-encoded extra attributes
28
-
29 27
 	mu      sync.RWMutex
30
-	buf     *bytes.Buffer // avoids allocating a new buffer on each call to `Log()`
31 28
 	closed  bool
32 29
 	writer  *loggerutils.RotateFileWriter
33 30
 	readers map[*logger.LogWatcher]struct{} // stores the active log followers
... ...
@@ -65,11 +61,6 @@ func New(info logger.Info) (logger.Logger, error) {
65 65
 		}
66 66
 	}
67 67
 
68
-	writer, err := loggerutils.NewRotateFileWriter(info.LogPath, capval, maxFiles)
69
-	if err != nil {
70
-		return nil, err
71
-	}
72
-
73 68
 	var extra []byte
74 69
 	attrs, err := info.ExtraAttributes(nil)
75 70
 	if err != nil {
... ...
@@ -83,33 +74,34 @@ func New(info logger.Info) (logger.Logger, error) {
83 83
 		}
84 84
 	}
85 85
 
86
+	buf := bytes.NewBuffer(nil)
87
+	marshalFunc := func(msg *logger.Message) ([]byte, error) {
88
+		if err := marshalMessage(msg, extra, buf); err != nil {
89
+			return nil, err
90
+		}
91
+		b := buf.Bytes()
92
+		buf.Reset()
93
+		return b, nil
94
+	}
95
+	writer, err := loggerutils.NewRotateFileWriter(info.LogPath, capval, maxFiles, marshalFunc)
96
+	if err != nil {
97
+		return nil, err
98
+	}
99
+
86 100
 	return &JSONFileLogger{
87
-		buf:     bytes.NewBuffer(nil),
88 101
 		writer:  writer,
89 102
 		readers: make(map[*logger.LogWatcher]struct{}),
90
-		extra:   extra,
91 103
 	}, nil
92 104
 }
93 105
 
94 106
 // Log converts logger.Message to jsonlog.JSONLog and serializes it to file.
95 107
 func (l *JSONFileLogger) Log(msg *logger.Message) error {
96 108
 	l.mu.Lock()
97
-	err := writeMessageBuf(l.writer, msg, l.extra, l.buf)
98
-	l.buf.Reset()
109
+	err := l.writer.WriteLogEntry(msg)
99 110
 	l.mu.Unlock()
100 111
 	return err
101 112
 }
102 113
 
103
-func writeMessageBuf(w io.Writer, m *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error {
104
-	if err := marshalMessage(m, extra, buf); err != nil {
105
-		logger.PutMessage(m)
106
-		return err
107
-	}
108
-	logger.PutMessage(m)
109
-	_, err := w.Write(buf.Bytes())
110
-	return errors.Wrap(err, "error writing log entry")
111
-}
112
-
113 114
 func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error {
114 115
 	logLine := msg.Line
115 116
 	if !msg.Partial {
... ...
@@ -140,3 +140,6 @@ type Capability struct {
140 140
 	// Determines if a log driver can read back logs
141 141
 	ReadLogs bool
142 142
 }
143
+
144
+// MarshalFunc is a func that marshals a message into an arbitrary format
145
+type MarshalFunc func(*Message) ([]byte, error)
... ...
@@ -1,12 +1,13 @@
1 1
 package loggerutils
2 2
 
3 3
 import (
4
-	"errors"
5 4
 	"os"
6 5
 	"strconv"
7 6
 	"sync"
8 7
 
8
+	"github.com/docker/docker/daemon/logger"
9 9
 	"github.com/docker/docker/pkg/pubsub"
10
+	"github.com/pkg/errors"
10 11
 )
11 12
 
12 13
 // RotateFileWriter is Logger implementation for default Docker logging.
... ...
@@ -18,10 +19,11 @@ type RotateFileWriter struct {
18 18
 	currentSize  int64 // current size of the latest file
19 19
 	maxFiles     int   //maximum number of files
20 20
 	notifyRotate *pubsub.Publisher
21
+	marshal      logger.MarshalFunc
21 22
 }
22 23
 
23 24
 //NewRotateFileWriter creates new RotateFileWriter
24
-func NewRotateFileWriter(logPath string, capacity int64, maxFiles int) (*RotateFileWriter, error) {
25
+func NewRotateFileWriter(logPath string, capacity int64, maxFiles int, marshaller logger.MarshalFunc) (*RotateFileWriter, error) {
25 26
 	log, err := os.OpenFile(logPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640)
26 27
 	if err != nil {
27 28
 		return nil, err
... ...
@@ -38,27 +40,37 @@ func NewRotateFileWriter(logPath string, capacity int64, maxFiles int) (*RotateF
38 38
 		currentSize:  size,
39 39
 		maxFiles:     maxFiles,
40 40
 		notifyRotate: pubsub.NewPublisher(0, 1),
41
+		marshal:      marshaller,
41 42
 	}, nil
42 43
 }
43 44
 
44
-//WriteLog write log message to File
45
-func (w *RotateFileWriter) Write(message []byte) (int, error) {
45
+// WriteLogEntry writes the provided log message to the current log file.
46
+// This may trigger a rotation event if the max file/capacity limits are hit.
47
+func (w *RotateFileWriter) WriteLogEntry(msg *logger.Message) error {
48
+	b, err := w.marshal(msg)
49
+	if err != nil {
50
+		return errors.Wrap(err, "error marshalling log message")
51
+	}
52
+
53
+	logger.PutMessage(msg)
54
+
46 55
 	w.mu.Lock()
47 56
 	if w.closed {
48 57
 		w.mu.Unlock()
49
-		return -1, errors.New("cannot write because the output file was closed")
58
+		return errors.New("cannot write because the output file was closed")
50 59
 	}
60
+
51 61
 	if err := w.checkCapacityAndRotate(); err != nil {
52 62
 		w.mu.Unlock()
53
-		return -1, err
63
+		return err
54 64
 	}
55 65
 
56
-	n, err := w.f.Write(message)
66
+	n, err := w.f.Write(b)
57 67
 	if err == nil {
58 68
 		w.currentSize += int64(n)
59 69
 	}
60 70
 	w.mu.Unlock()
61
-	return n, err
71
+	return err
62 72
 }
63 73
 
64 74
 func (w *RotateFileWriter) checkCapacityAndRotate() error {
... ...
@@ -69,7 +81,7 @@ func (w *RotateFileWriter) checkCapacityAndRotate() error {
69 69
 	if w.currentSize >= w.capacity {
70 70
 		name := w.f.Name()
71 71
 		if err := w.f.Close(); err != nil {
72
-			return err
72
+			return errors.Wrap(err, "error closing file")
73 73
 		}
74 74
 		if err := rotate(name, w.maxFiles); err != nil {
75 75
 			return err
... ...
@@ -94,12 +106,12 @@ func rotate(name string, maxFiles int) error {
94 94
 		toPath := name + "." + strconv.Itoa(i)
95 95
 		fromPath := name + "." + strconv.Itoa(i-1)
96 96
 		if err := os.Rename(fromPath, toPath); err != nil && !os.IsNotExist(err) {
97
-			return err
97
+			return errors.Wrap(err, "error rotating old log entries")
98 98
 		}
99 99
 	}
100 100
 
101 101
 	if err := os.Rename(name, name+".1"); err != nil && !os.IsNotExist(err) {
102
-		return err
102
+		return errors.Wrap(err, "error rotating current log")
103 103
 	}
104 104
 	return nil
105 105
 }