Browse code

add labels/env log option for jsonfile

this allows jsonfile logger to collect extra metadata from containers with
`--log-opt labels=label1,label2 --log-opt env=env1,env2`.

Extra attributes are saved into `attrs` attributes for each log data.

Signed-off-by: Daniel Dao <dqminh@cloudflare.com>

Daniel Dao authored on 2015/10/05 06:07:09
Showing 4 changed files
... ...
@@ -41,6 +41,7 @@ type JSONFileLogger struct {
41 41
 	ctx          logger.Context
42 42
 	readers      map[*logger.LogWatcher]struct{} // stores the active log followers
43 43
 	notifyRotate *pubsub.Publisher
44
+	extra        []byte // json-encoded extra attributes
44 45
 }
45 46
 
46 47
 func init() {
... ...
@@ -77,6 +78,16 @@ func New(ctx logger.Context) (logger.Logger, error) {
77 77
 			return nil, fmt.Errorf("max-file cannot be less than 1")
78 78
 		}
79 79
 	}
80
+
81
+	var extra []byte
82
+	if attrs := ctx.ExtraAttributes(nil); len(attrs) > 0 {
83
+		var err error
84
+		extra, err = json.Marshal(attrs)
85
+		if err != nil {
86
+			return nil, err
87
+		}
88
+	}
89
+
80 90
 	return &JSONFileLogger{
81 91
 		f:            log,
82 92
 		buf:          bytes.NewBuffer(nil),
... ...
@@ -85,6 +96,7 @@ func New(ctx logger.Context) (logger.Logger, error) {
85 85
 		n:            maxFiles,
86 86
 		readers:      make(map[*logger.LogWatcher]struct{}),
87 87
 		notifyRotate: pubsub.NewPublisher(0, 1),
88
+		extra:        extra,
88 89
 	}, nil
89 90
 }
90 91
 
... ...
@@ -97,7 +109,12 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error {
97 97
 	if err != nil {
98 98
 		return err
99 99
 	}
100
-	err = (&jsonlog.JSONLogs{Log: append(msg.Line, '\n'), Stream: msg.Source, Created: timestamp}).MarshalJSONBuf(l.buf)
100
+	err = (&jsonlog.JSONLogs{
101
+		Log:      append(msg.Line, '\n'),
102
+		Stream:   msg.Source,
103
+		Created:  timestamp,
104
+		RawAttrs: l.extra,
105
+	}).MarshalJSONBuf(l.buf)
101 106
 	if err != nil {
102 107
 		return err
103 108
 	}
... ...
@@ -181,6 +198,8 @@ func ValidateLogOpt(cfg map[string]string) error {
181 181
 		switch key {
182 182
 		case "max-file":
183 183
 		case "max-size":
184
+		case "labels":
185
+		case "env":
184 186
 		default:
185 187
 			return fmt.Errorf("unknown log opt '%s' for json-file log driver", key)
186 188
 		}
... ...
@@ -1,9 +1,11 @@
1 1
 package jsonfilelog
2 2
 
3 3
 import (
4
+	"encoding/json"
4 5
 	"io/ioutil"
5 6
 	"os"
6 7
 	"path/filepath"
8
+	"reflect"
7 9
 	"strconv"
8 10
 	"testing"
9 11
 	"time"
... ...
@@ -149,3 +151,51 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
149 149
 	}
150 150
 
151 151
 }
152
+
153
+func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
154
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
155
+	tmp, err := ioutil.TempDir("", "docker-logger-")
156
+	if err != nil {
157
+		t.Fatal(err)
158
+	}
159
+	defer os.RemoveAll(tmp)
160
+	filename := filepath.Join(tmp, "container.log")
161
+	config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl"}
162
+	l, err := New(logger.Context{
163
+		ContainerID:     cid,
164
+		LogPath:         filename,
165
+		Config:          config,
166
+		ContainerLabels: map[string]string{"rack": "101", "dc": "lhr"},
167
+		ContainerEnv:    []string{"environ=production", "debug=false", "port=10001", "ssl=true"},
168
+	})
169
+	if err != nil {
170
+		t.Fatal(err)
171
+	}
172
+	defer l.Close()
173
+	if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line"), Source: "src1"}); err != nil {
174
+		t.Fatal(err)
175
+	}
176
+	res, err := ioutil.ReadFile(filename)
177
+	if err != nil {
178
+		t.Fatal(err)
179
+	}
180
+
181
+	var jsonLog jsonlog.JSONLogs
182
+	if err := json.Unmarshal(res, &jsonLog); err != nil {
183
+		t.Fatal(err)
184
+	}
185
+	extra := make(map[string]string)
186
+	if err := json.Unmarshal(jsonLog.RawAttrs, &extra); err != nil {
187
+		t.Fatal(err)
188
+	}
189
+	expected := map[string]string{
190
+		"rack":    "101",
191
+		"dc":      "lhr",
192
+		"environ": "production",
193
+		"debug":   "false",
194
+		"ssl":     "true",
195
+	}
196
+	if !reflect.DeepEqual(extra, expected) {
197
+		t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected)
198
+	}
199
+}
... ...
@@ -2,6 +2,7 @@ package jsonlog
2 2
 
3 3
 import (
4 4
 	"bytes"
5
+	"encoding/json"
5 6
 	"unicode/utf8"
6 7
 )
7 8
 
... ...
@@ -12,6 +13,9 @@ type JSONLogs struct {
12 12
 	Log     []byte `json:"log,omitempty"`
13 13
 	Stream  string `json:"stream,omitempty"`
14 14
 	Created string `json:"time"`
15
+
16
+	// json-encoded bytes
17
+	RawAttrs json.RawMessage `json:"attrs,omitempty"`
15 18
 }
16 19
 
17 20
 // MarshalJSONBuf is based on the same method from JSONLog
... ...
@@ -34,6 +38,15 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
34 34
 		buf.WriteString(`"stream":`)
35 35
 		ffjsonWriteJSONString(buf, mj.Stream)
36 36
 	}
37
+	if len(mj.RawAttrs) > 0 {
38
+		if first == true {
39
+			first = false
40
+		} else {
41
+			buf.WriteString(`,`)
42
+		}
43
+		buf.WriteString(`"attrs":`)
44
+		buf.Write(mj.RawAttrs)
45
+	}
37 46
 	if first == true {
38 47
 		first = false
39 48
 	} else {
... ...
@@ -21,6 +21,8 @@ func TestJSONLogsMarshalJSONBuf(t *testing.T) {
21 21
 		&JSONLogs{Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`,
22 22
 		&JSONLogs{Log: []byte{0xaF}}:            `^{\"log\":\"\\ufffd\",\"time\":}$`,
23 23
 		&JSONLogs{Log: []byte{0x7F}}:            `^{\"log\":\"\x7f\",\"time\":}$`,
24
+		// with raw attributes
25
+		&JSONLogs{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`,
24 26
 	}
25 27
 	for jsonLog, expression := range logs {
26 28
 		var buf bytes.Buffer