Browse code

Added tag log option to json-logger

Fixes #19803
Updated the json-logger to utilize the common log option
'tag' that can define container/image information to include
as part of logging.

When the 'tag' log option is not included, there is no change
to the log content via the json-logger. When the 'tag' log option
is included, the tag will be parsed as a template and the result
will be stored within each log entry as the attribute 'tag'.

Update: Removing test added to integration_cli as those have been deprecated.
Update: Using proper test calls (require and assert) in jsonfilelog_test.go based on review.
Update: Added new unit test configs for logs with tag. Updated unit test error checking.
Update: Cleanup check in jsonlogbytes_test.go to match pending changes in PR #34946.
Update: Merging to correct conflicts from PR #34946.

Signed-off-by: bonczj <josh.bonczkowski@gmail.com>

bonczj authored on 2017/09/26 23:22:10
Showing 6 changed files
... ...
@@ -27,6 +27,7 @@ type JSONFileLogger struct {
27 27
 	closed  bool
28 28
 	writer  *loggerutils.LogFile
29 29
 	readers map[*logger.LogWatcher]struct{} // stores the active log followers
30
+	tag     string                          // tag values requested by the user to log
30 31
 }
31 32
 
32 33
 func init() {
... ...
@@ -61,6 +62,12 @@ func New(info logger.Info) (logger.Logger, error) {
61 61
 		}
62 62
 	}
63 63
 
64
+	// no default template. only use a tag if the user asked for it
65
+	tag, err := loggerutils.ParseLogTag(info, "")
66
+	if err != nil {
67
+		return nil, err
68
+	}
69
+
64 70
 	var extra []byte
65 71
 	attrs, err := info.ExtraAttributes(nil)
66 72
 	if err != nil {
... ...
@@ -76,7 +83,7 @@ func New(info logger.Info) (logger.Logger, error) {
76 76
 
77 77
 	buf := bytes.NewBuffer(nil)
78 78
 	marshalFunc := func(msg *logger.Message) ([]byte, error) {
79
-		if err := marshalMessage(msg, extra, buf); err != nil {
79
+		if err := marshalMessage(msg, extra, buf, tag); err != nil {
80 80
 			return nil, err
81 81
 		}
82 82
 		b := buf.Bytes()
... ...
@@ -92,6 +99,7 @@ func New(info logger.Info) (logger.Logger, error) {
92 92
 	return &JSONFileLogger{
93 93
 		writer:  writer,
94 94
 		readers: make(map[*logger.LogWatcher]struct{}),
95
+		tag:     tag,
95 96
 	}, nil
96 97
 }
97 98
 
... ...
@@ -103,7 +111,7 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error {
103 103
 	return err
104 104
 }
105 105
 
106
-func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error {
106
+func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer, tag string) error {
107 107
 	logLine := msg.Line
108 108
 	if !msg.Partial {
109 109
 		logLine = append(msg.Line, '\n')
... ...
@@ -113,6 +121,7 @@ func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffe
113 113
 		Stream:   msg.Source,
114 114
 		Created:  msg.Timestamp,
115 115
 		RawAttrs: extra,
116
+		Tag:      tag,
116 117
 	}).MarshalJSONBuf(buf)
117 118
 	if err != nil {
118 119
 		return errors.Wrap(err, "error writing log message to buffer")
... ...
@@ -130,6 +139,7 @@ func ValidateLogOpt(cfg map[string]string) error {
130 130
 		case "labels":
131 131
 		case "env":
132 132
 		case "env-regex":
133
+		case "tag":
133 134
 		default:
134 135
 			return fmt.Errorf("unknown log opt '%s' for json-file log driver", key)
135 136
 		}
... ...
@@ -57,6 +57,49 @@ func TestJSONFileLogger(t *testing.T) {
57 57
 	}
58 58
 }
59 59
 
60
+func TestJSONFileLoggerWithTags(t *testing.T) {
61
+	cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657"
62
+	cname := "test-container"
63
+	tmp, err := ioutil.TempDir("", "docker-logger-")
64
+
65
+	require.NoError(t, err)
66
+
67
+	defer os.RemoveAll(tmp)
68
+	filename := filepath.Join(tmp, "container.log")
69
+	l, err := New(logger.Info{
70
+		Config: map[string]string{
71
+			"tag": "{{.ID}}/{{.Name}}", // first 12 characters of ContainerID and full ContainerName
72
+		},
73
+		ContainerID:   cid,
74
+		ContainerName: cname,
75
+		LogPath:       filename,
76
+	})
77
+
78
+	require.NoError(t, err)
79
+	defer l.Close()
80
+
81
+	err = l.Log(&logger.Message{Line: []byte("line1"), Source: "src1"})
82
+	require.NoError(t, err)
83
+
84
+	err = l.Log(&logger.Message{Line: []byte("line2"), Source: "src2"})
85
+	require.NoError(t, err)
86
+
87
+	err = l.Log(&logger.Message{Line: []byte("line3"), Source: "src3"})
88
+	require.NoError(t, err)
89
+
90
+	res, err := ioutil.ReadFile(filename)
91
+	require.NoError(t, err)
92
+
93
+	expected := `{"log":"line1\n","stream":"src1","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"}
94
+{"log":"line2\n","stream":"src2","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"}
95
+{"log":"line3\n","stream":"src3","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"}
96
+`
97
+
98
+	if string(res) != expected {
99
+		t.Fatalf("Wrong log content: %q, expected %q", res, expected)
100
+	}
101
+}
102
+
60 103
 func BenchmarkJSONFileLoggerLog(b *testing.B) {
61 104
 	tmp := fs.NewDir(b, "bench-jsonfilelog")
62 105
 	defer tmp.Remove()
... ...
@@ -82,7 +125,7 @@ func BenchmarkJSONFileLoggerLog(b *testing.B) {
82 82
 	}
83 83
 
84 84
 	buf := bytes.NewBuffer(nil)
85
-	require.NoError(b, marshalMessage(msg, nil, buf))
85
+	require.NoError(b, marshalMessage(msg, nil, buf, ""))
86 86
 	b.SetBytes(int64(buf.Len()))
87 87
 
88 88
 	b.ResetTimer()
... ...
@@ -14,6 +14,8 @@ type JSONLog struct {
14 14
 	Created time.Time `json:"time"`
15 15
 	// Attrs is the list of extra attributes provided by the user
16 16
 	Attrs map[string]string `json:"attrs,omitempty"`
17
+	// Tags requested the operator
18
+	Tag string `json:"tag,omitempty"`
17 19
 }
18 20
 
19 21
 // Reset all fields to their zero value.
... ...
@@ -22,4 +24,5 @@ func (jl *JSONLog) Reset() {
22 22
 	jl.Stream = ""
23 23
 	jl.Created = time.Time{}
24 24
 	jl.Attrs = make(map[string]string)
25
+	jl.Tag = ""
25 26
 }
... ...
@@ -12,6 +12,7 @@ type JSONLogs struct {
12 12
 	Log     []byte    `json:"log,omitempty"`
13 13
 	Stream  string    `json:"stream,omitempty"`
14 14
 	Created time.Time `json:"time"`
15
+	Tag     string    `json:"tag,omitempty"`
15 16
 
16 17
 	// json-encoded bytes
17 18
 	RawAttrs json.RawMessage `json:"attrs,omitempty"`
... ...
@@ -37,6 +38,15 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
37 37
 		buf.WriteString(`"stream":`)
38 38
 		ffjsonWriteJSONBytesAsString(buf, []byte(mj.Stream))
39 39
 	}
40
+	if len(mj.Tag) > 0 {
41
+		if first {
42
+			first = false
43
+		} else {
44
+			buf.WriteString(`,`)
45
+		}
46
+		buf.WriteString(`"tag":`)
47
+		ffjsonWriteJSONBytesAsString(buf, []byte(mj.Tag))
48
+	}
40 49
 	if len(mj.RawAttrs) > 0 {
41 50
 		if first {
42 51
 			first = false
... ...
@@ -29,6 +29,8 @@ func TestJSONLogsMarshalJSONBuf(t *testing.T) {
29 29
 		{Log: []byte{0x7F}}:            `^{\"log\":\"\x7f\",\"time\":`,
30 30
 		// with raw attributes
31 31
 		{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`,
32
+		// with Tag set
33
+		{Log: []byte("A log line with tag"), Tag: "test-tag"}: `^{\"log\":\"A log line with tag\",\"tag\":\"test-tag\",\"time\":`,
32 34
 	}
33 35
 	for jsonLog, expression := range logs {
34 36
 		var buf bytes.Buffer
... ...
@@ -35,7 +35,7 @@ func BenchmarkJSONFileLoggerReadLogs(b *testing.B) {
35 35
 	}
36 36
 
37 37
 	buf := bytes.NewBuffer(nil)
38
-	require.NoError(b, marshalMessage(msg, nil, buf))
38
+	require.NoError(b, marshalMessage(msg, nil, buf, ""))
39 39
 	b.SetBytes(int64(buf.Len()))
40 40
 
41 41
 	b.ResetTimer()