Fixes #19803
Updated the json-logger to utilize the common log option
'tag' that can define container/image information to include
as part of logging.
When the 'tag' log option is not included, there is no change
to the log content via the json-logger. When the 'tag' log option
is included, the tag will be parsed as a template and the result
will be stored within each log entry as the attribute 'tag'.
Update: Removing test added to integration_cli as those have been deprecated.
Update: Using proper test calls (require and assert) in jsonfilelog_test.go based on review.
Update: Added new unit test configs for logs with tag. Updated unit test error checking.
Update: Cleanup check in jsonlogbytes_test.go to match pending changes in PR #34946.
Update: Merging to correct conflicts from PR #34946.
Signed-off-by: bonczj <josh.bonczkowski@gmail.com>
| ... | ... |
@@ -27,6 +27,7 @@ type JSONFileLogger struct {
|
| 27 | 27 |
closed bool |
| 28 | 28 |
writer *loggerutils.LogFile |
| 29 | 29 |
readers map[*logger.LogWatcher]struct{} // stores the active log followers
|
| 30 |
+ tag string // tag values requested by the user to log |
|
| 30 | 31 |
} |
| 31 | 32 |
|
| 32 | 33 |
func init() {
|
| ... | ... |
@@ -61,6 +62,12 @@ func New(info logger.Info) (logger.Logger, error) {
|
| 61 | 61 |
} |
| 62 | 62 |
} |
| 63 | 63 |
|
| 64 |
+ // no default template. only use a tag if the user asked for it |
|
| 65 |
+ tag, err := loggerutils.ParseLogTag(info, "") |
|
| 66 |
+ if err != nil {
|
|
| 67 |
+ return nil, err |
|
| 68 |
+ } |
|
| 69 |
+ |
|
| 64 | 70 |
var extra []byte |
| 65 | 71 |
attrs, err := info.ExtraAttributes(nil) |
| 66 | 72 |
if err != nil {
|
| ... | ... |
@@ -76,7 +83,7 @@ func New(info logger.Info) (logger.Logger, error) {
|
| 76 | 76 |
|
| 77 | 77 |
buf := bytes.NewBuffer(nil) |
| 78 | 78 |
marshalFunc := func(msg *logger.Message) ([]byte, error) {
|
| 79 |
- if err := marshalMessage(msg, extra, buf); err != nil {
|
|
| 79 |
+ if err := marshalMessage(msg, extra, buf, tag); err != nil {
|
|
| 80 | 80 |
return nil, err |
| 81 | 81 |
} |
| 82 | 82 |
b := buf.Bytes() |
| ... | ... |
@@ -92,6 +99,7 @@ func New(info logger.Info) (logger.Logger, error) {
|
| 92 | 92 |
return &JSONFileLogger{
|
| 93 | 93 |
writer: writer, |
| 94 | 94 |
readers: make(map[*logger.LogWatcher]struct{}),
|
| 95 |
+ tag: tag, |
|
| 95 | 96 |
}, nil |
| 96 | 97 |
} |
| 97 | 98 |
|
| ... | ... |
@@ -103,7 +111,7 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error {
|
| 103 | 103 |
return err |
| 104 | 104 |
} |
| 105 | 105 |
|
| 106 |
-func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer) error {
|
|
| 106 |
+func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffer, tag string) error {
|
|
| 107 | 107 |
logLine := msg.Line |
| 108 | 108 |
if !msg.Partial {
|
| 109 | 109 |
logLine = append(msg.Line, '\n') |
| ... | ... |
@@ -113,6 +121,7 @@ func marshalMessage(msg *logger.Message, extra json.RawMessage, buf *bytes.Buffe |
| 113 | 113 |
Stream: msg.Source, |
| 114 | 114 |
Created: msg.Timestamp, |
| 115 | 115 |
RawAttrs: extra, |
| 116 |
+ Tag: tag, |
|
| 116 | 117 |
}).MarshalJSONBuf(buf) |
| 117 | 118 |
if err != nil {
|
| 118 | 119 |
return errors.Wrap(err, "error writing log message to buffer") |
| ... | ... |
@@ -130,6 +139,7 @@ func ValidateLogOpt(cfg map[string]string) error {
|
| 130 | 130 |
case "labels": |
| 131 | 131 |
case "env": |
| 132 | 132 |
case "env-regex": |
| 133 |
+ case "tag": |
|
| 133 | 134 |
default: |
| 134 | 135 |
return fmt.Errorf("unknown log opt '%s' for json-file log driver", key)
|
| 135 | 136 |
} |
| ... | ... |
@@ -57,6 +57,49 @@ func TestJSONFileLogger(t *testing.T) {
|
| 57 | 57 |
} |
| 58 | 58 |
} |
| 59 | 59 |
|
| 60 |
+func TestJSONFileLoggerWithTags(t *testing.T) {
|
|
| 61 |
+ cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" |
|
| 62 |
+ cname := "test-container" |
|
| 63 |
+ tmp, err := ioutil.TempDir("", "docker-logger-")
|
|
| 64 |
+ |
|
| 65 |
+ require.NoError(t, err) |
|
| 66 |
+ |
|
| 67 |
+ defer os.RemoveAll(tmp) |
|
| 68 |
+ filename := filepath.Join(tmp, "container.log") |
|
| 69 |
+ l, err := New(logger.Info{
|
|
| 70 |
+ Config: map[string]string{
|
|
| 71 |
+ "tag": "{{.ID}}/{{.Name}}", // first 12 characters of ContainerID and full ContainerName
|
|
| 72 |
+ }, |
|
| 73 |
+ ContainerID: cid, |
|
| 74 |
+ ContainerName: cname, |
|
| 75 |
+ LogPath: filename, |
|
| 76 |
+ }) |
|
| 77 |
+ |
|
| 78 |
+ require.NoError(t, err) |
|
| 79 |
+ defer l.Close() |
|
| 80 |
+ |
|
| 81 |
+ err = l.Log(&logger.Message{Line: []byte("line1"), Source: "src1"})
|
|
| 82 |
+ require.NoError(t, err) |
|
| 83 |
+ |
|
| 84 |
+ err = l.Log(&logger.Message{Line: []byte("line2"), Source: "src2"})
|
|
| 85 |
+ require.NoError(t, err) |
|
| 86 |
+ |
|
| 87 |
+ err = l.Log(&logger.Message{Line: []byte("line3"), Source: "src3"})
|
|
| 88 |
+ require.NoError(t, err) |
|
| 89 |
+ |
|
| 90 |
+ res, err := ioutil.ReadFile(filename) |
|
| 91 |
+ require.NoError(t, err) |
|
| 92 |
+ |
|
| 93 |
+ expected := `{"log":"line1\n","stream":"src1","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"}
|
|
| 94 |
+{"log":"line2\n","stream":"src2","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"}
|
|
| 95 |
+{"log":"line3\n","stream":"src3","tag":"a7317399f3f8/test-container","time":"0001-01-01T00:00:00Z"}
|
|
| 96 |
+` |
|
| 97 |
+ |
|
| 98 |
+ if string(res) != expected {
|
|
| 99 |
+ t.Fatalf("Wrong log content: %q, expected %q", res, expected)
|
|
| 100 |
+ } |
|
| 101 |
+} |
|
| 102 |
+ |
|
| 60 | 103 |
func BenchmarkJSONFileLoggerLog(b *testing.B) {
|
| 61 | 104 |
tmp := fs.NewDir(b, "bench-jsonfilelog") |
| 62 | 105 |
defer tmp.Remove() |
| ... | ... |
@@ -82,7 +125,7 @@ func BenchmarkJSONFileLoggerLog(b *testing.B) {
|
| 82 | 82 |
} |
| 83 | 83 |
|
| 84 | 84 |
buf := bytes.NewBuffer(nil) |
| 85 |
- require.NoError(b, marshalMessage(msg, nil, buf)) |
|
| 85 |
+ require.NoError(b, marshalMessage(msg, nil, buf, "")) |
|
| 86 | 86 |
b.SetBytes(int64(buf.Len())) |
| 87 | 87 |
|
| 88 | 88 |
b.ResetTimer() |
| ... | ... |
@@ -14,6 +14,8 @@ type JSONLog struct {
|
| 14 | 14 |
Created time.Time `json:"time"` |
| 15 | 15 |
// Attrs is the list of extra attributes provided by the user |
| 16 | 16 |
Attrs map[string]string `json:"attrs,omitempty"` |
| 17 |
+ // Tags requested the operator |
|
| 18 |
+ Tag string `json:"tag,omitempty"` |
|
| 17 | 19 |
} |
| 18 | 20 |
|
| 19 | 21 |
// Reset all fields to their zero value. |
| ... | ... |
@@ -22,4 +24,5 @@ func (jl *JSONLog) Reset() {
|
| 22 | 22 |
jl.Stream = "" |
| 23 | 23 |
jl.Created = time.Time{}
|
| 24 | 24 |
jl.Attrs = make(map[string]string) |
| 25 |
+ jl.Tag = "" |
|
| 25 | 26 |
} |
| ... | ... |
@@ -12,6 +12,7 @@ type JSONLogs struct {
|
| 12 | 12 |
Log []byte `json:"log,omitempty"` |
| 13 | 13 |
Stream string `json:"stream,omitempty"` |
| 14 | 14 |
Created time.Time `json:"time"` |
| 15 |
+ Tag string `json:"tag,omitempty"` |
|
| 15 | 16 |
|
| 16 | 17 |
// json-encoded bytes |
| 17 | 18 |
RawAttrs json.RawMessage `json:"attrs,omitempty"` |
| ... | ... |
@@ -37,6 +38,15 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
|
| 37 | 37 |
buf.WriteString(`"stream":`) |
| 38 | 38 |
ffjsonWriteJSONBytesAsString(buf, []byte(mj.Stream)) |
| 39 | 39 |
} |
| 40 |
+ if len(mj.Tag) > 0 {
|
|
| 41 |
+ if first {
|
|
| 42 |
+ first = false |
|
| 43 |
+ } else {
|
|
| 44 |
+ buf.WriteString(`,`) |
|
| 45 |
+ } |
|
| 46 |
+ buf.WriteString(`"tag":`) |
|
| 47 |
+ ffjsonWriteJSONBytesAsString(buf, []byte(mj.Tag)) |
|
| 48 |
+ } |
|
| 40 | 49 |
if len(mj.RawAttrs) > 0 {
|
| 41 | 50 |
if first {
|
| 42 | 51 |
first = false |
| ... | ... |
@@ -29,6 +29,8 @@ func TestJSONLogsMarshalJSONBuf(t *testing.T) {
|
| 29 | 29 |
{Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":`,
|
| 30 | 30 |
// with raw attributes |
| 31 | 31 |
{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":`,
|
| 32 |
+ // with Tag set |
|
| 33 |
+ {Log: []byte("A log line with tag"), Tag: "test-tag"}: `^{\"log\":\"A log line with tag\",\"tag\":\"test-tag\",\"time\":`,
|
|
| 32 | 34 |
} |
| 33 | 35 |
for jsonLog, expression := range logs {
|
| 34 | 36 |
var buf bytes.Buffer |
| ... | ... |
@@ -35,7 +35,7 @@ func BenchmarkJSONFileLoggerReadLogs(b *testing.B) {
|
| 35 | 35 |
} |
| 36 | 36 |
|
| 37 | 37 |
buf := bytes.NewBuffer(nil) |
| 38 |
- require.NoError(b, marshalMessage(msg, nil, buf)) |
|
| 38 |
+ require.NoError(b, marshalMessage(msg, nil, buf, "")) |
|
| 39 | 39 |
b.SetBytes(int64(buf.Len())) |
| 40 | 40 |
|
| 41 | 41 |
b.ResetTimer() |