this allows jsonfile logger to collect extra metadata from containers with
`--log-opt labels=label1,label2 --log-opt env=env1,env2`.
Extra attributes are saved into `attrs` attributes for each log data.
Signed-off-by: Daniel Dao <dqminh@cloudflare.com>
| ... | ... |
@@ -41,6 +41,7 @@ type JSONFileLogger struct {
|
| 41 | 41 |
ctx logger.Context |
| 42 | 42 |
readers map[*logger.LogWatcher]struct{} // stores the active log followers
|
| 43 | 43 |
notifyRotate *pubsub.Publisher |
| 44 |
+ extra []byte // json-encoded extra attributes |
|
| 44 | 45 |
} |
| 45 | 46 |
|
| 46 | 47 |
func init() {
|
| ... | ... |
@@ -77,6 +78,16 @@ func New(ctx logger.Context) (logger.Logger, error) {
|
| 77 | 77 |
return nil, fmt.Errorf("max-file cannot be less than 1")
|
| 78 | 78 |
} |
| 79 | 79 |
} |
| 80 |
+ |
|
| 81 |
+ var extra []byte |
|
| 82 |
+ if attrs := ctx.ExtraAttributes(nil); len(attrs) > 0 {
|
|
| 83 |
+ var err error |
|
| 84 |
+ extra, err = json.Marshal(attrs) |
|
| 85 |
+ if err != nil {
|
|
| 86 |
+ return nil, err |
|
| 87 |
+ } |
|
| 88 |
+ } |
|
| 89 |
+ |
|
| 80 | 90 |
return &JSONFileLogger{
|
| 81 | 91 |
f: log, |
| 82 | 92 |
buf: bytes.NewBuffer(nil), |
| ... | ... |
@@ -85,6 +96,7 @@ func New(ctx logger.Context) (logger.Logger, error) {
|
| 85 | 85 |
n: maxFiles, |
| 86 | 86 |
readers: make(map[*logger.LogWatcher]struct{}),
|
| 87 | 87 |
notifyRotate: pubsub.NewPublisher(0, 1), |
| 88 |
+ extra: extra, |
|
| 88 | 89 |
}, nil |
| 89 | 90 |
} |
| 90 | 91 |
|
| ... | ... |
@@ -97,7 +109,12 @@ func (l *JSONFileLogger) Log(msg *logger.Message) error {
|
| 97 | 97 |
if err != nil {
|
| 98 | 98 |
return err |
| 99 | 99 |
} |
| 100 |
- err = (&jsonlog.JSONLogs{Log: append(msg.Line, '\n'), Stream: msg.Source, Created: timestamp}).MarshalJSONBuf(l.buf)
|
|
| 100 |
+ err = (&jsonlog.JSONLogs{
|
|
| 101 |
+ Log: append(msg.Line, '\n'), |
|
| 102 |
+ Stream: msg.Source, |
|
| 103 |
+ Created: timestamp, |
|
| 104 |
+ RawAttrs: l.extra, |
|
| 105 |
+ }).MarshalJSONBuf(l.buf) |
|
| 101 | 106 |
if err != nil {
|
| 102 | 107 |
return err |
| 103 | 108 |
} |
| ... | ... |
@@ -181,6 +198,8 @@ func ValidateLogOpt(cfg map[string]string) error {
|
| 181 | 181 |
switch key {
|
| 182 | 182 |
case "max-file": |
| 183 | 183 |
case "max-size": |
| 184 |
+ case "labels": |
|
| 185 |
+ case "env": |
|
| 184 | 186 |
default: |
| 185 | 187 |
return fmt.Errorf("unknown log opt '%s' for json-file log driver", key)
|
| 186 | 188 |
} |
| ... | ... |
@@ -1,9 +1,11 @@ |
| 1 | 1 |
package jsonfilelog |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "encoding/json" |
|
| 4 | 5 |
"io/ioutil" |
| 5 | 6 |
"os" |
| 6 | 7 |
"path/filepath" |
| 8 |
+ "reflect" |
|
| 7 | 9 |
"strconv" |
| 8 | 10 |
"testing" |
| 9 | 11 |
"time" |
| ... | ... |
@@ -149,3 +151,51 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
|
| 149 | 149 |
} |
| 150 | 150 |
|
| 151 | 151 |
} |
| 152 |
+ |
|
| 153 |
+func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
|
|
| 154 |
+ cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" |
|
| 155 |
+ tmp, err := ioutil.TempDir("", "docker-logger-")
|
|
| 156 |
+ if err != nil {
|
|
| 157 |
+ t.Fatal(err) |
|
| 158 |
+ } |
|
| 159 |
+ defer os.RemoveAll(tmp) |
|
| 160 |
+ filename := filepath.Join(tmp, "container.log") |
|
| 161 |
+ config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl"}
|
|
| 162 |
+ l, err := New(logger.Context{
|
|
| 163 |
+ ContainerID: cid, |
|
| 164 |
+ LogPath: filename, |
|
| 165 |
+ Config: config, |
|
| 166 |
+ ContainerLabels: map[string]string{"rack": "101", "dc": "lhr"},
|
|
| 167 |
+ ContainerEnv: []string{"environ=production", "debug=false", "port=10001", "ssl=true"},
|
|
| 168 |
+ }) |
|
| 169 |
+ if err != nil {
|
|
| 170 |
+ t.Fatal(err) |
|
| 171 |
+ } |
|
| 172 |
+ defer l.Close() |
|
| 173 |
+ if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line"), Source: "src1"}); err != nil {
|
|
| 174 |
+ t.Fatal(err) |
|
| 175 |
+ } |
|
| 176 |
+ res, err := ioutil.ReadFile(filename) |
|
| 177 |
+ if err != nil {
|
|
| 178 |
+ t.Fatal(err) |
|
| 179 |
+ } |
|
| 180 |
+ |
|
| 181 |
+ var jsonLog jsonlog.JSONLogs |
|
| 182 |
+ if err := json.Unmarshal(res, &jsonLog); err != nil {
|
|
| 183 |
+ t.Fatal(err) |
|
| 184 |
+ } |
|
| 185 |
+ extra := make(map[string]string) |
|
| 186 |
+ if err := json.Unmarshal(jsonLog.RawAttrs, &extra); err != nil {
|
|
| 187 |
+ t.Fatal(err) |
|
| 188 |
+ } |
|
| 189 |
+ expected := map[string]string{
|
|
| 190 |
+ "rack": "101", |
|
| 191 |
+ "dc": "lhr", |
|
| 192 |
+ "environ": "production", |
|
| 193 |
+ "debug": "false", |
|
| 194 |
+ "ssl": "true", |
|
| 195 |
+ } |
|
| 196 |
+ if !reflect.DeepEqual(extra, expected) {
|
|
| 197 |
+ t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected)
|
|
| 198 |
+ } |
|
| 199 |
+} |
| ... | ... |
@@ -2,6 +2,7 @@ package jsonlog |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"bytes" |
| 5 |
+ "encoding/json" |
|
| 5 | 6 |
"unicode/utf8" |
| 6 | 7 |
) |
| 7 | 8 |
|
| ... | ... |
@@ -12,6 +13,9 @@ type JSONLogs struct {
|
| 12 | 12 |
Log []byte `json:"log,omitempty"` |
| 13 | 13 |
Stream string `json:"stream,omitempty"` |
| 14 | 14 |
Created string `json:"time"` |
| 15 |
+ |
|
| 16 |
+ // json-encoded bytes |
|
| 17 |
+ RawAttrs json.RawMessage `json:"attrs,omitempty"` |
|
| 15 | 18 |
} |
| 16 | 19 |
|
| 17 | 20 |
// MarshalJSONBuf is based on the same method from JSONLog |
| ... | ... |
@@ -34,6 +38,15 @@ func (mj *JSONLogs) MarshalJSONBuf(buf *bytes.Buffer) error {
|
| 34 | 34 |
buf.WriteString(`"stream":`) |
| 35 | 35 |
ffjsonWriteJSONString(buf, mj.Stream) |
| 36 | 36 |
} |
| 37 |
+ if len(mj.RawAttrs) > 0 {
|
|
| 38 |
+ if first == true {
|
|
| 39 |
+ first = false |
|
| 40 |
+ } else {
|
|
| 41 |
+ buf.WriteString(`,`) |
|
| 42 |
+ } |
|
| 43 |
+ buf.WriteString(`"attrs":`) |
|
| 44 |
+ buf.Write(mj.RawAttrs) |
|
| 45 |
+ } |
|
| 37 | 46 |
if first == true {
|
| 38 | 47 |
first = false |
| 39 | 48 |
} else {
|
| ... | ... |
@@ -21,6 +21,8 @@ func TestJSONLogsMarshalJSONBuf(t *testing.T) {
|
| 21 | 21 |
&JSONLogs{Log: []byte("\u2028 \u2029")}: `^{\"log\":\"\\u2028 \\u2029\",\"time\":}$`,
|
| 22 | 22 |
&JSONLogs{Log: []byte{0xaF}}: `^{\"log\":\"\\ufffd\",\"time\":}$`,
|
| 23 | 23 |
&JSONLogs{Log: []byte{0x7F}}: `^{\"log\":\"\x7f\",\"time\":}$`,
|
| 24 |
+ // with raw attributes |
|
| 25 |
+ &JSONLogs{Log: []byte("A log line"), RawAttrs: []byte(`{"hello":"world","value":1234}`)}: `^{\"log\":\"A log line\",\"attrs\":{\"hello\":\"world\",\"value\":1234},\"time\":}$`,
|
|
| 24 | 26 |
} |
| 25 | 27 |
for jsonLog, expression := range logs {
|
| 26 | 28 |
var buf bytes.Buffer |