Browse code

Avoid using a map for log attributes

Having a map per log entry seemed heavier than necessary. These
attributes end up being sorted and serialized, so storing them in a map
doesn't add anything (there's no random access element). In SwarmKit,
they originate as a slice, so there's an unnecessary conversion to a map
and back.

This also fixes the sort comparator, which used to inefficiently split
the string on each comparison.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

Aaron Lehmann authored on 2017/07/19 11:01:20
Showing 8 changed files
... ...
@@ -5,7 +5,6 @@ import (
5 5
 	"io"
6 6
 	"net/url"
7 7
 	"sort"
8
-	"strings"
9 8
 
10 9
 	"golang.org/x/net/context"
11 10
 
... ...
@@ -53,7 +52,8 @@ func WriteLogStream(ctx context.Context, w io.Writer, msgs <-chan *backend.LogMe
53 53
 		}
54 54
 		logLine := msg.Line
55 55
 		if config.Details {
56
-			logLine = append([]byte(stringAttrs(msg.Attrs)+" "), logLine...)
56
+			logLine = append(attrsByteSlice(msg.Attrs), ' ')
57
+			logLine = append(logLine, msg.Line...)
57 58
 		}
58 59
 		if config.Timestamps {
59 60
 			// TODO(dperny) the format is defined in
... ...
@@ -71,24 +71,26 @@ func WriteLogStream(ctx context.Context, w io.Writer, msgs <-chan *backend.LogMe
71 71
 	}
72 72
 }
73 73
 
74
-type byKey []string
74
+type byKey []backend.LogAttr
75 75
 
76
-func (s byKey) Len() int { return len(s) }
77
-func (s byKey) Less(i, j int) bool {
78
-	keyI := strings.Split(s[i], "=")
79
-	keyJ := strings.Split(s[j], "=")
80
-	return keyI[0] < keyJ[0]
81
-}
82
-func (s byKey) Swap(i, j int) {
83
-	s[i], s[j] = s[j], s[i]
84
-}
76
+func (b byKey) Len() int           { return len(b) }
77
+func (b byKey) Less(i, j int) bool { return b[i].Key < b[j].Key }
78
+func (b byKey) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
85 79
 
86
-func stringAttrs(a backend.LogAttributes) string {
87
-	var ss byKey
88
-	for k, v := range a {
89
-		k, v := url.QueryEscape(k), url.QueryEscape(v)
90
-		ss = append(ss, k+"="+v)
80
+func attrsByteSlice(a []backend.LogAttr) []byte {
81
+	// Note this sorts "a" in-place. That is fine here - nothing else is
82
+	// going to use Attrs or care about the order.
83
+	sort.Sort(byKey(a))
84
+
85
+	var ret []byte
86
+	for i, pair := range a {
87
+		k, v := url.QueryEscape(pair.Key), url.QueryEscape(pair.Value)
88
+		ret = append(ret, []byte(k)...)
89
+		ret = append(ret, '=')
90
+		ret = append(ret, []byte(v)...)
91
+		if i != len(a)-1 {
92
+			ret = append(ret, ',')
93
+		}
91 94
 	}
92
-	sort.Sort(ss)
93
-	return strings.Join(ss, ",")
95
+	return ret
94 96
 }
... ...
@@ -35,7 +35,7 @@ type LogMessage struct {
35 35
 	Line      []byte
36 36
 	Source    string
37 37
 	Timestamp time.Time
38
-	Attrs     LogAttributes
38
+	Attrs     []LogAttr
39 39
 	Partial   bool
40 40
 
41 41
 	// Err is an error associated with a message. Completeness of a message
... ...
@@ -44,9 +44,11 @@ type LogMessage struct {
44 44
 	Err error
45 45
 }
46 46
 
47
-// LogAttributes is used to hold the extra attributes available in the log message
48
-// Primarily used for converting the map type to string and sorting.
49
-type LogAttributes map[string]string
47
+// LogAttr is used to hold the extra attributes available in the log message.
48
+type LogAttr struct {
49
+	Key   string
50
+	Value string
51
+}
50 52
 
51 53
 // LogSelector is a list of services and tasks that should be returned as part
52 54
 // of a log stream. It is similar to swarmapi.LogSelector, with the difference
... ...
@@ -524,10 +524,12 @@ func (r *controller) Logs(ctx context.Context, publisher exec.LogPublisher, opti
524 524
 		}
525 525
 
526 526
 		// parse the details out of the Attrs map
527
-		attrs := []api.LogAttr{}
528
-		for k, v := range msg.Attrs {
529
-			attr := api.LogAttr{Key: k, Value: v}
530
-			attrs = append(attrs, attr)
527
+		var attrs []api.LogAttr
528
+		if len(msg.Attrs) != 0 {
529
+			attrs = make([]api.LogAttr, 0, len(msg.Attrs))
530
+			for _, attr := range msg.Attrs {
531
+				attrs = append(attrs, api.LogAttr{Key: attr.Key, Value: attr.Value})
532
+			}
531 533
 		}
532 534
 
533 535
 		if err := publisher.Publish(ctx, api.LogMessage{
... ...
@@ -458,22 +458,33 @@ func (c *Cluster) ServiceLogs(ctx context.Context, selector *backend.LogSelector
458 458
 			for _, msg := range subscribeMsg.Messages {
459 459
 				// make a new message
460 460
 				m := new(backend.LogMessage)
461
-				m.Attrs = make(backend.LogAttributes)
461
+				m.Attrs = make([]backend.LogAttr, 0, len(msg.Attrs)+3)
462 462
 				// add the timestamp, adding the error if it fails
463 463
 				m.Timestamp, err = gogotypes.TimestampFromProto(msg.Timestamp)
464 464
 				if err != nil {
465 465
 					m.Err = err
466 466
 				}
467
+
468
+				nodeKey := contextPrefix + ".node.id"
469
+				serviceKey := contextPrefix + ".service.id"
470
+				taskKey := contextPrefix + ".task.id"
471
+
467 472
 				// copy over all of the details
468 473
 				for _, d := range msg.Attrs {
469
-					m.Attrs[d.Key] = d.Value
474
+					switch d.Key {
475
+					case nodeKey, serviceKey, taskKey:
476
+						// we have the final say over context details (in case there
477
+						// is a conflict (if the user added a detail with a context's
478
+						// key for some reason))
479
+					default:
480
+						m.Attrs = append(m.Attrs, backend.LogAttr{Key: d.Key, Value: d.Value})
481
+					}
470 482
 				}
471
-				// we have the final say over context details (in case there
472
-				// is a conflict (if the user added a detail with a context's
473
-				// key for some reason))
474
-				m.Attrs[contextPrefix+".node.id"] = msg.Context.NodeID
475
-				m.Attrs[contextPrefix+".service.id"] = msg.Context.ServiceID
476
-				m.Attrs[contextPrefix+".task.id"] = msg.Context.TaskID
483
+				m.Attrs = append(m.Attrs,
484
+					backend.LogAttr{Key: nodeKey, Value: msg.Context.NodeID},
485
+					backend.LogAttr{Key: serviceKey, Value: msg.Context.ServiceID},
486
+					backend.LogAttr{Key: taskKey, Value: msg.Context.TaskID},
487
+				)
477 488
 
478 489
 				switch msg.Stream {
479 490
 				case swarmapi.LogStreamStdout:
... ...
@@ -157,6 +157,7 @@ import (
157 157
 
158 158
 	"github.com/Sirupsen/logrus"
159 159
 	"github.com/coreos/go-systemd/journal"
160
+	"github.com/docker/docker/api/types/backend"
160 161
 	"github.com/docker/docker/daemon/logger"
161 162
 )
162 163
 
... ...
@@ -213,14 +214,11 @@ drain:
213 213
 				source = "stdout"
214 214
 			}
215 215
 			// Retrieve the values of any variables we're adding to the journal.
216
-			attrs := make(map[string]string)
216
+			var attrs []backend.LogAttr
217 217
 			C.sd_journal_restart_data(j)
218 218
 			for C.get_attribute_field(j, &data, &length) > C.int(0) {
219 219
 				kv := strings.SplitN(C.GoStringN(data, C.int(length)), "=", 2)
220
-				attrs[kv[0]] = kv[1]
221
-			}
222
-			if len(attrs) == 0 {
223
-				attrs = nil
220
+				attrs = append(attrs, backend.LogAttr{Key: kv[0], Value: kv[1]})
224 221
 			}
225 222
 			// Send the log message.
226 223
 			logWatcher.Msg <- &logger.Message{
... ...
@@ -12,6 +12,7 @@ import (
12 12
 	"golang.org/x/net/context"
13 13
 
14 14
 	"github.com/Sirupsen/logrus"
15
+	"github.com/docker/docker/api/types/backend"
15 16
 	"github.com/docker/docker/daemon/logger"
16 17
 	"github.com/docker/docker/daemon/logger/jsonfilelog/multireader"
17 18
 	"github.com/docker/docker/pkg/filenotify"
... ...
@@ -27,11 +28,18 @@ func decodeLogLine(dec *json.Decoder, l *jsonlog.JSONLog) (*logger.Message, erro
27 27
 	if err := dec.Decode(l); err != nil {
28 28
 		return nil, err
29 29
 	}
30
+	var attrs []backend.LogAttr
31
+	if len(l.Attrs) != 0 {
32
+		attrs = make([]backend.LogAttr, 0, len(l.Attrs))
33
+		for k, v := range l.Attrs {
34
+			attrs = append(attrs, backend.LogAttr{Key: k, Value: v})
35
+		}
36
+	}
30 37
 	msg := &logger.Message{
31 38
 		Source:    l.Stream,
32 39
 		Timestamp: l.Created,
33 40
 		Line:      []byte(l.Log),
34
-		Attrs:     l.Attrs,
41
+		Attrs:     attrs,
35 42
 	}
36 43
 	return msg, nil
37 44
 }
... ...
@@ -68,11 +68,6 @@ func (m *Message) AsLogMessage() *backend.LogMessage {
68 68
 	return (*backend.LogMessage)(m)
69 69
 }
70 70
 
71
-// LogAttributes is used to hold the extra attributes available in the log message
72
-// Primarily used for converting the map type to string and sorting.
73
-// Imported here so it can be used internally with less refactoring
74
-type LogAttributes backend.LogAttributes
75
-
76 71
 // Logger is the interface for docker logging drivers.
77 72
 type Logger interface {
78 73
 	Log(*Message) error
... ...
@@ -1,5 +1,9 @@
1 1
 package logger
2 2
 
3
+import (
4
+	"github.com/docker/docker/api/types/backend"
5
+)
6
+
3 7
 func (m *Message) copy() *Message {
4 8
 	msg := &Message{
5 9
 		Source:    m.Source,
... ...
@@ -8,10 +12,8 @@ func (m *Message) copy() *Message {
8 8
 	}
9 9
 
10 10
 	if m.Attrs != nil {
11
-		msg.Attrs = make(map[string]string, len(m.Attrs))
12
-		for k, v := range m.Attrs {
13
-			msg.Attrs[k] = v
14
-		}
11
+		msg.Attrs = make([]backend.LogAttr, len(m.Attrs))
12
+		copy(msg.Attrs, m.Attrs)
15 13
 	}
16 14
 
17 15
 	msg.Line = append(make([]byte, 0, len(m.Line)), m.Line...)