Browse code

Change log details encoding format and support details on service logs

URL query encode log details, so that characters like spaces don't make
log parsing ambiguous. Add a helper function to parse these details to a
map, if needed

Add support for details on service logs

Signed-off-by: Drew Erny <drew.erny@docker.com>

Drew Erny authored on 2017/04/05 07:52:19
Showing 8 changed files
... ...
@@ -3,6 +3,7 @@ package httputils
3 3
 import (
4 4
 	"fmt"
5 5
 	"io"
6
+	"net/url"
6 7
 	"sort"
7 8
 	"strings"
8 9
 
... ...
@@ -85,6 +86,7 @@ func (s byKey) Swap(i, j int) {
85 85
 func stringAttrs(a backend.LogAttributes) string {
86 86
 	var ss byKey
87 87
 	for k, v := range a {
88
+		k, v := url.QueryEscape(k), url.QueryEscape(v)
88 89
 		ss = append(ss, k+"="+v)
89 90
 	}
90 91
 	sort.Sort(ss)
91 92
new file mode 100644
... ...
@@ -0,0 +1,41 @@
0
+package client
1
+
2
+// parse_logs.go contains utility helpers for getting information out of docker
3
+// log lines. really, it only contains ParseDetails right now. maybe in the
4
+// future there will be some desire to parse log messages back into a struct?
5
+// that would go here if we did
6
+
7
+import (
8
+	"net/url"
9
+	"strings"
10
+
11
+	"github.com/pkg/errors"
12
+)
13
+
14
+// ParseLogDetails takes a details string of key value pairs in the form
15
+// "k=v,l=w", where the keys and values are url query escaped, and each pair
16
+// is separated by a comma, returns a map. returns an error if the details
17
+// string is not in a valid format
18
+// the exact form of details encoding is implemented in
19
+// api/server/httputils/write_log_stream.go
20
+func ParseLogDetails(details string) (map[string]string, error) {
21
+	pairs := strings.Split(details, ",")
22
+	detailsMap := make(map[string]string, len(pairs))
23
+	for _, pair := range pairs {
24
+		p := strings.SplitN(pair, "=", 2)
25
+		// if there is no equals sign, we will only get 1 part back
26
+		if len(p) != 2 {
27
+			return nil, errors.New("invalid details format")
28
+		}
29
+		k, err := url.QueryUnescape(p[0])
30
+		if err != nil {
31
+			return nil, err
32
+		}
33
+		v, err := url.QueryUnescape(p[1])
34
+		if err != nil {
35
+			return nil, err
36
+		}
37
+		detailsMap[k] = v
38
+	}
39
+	return detailsMap, nil
40
+}
0 41
new file mode 100644
... ...
@@ -0,0 +1,36 @@
0
+package client
1
+
2
+import (
3
+	"reflect"
4
+	"testing"
5
+
6
+	"github.com/pkg/errors"
7
+)
8
+
9
+func TestParseLogDetails(t *testing.T) {
10
+	testCases := []struct {
11
+		line     string
12
+		expected map[string]string
13
+		err      error
14
+	}{
15
+		{"key=value", map[string]string{"key": "value"}, nil},
16
+		{"key1=value1,key2=value2", map[string]string{"key1": "value1", "key2": "value2"}, nil},
17
+		{"key+with+spaces=value%3Dequals,asdf%2C=", map[string]string{"key with spaces": "value=equals", "asdf,": ""}, nil},
18
+		{"key=,=nothing", map[string]string{"key": "", "": "nothing"}, nil},
19
+		{"=", map[string]string{"": ""}, nil},
20
+		{"errors", nil, errors.New("invalid details format")},
21
+	}
22
+	for _, tc := range testCases {
23
+		tc := tc // capture range variable
24
+		t.Run(tc.line, func(t *testing.T) {
25
+			t.Parallel()
26
+			res, err := ParseLogDetails(tc.line)
27
+			if err != nil && (err.Error() != tc.err.Error()) {
28
+				t.Fatalf("unexpected error parsing logs:\nExpected:\n\t%v\nActual:\n\t%v", tc.err, err)
29
+			}
30
+			if !reflect.DeepEqual(tc.expected, res) {
31
+				t.Errorf("result does not match expected:\nExpected:\n\t%#v\nActual:\n\t%#v", tc.expected, res)
32
+			}
33
+		})
34
+	}
35
+}
... ...
@@ -405,11 +405,11 @@ func (c *containerAdapter) logs(ctx context.Context, options api.LogSubscription
405 405
 	apiOptions := &types.ContainerLogsOptions{
406 406
 		Follow: options.Follow,
407 407
 
408
-		// TODO(stevvooe): Parse timestamp out of message. This
409
-		// absolutely needs to be done before going to production with
410
-		// this, at it is completely redundant.
408
+		// Always say yes to Timestamps and Details. we make the decision
409
+		// of whether to return these to the user or not way higher up the
410
+		// stack.
411 411
 		Timestamps: true,
412
-		Details:    false, // no clue what to do with this, let's just deprecate it.
412
+		Details:    true,
413 413
 	}
414 414
 
415 415
 	if options.Since != nil {
... ...
@@ -523,10 +523,18 @@ func (r *controller) Logs(ctx context.Context, publisher exec.LogPublisher, opti
523 523
 			stream = api.LogStreamStderr
524 524
 		}
525 525
 
526
+		// parse the details out of the Attrs map
527
+		attrs := []api.LogAttr{}
528
+		for k, v := range msg.Attrs {
529
+			attr := api.LogAttr{Key: k, Value: v}
530
+			attrs = append(attrs, attr)
531
+		}
532
+
526 533
 		if err := publisher.Publish(ctx, api.LogMessage{
527 534
 			Context:   msgctx,
528 535
 			Timestamp: tsp,
529 536
 			Stream:    stream,
537
+			Attrs:     attrs,
530 538
 			Data:      msg.Line,
531 539
 		}); err != nil {
532 540
 			return errors.Wrap(err, "failed to publish log message")
... ...
@@ -430,9 +430,17 @@ func (c *Cluster) ServiceLogs(ctx context.Context, selector *backend.LogSelector
430 430
 				if err != nil {
431 431
 					m.Err = err
432 432
 				}
433
+				// copy over all of the details
434
+				for _, d := range msg.Attrs {
435
+					m.Attrs[d.Key] = d.Value
436
+				}
437
+				// we have the final say over context details (in case there
438
+				// is a conflict (if the user added a detail with a context's
439
+				// key for some reason))
433 440
 				m.Attrs[contextPrefix+".node.id"] = msg.Context.NodeID
434 441
 				m.Attrs[contextPrefix+".service.id"] = msg.Context.ServiceID
435 442
 				m.Attrs[contextPrefix+".task.id"] = msg.Context.TaskID
443
+
436 444
 				switch msg.Stream {
437 445
 				case swarmapi.LogStreamStdout:
438 446
 					m.Source = "stdout"
... ...
@@ -11,5 +11,5 @@ VNDR_COMMIT=c56e082291115e369f77601f9c071dd0b87c7120
11 11
 BINDATA_COMMIT=a0ff2567cfb70903282db057e799fd826784d41d
12 12
 
13 13
 # CLI
14
-DOCKERCLI_REPO=https://github.com/aaronlehmann/cli
15
-DOCKERCLI_COMMIT=0e2b9d30814eaecd6518bfb112bb4adef0265d4b
14
+DOCKERCLI_REPO=https://github.com/dperny/cli
15
+DOCKERCLI_COMMIT=7230906e0e297999eb33da74e0279c5cf41a119e
... ...
@@ -55,8 +55,14 @@ func (s *DockerSwarmSuite) TestServiceLogs(c *check.C) {
55 55
 // output.
56 56
 func countLogLines(d *daemon.Swarm, name string) func(*check.C) (interface{}, check.CommentInterface) {
57 57
 	return func(c *check.C) (interface{}, check.CommentInterface) {
58
-		result := icmd.RunCmd(d.Command("service", "logs", "-t", name))
58
+		result := icmd.RunCmd(d.Command("service", "logs", "-t", "--raw", name))
59 59
 		result.Assert(c, icmd.Expected{})
60
+		// if this returns an emptystring, trying to split it later will return
61
+		// an array containing emptystring. a valid log line will NEVER be
62
+		// emptystring because we ask for the timestamp.
63
+		if result.Stdout() == "" {
64
+			return 0, check.Commentf("Empty stdout")
65
+		}
60 66
 		lines := strings.Split(strings.TrimSpace(result.Stdout()), "\n")
61 67
 		return len(lines), check.Commentf("output, %q", string(result.Stdout()))
62 68
 	}
... ...
@@ -277,7 +283,7 @@ func (s *DockerSwarmSuite) TestServiceLogsTTY(c *check.C) {
277 277
 	// and make sure we have all the log lines
278 278
 	waitAndAssert(c, defaultReconciliationTimeout, countLogLines(d, name), checker.Equals, 2)
279 279
 
280
-	cmd := d.Command("service", "logs", name)
280
+	cmd := d.Command("service", "logs", "--raw", name)
281 281
 	result = icmd.RunCmd(cmd)
282 282
 	// for some reason there is carriage return in the output. i think this is
283 283
 	// just expected.
... ...
@@ -332,3 +338,50 @@ func (s *DockerSwarmSuite) TestServiceLogsNoHangDeletedContainer(c *check.C) {
332 332
 	// Expected defaults to false
333 333
 	c.Assert(result, icmd.Matches, icmd.Expected{})
334 334
 }
335
+
336
+func (s *DockerSwarmSuite) TestServiceLogsDetails(c *check.C) {
337
+	d := s.AddDaemon(c, true, true)
338
+
339
+	name := "TestServiceLogsDetails"
340
+
341
+	result := icmd.RunCmd(d.Command(
342
+		// create a service
343
+		"service", "create",
344
+		// name it $name
345
+		"--name", name,
346
+		// add an environment variable
347
+		"--env", "asdf=test1",
348
+		// add a log driver (without explicitly setting a driver, log-opt doesn't work)
349
+		"--log-driver", "json-file",
350
+		// add a log option to print the environment variable
351
+		"--log-opt", "env=asdf",
352
+		// busybox image, shell string
353
+		"busybox", "sh", "-c",
354
+		// make a log line
355
+		"echo LogLine; while true; do sleep 1; done;",
356
+	))
357
+
358
+	result.Assert(c, icmd.Expected{})
359
+	id := strings.TrimSpace(result.Stdout())
360
+	c.Assert(id, checker.Not(checker.Equals), "")
361
+
362
+	// make sure task has been deployed
363
+	waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1)
364
+	// and make sure we have all the log lines
365
+	waitAndAssert(c, defaultReconciliationTimeout, countLogLines(d, name), checker.Equals, 1)
366
+
367
+	// First, test without pretty printing
368
+	// call service logs with details. set raw to skip pretty printing
369
+	result = icmd.RunCmd(d.Command("service", "logs", "--raw", "--details", name))
370
+	// in this case, we should get details and we should get log message, but
371
+	// there will also be context as details (which will fall after the detail
372
+	// we inserted in alphabetical order
373
+	c.Assert(result, icmd.Matches, icmd.Expected{Out: "asdf=test1"})
374
+	c.Assert(result, icmd.Matches, icmd.Expected{Out: "LogLine"})
375
+
376
+	// call service logs with details. this time, don't pass raw
377
+	result = icmd.RunCmd(d.Command("service", "logs", "--details", id))
378
+	// in this case, we should get details space logmessage as well. the context
379
+	// is part of the pretty part of the logline
380
+	c.Assert(result, icmd.Matches, icmd.Expected{Out: "asdf=test1 LogLine"})
381
+}