1. docker
  2. Commit 342b44bf20c49ec1b0da9d1973e1043f672638f6
Browse code

logger/journald: rewrite reader w/o cursors

Careful management of the journal read pointer is sufficient to ensure
that no entry is read more than once.

Unit test the journald logger without requiring a running journald by
using the systemd-journal-remote command to write arbitrary entries to
journal files.

Signed-off-by: Cory Snider <csnider@mirantis.com>

Cory Snider authored on 2022/02/03 06:39:35
Showing 16 changed files
Dockerfile
History View file @ 342b44b
... ... 
@@ -333,6 +333,7 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
333 333 
             python3-setuptools \
334 334 
             python3-wheel \
335 335 
             sudo \
336 
+            systemd-journal-remote \
336 337 
             thin-provisioning-tools \
337 338 
             uidmap \
338 339 
             vim \
daemon/logger/journald/internal/export/export.go
History View file @ 342b44b
339 340 
new file mode 100644
... ... 
@@ -0,0 +1,50 @@
0 
+// Package export implements a serializer for the systemd Journal Export Format
1 
+// as documented at https://systemd.io/JOURNAL_EXPORT_FORMATS/
2 
+package export // import "github.com/docker/docker/daemon/logger/journald/internal/export"
3 
+
4 
+import (
5 
+	"encoding/binary"
6 
+	"fmt"
7 
+	"io"
8 
+	"unicode/utf8"
9 
+)
10 
+
11 
+// Returns whether s can be serialized as a field value "as they are" without
12 
+// the special binary safe serialization.
13 
+func isSerializableAsIs(s string) bool {
14 
+	if !utf8.ValidString(s) {
15 
+		return false
16 
+	}
17 
+	for _, c := range s {
18 
+		if c < ' ' && c != '\t' {
19 
+			return false
20 
+		}
21 
+	}
22 
+	return true
23 
+}
24 
+
25 
+// WriteField writes the field serialized to Journal Export format to w.
26 
+//
27 
+// The variable name must consist only of uppercase characters, numbers and
28 
+// underscores. No validation or sanitization is performed.
29 
+func WriteField(w io.Writer, variable, value string) error {
30 
+	if isSerializableAsIs(value) {
31 
+		_, err := fmt.Fprintf(w, "%s=%s\n", variable, value)
32 
+		return err
33 
+	}
34 
+
35 
+	if _, err := fmt.Fprintln(w, variable); err != nil {
36 
+		return err
37 
+	}
38 
+	if err := binary.Write(w, binary.LittleEndian, uint64(len(value))); err != nil {
39 
+		return err
40 
+	}
41 
+	_, err := fmt.Fprintln(w, value)
42 
+	return err
43 
+}
44 
+
45 
+// WriteEndOfEntry terminates the journal entry.
46 
+func WriteEndOfEntry(w io.Writer) error {
47 
+	_, err := fmt.Fprintln(w)
48 
+	return err
49 
+}
daemon/logger/journald/internal/export/export_test.go
History View file @ 342b44b
0 50 
new file mode 100644
... ... 
@@ -0,0 +1,27 @@
0 
+package export_test
1 
+
2 
+import (
3 
+	"bytes"
4 
+	"testing"
5 
+
6 
+	"github.com/docker/docker/daemon/logger/journald/internal/export"
7 
+	"gotest.tools/v3/assert"
8 
+	"gotest.tools/v3/golden"
9 
+)
10 
+
11 
+func TestExportSerialization(t *testing.T) {
12 
+	must := func(err error) { t.Helper(); assert.NilError(t, err) }
13 
+	var buf bytes.Buffer
14 
+	must(export.WriteField(&buf, "_TRANSPORT", "journal"))
15 
+	must(export.WriteField(&buf, "MESSAGE", "this is a single-line message.\tšŸš€"))
16 
+	must(export.WriteField(&buf, "EMPTY_VALUE", ""))
17 
+	must(export.WriteField(&buf, "NEWLINE", "\n"))
18 
+	must(export.WriteEndOfEntry(&buf))
19 
+
20 
+	must(export.WriteField(&buf, "MESSAGE", "this is a\nmulti line\nmessage"))
21 
+	must(export.WriteField(&buf, "INVALID_UTF8", "a\x80b"))
22 
+	must(export.WriteField(&buf, "BINDATA", "\x00\x01\x02\x03"))
23 
+	must(export.WriteEndOfEntry(&buf))
24 
+
25 
+	golden.Assert(t, buf.String(), "export-serialization.golden")
26 
+}
daemon/logger/journald/internal/export/testdata/export-serialization.golden
History View file @ 342b44b
0 27 
new file mode 100644
1 28 
Binary files /dev/null and b/daemon/logger/journald/internal/export/testdata/export-serialization.golden differ
daemon/logger/journald/internal/fake/sender.go
History View file @ 342b44b
2 29 
new file mode 100644
... ... 
@@ -0,0 +1,149 @@
0 
+// Package fake implements a journal writer for testing which is decoupled from
1 
+// the system's journald.
2 
+//
3 
+// The systemd project does not have any facilities to support testing of
4 
+// journal reader clients (although it has been requested:
5 
+// https://github.com/systemd/systemd/issues/14120) so we have to get creative.
6 
+// The systemd-journal-remote command reads serialized journal entries in the
7 
+// Journal Export Format and writes them to journal files. This format is
8 
+// well-documented and straightforward to generate.
9 
+package fake // import "github.com/docker/docker/daemon/logger/journald/internal/fake"
10 
+
11 
+import (
12 
+	"bytes"
13 
+	"errors"
14 
+	"fmt"
15 
+	"os"
16 
+	"os/exec"
17 
+	"regexp"
18 
+	"strconv"
19 
+	"testing"
20 
+	"time"
21 
+
22 
+	"code.cloudfoundry.org/clock"
23 
+	"github.com/coreos/go-systemd/v22/journal"
24 
+	"gotest.tools/v3/assert"
25 
+
26 
+	"github.com/docker/docker/daemon/logger/journald/internal/export"
27 
+)
28 
+
29 
+// The systemd-journal-remote command is not conventionally installed on $PATH.
30 
+// The manpage from upstream systemd lists the command as
31 
+// /usr/lib/systemd/systemd-journal-remote, but Debian installs it to
32 
+// /lib/systemd instead.
33 
+var cmdPaths = []string{
34 
+	"/usr/lib/systemd/systemd-journal-remote",
35 
+	"/lib/systemd/systemd-journal-remote",
36 
+	"systemd-journal-remote", // Check $PATH anyway, just in case.
37 
+}
38 
+
39 
+// ErrCommandNotFound is returned when the systemd-journal-remote command could
40 
+// not be located at the well-known paths or $PATH.
41 
+var ErrCommandNotFound = errors.New("systemd-journal-remote command not found")
42 
+
43 
+// JournalRemoteCmdPath searches for the systemd-journal-remote command in
44 
+// well-known paths and the directories named in the $PATH environment variable.
45 
+func JournalRemoteCmdPath() (string, error) {
46 
+	for _, p := range cmdPaths {
47 
+		if path, err := exec.LookPath(p); err == nil {
48 
+			return path, nil
49 
+		}
50 
+	}
51 
+	return "", ErrCommandNotFound
52 
+}
53 
+
54 
+// Sender fakes github.com/coreos/go-systemd/v22/journal.Send, writing journal
55 
+// entries to an arbitrary journal file without depending on a running journald
56 
+// process.
57 
+type Sender struct {
58 
+	CmdName    string
59 
+	OutputPath string
60 
+
61 
+	// Clock for timestamping sent messages.
62 
+	Clock clock.Clock
63 
+	// Whether to assign the event's realtime timestamp to the time
64 
+	// specified by the SYSLOG_TIMESTAMP variable value. This is roughly
65 
+	// analogous to journald receiving the event and assigning it a
66 
+	// timestamp in zero time after the SYSLOG_TIMESTAMP value was set,
67 
+	// which is higly unrealistic in practice.
68 
+	AssignEventTimestampFromSyslogTimestamp bool
69 
+}
70 
+
71 
+// New constructs a new Sender which will write journal entries to outpath. The
72 
+// file name must end in '.journal' and the directory must already exist. The
73 
+// journal file will be created if it does not exist. An existing journal file
74 
+// will be appended to.
75 
+func New(outpath string) (*Sender, error) {
76 
+	p, err := JournalRemoteCmdPath()
77 
+	if err != nil {
78 
+		return nil, err
79 
+	}
80 
+	sender := &Sender{
81 
+		CmdName:    p,
82 
+		OutputPath: outpath,
83 
+		Clock:      clock.NewClock(),
84 
+	}
85 
+	return sender, nil
86 
+}
87 
+
88 
+// NewT is like New but will skip the test if the systemd-journal-remote command
89 
+// is not available.
90 
+func NewT(t *testing.T, outpath string) *Sender {
91 
+	t.Helper()
92 
+	s, err := New(outpath)
93 
+	if errors.Is(err, ErrCommandNotFound) {
94 
+		t.Skip(err)
95 
+	}
96 
+	assert.NilError(t, err)
97 
+	return s
98 
+}
99 
+
100 
+var validVarName = regexp.MustCompile("^[A-Z0-9][A-Z0-9_]*$")
101 
+
102 
+// Send is a drop-in replacement for
103 
+// github.com/coreos/go-systemd/v22/journal.Send.
104 
+func (s *Sender) Send(message string, priority journal.Priority, vars map[string]string) error {
105 
+	var buf bytes.Buffer
106 
+	// https://systemd.io/JOURNAL_EXPORT_FORMATS/ says "if you are
107 
+	// generating this format you shouldnā€™t care about these special
108 
+	// double-underscore fields," yet systemd-journal-remote treats entries
109 
+	// without a __REALTIME_TIMESTAMP as invalid and discards them.
110 
+	// Reported upstream: https://github.com/systemd/systemd/issues/22411
111 
+	var ts time.Time
112 
+	if sts := vars["SYSLOG_TIMESTAMP"]; s.AssignEventTimestampFromSyslogTimestamp && sts != "" {
113 
+		var err error
114 
+		if ts, err = time.Parse(time.RFC3339Nano, sts); err != nil {
115 
+			return fmt.Errorf("fake: error parsing SYSLOG_TIMESTAMP value %q: %w", ts, err)
116 
+		}
117 
+	} else {
118 
+		ts = s.Clock.Now()
119 
+	}
120 
+	if err := export.WriteField(&buf, "__REALTIME_TIMESTAMP", strconv.FormatInt(ts.UnixMicro(), 10)); err != nil {
121 
+		return fmt.Errorf("fake: error writing entry to systemd-journal-remote: %w", err)
122 
+	}
123 
+	if err := export.WriteField(&buf, "MESSAGE", message); err != nil {
124 
+		return fmt.Errorf("fake: error writing entry to systemd-journal-remote: %w", err)
125 
+	}
126 
+	if err := export.WriteField(&buf, "PRIORITY", strconv.Itoa(int(priority))); err != nil {
127 
+		return fmt.Errorf("fake: error writing entry to systemd-journal-remote: %w", err)
128 
+	}
129 
+	for k, v := range vars {
130 
+		if !validVarName.MatchString(k) {
131 
+			return fmt.Errorf("fake: invalid journal-entry variable name %q", k)
132 
+		}
133 
+		if err := export.WriteField(&buf, k, v); err != nil {
134 
+			return fmt.Errorf("fake: error writing entry to systemd-journal-remote: %w", err)
135 
+		}
136 
+	}
137 
+	if err := export.WriteEndOfEntry(&buf); err != nil {
138 
+		return fmt.Errorf("fake: error writing entry to systemd-journal-remote: %w", err)
139 
+	}
140 
+
141 
+	// Invoke the command separately for each entry to ensure that the entry
142 
+	// has been flushed to disk when Send returns.
143 
+	cmd := exec.Command(s.CmdName, "--output", s.OutputPath, "-")
144 
+	cmd.Stdin = &buf
145 
+	cmd.Stdout = os.Stdout
146 
+	cmd.Stderr = os.Stderr
147 
+	return cmd.Run()
148 
+}
daemon/logger/journald/internal/sdjournal/cursor.go
History View file @ 342b44b
0 149 
deleted file mode 100644
... ... 
@@ -1,45 +0,0 @@
1 
-//go:build linux && cgo && !static_build && journald
2 
-// +build linux,cgo,!static_build,journald
3 
-
4 
-package sdjournal // import "github.com/docker/docker/daemon/logger/journald/internal/sdjournal"
5 
-
6 
-// #include <stdlib.h>
7 
-import "C"
8 
-import (
9 
-	"runtime"
10 
-	"unsafe"
11 
-)
12 
-
13 
-// Cursor is a reference to a journal cursor. A Cursor must not be copied.
14 
-type Cursor struct {
15 
-	c      *C.char
16 
-	noCopy noCopy //nolint:structcheck,unused // Exists only to mark values uncopyable for `go vet`.
17 
-}
18 
-
19 
-func wrapCursor(cur *C.char) *Cursor {
20 
-	c := &Cursor{c: cur}
21 
-	runtime.SetFinalizer(c, (*Cursor).Free)
22 
-	return c
23 
-}
24 
-
25 
-func (c *Cursor) String() string {
26 
-	if c.c == nil {
27 
-		return "<nil>"
28 
-	}
29 
-	return C.GoString(c.c)
30 
-}
31 
-
32 
-// Free invalidates the cursor and frees any associated resources on the C heap.
33 
-func (c *Cursor) Free() {
34 
-	if c == nil {
35 
-		return
36 
-	}
37 
-	C.free(unsafe.Pointer(c.c))
38 
-	runtime.SetFinalizer(c, nil)
39 
-	c.c = nil
40 
-}
41 
-
42 
-type noCopy struct{}
43 
-
44 
-func (*noCopy) Lock()   {}
45 
-func (*noCopy) Unlock() {}
daemon/logger/journald/internal/sdjournal/sdjournal.go
History View file @ 342b44b
... ... 
@@ -56,6 +56,23 @@ func Open(flags int) (*Journal, error) {
56 56 
 	return j, nil
57 57 
 }
58 58
59 
+// OpenDir opens the journal files at the specified absolute directory path for
60 
+// reading.
61 
+//
62 
+// The returned Journal value may only be used from the same operating system
63 
+// thread which Open was called from. Using it from only a single goroutine is
64 
+// not sufficient; runtime.LockOSThread must also be used.
65 
+func OpenDir(path string, flags int) (*Journal, error) {
66 
+	j := &Journal{}
67 
+	cpath := C.CString(path)
68 
+	defer C.free(unsafe.Pointer(cpath))
69 
+	if rc := C.sd_journal_open_directory(&j.j, cpath, C.int(flags)); rc != 0 {
70 
+		return nil, fmt.Errorf("journald: error opening journal: %w", syscall.Errno(-rc))
71 
+	}
72 
+	runtime.SetFinalizer(j, (*Journal).Close)
73 
+	return j, nil
74 
+}
75 
+
59 76 
 // Close closes the journal. The return value is always nil.
60 77 
 func (j *Journal) Close() error {
61 78 
 	if j.j != nil {
... ... 
@@ -105,10 +122,27 @@ func (j *Journal) Next() (bool, error) {
105 105 
 	return rc > 0, nil
106 106 
 }
107 107
108 
-// PreviousSkip sets back the read pointer by n entries. The number of entries
109 
-// must be less than or equal to 2147483647 (2**31 - 1).
110 
-func (j *Journal) PreviousSkip(n uint) (int, error) {
111 
-	rc := C.sd_journal_previous_skip(j.j, C.uint64_t(n))
108 
+// Previous sets back the read pointer to the previous entry.
109 
+func (j *Journal) Previous() (bool, error) {
110 
+	rc := C.sd_journal_previous(j.j)
111 
+	if rc < 0 {
112 
+		return false, fmt.Errorf("journald: error setting back read pointer: %w", syscall.Errno(-rc))
113 
+	}
114 
+	return rc > 0, nil
115 
+}
116 
+
117 
+// PreviousSkip sets back the read pointer by skip entries, returning the number
118 
+// of entries set back. skip must be less than or equal to 2147483647
119 
+// (2**31 - 1).
120 
+//
121 
+// skip == 0 is a special case: PreviousSkip(0) resolves the read pointer to a
122 
+// discrete position without setting it back to a different entry. The trouble
123 
+// is, it always returns zero on recent libsystemd versions. There is no way to
124 
+// tell from the return values whether or not it successfully resolved the read
125 
+// pointer to a discrete entry.
126 
+// https://github.com/systemd/systemd/pull/5930#issuecomment-300878104
127 
+func (j *Journal) PreviousSkip(skip uint) (int, error) {
128 
+	rc := C.sd_journal_previous_skip(j.j, C.uint64_t(skip))
112 129 
 	if rc < 0 {
113 130 
 		return 0, fmt.Errorf("journald: error setting back read pointer: %w", syscall.Errno(-rc))
114 131 
 	}
... ... 
@@ -116,6 +150,9 @@ func (j *Journal) PreviousSkip(n uint) (int, error) {
116 116 
 }
117 117
118 118 
 // SeekHead sets the read pointer to the position before the oldest available entry.
119 
+//
120 
+// BUG: SeekHead() followed by Previous() has unexpected behavior.
121 
+// https://github.com/systemd/systemd/issues/17662
119 122 
 func (j *Journal) SeekHead() error {
120 123 
 	if rc := C.sd_journal_seek_head(j.j); rc != 0 {
121 124 
 		return fmt.Errorf("journald: error seeking to head of journal: %w", syscall.Errno(-rc))
... ... 
@@ -124,6 +161,9 @@ func (j *Journal) SeekHead() error {
124 124 
 }
125 125
126 126 
 // SeekTail sets the read pointer to the position after the most recent available entry.
127 
+//
128 
+// BUG: SeekTail() followed by Next() has unexpected behavior.
129 
+// https://github.com/systemd/systemd/issues/9934
127 130 
 func (j *Journal) SeekTail() error {
128 131 
 	if rc := C.sd_journal_seek_tail(j.j); rc != 0 {
129 132 
 		return fmt.Errorf("journald: error seeking to tail of journal: %w", syscall.Errno(-rc))
... ... 
@@ -142,24 +182,6 @@ func (j *Journal) SeekRealtime(t time.Time) error {
142 142 
 	return nil
143 143 
 }
144 144
145 
-// Cursor returns a serialization of the journal read pointer's current position.
146 
-func (j *Journal) Cursor() (*Cursor, error) {
147 
-	var c *C.char
148 
-	if rc := C.sd_journal_get_cursor(j.j, &c); rc != 0 {
149 
-		return nil, fmt.Errorf("journald: error getting cursor: %w", syscall.Errno(-rc))
150 
-	}
151 
-	return wrapCursor(c), nil
152 
-}
153 
-
154 
-// TestCursor checks whether the current position of the journal read pointer matches c.
155 
-func (j *Journal) TestCursor(c *Cursor) (bool, error) {
156 
-	rc := C.sd_journal_test_cursor(j.j, c.c)
157 
-	if rc < 0 {
158 
-		return false, fmt.Errorf("journald: error testing cursor: %w", syscall.Errno(-rc))
159 
-	}
160 
-	return rc > 0, nil
161 
-}
162 
-
163 145 
 // Wait blocks until the journal gets changed or timeout has elapsed.
164 146 
 // Pass a negative timeout to wait indefinitely.
165 147 
 func (j *Journal) Wait(timeout time.Duration) (Status, error) {
... ... 
@@ -235,3 +257,8 @@ func (j *Journal) SetDataThreshold(v uint) error {
235 235 
 	}
236 236 
 	return nil
237 237 
 }
238 
+
239 
+type noCopy struct{}
240 
+
241 
+func (noCopy) Lock()   {}
242 
+func (noCopy) Unlock() {}
daemon/logger/journald/journald.go
History View file @ 342b44b
... ... 
@@ -6,6 +6,7 @@ package journald // import "github.com/docker/docker/daemon/logger/journald"
6 6 
 import (
7 7 
 	"fmt"
8 8 
 	"strconv"
9 
+	"time"
9 10 
 	"unicode"
10 11
11 12 
 	"github.com/coreos/go-systemd/v22/journal"
... ... 
@@ -15,10 +16,38 @@ import (
15 15
16 16 
 const name = "journald"
17 17
18 
+// Well-known user journal fields.
19 
+// https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
20 
+const (
21 
+	fieldSyslogIdentifier = "SYSLOG_IDENTIFIER"
22 
+	fieldSyslogTimestamp  = "SYSLOG_TIMESTAMP"
23 
+)
24 
+
25 
+// User journal fields used by the log driver.
26 
+const (
27 
+	fieldContainerID     = "CONTAINER_ID"
28 
+	fieldContainerIDFull = "CONTAINER_ID_FULL"
29 
+	fieldContainerName   = "CONTAINER_NAME"
30 
+	fieldContainerTag    = "CONTAINER_TAG"
31 
+	fieldImageName       = "IMAGE_NAME"
32 
+
33 
+	// Fields used to serialize PLogMetaData.
34 
+
35 
+	fieldPLogID         = "CONTAINER_PARTIAL_ID"
36 
+	fieldPLogOrdinal    = "CONTAINER_PARTIAL_ORDINAL"
37 
+	fieldPLogLast       = "CONTAINER_PARTIAL_LAST"
38 
+	fieldPartialMessage = "CONTAINER_PARTIAL_MESSAGE"
39 
+)
40 
+
18 41 
 type journald struct {
19 42 
 	vars map[string]string // additional variables and values to send to the journal along with the log message
20 43
21 44 
 	closed chan struct{}
45 
+
46 
+	// Overrides for unit tests.
47 
+
48 
+	sendToJournal  func(message string, priority journal.Priority, vars map[string]string) error
49 
+	journalReadDir string //nolint:structcheck,unused // Referenced in read.go, which has more restrictive build constraints.
22 50 
 }
23 51
24 52 
 func init() {
... ... 
@@ -57,6 +86,10 @@ func New(info logger.Info) (logger.Logger, error) {
57 57 
 		return nil, fmt.Errorf("journald is not enabled on this host")
58 58 
 	}
59 59
60 
+	return new(info)
61 
+}
62 
+
63 
+func new(info logger.Info) (*journald, error) {
60 64 
 	// parse log tag
61 65 
 	tag, err := loggerutils.ParseLogTag(info, loggerutils.DefaultTemplate)
62 66 
 	if err != nil {
... ... 
@@ -64,12 +97,12 @@ func New(info logger.Info) (logger.Logger, error) {
64 64 
 	}
65 65
66 66 
 	vars := map[string]string{
67 
-		"CONTAINER_ID":      info.ContainerID[:12],
68 
-		"CONTAINER_ID_FULL": info.ContainerID,
69 
-		"CONTAINER_NAME":    info.Name(),
70 
-		"CONTAINER_TAG":     tag,
71 
-		"IMAGE_NAME":        info.ImageName(),
72 
-		"SYSLOG_IDENTIFIER": tag,
67 
+		fieldContainerID:      info.ContainerID[:12],
68 
+		fieldContainerIDFull:  info.ContainerID,
69 
+		fieldContainerName:    info.Name(),
70 
+		fieldContainerTag:     tag,
71 
+		fieldImageName:        info.ImageName(),
72 
+		fieldSyslogIdentifier: tag,
73 73 
 	}
74 74 
 	extraAttrs, err := info.ExtraAttributes(sanitizeKeyMod)
75 75 
 	if err != nil {
... ... 
@@ -78,7 +111,7 @@ func New(info logger.Info) (logger.Logger, error) {
78 78 
 	for k, v := range extraAttrs {
79 79 
 		vars[k] = v
80 80 
 	}
81 
-	return &journald{vars: vars, closed: make(chan struct{})}, nil
81 
+	return &journald{vars: vars, closed: make(chan struct{}), sendToJournal: journal.Send}, nil
82 82 
 }
83 83
84 84 
 // We don't actually accept any options, but we have to supply a callback for
... ... 
@@ -103,12 +136,15 @@ func (s *journald) Log(msg *logger.Message) error {
103 103 
 	for k, v := range s.vars {
104 104 
 		vars[k] = v
105 105 
 	}
106 
+	if !msg.Timestamp.IsZero() {
107 
+		vars[fieldSyslogTimestamp] = msg.Timestamp.Format(time.RFC3339Nano)
108 
+	}
106 109 
 	if msg.PLogMetaData != nil {
107 
-		vars["CONTAINER_PARTIAL_ID"] = msg.PLogMetaData.ID
108 
-		vars["CONTAINER_PARTIAL_ORDINAL"] = strconv.Itoa(msg.PLogMetaData.Ordinal)
109 
-		vars["CONTAINER_PARTIAL_LAST"] = strconv.FormatBool(msg.PLogMetaData.Last)
110 
+		vars[fieldPLogID] = msg.PLogMetaData.ID
111 
+		vars[fieldPLogOrdinal] = strconv.Itoa(msg.PLogMetaData.Ordinal)
112 
+		vars[fieldPLogLast] = strconv.FormatBool(msg.PLogMetaData.Last)
110 113 
 		if !msg.PLogMetaData.Last {
111 
-			vars["CONTAINER_PARTIAL_MESSAGE"] = "true"
114 
+			vars[fieldPartialMessage] = "true"
112 115 
 		}
113 116 
 	}
114 117
... ... 
@@ -117,9 +153,9 @@ func (s *journald) Log(msg *logger.Message) error {
117 117 
 	logger.PutMessage(msg)
118 118
119 119 
 	if source == "stderr" {
120 
-		return journal.Send(line, journal.PriErr, vars)
120 
+		return s.sendToJournal(line, journal.PriErr, vars)
121 121 
 	}
122 
-	return journal.Send(line, journal.PriInfo, vars)
122 
+	return s.sendToJournal(line, journal.PriInfo, vars)
123 123 
 }
124 124
125 125 
 func (s *journald) Name() string {
daemon/logger/journald/read.go
History View file @ 342b44b
... ... 
@@ -17,26 +17,39 @@ import (
17 17 
 	"github.com/docker/docker/daemon/logger/journald/internal/sdjournal"
18 18 
 )
19 19
20 
+// Fields which we know are not user-provided attribute fields.
20 21 
 var wellKnownFields = map[string]bool{
21 
-	"MESSAGE":           true,
22 
-	"MESSAGE_ID":        true,
23 
-	"PRIORITY":          true,
24 
-	"CODE_FILE":         true,
25 
-	"CODE_LINE":         true,
26 
-	"CODE_FUNC":         true,
27 
-	"ERRNO":             true,
28 
-	"SYSLOG_FACILITY":   true,
29 
-	"SYSLOG_IDENTIFIER": true,
30 
-	"SYSLOG_PID":        true,
31 
-	"CONTAINER_NAME":    true,
32 
-	"CONTAINER_ID":      true,
33 
-	"CONTAINER_ID_FULL": true,
34 
-	"CONTAINER_TAG":     true,
22 
+	"MESSAGE":             true,
23 
+	"MESSAGE_ID":          true,
24 
+	"PRIORITY":            true,
25 
+	"CODE_FILE":           true,
26 
+	"CODE_LINE":           true,
27 
+	"CODE_FUNC":           true,
28 
+	"ERRNO":               true,
29 
+	"SYSLOG_FACILITY":     true,
30 
+	fieldSyslogIdentifier: true,
31 
+	"SYSLOG_PID":          true,
32 
+	fieldSyslogTimestamp:  true,
33 
+	fieldContainerName:    true,
34 
+	fieldContainerID:      true,
35 
+	fieldContainerIDFull:  true,
36 
+	fieldContainerTag:     true,
37 
+	fieldImageName:        true,
38 
+	fieldPLogID:           true,
39 
+	fieldPLogOrdinal:      true,
40 
+	fieldPLogLast:         true,
41 
+	fieldPartialMessage:   true,
35 42 
 }
36 43
37 
-func getMessage(d map[string]string) (line []byte, partial, ok bool) {
44 
+func getMessage(d map[string]string) (line []byte, ok bool) {
38 45 
 	m, ok := d["MESSAGE"]
39 
-	return []byte(m), d["CONTAINER_PARTIAL_MESSAGE"] == "true", ok
46 
+	if ok {
47 
+		line = []byte(m)
48 
+		if d[fieldPartialMessage] != "true" {
49 
+			line = append(line, "\n"...)
50 
+		}
51 
+	}
52 
+	return line, ok
40 53 
 }
41 54
42 55 
 func getPriority(d map[string]string) (journal.Priority, bool) {
... ... 
@@ -47,106 +60,193 @@ func getPriority(d map[string]string) (journal.Priority, bool) {
47 47 
 	return -1, false
48 48 
 }
49 49
50 
-func (s *journald) drainJournal(logWatcher *logger.LogWatcher, j *sdjournal.Journal, oldCursor *sdjournal.Cursor, until time.Time) (*sdjournal.Cursor, bool, int) {
51 
-	var (
52 
-		done  bool
53 
-		shown int
54 
-	)
55 
-
56 
-	// Walk the journal from here forward until we run out of new entries
57 
-	// or we reach the until value (if provided).
58 
-drain:
59 
-	for ok := true; ok; ok, _ = j.Next() {
60 
-		// Try not to send a given entry twice.
61 
-		if oldCursor != nil {
62 
-			if ok, _ := j.TestCursor(oldCursor); ok {
63 
-				if ok, _ := j.Next(); !ok {
64 
-					break drain
65 
-				}
66 
-			}
50 
+// getSource recovers the stream name from the entry data by mapping from the
51 
+// journal priority field back to the stream that we would have assigned that
52 
+// value.
53 
+func getSource(d map[string]string) string {
54 
+	source := ""
55 
+	if priority, ok := getPriority(d); ok {
56 
+		if priority == journal.PriErr {
57 
+			source = "stderr"
58 
+		} else if priority == journal.PriInfo {
59 
+			source = "stdout"
67 60 
 		}
68 
-		// Read and send the logged message, if there is one to read.
69 
-		data, err := j.Data()
70 
-		if errors.Is(err, sdjournal.ErrInvalidReadPointer) {
71 
-			continue
61 
+	}
62 
+	return source
63 
+}
64 
+
65 
+func getAttrs(d map[string]string) []backend.LogAttr {
66 
+	var attrs []backend.LogAttr
67 
+	for k, v := range d {
68 
+		if k[0] != '_' && !wellKnownFields[k] {
69 
+			attrs = append(attrs, backend.LogAttr{Key: k, Value: v})
72 70 
 		}
73 
-		if line, partial, ok := getMessage(data); ok {
74 
-			// Read the entry's timestamp.
75 
-			timestamp, err := j.Realtime()
76 
-			if err != nil {
77 
-				break
71 
+	}
72 
+	return attrs
73 
+}
74 
+
75 
+// errDrainDone is the error returned by drainJournal to signal that there are
76 
+// no more log entries to send to the log watcher.
77 
+var errDrainDone = errors.New("journald drain done")
78 
+
79 
+// drainJournal reads and sends log messages from the journal. It returns the
80 
+// number of log messages sent and any error encountered. When initial != nil
81 
+// it initializes the journal read position to the position specified by config
82 
+// before reading. Otherwise it continues to read from the current position.
83 
+//
84 
+// drainJournal returns err == errDrainDone when a terminal stopping condition
85 
+// has been reached: either the watch consumer is gone or a log entry is read
86 
+// which has a timestamp after until (if until is nonzero). If the end of the
87 
+// journal is reached without encountering a terminal stopping condition,
88 
+// err == nil is returned.
89 
+func (s *journald) drainJournal(logWatcher *logger.LogWatcher, j *sdjournal.Journal, config logger.ReadConfig, initial chan struct{}) (int, error) {
90 
+	if initial != nil {
91 
+		defer func() {
92 
+			if initial != nil {
93 
+				close(initial)
78 94 
 			}
79 
-			// Break if the timestamp exceeds any provided until flag.
80 
-			if !until.IsZero() && until.Before(timestamp) {
81 
-				done = true
82 
-				break
95 
+		}()
96 
+
97 
+		var (
98 
+			err          error
99 
+			seekedToTail bool
100 
+		)
101 
+		if config.Tail >= 0 {
102 
+			if config.Until.IsZero() {
103 
+				err = j.SeekTail()
104 
+				seekedToTail = true
105 
+			} else {
106 
+				err = j.SeekRealtime(config.Until)
107 
+			}
108 
+		} else {
109 
+			if config.Since.IsZero() {
110 
+				err = j.SeekHead()
111 
+			} else {
112 
+				err = j.SeekRealtime(config.Since)
83 113 
 			}
114 
+		}
115 
+		if err != nil {
116 
+			return 0, err
117 
+		}
84 118
85 
-			// Set up the text of the entry.
86 
-			if !partial {
87 
-				line = append(line, "\n"...)
119 
+		// SeekTail() followed by Next() behaves incorrectly, so we need
120 
+		// to work around the bug by ensuring the first discrete
121 
+		// movement of the read pointer is Previous() or PreviousSkip().
122 
+		// PreviousSkip() is called inside the loop when config.Tail > 0
123 
+		// so the only special case requiring special handling is
124 
+		// config.Tail == 0.
125 
+		// https://github.com/systemd/systemd/issues/9934
126 
+		if seekedToTail && config.Tail == 0 {
127 
+			// Resolve the read pointer to the last entry in the
128 
+			// journal so that the call to Next() inside the loop
129 
+			// advances past it.
130 
+			if ok, err := j.Previous(); err != nil || !ok {
131 
+				return 0, err
88 132 
 			}
89 
-			// Recover the stream name by mapping
90 
-			// from the journal priority back to
91 
-			// the stream that we would have
92 
-			// assigned that value.
93 
-			source := ""
94 
-			if priority, ok := getPriority(data); ok {
95 
-				if priority == journal.PriErr {
96 
-					source = "stderr"
97 
-				} else if priority == journal.PriInfo {
98 
-					source = "stdout"
99 
-				}
133 
+		}
134 
+	}
135 
+
136 
+	var sent int
137 
+	for i := 0; ; i++ {
138 
+		if initial != nil && i == 0 && config.Tail > 0 {
139 
+			if n, err := j.PreviousSkip(uint(config.Tail)); err != nil || n == 0 {
140 
+				return sent, err
100 141 
 			}
101 
-			// Retrieve the values of any variables we're adding to the journal.
102 
-			var attrs []backend.LogAttr
103 
-			for k, v := range data {
104 
-				if k[0] != '_' && !wellKnownFields[k] {
105 
-					attrs = append(attrs, backend.LogAttr{Key: k, Value: v})
106 
-				}
142 
+		} else if ok, err := j.Next(); err != nil || !ok {
143 
+			return sent, err
144 
+		}
145 
+
146 
+		if initial != nil && i == 0 {
147 
+			// The cursor is in position. Signal that the watcher is
148 
+			// initialized.
149 
+			close(initial)
150 
+			initial = nil // Prevent double-closing.
151 
+		}
152 
+
153 
+		// Read the entry's timestamp.
154 
+		timestamp, err := j.Realtime()
155 
+		if err != nil {
156 
+			return sent, err
157 
+		}
158 
+		if timestamp.Before(config.Since) {
159 
+			if initial != nil && i == 0 && config.Tail > 0 {
160 
+				// PreviousSkip went too far back. Seek forwards.
161 
+				j.SeekRealtime(config.Since)
107 162 
 			}
163 
+			continue
164 
+		}
165 
+		if !config.Until.IsZero() && config.Until.Before(timestamp) {
166 
+			return sent, errDrainDone
167 
+		}
108 168
169 
+		// Read and send the logged message, if there is one to read.
170 
+		data, err := j.Data()
171 
+		if err != nil {
172 
+			return sent, err
173 
+		}
174 
+		if line, ok := getMessage(data); ok {
109 175 
 			// Send the log message, unless the consumer is gone
110 
-			select {
111 
-			case <-logWatcher.WatchConsumerGone():
112 
-				done = true // we won't be able to write anything anymore
113 
-				break drain
114 
-			case logWatcher.Msg <- &logger.Message{
176 
+			msg := &logger.Message{
115 177 
 				Line:      line,
116 
-				Source:    source,
178 
+				Source:    getSource(data),
117 179 
 				Timestamp: timestamp.In(time.UTC),
118 
-				Attrs:     attrs,
119 
-			}:
120 
-				shown++
180 
+				Attrs:     getAttrs(data),
121 181 
 			}
122 
-			// Call sd_journal_process() periodically during the processing loop
123 
-			// to close any opened file descriptors for rotated (deleted) journal files.
124 
-			if shown%1024 == 0 {
125 
-				if _, err := j.Process(); err != nil {
126 
-					// log a warning but ignore it for now
127 
-					logrus.WithField("container", s.vars["CONTAINER_ID_FULL"]).
128 
-						WithField("error", err).
129 
-						Warn("journald: error processing journal")
182 
+			// The daemon timestamp will differ from the "trusted"
183 
+			// timestamp of when the event was received by journald.
184 
+			// We can efficiently seek around the journal by the
185 
+			// event timestamp, and the event timestamp is what
186 
+			// journalctl displays. The daemon timestamp is just an
187 
+			// application-supplied field with no special
188 
+			// significance; libsystemd won't help us seek to the
189 
+			// entry with the closest timestamp.
190 
+			/*
191 
+				if sts := data["SYSLOG_TIMESTAMP"]; sts != "" {
192 
+					if tv, err := time.Parse(time.RFC3339Nano, sts); err == nil {
193 
+						msg.Timestamp = tv
194 
+					}
130 195 
 				}
196 
+			*/
197 
+			select {
198 
+			case <-logWatcher.WatchConsumerGone():
199 
+				return sent, errDrainDone
200 
+			case logWatcher.Msg <- msg:
201 
+				sent++
131 202 
 			}
132 203 
 		}
133 
-	}
134 204
135 
-	cursor, _ := j.Cursor()
136 
-	return cursor, done, shown
205 
+		// Call sd_journal_process() periodically during the processing loop
206 
+		// to close any opened file descriptors for rotated (deleted) journal files.
207 
+		if i != 0 && i%1024 == 0 {
208 
+			if _, err := j.Process(); err != nil {
209 
+				// log a warning but ignore it for now
210 
+				logrus.WithField("container", s.vars[fieldContainerIDFull]).
211 
+					WithField("error", err).
212 
+					Warn("journald: error processing journal")
213 
+			}
214 
+		}
215 
+	}
137 216 
 }
138 217
139 
-func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *sdjournal.Journal, cursor *sdjournal.Cursor, until time.Time) *sdjournal.Cursor {
140 
-LOOP:
218 
+func (s *journald) readJournal(logWatcher *logger.LogWatcher, j *sdjournal.Journal, config logger.ReadConfig, ready chan struct{}) error {
219 
+	if _, err := s.drainJournal(logWatcher, j, config, ready /* initial */); err != nil {
220 
+		if err != errDrainDone {
221 
+			return err
222 
+		}
223 
+		return nil
224 
+	}
225 
+	if !config.Follow {
226 
+		return nil
227 
+	}
228 
+
141 229 
 	for {
142 230 
 		status, err := j.Wait(250 * time.Millisecond)
143 231 
 		if err != nil {
144 
-			logWatcher.Err <- err
145 
-			break
232 
+			return err
146 233 
 		}
147 234 
 		select {
148 235 
 		case <-logWatcher.WatchConsumerGone():
149 
-			break LOOP // won't be able to write anything anymore
236 
+			return nil // won't be able to write anything anymore
150 237 
 		case <-s.closed:
151 238 
 			// container is gone, drain journal
152 239 
 		default:
... ... 
@@ -156,20 +256,31 @@ LOOP:
156 156 
 				continue
157 157 
 			}
158 158 
 		}
159 
-		newCursor, done, recv := s.drainJournal(logWatcher, j, cursor, until)
160 
-		cursor.Free()
161 
-		cursor = newCursor
162 
-		if done || (status == sdjournal.StatusNOP && recv == 0) {
163 
-			break
159 
+		n, err := s.drainJournal(logWatcher, j, config, nil /* initial */)
160 
+		if err != nil {
161 
+			if err != errDrainDone {
162 
+				return err
163 
+			}
164 
+			return nil
165 
+		} else if status == sdjournal.StatusNOP && n == 0 {
166 
+			return nil
164 167 
 		}
165 168 
 	}
166 
-
167 
-	return cursor
168 169 
 }
169 170
170 
-func (s *journald) readLogs(logWatcher *logger.LogWatcher, config logger.ReadConfig) {
171 
+func (s *journald) readLogs(logWatcher *logger.LogWatcher, config logger.ReadConfig, ready chan struct{}) {
171 172 
 	defer close(logWatcher.Msg)
172 173
174 
+	// Make sure the ready channel is closed in the event of an early
175 
+	// return.
176 
+	defer func() {
177 
+		select {
178 
+		case <-ready:
179 
+		default:
180 
+			close(ready)
181 
+		}
182 
+	}()
183 
+
173 184 
 	// Quoting https://www.freedesktop.org/software/systemd/man/sd-journal.html:
174 185 
 	//     Functions that operate on sd_journal objects are thread
175 186 
 	//     agnostic ā€” given sd_journal pointer may only be used from one
... ... 
@@ -183,7 +294,15 @@ func (s *journald) readLogs(logWatcher *logger.LogWatcher, config logger.ReadCon
183 183 
 	defer runtime.UnlockOSThread()
184 184
185 185 
 	// Get a handle to the journal.
186 
-	j, err := sdjournal.Open(0)
186 
+	var (
187 
+		j   *sdjournal.Journal
188 
+		err error
189 
+	)
190 
+	if s.journalReadDir != "" {
191 
+		j, err = sdjournal.OpenDir(s.journalReadDir, 0)
192 
+	} else {
193 
+		j, err = sdjournal.Open(0)
194 
+	}
187 195 
 	if err != nil {
188 196 
 		logWatcher.Err <- err
189 197 
 		return
... ... 
@@ -204,61 +323,23 @@ func (s *journald) readLogs(logWatcher *logger.LogWatcher, config logger.ReadCon
204 204 
 		return
205 205 
 	}
206 206 
 	// Add a match to have the library do the searching for us.
207 
-	if err := j.AddMatch("CONTAINER_ID_FULL", s.vars["CONTAINER_ID_FULL"]); err != nil {
207 
+	if err := j.AddMatch(fieldContainerIDFull, s.vars[fieldContainerIDFull]); err != nil {
208 208 
 		logWatcher.Err <- err
209 209 
 		return
210 210 
 	}
211 
-	if config.Tail >= 0 {
212 
-		// If until time provided, start from there.
213 
-		// Otherwise start at the end of the journal.
214 
-		if !config.Until.IsZero() {
215 
-			if err := j.SeekRealtime(config.Until); err != nil {
216 
-				logWatcher.Err <- err
217 
-				return
218 
-			}
219 
-		} else if err := j.SeekTail(); err != nil {
220 
-			logWatcher.Err <- err
221 
-			return
222 
-		}
223 
-		// (Try to) skip backwards by the requested number of lines...
224 
-		if _, err := j.PreviousSkip(uint(config.Tail)); err == nil {
225 
-			// ...but not before "since"
226 
-			if !config.Since.IsZero() {
227 
-				if stamp, err := j.Realtime(); err == nil && stamp.Before(config.Since) {
228 
-					_ = j.SeekRealtime(config.Since)
229 
-				}
230 
-			}
231 
-		}
232 
-	} else {
233 
-		// Start at the beginning of the journal.
234 
-		if err := j.SeekHead(); err != nil {
235 
-			logWatcher.Err <- err
236 
-			return
237 
-		}
238 
-		// If we have a cutoff date, fast-forward to it.
239 
-		if !config.Since.IsZero() {
240 
-			if err := j.SeekRealtime(config.Since); err != nil {
241 
-				logWatcher.Err <- err
242 
-				return
243 
-			}
244 
-		}
245 
-		if _, err := j.Next(); err != nil {
246 
-			logWatcher.Err <- err
247 
-			return
248 
-		}
249 
-	}
250 
-	var cursor *sdjournal.Cursor
251 
-	if config.Tail != 0 { // special case for --tail 0
252 
-		cursor, _, _ = s.drainJournal(logWatcher, j, nil, config.Until)
253 
-	}
254 
-	if config.Follow {
255 
-		cursor = s.followJournal(logWatcher, j, cursor, config.Until)
211 
+
212 
+	if err := s.readJournal(logWatcher, j, config, ready); err != nil {
213 
+		logWatcher.Err <- err
214 
+		return
256 215 
 	}
257 
-	cursor.Free()
258 216 
 }
259 217
260 218 
 func (s *journald) ReadLogs(config logger.ReadConfig) *logger.LogWatcher {
261 219 
 	logWatcher := logger.NewLogWatcher()
262 
-	go s.readLogs(logWatcher, config)
220 
+	ready := make(chan struct{})
221 
+	go s.readLogs(logWatcher, config, ready)
222 
+	// Block until the reader is in position to read from the current config
223 
+	// location to prevent race conditions in tests.
224 
+	<-ready
263 225 
 	return logWatcher
264 226 
 }
daemon/logger/journald/read_test.go
History View file @ 342b44b
265 227 
new file mode 100644
... ... 
@@ -0,0 +1,60 @@
0 
+//go:build linux && cgo && !static_build && journald
1 
+// +build linux,cgo,!static_build,journald
2 
+
3 
+package journald // import "github.com/docker/docker/daemon/logger/journald"
4 
+
5 
+import (
6 
+	"testing"
7 
+	"time"
8 
+
9 
+	"github.com/coreos/go-systemd/v22/journal"
10 
+	"gotest.tools/v3/assert"
11 
+
12 
+	"github.com/docker/docker/daemon/logger"
13 
+	"github.com/docker/docker/daemon/logger/journald/internal/fake"
14 
+	"github.com/docker/docker/daemon/logger/loggertest"
15 
+)
16 
+
17 
+func TestLogRead(t *testing.T) {
18 
+	r := loggertest.Reader{
19 
+		Factory: func(t *testing.T, info logger.Info) func(*testing.T) logger.Logger {
20 
+			journalDir := t.TempDir()
21 
+
22 
+			// Fill the journal with irrelevant events which the
23 
+			// LogReader needs to filter out.
24 
+			rotatedJournal := fake.NewT(t, journalDir+"/rotated.journal")
25 
+			rotatedJournal.AssignEventTimestampFromSyslogTimestamp = true
26 
+			l, err := new(logger.Info{
27 
+				ContainerID:   "wrongone0001",
28 
+				ContainerName: "fake",
29 
+			})
30 
+			assert.NilError(t, err)
31 
+			l.sendToJournal = rotatedJournal.Send
32 
+			assert.NilError(t, l.Log(&logger.Message{Source: "stdout", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("stdout of a different container in a rotated journal file")}))
33 
+			assert.NilError(t, l.Log(&logger.Message{Source: "stderr", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("stderr of a different container in a rotated journal file")}))
34 
+			assert.NilError(t, rotatedJournal.Send("a log message from a totally different process in a rotated journal", journal.PriInfo, nil))
35 
+
36 
+			activeJournal := fake.NewT(t, journalDir+"/fake.journal")
37 
+			activeJournal.AssignEventTimestampFromSyslogTimestamp = true
38 
+			l, err = new(logger.Info{
39 
+				ContainerID:   "wrongone0002",
40 
+				ContainerName: "fake",
41 
+			})
42 
+			assert.NilError(t, err)
43 
+			l.sendToJournal = activeJournal.Send
44 
+			assert.NilError(t, l.Log(&logger.Message{Source: "stdout", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("stdout of a different container in the active journal file")}))
45 
+			assert.NilError(t, l.Log(&logger.Message{Source: "stderr", Timestamp: time.Now().Add(-1 * 30 * time.Minute), Line: []byte("stderr of a different container in the active journal file")}))
46 
+			assert.NilError(t, rotatedJournal.Send("a log message from a totally different process in the active journal", journal.PriInfo, nil))
47 
+
48 
+			return func(t *testing.T) logger.Logger {
49 
+				l, err := new(info)
50 
+				assert.NilError(t, err)
51 
+				l.journalReadDir = journalDir
52 
+				l.sendToJournal = activeJournal.Send
53 
+				return l
54 
+			}
55 
+		},
56 
+	}
57 
+	t.Run("Tail", r.TestTail)
58 
+	t.Run("Follow", r.TestFollow)
59 
+}
daemon/logger/loggertest/logreader.go
History View file @ 342b44b
... ... 
@@ -10,6 +10,7 @@ import (
10 10 
 	"github.com/google/go-cmp/cmp"
11 11 
 	"github.com/google/go-cmp/cmp/cmpopts"
12 12 
 	"gotest.tools/v3/assert"
13 
+	"gotest.tools/v3/assert/opt"
13 14
14 15 
 	"github.com/docker/docker/api/types/backend"
15 16 
 	"github.com/docker/docker/daemon/logger"
... ... 
@@ -25,6 +26,9 @@ type Reader struct {
25 25 
 }
26 26
27 27 
 var compareLog cmp.Options = []cmp.Option{
28 
+	// Not all log drivers can round-trip timestamps at full nanosecond
29 
+	// precision.
30 
+	opt.TimeWithThreshold(time.Millisecond),
28 31 
 	// The json-log driver does not round-trip PLogMetaData and API users do
29 32 
 	// not expect it.
30 33 
 	cmpopts.IgnoreFields(logger.Message{}, "PLogMetaData"),
hack/test/unit
History View file @ 342b44b
... ... 
@@ -12,7 +12,7 @@
12 12 
 #
13 13 
 set -eux -o pipefail
14 14
15 
-BUILDFLAGS=(-tags 'netgo libdm_no_deferred_remove')
15 
+BUILDFLAGS=(-tags 'netgo libdm_no_deferred_remove journald')
16 16 
 TESTFLAGS+=" -test.timeout=${TIMEOUT:-5m}"
17 17 
 TESTDIRS="${TESTDIRS:-./...}"
18 18 
 exclude_paths='/vendor/|/integration'
vendor.mod
History View file @ 342b44b
... ... 
@@ -9,6 +9,7 @@ go 1.17
9 9 
 require (
10 10 
 	cloud.google.com/go v0.93.3
11 11 
 	cloud.google.com/go/logging v1.4.2
12 
+	code.cloudfoundry.org/clock v1.0.0
12 13 
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1
13 14 
 	github.com/Graylog2/go-gelf v0.0.0-20191017102106-1550ee647df0
14 15 
 	github.com/Microsoft/go-winio v0.5.2
... ... 
@@ -87,7 +88,6 @@ require (
87 87 
 )
88 88
89 89 
 require (
90 
-	code.cloudfoundry.org/clock v1.0.0 // indirect
91 90 
 	github.com/agext/levenshtein v1.2.3 // indirect
92 91 
 	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect
93 92 
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
vendor/gotest.tools/v3/assert/opt/opt.go
History View file @ 342b44b
94 93 
new file mode 100644
... ... 
@@ -0,0 +1,118 @@
0 
+/*Package opt provides common go-cmp.Options for use with assert.DeepEqual.
1 
+ */
2 
+package opt // import "gotest.tools/v3/assert/opt"
3 
+
4 
+import (
5 
+	"fmt"
6 
+	"reflect"
7 
+	"strings"
8 
+	"time"
9 
+
10 
+	gocmp "github.com/google/go-cmp/cmp"
11 
+)
12 
+
13 
+// DurationWithThreshold returns a gocmp.Comparer for comparing time.Duration. The
14 
+// Comparer returns true if the difference between the two Duration values is
15 
+// within the threshold and neither value is zero.
16 
+func DurationWithThreshold(threshold time.Duration) gocmp.Option {
17 
+	return gocmp.Comparer(cmpDuration(threshold))
18 
+}
19 
+
20 
+func cmpDuration(threshold time.Duration) func(x, y time.Duration) bool {
21 
+	return func(x, y time.Duration) bool {
22 
+		if x == 0 || y == 0 {
23 
+			return false
24 
+		}
25 
+		delta := x - y
26 
+		return delta <= threshold && delta >= -threshold
27 
+	}
28 
+}
29 
+
30 
+// TimeWithThreshold returns a gocmp.Comparer for comparing time.Time. The
31 
+// Comparer returns true if the difference between the two Time values is
32 
+// within the threshold and neither value is zero.
33 
+func TimeWithThreshold(threshold time.Duration) gocmp.Option {
34 
+	return gocmp.Comparer(cmpTime(threshold))
35 
+}
36 
+
37 
+func cmpTime(threshold time.Duration) func(x, y time.Time) bool {
38 
+	return func(x, y time.Time) bool {
39 
+		if x.IsZero() || y.IsZero() {
40 
+			return false
41 
+		}
42 
+		delta := x.Sub(y)
43 
+		return delta <= threshold && delta >= -threshold
44 
+	}
45 
+}
46 
+
47 
+// PathString is a gocmp.FilterPath filter that returns true when path.String()
48 
+// matches any of the specs.
49 
+//
50 
+// The path spec is a dot separated string where each segment is a field name.
51 
+// Slices, Arrays, and Maps are always matched against every element in the
52 
+// sequence. gocmp.Indirect, gocmp.Transform, and gocmp.TypeAssertion are always
53 
+// ignored.
54 
+//
55 
+// Note: this path filter is not type safe. Incorrect paths will be silently
56 
+// ignored. Consider using a type safe path filter for more complex paths.
57 
+func PathString(specs ...string) func(path gocmp.Path) bool {
58 
+	return func(path gocmp.Path) bool {
59 
+		for _, spec := range specs {
60 
+			if path.String() == spec {
61 
+				return true
62 
+			}
63 
+		}
64 
+		return false
65 
+	}
66 
+}
67 
+
68 
+// PathDebug is a gocmp.FilerPath filter that always returns false. It prints
69 
+// each path it receives. It can be used to debug path matching problems.
70 
+func PathDebug(path gocmp.Path) bool {
71 
+	fmt.Printf("PATH string=%s gostring=%s\n", path, path.GoString())
72 
+	for _, step := range path {
73 
+		fmt.Printf("  STEP %s\ttype=%s\t%s\n",
74 
+			formatStepType(step), step.Type(), stepTypeFields(step))
75 
+	}
76 
+	return false
77 
+}
78 
+
79 
+func formatStepType(step gocmp.PathStep) string {
80 
+	return strings.Title(strings.TrimPrefix(reflect.TypeOf(step).String(), "*cmp."))
81 
+}
82 
+
83 
+func stepTypeFields(step gocmp.PathStep) string {
84 
+	switch typed := step.(type) {
85 
+	case gocmp.StructField:
86 
+		return fmt.Sprintf("name=%s", typed.Name())
87 
+	case gocmp.MapIndex:
88 
+		return fmt.Sprintf("key=%s", typed.Key().Interface())
89 
+	case gocmp.Transform:
90 
+		return fmt.Sprintf("name=%s", typed.Name())
91 
+	case gocmp.SliceIndex:
92 
+		return fmt.Sprintf("name=%d", typed.Key())
93 
+	}
94 
+	return ""
95 
+}
96 
+
97 
+// PathField is a gocmp.FilerPath filter that matches a struct field by name.
98 
+// PathField will match every instance of the field in a recursive or nested
99 
+// structure.
100 
+func PathField(structType interface{}, field string) func(gocmp.Path) bool {
101 
+	typ := reflect.TypeOf(structType)
102 
+	if typ.Kind() != reflect.Struct {
103 
+		panic(fmt.Sprintf("type %s is not a struct", typ))
104 
+	}
105 
+	if _, ok := typ.FieldByName(field); !ok {
106 
+		panic(fmt.Sprintf("type %s does not have field %s", typ, field))
107 
+	}
108 
+
109 
+	return func(path gocmp.Path) bool {
110 
+		return path.Index(-2).Type() == typ && isStructField(path.Index(-1), field)
111 
+	}
112 
+}
113 
+
114 
+func isStructField(step gocmp.PathStep, name string) bool {
115 
+	field, ok := step.(gocmp.StructField)
116 
+	return ok && field.Name() == name
117 
+}
vendor/gotest.tools/v3/golden/golden.go
History View file @ 342b44b
0 118 
new file mode 100644
... ... 
@@ -0,0 +1,187 @@
0 
+/*Package golden provides tools for comparing large mutli-line strings.
1 
+
2 
+Golden files are files in the ./testdata/ subdirectory of the package under test.
3 
+Golden files can be automatically updated to match new values by running
4 
+`go test pkgname -test.update-golden`. To ensure the update is correct
5 
+compare the diff of the old expected value to the new expected value.
6 
+*/
7 
+package golden // import "gotest.tools/v3/golden"
8 
+
9 
+import (
10 
+	"bytes"
11 
+	"flag"
12 
+	"fmt"
13 
+	"io/ioutil"
14 
+	"os"
15 
+	"path/filepath"
16 
+
17 
+	"gotest.tools/v3/assert"
18 
+	"gotest.tools/v3/assert/cmp"
19 
+	"gotest.tools/v3/internal/format"
20 
+)
21 
+
22 
+var flagUpdate = flag.Bool("test.update-golden", false, "update golden file")
23 
+
24 
+type helperT interface {
25 
+	Helper()
26 
+}
27 
+
28 
+// NormalizeCRLFToLF enables end-of-line normalization for actual values passed
29 
+// to Assert and String, as well as the values saved to golden files with
30 
+// -test.update-golden.
31 
+//
32 
+// Defaults to true. If you use the core.autocrlf=true git setting on windows
33 
+// you will need to set this to false.
34 
+//
35 
+// The value may be set to false by setting GOTESTTOOLS_GOLDEN_NormalizeCRLFToLF=false
36 
+// in the environment before running tests.
37 
+//
38 
+// The default value may change in a future major release.
39 
+var NormalizeCRLFToLF = os.Getenv("GOTESTTOOLS_GOLDEN_NormalizeCRLFToLF") != "false"
40 
+
41 
+// FlagUpdate returns true when the -test.update-golden flag has been set.
42 
+func FlagUpdate() bool {
43 
+	return *flagUpdate
44 
+}
45 
+
46 
+// Open opens the file in ./testdata
47 
+func Open(t assert.TestingT, filename string) *os.File {
48 
+	if ht, ok := t.(helperT); ok {
49 
+		ht.Helper()
50 
+	}
51 
+	f, err := os.Open(Path(filename))
52 
+	assert.NilError(t, err)
53 
+	return f
54 
+}
55 
+
56 
+// Get returns the contents of the file in ./testdata
57 
+func Get(t assert.TestingT, filename string) []byte {
58 
+	if ht, ok := t.(helperT); ok {
59 
+		ht.Helper()
60 
+	}
61 
+	expected, err := ioutil.ReadFile(Path(filename))
62 
+	assert.NilError(t, err)
63 
+	return expected
64 
+}
65 
+
66 
+// Path returns the full path to a file in ./testdata
67 
+func Path(filename string) string {
68 
+	if filepath.IsAbs(filename) {
69 
+		return filename
70 
+	}
71 
+	return filepath.Join("testdata", filename)
72 
+}
73 
+
74 
+func removeCarriageReturn(in []byte) []byte {
75 
+	if !NormalizeCRLFToLF {
76 
+		return in
77 
+	}
78 
+	return bytes.Replace(in, []byte("\r\n"), []byte("\n"), -1)
79 
+}
80 
+
81 
+// Assert compares actual to the expected value in the golden file.
82 
+//
83 
+// Running `go test pkgname -test.update-golden` will write the value of actual
84 
+// to the golden file.
85 
+//
86 
+// This is equivalent to assert.Assert(t, String(actual, filename))
87 
+func Assert(t assert.TestingT, actual string, filename string, msgAndArgs ...interface{}) {
88 
+	if ht, ok := t.(helperT); ok {
89 
+		ht.Helper()
90 
+	}
91 
+	assert.Assert(t, String(actual, filename), msgAndArgs...)
92 
+}
93 
+
94 
+// String compares actual to the contents of filename and returns success
95 
+// if the strings are equal.
96 
+//
97 
+// Running `go test pkgname -test.update-golden` will write the value of actual
98 
+// to the golden file.
99 
+//
100 
+// Any \r\n substrings in actual are converted to a single \n character
101 
+// before comparing it to the expected string. When updating the golden file the
102 
+// normalized version will be written to the file. This allows Windows to use
103 
+// the same golden files as other operating systems.
104 
+func String(actual string, filename string) cmp.Comparison {
105 
+	return func() cmp.Result {
106 
+		actualBytes := removeCarriageReturn([]byte(actual))
107 
+		result, expected := compare(actualBytes, filename)
108 
+		if result != nil {
109 
+			return result
110 
+		}
111 
+		diff := format.UnifiedDiff(format.DiffConfig{
112 
+			A:    string(expected),
113 
+			B:    string(actualBytes),
114 
+			From: "expected",
115 
+			To:   "actual",
116 
+		})
117 
+		return cmp.ResultFailure("\n" + diff + failurePostamble(filename))
118 
+	}
119 
+}
120 
+
121 
+func failurePostamble(filename string) string {
122 
+	return fmt.Sprintf(`
123 
+
124 
+You can run 'go test . -test.update-golden' to automatically update %s to the new expected value.'
125 
+`, Path(filename))
126 
+}
127 
+
128 
+// AssertBytes compares actual to the expected value in the golden.
129 
+//
130 
+// Running `go test pkgname -test.update-golden` will write the value of actual
131 
+// to the golden file.
132 
+//
133 
+// This is equivalent to assert.Assert(t, Bytes(actual, filename))
134 
+func AssertBytes(
135 
+	t assert.TestingT,
136 
+	actual []byte,
137 
+	filename string,
138 
+	msgAndArgs ...interface{},
139 
+) {
140 
+	if ht, ok := t.(helperT); ok {
141 
+		ht.Helper()
142 
+	}
143 
+	assert.Assert(t, Bytes(actual, filename), msgAndArgs...)
144 
+}
145 
+
146 
+// Bytes compares actual to the contents of filename and returns success
147 
+// if the bytes are equal.
148 
+//
149 
+// Running `go test pkgname -test.update-golden` will write the value of actual
150 
+// to the golden file.
151 
+func Bytes(actual []byte, filename string) cmp.Comparison {
152 
+	return func() cmp.Result {
153 
+		result, expected := compare(actual, filename)
154 
+		if result != nil {
155 
+			return result
156 
+		}
157 
+		msg := fmt.Sprintf("%v (actual) != %v (expected)", actual, expected)
158 
+		return cmp.ResultFailure(msg + failurePostamble(filename))
159 
+	}
160 
+}
161 
+
162 
+func compare(actual []byte, filename string) (cmp.Result, []byte) {
163 
+	if err := update(filename, actual); err != nil {
164 
+		return cmp.ResultFromError(err), nil
165 
+	}
166 
+	expected, err := ioutil.ReadFile(Path(filename))
167 
+	if err != nil {
168 
+		return cmp.ResultFromError(err), nil
169 
+	}
170 
+	if bytes.Equal(expected, actual) {
171 
+		return cmp.ResultSuccess, nil
172 
+	}
173 
+	return nil, expected
174 
+}
175 
+
176 
+func update(filename string, actual []byte) error {
177 
+	if !*flagUpdate {
178 
+		return nil
179 
+	}
180 
+	if dir := filepath.Dir(Path(filename)); dir != "." {
181 
+		if err := os.MkdirAll(dir, 0755); err != nil {
182 
+			return err
183 
+		}
184 
+	}
185 
+	return ioutil.WriteFile(Path(filename), actual, 0644)
186 
+}
vendor/modules.txt
History View file @ 342b44b
... ... 
@@ -1099,8 +1099,10 @@ google.golang.org/protobuf/types/known/wrapperspb
1099 1099 
 ## explicit; go 1.13
1100 1100 
 gotest.tools/v3/assert
1101 1101 
 gotest.tools/v3/assert/cmp
1102 
+gotest.tools/v3/assert/opt
1102 1103 
 gotest.tools/v3/env
1103 1104 
 gotest.tools/v3/fs
1105 
+gotest.tools/v3/golden
1104 1106 
 gotest.tools/v3/icmd
1105 1107 
 gotest.tools/v3/internal/assert
1106 1108 
 gotest.tools/v3/internal/cleanup