Browse code

Merge pull request #50953 from ndeloof/ndjson

fix content-type declared by /events API

Sebastiaan van Stijn authored on 2025/10/04 06:50:58
Showing 12 changed files
... ...
@@ -45,6 +45,8 @@ keywords: "API, Docker, rcli, REST, documentation"
45 45
   on API version `v1.52` and up. Older API versions still accept this field, but
46 46
   may take no effect, depending on the kernel version and OCI runtime in use.
47 47
 * Removed the `KernelMemoryTCP` field from the `GET /info` endpoint.
48
+* `GET /events` supports content-type negotiation and can produce either `application/x-ndjson` 
49
+  (Newline delimited JSON object stream) or `application/json-seq` (RFC7464).
48 50
 
49 51
 ## v1.51 API changes
50 52
 
... ...
@@ -10235,7 +10235,8 @@ paths:
10235 10235
 
10236 10236
       operationId: "SystemEvents"
10237 10237
       produces:
10238
-        - "application/json"
10238
+        - "application/x-ndjson"
10239
+        - "application/json-seq"
10239 10240
       responses:
10240 10241
         200:
10241 10242
           description: "no error"
... ...
@@ -11,6 +11,15 @@ const (
11 11
 
12 12
 	// MediaTypeMultiplexedStream is vendor specific MIME-Type set for stdin/stdout/stderr multiplexed streams
13 13
 	MediaTypeMultiplexedStream = "application/vnd.docker.multiplexed-stream"
14
+
15
+	// MediaTypeJSON is the MIME-Type for JSON objects
16
+	MediaTypeJSON = "application/json"
17
+
18
+	// MediaTypeNDJson is the MIME-Type for Newline Delimited JSON objects streams
19
+	MediaTypeNDJSON = "application/x-ndjson"
20
+
21
+	// MediaTypeJsonSequence is the MIME-Type for JSON Text Sequences (RFC7464)
22
+	MediaTypeJSONSequence = "application/json-seq"
14 23
 )
15 24
 
16 25
 // Ping contains response of Engine API:
17 26
new file mode 100644
... ...
@@ -0,0 +1,50 @@
0
+package internal
1
+
2
+import (
3
+	"encoding/json"
4
+	"io"
5
+	"slices"
6
+
7
+	"github.com/moby/moby/api/types"
8
+)
9
+
10
+const rs = 0x1E
11
+
12
+type DecoderFn func(v any) error
13
+
14
+// NewJSONStreamDecoder builds adequate DecoderFn to read json records formatted with specified content-type
15
+func NewJSONStreamDecoder(r io.Reader, contentType string) DecoderFn {
16
+	switch contentType {
17
+	case types.MediaTypeJSONSequence:
18
+		return json.NewDecoder(NewRSFilterReader(r)).Decode
19
+	case types.MediaTypeJSON, types.MediaTypeNDJSON:
20
+		fallthrough
21
+	default:
22
+		return json.NewDecoder(r).Decode
23
+	}
24
+}
25
+
26
+// RSFilterReader wraps an io.Reader and filters out ASCII RS characters
27
+type RSFilterReader struct {
28
+	reader io.Reader
29
+	buffer []byte
30
+}
31
+
32
+// NewRSFilterReader creates a new RSFilterReader that filters out RS characters
33
+func NewRSFilterReader(r io.Reader) *RSFilterReader {
34
+	return &RSFilterReader{
35
+		reader: r,
36
+		buffer: make([]byte, 4096), // Internal buffer for reading chunks
37
+	}
38
+}
39
+
40
+// Read implements the io.Reader interface, filtering out RS characters
41
+func (r *RSFilterReader) Read(p []byte) (n int, err error) {
42
+	if len(p) == 0 {
43
+		return 0, nil
44
+	}
45
+
46
+	n, err = r.reader.Read(p)
47
+	filtered := slices.DeleteFunc(p[:n], func(b byte) bool { return b == rs })
48
+	return len(filtered), err
49
+}
0 50
new file mode 100644
... ...
@@ -0,0 +1,29 @@
0
+package internal
1
+
2
+import (
3
+	"fmt"
4
+	"strings"
5
+	"testing"
6
+
7
+	"github.com/moby/moby/api/types"
8
+	"gotest.tools/v3/assert"
9
+)
10
+
11
+func Test_JsonSeqDecoder(t *testing.T) {
12
+	separator := string(rune(rs))
13
+	lf := "\n"
14
+	input := fmt.Sprintf(`%s{"hello":"world"}%s%s{ "hello": "again" }%s`, separator, lf, separator, lf)
15
+	decoder := NewJSONStreamDecoder(strings.NewReader(input), types.MediaTypeJSONSequence)
16
+	type Hello struct {
17
+		Hello string `json:"hello"`
18
+	}
19
+	var hello Hello
20
+	err := decoder(&hello)
21
+	assert.NilError(t, err)
22
+	assert.Equal(t, "world", hello.Hello)
23
+
24
+	var again Hello
25
+	err = decoder(&again)
26
+	assert.NilError(t, err)
27
+	assert.Equal(t, "again", again.Hello)
28
+}
... ...
@@ -2,12 +2,14 @@ package client
2 2
 
3 3
 import (
4 4
 	"context"
5
-	"encoding/json"
5
+	"net/http"
6 6
 	"net/url"
7 7
 	"time"
8 8
 
9
+	"github.com/moby/moby/api/types"
9 10
 	"github.com/moby/moby/api/types/events"
10 11
 	"github.com/moby/moby/api/types/filters"
12
+	"github.com/moby/moby/client/internal"
11 13
 	"github.com/moby/moby/client/internal/timestamp"
12 14
 )
13 15
 
... ...
@@ -37,7 +39,10 @@ func (cli *Client) Events(ctx context.Context, options EventsListOptions) (<-cha
37 37
 			return
38 38
 		}
39 39
 
40
-		resp, err := cli.get(ctx, "/events", query, nil)
40
+		headers := http.Header{}
41
+		headers.Add("Accept", types.MediaTypeJSONSequence)
42
+		headers.Add("Accept", types.MediaTypeNDJSON)
43
+		resp, err := cli.get(ctx, "/events", query, headers)
41 44
 		if err != nil {
42 45
 			close(started)
43 46
 			errs <- err
... ...
@@ -45,7 +50,8 @@ func (cli *Client) Events(ctx context.Context, options EventsListOptions) (<-cha
45 45
 		}
46 46
 		defer resp.Body.Close()
47 47
 
48
-		decoder := json.NewDecoder(resp.Body)
48
+		contentType := resp.Header.Get("Content-Type")
49
+		decoder := internal.NewJSONStreamDecoder(resp.Body, contentType)
49 50
 
50 51
 		close(started)
51 52
 		for {
... ...
@@ -55,7 +61,7 @@ func (cli *Client) Events(ctx context.Context, options EventsListOptions) (<-cha
55 55
 				return
56 56
 			default:
57 57
 				var event events.Message
58
-				if err := decoder.Decode(&event); err != nil {
58
+				if err := decoder(&event); err != nil {
59 59
 					errs <- err
60 60
 					return
61 61
 				}
62 62
new file mode 100644
... ...
@@ -0,0 +1,44 @@
0
+package httputils
1
+
2
+import (
3
+	"encoding/json"
4
+	"io"
5
+
6
+	"github.com/moby/moby/api/types"
7
+)
8
+
9
+const rs = 0x1E
10
+
11
+type EncoderFn func(any) error
12
+
13
+// NewJSONStreamEncoder builds adequate EncoderFn to write json records using selected content-type formalism
14
+func NewJSONStreamEncoder(w io.Writer, contentType string) EncoderFn {
15
+	jsonEncoder := json.NewEncoder(w)
16
+	switch contentType {
17
+	case types.MediaTypeJSONSequence:
18
+		jseq := &jsonSeq{
19
+			w:    w,
20
+			json: jsonEncoder,
21
+		}
22
+		return jseq.Encode
23
+	case types.MediaTypeNDJSON, types.MediaTypeJSON:
24
+		fallthrough
25
+	default:
26
+		return jsonEncoder.Encode
27
+	}
28
+}
29
+
30
+type jsonSeq struct {
31
+	w    io.Writer
32
+	json *json.Encoder
33
+}
34
+
35
+// Encode prefixes every written record with an ASCII record separator.
36
+func (js *jsonSeq) Encode(record any) error {
37
+	_, err := js.w.Write([]byte{rs})
38
+	if err != nil {
39
+		return err
40
+	}
41
+	// JSON-seq also requires a LF character, bu json.Encoder already adds one
42
+	return js.json.Encode(record)
43
+}
... ...
@@ -8,7 +8,9 @@ import (
8 8
 	"time"
9 9
 
10 10
 	"github.com/containerd/log"
11
+	"github.com/golang/gddo/httputil"
11 12
 	"github.com/moby/moby/api/pkg/authconfig"
13
+	"github.com/moby/moby/api/types"
12 14
 	buildtypes "github.com/moby/moby/api/types/build"
13 15
 	"github.com/moby/moby/api/types/events"
14 16
 	"github.com/moby/moby/api/types/filters"
... ...
@@ -296,13 +298,17 @@ func (s *systemRouter) getEvents(ctx context.Context, w http.ResponseWriter, r *
296 296
 		return err
297 297
 	}
298 298
 
299
-	w.Header().Set("Content-Type", "application/json")
299
+	contentType := httputil.NegotiateContentType(r, []string{
300
+		types.MediaTypeNDJSON,
301
+		types.MediaTypeJSONSequence,
302
+	}, types.MediaTypeJSON) // output isn't actually JSON but API used to  this content-type
303
+	w.Header().Set("Content-Type", contentType)
300 304
 	w.WriteHeader(http.StatusOK)
301 305
 	output := ioutils.NewWriteFlusher(w)
302 306
 	defer output.Close()
303 307
 	output.Flush()
304 308
 
305
-	enc := json.NewEncoder(output)
309
+	encode := httputils.NewJSONStreamEncoder(output, contentType)
306 310
 
307 311
 	buffered, l := s.backend.SubscribeToEvents(since, until, ef)
308 312
 	defer s.backend.UnsubscribeFromEvents(l)
... ...
@@ -325,12 +331,12 @@ func (s *systemRouter) getEvents(ctx context.Context, w http.ResponseWriter, r *
325 325
 			continue
326 326
 		}
327 327
 		if includeLegacyFields {
328
-			if err := enc.Encode(backFillLegacy(&ev)); err != nil {
328
+			if err := encode(backFillLegacy(&ev)); err != nil {
329 329
 				return err
330 330
 			}
331 331
 			continue
332 332
 		}
333
-		if err := enc.Encode(ev); err != nil {
333
+		if err := encode(ev); err != nil {
334 334
 			return err
335 335
 		}
336 336
 	}
... ...
@@ -351,12 +357,12 @@ func (s *systemRouter) getEvents(ctx context.Context, w http.ResponseWriter, r *
351 351
 				continue
352 352
 			}
353 353
 			if includeLegacyFields {
354
-				if err := enc.Encode(backFillLegacy(&jev)); err != nil {
354
+				if err := encode(backFillLegacy(&jev)); err != nil {
355 355
 					return err
356 356
 				}
357 357
 				continue
358 358
 			}
359
-			if err := enc.Encode(jev); err != nil {
359
+			if err := encode(jev); err != nil {
360 360
 				return err
361 361
 			}
362 362
 		case <-timeout:
... ...
@@ -11,6 +11,15 @@ const (
11 11
 
12 12
 	// MediaTypeMultiplexedStream is vendor specific MIME-Type set for stdin/stdout/stderr multiplexed streams
13 13
 	MediaTypeMultiplexedStream = "application/vnd.docker.multiplexed-stream"
14
+
15
+	// MediaTypeJSON is the MIME-Type for JSON objects
16
+	MediaTypeJSON = "application/json"
17
+
18
+	// MediaTypeNDJson is the MIME-Type for Newline Delimited JSON objects streams
19
+	MediaTypeNDJSON = "application/x-ndjson"
20
+
21
+	// MediaTypeJsonSequence is the MIME-Type for JSON Text Sequences (RFC7464)
22
+	MediaTypeJSONSequence = "application/json-seq"
14 23
 )
15 24
 
16 25
 // Ping contains response of Engine API:
17 26
new file mode 100644
... ...
@@ -0,0 +1,50 @@
0
+package internal
1
+
2
+import (
3
+	"encoding/json"
4
+	"io"
5
+	"slices"
6
+
7
+	"github.com/moby/moby/api/types"
8
+)
9
+
10
+const rs = 0x1E
11
+
12
+type DecoderFn func(v any) error
13
+
14
+// NewJSONStreamDecoder builds adequate DecoderFn to read json records formatted with specified content-type
15
+func NewJSONStreamDecoder(r io.Reader, contentType string) DecoderFn {
16
+	switch contentType {
17
+	case types.MediaTypeJSONSequence:
18
+		return json.NewDecoder(NewRSFilterReader(r)).Decode
19
+	case types.MediaTypeJSON, types.MediaTypeNDJSON:
20
+		fallthrough
21
+	default:
22
+		return json.NewDecoder(r).Decode
23
+	}
24
+}
25
+
26
+// RSFilterReader wraps an io.Reader and filters out ASCII RS characters
27
+type RSFilterReader struct {
28
+	reader io.Reader
29
+	buffer []byte
30
+}
31
+
32
+// NewRSFilterReader creates a new RSFilterReader that filters out RS characters
33
+func NewRSFilterReader(r io.Reader) *RSFilterReader {
34
+	return &RSFilterReader{
35
+		reader: r,
36
+		buffer: make([]byte, 4096), // Internal buffer for reading chunks
37
+	}
38
+}
39
+
40
+// Read implements the io.Reader interface, filtering out RS characters
41
+func (r *RSFilterReader) Read(p []byte) (n int, err error) {
42
+	if len(p) == 0 {
43
+		return 0, nil
44
+	}
45
+
46
+	n, err = r.reader.Read(p)
47
+	filtered := slices.DeleteFunc(p[:n], func(b byte) bool { return b == rs })
48
+	return len(filtered), err
49
+}
... ...
@@ -2,12 +2,14 @@ package client
2 2
 
3 3
 import (
4 4
 	"context"
5
-	"encoding/json"
5
+	"net/http"
6 6
 	"net/url"
7 7
 	"time"
8 8
 
9
+	"github.com/moby/moby/api/types"
9 10
 	"github.com/moby/moby/api/types/events"
10 11
 	"github.com/moby/moby/api/types/filters"
12
+	"github.com/moby/moby/client/internal"
11 13
 	"github.com/moby/moby/client/internal/timestamp"
12 14
 )
13 15
 
... ...
@@ -37,7 +39,10 @@ func (cli *Client) Events(ctx context.Context, options EventsListOptions) (<-cha
37 37
 			return
38 38
 		}
39 39
 
40
-		resp, err := cli.get(ctx, "/events", query, nil)
40
+		headers := http.Header{}
41
+		headers.Add("Accept", types.MediaTypeJSONSequence)
42
+		headers.Add("Accept", types.MediaTypeNDJSON)
43
+		resp, err := cli.get(ctx, "/events", query, headers)
41 44
 		if err != nil {
42 45
 			close(started)
43 46
 			errs <- err
... ...
@@ -45,7 +50,8 @@ func (cli *Client) Events(ctx context.Context, options EventsListOptions) (<-cha
45 45
 		}
46 46
 		defer resp.Body.Close()
47 47
 
48
-		decoder := json.NewDecoder(resp.Body)
48
+		contentType := resp.Header.Get("Content-Type")
49
+		decoder := internal.NewJSONStreamDecoder(resp.Body, contentType)
49 50
 
50 51
 		close(started)
51 52
 		for {
... ...
@@ -55,7 +61,7 @@ func (cli *Client) Events(ctx context.Context, options EventsListOptions) (<-cha
55 55
 				return
56 56
 			default:
57 57
 				var event events.Message
58
-				if err := decoder.Decode(&event); err != nil {
58
+				if err := decoder(&event); err != nil {
59 59
 					errs <- err
60 60
 					return
61 61
 				}
... ...
@@ -966,6 +966,7 @@ github.com/moby/moby/api/types/volume
966 966
 # github.com/moby/moby/client v0.1.0-beta.0 => ./client
967 967
 ## explicit; go 1.23.0
968 968
 github.com/moby/moby/client
969
+github.com/moby/moby/client/internal
969 970
 github.com/moby/moby/client/internal/timestamp
970 971
 github.com/moby/moby/client/pkg/jsonmessage
971 972
 github.com/moby/moby/client/pkg/stringid