Browse code

Merge pull request #18840 from aaronlehmann/trust-messages

Send push information to trust code out-of-band

Michael Crosby authored on 2016/01/09 09:56:57
Showing 15 changed files
... ...
@@ -255,7 +255,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
255 255
 		return err
256 256
 	}
257 257
 
258
-	err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, cli.outFd, cli.isTerminalOut)
258
+	err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, cli.outFd, cli.isTerminalOut, nil)
259 259
 	if err != nil {
260 260
 		if jerr, ok := err.(*jsonmessage.JSONError); ok {
261 261
 			// If no error code is set, default to 1
... ...
@@ -58,7 +58,7 @@ func (cli *DockerCli) pullImageCustomOut(image string, out io.Writer) error {
58 58
 	}
59 59
 	defer responseBody.Close()
60 60
 
61
-	return jsonmessage.DisplayJSONMessagesStream(responseBody, out, cli.outFd, cli.isTerminalOut)
61
+	return jsonmessage.DisplayJSONMessagesStream(responseBody, out, cli.outFd, cli.isTerminalOut, nil)
62 62
 }
63 63
 
64 64
 type cidFile struct {
... ...
@@ -76,5 +76,5 @@ func (cli *DockerCli) CmdImport(args ...string) error {
76 76
 	}
77 77
 	defer responseBody.Close()
78 78
 
79
-	return jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut)
79
+	return jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut, nil)
80 80
 }
... ...
@@ -37,7 +37,7 @@ func (cli *DockerCli) CmdLoad(args ...string) error {
37 37
 	defer response.Body.Close()
38 38
 
39 39
 	if response.JSON {
40
-		return jsonmessage.DisplayJSONMessagesStream(response.Body, cli.out, cli.outFd, cli.isTerminalOut)
40
+		return jsonmessage.DisplayJSONMessagesStream(response.Body, cli.out, cli.outFd, cli.isTerminalOut, nil)
41 41
 	}
42 42
 
43 43
 	_, err = io.Copy(cli.out, response.Body)
... ...
@@ -83,5 +83,5 @@ func (cli *DockerCli) imagePullPrivileged(authConfig types.AuthConfig, imageID,
83 83
 	}
84 84
 	defer responseBody.Close()
85 85
 
86
-	return jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut)
86
+	return jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut, nil)
87 87
 }
... ...
@@ -49,13 +49,20 @@ func (cli *DockerCli) CmdPush(args ...string) error {
49 49
 		return cli.trustedPush(repoInfo, tag, authConfig, requestPrivilege)
50 50
 	}
51 51
 
52
-	return cli.imagePushPrivileged(authConfig, ref.Name(), tag, cli.out, requestPrivilege)
52
+	responseBody, err := cli.imagePushPrivileged(authConfig, ref.Name(), tag, requestPrivilege)
53
+	if err != nil {
54
+		return err
55
+	}
56
+
57
+	defer responseBody.Close()
58
+
59
+	return jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut, nil)
53 60
 }
54 61
 
55
-func (cli *DockerCli) imagePushPrivileged(authConfig types.AuthConfig, imageID, tag string, outputStream io.Writer, requestPrivilege client.RequestPrivilegeFunc) error {
62
+func (cli *DockerCli) imagePushPrivileged(authConfig types.AuthConfig, imageID, tag string, requestPrivilege client.RequestPrivilegeFunc) (io.ReadCloser, error) {
56 63
 	encodedAuth, err := encodeAuthToBase64(authConfig)
57 64
 	if err != nil {
58
-		return err
65
+		return nil, err
59 66
 	}
60 67
 	options := types.ImagePushOptions{
61 68
 		ImageID:      imageID,
... ...
@@ -63,11 +70,5 @@ func (cli *DockerCli) imagePushPrivileged(authConfig types.AuthConfig, imageID,
63 63
 		RegistryAuth: encodedAuth,
64 64
 	}
65 65
 
66
-	responseBody, err := cli.client.ImagePush(options, requestPrivilege)
67
-	if err != nil {
68
-		return err
69
-	}
70
-	defer responseBody.Close()
71
-
72
-	return jsonmessage.DisplayJSONMessagesStream(responseBody, outputStream, cli.outFd, cli.isTerminalOut)
66
+	return cli.client.ImagePush(options, requestPrivilege)
73 67
 }
... ...
@@ -1,19 +1,16 @@
1 1
 package client
2 2
 
3 3
 import (
4
-	"bufio"
5 4
 	"encoding/hex"
6 5
 	"encoding/json"
7 6
 	"errors"
8 7
 	"fmt"
9
-	"io"
10 8
 	"net"
11 9
 	"net/http"
12 10
 	"net/url"
13 11
 	"os"
14 12
 	"path"
15 13
 	"path/filepath"
16
-	"regexp"
17 14
 	"sort"
18 15
 	"strconv"
19 16
 	"time"
... ...
@@ -23,8 +20,8 @@ import (
23 23
 	"github.com/docker/distribution/registry/client/auth"
24 24
 	"github.com/docker/distribution/registry/client/transport"
25 25
 	"github.com/docker/docker/cliconfig"
26
-	"github.com/docker/docker/pkg/ansiescape"
27
-	"github.com/docker/docker/pkg/ioutils"
26
+	"github.com/docker/docker/distribution"
27
+	"github.com/docker/docker/pkg/jsonmessage"
28 28
 	flag "github.com/docker/docker/pkg/mflag"
29 29
 	"github.com/docker/docker/reference"
30 30
 	"github.com/docker/docker/registry"
... ...
@@ -64,8 +61,6 @@ func isTrusted() bool {
64 64
 	return !untrusted
65 65
 }
66 66
 
67
-var targetRegexp = regexp.MustCompile(`([\S]+): digest: ([\S]+) size: ([\d]+)`)
68
-
69 67
 type target struct {
70 68
 	reference registry.Reference
71 69
 	digest    digest.Digest
... ...
@@ -366,60 +361,31 @@ func (cli *DockerCli) trustedPull(repoInfo *registry.RepositoryInfo, ref registr
366 366
 	return nil
367 367
 }
368 368
 
369
-func targetStream(in io.Writer) (io.WriteCloser, <-chan []target) {
370
-	r, w := io.Pipe()
371
-	out := io.MultiWriter(in, w)
372
-	targetChan := make(chan []target)
373
-
374
-	go func() {
375
-		targets := []target{}
376
-		scanner := bufio.NewScanner(r)
377
-		scanner.Split(ansiescape.ScanANSILines)
378
-		for scanner.Scan() {
379
-			line := scanner.Bytes()
380
-			if matches := targetRegexp.FindSubmatch(line); len(matches) == 4 {
381
-				dgst, err := digest.ParseDigest(string(matches[2]))
382
-				if err != nil {
383
-					// Line does match what is expected, continue looking for valid lines
384
-					logrus.Debugf("Bad digest value %q in matched line, ignoring\n", string(matches[2]))
385
-					continue
386
-				}
387
-				s, err := strconv.ParseInt(string(matches[3]), 10, 64)
388
-				if err != nil {
389
-					// Line does match what is expected, continue looking for valid lines
390
-					logrus.Debugf("Bad size value %q in matched line, ignoring\n", string(matches[3]))
391
-					continue
392
-				}
393
-
394
-				targets = append(targets, target{
395
-					reference: registry.ParseReference(string(matches[1])),
396
-					digest:    dgst,
397
-					size:      s,
398
-				})
399
-			}
400
-		}
401
-		targetChan <- targets
402
-	}()
403
-
404
-	return ioutils.NewWriteCloserWrapper(out, w.Close), targetChan
405
-}
406
-
407 369
 func (cli *DockerCli) trustedPush(repoInfo *registry.RepositoryInfo, tag string, authConfig types.AuthConfig, requestPrivilege apiclient.RequestPrivilegeFunc) error {
408
-	streamOut, targetChan := targetStream(cli.out)
409
-
410
-	reqError := cli.imagePushPrivileged(authConfig, repoInfo.Name(), tag, streamOut, requestPrivilege)
411
-
412
-	// Close stream channel to finish target parsing
413
-	if err := streamOut.Close(); err != nil {
370
+	responseBody, err := cli.imagePushPrivileged(authConfig, repoInfo.Name(), tag, requestPrivilege)
371
+	if err != nil {
414 372
 		return err
415 373
 	}
416
-	// Check error from request
417
-	if reqError != nil {
418
-		return reqError
374
+
375
+	defer responseBody.Close()
376
+
377
+	targets := []target{}
378
+	handleTarget := func(aux *json.RawMessage) {
379
+		var pushResult distribution.PushResult
380
+		err := json.Unmarshal(*aux, &pushResult)
381
+		if err == nil && pushResult.Tag != "" && pushResult.Digest.Validate() == nil {
382
+			targets = append(targets, target{
383
+				reference: registry.ParseReference(pushResult.Tag),
384
+				digest:    pushResult.Digest,
385
+				size:      int64(pushResult.Size),
386
+			})
387
+		}
419 388
 	}
420 389
 
421
-	// Get target results
422
-	targets := <-targetChan
390
+	err = jsonmessage.DisplayJSONMessagesStream(responseBody, cli.out, cli.outFd, cli.isTerminalOut, handleTarget)
391
+	if err != nil {
392
+		return err
393
+	}
423 394
 
424 395
 	if tag == "" {
425 396
 		fmt.Fprintf(cli.out, "No tag specified, skipping trust metadata push\n")
... ...
@@ -26,6 +26,15 @@ import (
26 26
 	"golang.org/x/net/context"
27 27
 )
28 28
 
29
+// PushResult contains the tag, manifest digest, and manifest size from the
30
+// push. It's used to signal this information to the trust code in the client
31
+// so it can sign the manifest if necessary.
32
+type PushResult struct {
33
+	Tag    string
34
+	Digest digest.Digest
35
+	Size   int
36
+}
37
+
29 38
 type v2Pusher struct {
30 39
 	blobSumService *metadata.BlobSumService
31 40
 	ref            reference.Named
... ...
@@ -174,9 +183,10 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, association reference.Associat
174 174
 	}
175 175
 	if manifestDigest != "" {
176 176
 		if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
177
-			// NOTE: do not change this format without first changing the trust client
178
-			// code. This information is used to determine what was pushed and should be signed.
179 177
 			progress.Messagef(p.config.ProgressOutput, "", "%s: digest: %s size: %d", tagged.Tag(), manifestDigest, manifestSize)
178
+			// Signal digest to the trust client so it can sign the
179
+			// push, if appropriate.
180
+			progress.Aux(p.config.ProgressOutput, PushResult{Tag: tagged.Tag(), Digest: manifestDigest, Size: manifestSize})
180 181
 		}
181 182
 	}
182 183
 
183 184
deleted file mode 100644
... ...
@@ -1,89 +0,0 @@
1
-package ansiescape
2
-
3
-import "bytes"
4
-
5
-// dropCR drops a leading or terminal \r from the data.
6
-func dropCR(data []byte) []byte {
7
-	if len(data) > 0 && data[len(data)-1] == '\r' {
8
-		data = data[0 : len(data)-1]
9
-	}
10
-	if len(data) > 0 && data[0] == '\r' {
11
-		data = data[1:]
12
-	}
13
-	return data
14
-}
15
-
16
-// escapeSequenceLength calculates the length of an ANSI escape sequence
17
-// If there is not enough characters to match a sequence, -1 is returned,
18
-// if there is no valid sequence 0 is returned, otherwise the number
19
-// of bytes in the sequence is returned. Only returns length for
20
-// line moving sequences.
21
-func escapeSequenceLength(data []byte) int {
22
-	next := 0
23
-	if len(data) <= next {
24
-		return -1
25
-	}
26
-	if data[next] != '[' {
27
-		return 0
28
-	}
29
-	for {
30
-		next = next + 1
31
-		if len(data) <= next {
32
-			return -1
33
-		}
34
-		if (data[next] > '9' || data[next] < '0') && data[next] != ';' {
35
-			break
36
-		}
37
-	}
38
-	if len(data) <= next {
39
-		return -1
40
-	}
41
-	// Only match line moving codes
42
-	switch data[next] {
43
-	case 'A', 'B', 'E', 'F', 'H', 'h':
44
-		return next + 1
45
-	}
46
-
47
-	return 0
48
-}
49
-
50
-// ScanANSILines is a scanner function which splits the
51
-// input based on ANSI escape codes and new lines.
52
-func ScanANSILines(data []byte, atEOF bool) (advance int, token []byte, err error) {
53
-	if atEOF && len(data) == 0 {
54
-		return 0, nil, nil
55
-	}
56
-
57
-	// Look for line moving escape sequence
58
-	if i := bytes.IndexByte(data, '\x1b'); i >= 0 {
59
-		last := 0
60
-		for i >= 0 {
61
-			last = last + i
62
-
63
-			// get length of ANSI escape sequence
64
-			sl := escapeSequenceLength(data[last+1:])
65
-			if sl == -1 {
66
-				return 0, nil, nil
67
-			}
68
-			if sl == 0 {
69
-				// If no relevant sequence was found, skip
70
-				last = last + 1
71
-				i = bytes.IndexByte(data[last:], '\x1b')
72
-				continue
73
-			}
74
-
75
-			return last + 1 + sl, dropCR(data[0:(last)]), nil
76
-		}
77
-	}
78
-	if i := bytes.IndexByte(data, '\n'); i >= 0 {
79
-		// No escape sequence, check for new line
80
-		return i + 1, dropCR(data[0:i]), nil
81
-	}
82
-
83
-	// If we're at EOF, we have a final, non-terminated line. Return it.
84
-	if atEOF {
85
-		return len(data), dropCR(data), nil
86
-	}
87
-	// Request more data.
88
-	return 0, nil, nil
89
-}
90 1
deleted file mode 100644
... ...
@@ -1,53 +0,0 @@
1
-package ansiescape
2
-
3
-import (
4
-	"bufio"
5
-	"strings"
6
-	"testing"
7
-)
8
-
9
-func TestSplit(t *testing.T) {
10
-	lines := []string{
11
-		"test line 1",
12
-		"another test line",
13
-		"some test line",
14
-		"line with non-cursor moving sequence \x1b[1T", // Scroll Down
15
-		"line with \x1b[31;1mcolor\x1b[0m then reset",  // "color" in Bold Red
16
-		"cursor forward \x1b[1C and backward \x1b[1D",
17
-		"invalid sequence \x1babcd",
18
-		"",
19
-		"after empty",
20
-	}
21
-	splitSequences := []string{
22
-		"\x1b[1A",   // Cursor up
23
-		"\x1b[1B",   // Cursor down
24
-		"\x1b[1E",   // Cursor next line
25
-		"\x1b[1F",   // Cursor previous line
26
-		"\x1b[1;1H", // Move cursor to position
27
-		"\x1b[1;1h", // Move cursor to position
28
-		"\n",
29
-		"\r\n",
30
-		"\n\r",
31
-		"\x1b[1A\r",
32
-		"\r\x1b[1A",
33
-	}
34
-
35
-	for _, sequence := range splitSequences {
36
-		scanner := bufio.NewScanner(strings.NewReader(strings.Join(lines, sequence)))
37
-		scanner.Split(ScanANSILines)
38
-		i := 0
39
-		for scanner.Scan() {
40
-			if i >= len(lines) {
41
-				t.Fatalf("Too many scanned lines")
42
-			}
43
-			scanned := scanner.Text()
44
-			if scanned != lines[i] {
45
-				t.Fatalf("Wrong line scanned with sequence %q\n\tExpected: %q\n\tActual:   %q", sequence, lines[i], scanned)
46
-			}
47
-			i++
48
-		}
49
-		if i < len(lines) {
50
-			t.Errorf("Wrong number of lines for sequence %q: %d, expected %d", sequence, i, len(lines))
51
-		}
52
-	}
53
-}
... ...
@@ -102,6 +102,8 @@ type JSONMessage struct {
102 102
 	TimeNano        int64         `json:"timeNano,omitempty"`
103 103
 	Error           *JSONError    `json:"errorDetail,omitempty"`
104 104
 	ErrorMessage    string        `json:"error,omitempty"` //deprecated
105
+	// Aux contains out-of-band data, such as digests for push signing.
106
+	Aux *json.RawMessage `json:"aux,omitempty"`
105 107
 }
106 108
 
107 109
 // Display displays the JSONMessage to `out`. `isTerminal` describes if `out`
... ...
@@ -148,7 +150,7 @@ func (jm *JSONMessage) Display(out io.Writer, isTerminal bool) error {
148 148
 // DisplayJSONMessagesStream displays a json message stream from `in` to `out`, `isTerminal`
149 149
 // describes if `out` is a terminal. If this is the case, it will print `\n` at the end of
150 150
 // each line and move the cursor while displaying.
151
-func DisplayJSONMessagesStream(in io.Reader, out io.Writer, terminalFd uintptr, isTerminal bool) error {
151
+func DisplayJSONMessagesStream(in io.Reader, out io.Writer, terminalFd uintptr, isTerminal bool, auxCallback func(*json.RawMessage)) error {
152 152
 	var (
153 153
 		dec = json.NewDecoder(in)
154 154
 		ids = make(map[string]int)
... ...
@@ -163,6 +165,13 @@ func DisplayJSONMessagesStream(in io.Reader, out io.Writer, terminalFd uintptr,
163 163
 			return err
164 164
 		}
165 165
 
166
+		if jm.Aux != nil {
167
+			if auxCallback != nil {
168
+				auxCallback(jm.Aux)
169
+			}
170
+			continue
171
+		}
172
+
166 173
 		if jm.Progress != nil {
167 174
 			jm.Progress.terminalFd = terminalFd
168 175
 		}
... ...
@@ -168,7 +168,7 @@ func TestDisplayJSONMessagesStreamInvalidJSON(t *testing.T) {
168 168
 	reader := strings.NewReader("This is not a 'valid' JSON []")
169 169
 	inFd, _ = term.GetFdInfo(reader)
170 170
 
171
-	if err := DisplayJSONMessagesStream(reader, data, inFd, false); err == nil && err.Error()[:17] != "invalid character" {
171
+	if err := DisplayJSONMessagesStream(reader, data, inFd, false, nil); err == nil && err.Error()[:17] != "invalid character" {
172 172
 		t.Fatalf("Should have thrown an error (invalid character in ..), got [%v]", err)
173 173
 	}
174 174
 }
... ...
@@ -210,7 +210,7 @@ func TestDisplayJSONMessagesStream(t *testing.T) {
210 210
 		inFd, _ = term.GetFdInfo(reader)
211 211
 
212 212
 		// Without terminal
213
-		if err := DisplayJSONMessagesStream(reader, data, inFd, false); err != nil {
213
+		if err := DisplayJSONMessagesStream(reader, data, inFd, false, nil); err != nil {
214 214
 			t.Fatal(err)
215 215
 		}
216 216
 		if data.String() != expectedMessages[0] {
... ...
@@ -220,7 +220,7 @@ func TestDisplayJSONMessagesStream(t *testing.T) {
220 220
 		// With terminal
221 221
 		data = bytes.NewBuffer([]byte{})
222 222
 		reader = strings.NewReader(jsonMessage)
223
-		if err := DisplayJSONMessagesStream(reader, data, inFd, true); err != nil {
223
+		if err := DisplayJSONMessagesStream(reader, data, inFd, true, nil); err != nil {
224 224
 			t.Fatal(err)
225 225
 		}
226 226
 		if data.String() != expectedMessages[1] {
... ...
@@ -16,6 +16,10 @@ type Progress struct {
16 16
 	Current int64
17 17
 	Total   int64
18 18
 
19
+	// Aux contains extra information not presented to the user, such as
20
+	// digests for push signing.
21
+	Aux interface{}
22
+
19 23
 	LastUpdate bool
20 24
 }
21 25
 
... ...
@@ -61,3 +65,9 @@ func Message(out Output, id, message string) {
61 61
 func Messagef(out Output, id, format string, a ...interface{}) {
62 62
 	Message(out, id, fmt.Sprintf(format, a...))
63 63
 }
64
+
65
+// Aux sends auxiliary information over a progress interface, which will not be
66
+// formatted for the UI. This is used for things such as push signing.
67
+func Aux(out Output, a interface{}) {
68
+	out.WriteProgress(Progress{Aux: a})
69
+}
... ...
@@ -70,16 +70,26 @@ func (sf *StreamFormatter) FormatError(err error) []byte {
70 70
 }
71 71
 
72 72
 // FormatProgress formats the progress information for a specified action.
73
-func (sf *StreamFormatter) FormatProgress(id, action string, progress *jsonmessage.JSONProgress) []byte {
73
+func (sf *StreamFormatter) FormatProgress(id, action string, progress *jsonmessage.JSONProgress, aux interface{}) []byte {
74 74
 	if progress == nil {
75 75
 		progress = &jsonmessage.JSONProgress{}
76 76
 	}
77 77
 	if sf.json {
78
+		var auxJSON *json.RawMessage
79
+		if aux != nil {
80
+			auxJSONBytes, err := json.Marshal(aux)
81
+			if err != nil {
82
+				return nil
83
+			}
84
+			auxJSON = new(json.RawMessage)
85
+			*auxJSON = auxJSONBytes
86
+		}
78 87
 		b, err := json.Marshal(&jsonmessage.JSONMessage{
79 88
 			Status:          action,
80 89
 			ProgressMessage: progress.String(),
81 90
 			Progress:        progress,
82 91
 			ID:              id,
92
+			Aux:             auxJSON,
83 93
 		})
84 94
 		if err != nil {
85 95
 			return nil
... ...
@@ -116,7 +126,7 @@ func (out *progressOutput) WriteProgress(prog progress.Progress) error {
116 116
 		formatted = out.sf.FormatStatus(prog.ID, prog.Message)
117 117
 	} else {
118 118
 		jsonProgress := jsonmessage.JSONProgress{Current: prog.Current, Total: prog.Total}
119
-		formatted = out.sf.FormatProgress(prog.ID, prog.Action, &jsonProgress)
119
+		formatted = out.sf.FormatProgress(prog.ID, prog.Action, &jsonProgress, prog.Aux)
120 120
 	}
121 121
 	_, err := out.out.Write(formatted)
122 122
 	if err != nil {
... ...
@@ -73,7 +73,7 @@ func TestJSONFormatProgress(t *testing.T) {
73 73
 		Total:   30,
74 74
 		Start:   1,
75 75
 	}
76
-	res := sf.FormatProgress("id", "action", progress)
76
+	res := sf.FormatProgress("id", "action", progress, nil)
77 77
 	msg := &jsonmessage.JSONMessage{}
78 78
 	if err := json.Unmarshal(res, msg); err != nil {
79 79
 		t.Fatal(err)