Browse code

Merge pull request #8320 from dmcgowan/provenance_pull

Official image provenance pull flow

Michael Crosby authored on 2014/10/04 02:56:54
Showing 21 changed files
... ...
@@ -38,6 +38,7 @@ import (
38 38
 	"github.com/docker/docker/pkg/sysinfo"
39 39
 	"github.com/docker/docker/pkg/truncindex"
40 40
 	"github.com/docker/docker/runconfig"
41
+	"github.com/docker/docker/trust"
41 42
 	"github.com/docker/docker/utils"
42 43
 	"github.com/docker/docker/volumes"
43 44
 )
... ...
@@ -98,6 +99,7 @@ type Daemon struct {
98 98
 	containerGraph *graphdb.Database
99 99
 	driver         graphdriver.Driver
100 100
 	execDriver     execdriver.Driver
101
+	trustStore     *trust.TrustStore
101 102
 }
102 103
 
103 104
 // Install installs daemon capabilities to eng.
... ...
@@ -136,6 +138,9 @@ func (daemon *Daemon) Install(eng *engine.Engine) error {
136 136
 	if err := daemon.Repositories().Install(eng); err != nil {
137 137
 		return err
138 138
 	}
139
+	if err := daemon.trustStore.Install(eng); err != nil {
140
+		return err
141
+	}
139 142
 	// FIXME: this hack is necessary for legacy integration tests to access
140 143
 	// the daemon object.
141 144
 	eng.Hack_SetGlobalVar("httpapi.daemon", daemon)
... ...
@@ -835,6 +840,15 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
835 835
 		return nil, fmt.Errorf("Couldn't create Tag store: %s", err)
836 836
 	}
837 837
 
838
+	trustDir := path.Join(config.Root, "trust")
839
+	if err := os.MkdirAll(trustDir, 0700); err != nil && !os.IsExist(err) {
840
+		return nil, err
841
+	}
842
+	t, err := trust.NewTrustStore(trustDir)
843
+	if err != nil {
844
+		return nil, fmt.Errorf("could not create trust store: %s", err)
845
+	}
846
+
838 847
 	if !config.DisableNetwork {
839 848
 		job := eng.Job("init_networkdriver")
840 849
 
... ...
@@ -899,6 +913,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
899 899
 		sysInitPath:    sysInitPath,
900 900
 		execDriver:     ed,
901 901
 		eng:            eng,
902
+		trustStore:     t,
902 903
 	}
903 904
 	if err := daemon.checkLocaldns(); err != nil {
904 905
 		return nil, err
... ...
@@ -1,10 +1,14 @@
1 1
 package graph
2 2
 
3 3
 import (
4
+	"bytes"
5
+	"encoding/json"
4 6
 	"fmt"
5 7
 	"io"
8
+	"io/ioutil"
6 9
 	"net"
7 10
 	"net/url"
11
+	"os"
8 12
 	"strings"
9 13
 	"time"
10 14
 
... ...
@@ -13,8 +17,60 @@ import (
13 13
 	"github.com/docker/docker/pkg/log"
14 14
 	"github.com/docker/docker/registry"
15 15
 	"github.com/docker/docker/utils"
16
+	"github.com/docker/libtrust"
16 17
 )
17 18
 
19
+func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) {
20
+	sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures")
21
+	if err != nil {
22
+		return nil, false, fmt.Errorf("error parsing payload: %s", err)
23
+	}
24
+	keys, err := sig.Verify()
25
+	if err != nil {
26
+		return nil, false, fmt.Errorf("error verifying payload: %s", err)
27
+	}
28
+
29
+	payload, err := sig.Payload()
30
+	if err != nil {
31
+		return nil, false, fmt.Errorf("error retrieving payload: %s", err)
32
+	}
33
+
34
+	var manifest registry.ManifestData
35
+	if err := json.Unmarshal(payload, &manifest); err != nil {
36
+		return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
37
+	}
38
+
39
+	var verified bool
40
+	for _, key := range keys {
41
+		job := eng.Job("trust_key_check")
42
+		b, err := key.MarshalJSON()
43
+		if err != nil {
44
+			return nil, false, fmt.Errorf("error marshalling public key: %s", err)
45
+		}
46
+		namespace := manifest.Name
47
+		if namespace[0] != '/' {
48
+			namespace = "/" + namespace
49
+		}
50
+		stdoutBuffer := bytes.NewBuffer(nil)
51
+
52
+		job.Args = append(job.Args, namespace)
53
+		job.Setenv("PublicKey", string(b))
54
+		// Check key has read/write permission (0x03)
55
+		job.SetenvInt("Permission", 0x03)
56
+		job.Stdout.Add(stdoutBuffer)
57
+		if err = job.Run(); err != nil {
58
+			return nil, false, fmt.Errorf("error running key check: %s", err)
59
+		}
60
+		result := engine.Tail(stdoutBuffer, 1)
61
+		log.Debugf("Key check result: %q", result)
62
+		if result == "verified" {
63
+			verified = true
64
+		}
65
+	}
66
+
67
+	return &manifest, verified, nil
68
+}
69
+
18 70
 func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
19 71
 	if n := len(job.Args); n != 1 && n != 2 {
20 72
 		return job.Errorf("Usage: %s IMAGE [TAG]", job.Name)
... ...
@@ -52,7 +108,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
52 52
 		return job.Error(err)
53 53
 	}
54 54
 
55
-	endpoint, err := registry.ExpandAndVerifyRegistryUrl(hostname)
55
+	endpoint, err := registry.NewEndpoint(hostname)
56 56
 	if err != nil {
57 57
 		return job.Error(err)
58 58
 	}
... ...
@@ -62,14 +118,32 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
62 62
 		return job.Error(err)
63 63
 	}
64 64
 
65
-	if endpoint == registry.IndexServerAddress() {
65
+	var isOfficial bool
66
+	if endpoint.VersionString(1) == registry.IndexServerAddress() {
66 67
 		// If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar"
67 68
 		localName = remoteName
68 69
 
70
+		isOfficial = isOfficialName(remoteName)
71
+		if isOfficial && strings.IndexRune(remoteName, '/') == -1 {
72
+			remoteName = "library/" + remoteName
73
+		}
74
+
69 75
 		// Use provided mirrors, if any
70 76
 		mirrors = s.mirrors
71 77
 	}
72 78
 
79
+	if isOfficial || endpoint.Version == registry.APIVersion2 {
80
+		j := job.Eng.Job("trust_update_base")
81
+		if err = j.Run(); err != nil {
82
+			return job.Errorf("error updating trust base graph: %s", err)
83
+		}
84
+
85
+		if err := s.pullV2Repository(job.Eng, r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err == nil {
86
+			return engine.StatusOK
87
+		} else if err != registry.ErrDoesNotExist {
88
+			log.Errorf("Error from V2 registry: %s", err)
89
+		}
90
+	}
73 91
 	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil {
74 92
 		return job.Error(err)
75 93
 	}
... ...
@@ -337,3 +411,169 @@ func WriteStatus(requestedTag string, out io.Writer, sf *utils.StreamFormatter,
337 337
 		out.Write(sf.FormatStatus("", "Status: Image is up to date for %s", requestedTag))
338 338
 	}
339 339
 }
340
+
341
+// downloadInfo is used to pass information from download to extractor
342
+type downloadInfo struct {
343
+	imgJSON    []byte
344
+	img        *image.Image
345
+	tmpFile    *os.File
346
+	length     int64
347
+	downloaded bool
348
+	err        chan error
349
+}
350
+
351
+func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
352
+	if tag == "" {
353
+		log.Debugf("Pulling tag list from V2 registry for %s", remoteName)
354
+		tags, err := r.GetV2RemoteTags(remoteName, nil)
355
+		if err != nil {
356
+			return err
357
+		}
358
+		for _, t := range tags {
359
+			if err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil {
360
+				return err
361
+			}
362
+		}
363
+	} else {
364
+		if err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil {
365
+			return err
366
+		}
367
+	}
368
+
369
+	return nil
370
+}
371
+
372
+func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
373
+	log.Debugf("Pulling tag from V2 registry: %q", tag)
374
+	manifestBytes, err := r.GetV2ImageManifest(remoteName, tag, nil)
375
+	if err != nil {
376
+		return err
377
+	}
378
+
379
+	manifest, verified, err := s.verifyManifest(eng, manifestBytes)
380
+	if err != nil {
381
+		return fmt.Errorf("error verifying manifest: %s", err)
382
+	}
383
+
384
+	if len(manifest.BlobSums) != len(manifest.History) {
385
+		return fmt.Errorf("length of history not equal to number of layers")
386
+	}
387
+
388
+	if verified {
389
+		out.Write(sf.FormatStatus("", "The image you are pulling has been digitally signed by Docker, Inc."))
390
+	}
391
+	out.Write(sf.FormatStatus(tag, "Pulling from %s", localName))
392
+
393
+	downloads := make([]downloadInfo, len(manifest.BlobSums))
394
+
395
+	for i := len(manifest.BlobSums) - 1; i >= 0; i-- {
396
+		var (
397
+			sumStr  = manifest.BlobSums[i]
398
+			imgJSON = []byte(manifest.History[i])
399
+		)
400
+
401
+		img, err := image.NewImgJSON(imgJSON)
402
+		if err != nil {
403
+			return fmt.Errorf("failed to parse json: %s", err)
404
+		}
405
+		downloads[i].img = img
406
+
407
+		// Check if exists
408
+		if s.graph.Exists(img.ID) {
409
+			log.Debugf("Image already exists: %s", img.ID)
410
+			continue
411
+		}
412
+
413
+		chunks := strings.SplitN(sumStr, ":", 2)
414
+		if len(chunks) < 2 {
415
+			return fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)
416
+		}
417
+		sumType, checksum := chunks[0], chunks[1]
418
+		out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil))
419
+
420
+		downloadFunc := func(di *downloadInfo) error {
421
+			log.Infof("pulling blob %q to V1 img %s", sumStr, img.ID)
422
+
423
+			if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
424
+				if c != nil {
425
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil))
426
+					<-c
427
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil))
428
+				} else {
429
+					log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
430
+				}
431
+			} else {
432
+				tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob")
433
+				if err != nil {
434
+					return err
435
+				}
436
+
437
+				r, l, err := r.GetV2ImageBlobReader(remoteName, sumType, checksum, nil)
438
+				if err != nil {
439
+					return err
440
+				}
441
+				defer r.Close()
442
+				io.Copy(tmpFile, utils.ProgressReader(r, int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading"))
443
+
444
+				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil))
445
+
446
+				log.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name())
447
+				di.tmpFile = tmpFile
448
+				di.length = l
449
+				di.downloaded = true
450
+			}
451
+			di.imgJSON = imgJSON
452
+			defer s.poolRemove("pull", "img:"+img.ID)
453
+
454
+			return nil
455
+		}
456
+
457
+		if parallel {
458
+			downloads[i].err = make(chan error)
459
+			go func(di *downloadInfo) {
460
+				di.err <- downloadFunc(di)
461
+			}(&downloads[i])
462
+		} else {
463
+			err := downloadFunc(&downloads[i])
464
+			if err != nil {
465
+				return err
466
+			}
467
+		}
468
+	}
469
+
470
+	for i := len(downloads) - 1; i >= 0; i-- {
471
+		d := &downloads[i]
472
+		if d.err != nil {
473
+			err := <-d.err
474
+			if err != nil {
475
+				return err
476
+			}
477
+		}
478
+		if d.downloaded {
479
+			// if tmpFile is empty assume download and extracted elsewhere
480
+			defer os.Remove(d.tmpFile.Name())
481
+			defer d.tmpFile.Close()
482
+			d.tmpFile.Seek(0, 0)
483
+			if d.tmpFile != nil {
484
+				err = s.graph.Register(d.img, d.imgJSON,
485
+					utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting"))
486
+				if err != nil {
487
+					return err
488
+				}
489
+
490
+				// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
491
+			}
492
+			out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Pull complete", nil))
493
+
494
+		} else {
495
+			out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Already exists", nil))
496
+		}
497
+
498
+	}
499
+
500
+	if err = s.Set(localName, tag, downloads[0].img.ID, true); err != nil {
501
+		return err
502
+	}
503
+
504
+	return nil
505
+}
... ...
@@ -214,7 +214,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status {
214 214
 		return job.Error(err)
215 215
 	}
216 216
 
217
-	endpoint, err := registry.ExpandAndVerifyRegistryUrl(hostname)
217
+	endpoint, err := registry.NewEndpoint(hostname)
218 218
 	if err != nil {
219 219
 		return job.Error(err)
220 220
 	}
... ...
@@ -243,7 +243,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status {
243 243
 
244 244
 	var token []string
245 245
 	job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName))
246
-	if _, err := s.pushImage(r, job.Stdout, remoteName, img.ID, endpoint, token, sf); err != nil {
246
+	if _, err := s.pushImage(r, job.Stdout, remoteName, img.ID, endpoint.String(), token, sf); err != nil {
247 247
 		return job.Error(err)
248 248
 	}
249 249
 	return engine.StatusOK
... ...
@@ -276,6 +276,20 @@ func (store *TagStore) GetRepoRefs() map[string][]string {
276 276
 	return reporefs
277 277
 }
278 278
 
279
+// isOfficialName returns whether a repo name is considered an official
280
+// repository.  Official repositories are repos with names within
281
+// the library namespace or which default to the library namespace
282
+// by not providing one.
283
+func isOfficialName(name string) bool {
284
+	if strings.HasPrefix(name, "library/") {
285
+		return true
286
+	}
287
+	if strings.IndexRune(name, '/') == -1 {
288
+		return true
289
+	}
290
+	return false
291
+}
292
+
279 293
 // Validate the name of a repository
280 294
 func validateRepoName(name string) error {
281 295
 	if name == "" {
... ...
@@ -2,15 +2,16 @@ package graph
2 2
 
3 3
 import (
4 4
 	"bytes"
5
+	"io"
6
+	"os"
7
+	"path"
8
+	"testing"
9
+
5 10
 	"github.com/docker/docker/daemon/graphdriver"
6 11
 	_ "github.com/docker/docker/daemon/graphdriver/vfs" // import the vfs driver so it is used in the tests
7 12
 	"github.com/docker/docker/image"
8 13
 	"github.com/docker/docker/utils"
9 14
 	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
10
-	"io"
11
-	"os"
12
-	"path"
13
-	"testing"
14 15
 )
15 16
 
16 17
 const (
... ...
@@ -132,3 +133,18 @@ func TestInvalidTagName(t *testing.T) {
132 132
 		}
133 133
 	}
134 134
 }
135
+
136
+func TestOfficialName(t *testing.T) {
137
+	names := map[string]bool{
138
+		"library/ubuntu":    true,
139
+		"nonlibrary/ubuntu": false,
140
+		"ubuntu":            true,
141
+		"other/library":     false,
142
+	}
143
+	for name, isOfficial := range names {
144
+		result := isOfficialName(name)
145
+		if result != isOfficial {
146
+			t.Errorf("Unexpected result for %s\n\tExpecting: %v\n\tActual: %v", name, isOfficial, result)
147
+		}
148
+	}
149
+}
... ...
@@ -51,7 +51,7 @@ clone hg code.google.com/p/go.net 84a4013f96e0
51 51
 
52 52
 clone hg code.google.com/p/gosqlite 74691fb6f837
53 53
 
54
-clone git github.com/docker/libtrust 136d534cc940
54
+clone git github.com/docker/libtrust d273ef2565ca
55 55
 
56 56
 # get Go tip's archive/tar, for xattr support and improved performance
57 57
 # TODO after Go 1.4 drops, bump our minimum supported version and drop this vendored dep
58 58
new file mode 100644
... ...
@@ -0,0 +1,129 @@
0
+package registry
1
+
2
+import (
3
+	"encoding/json"
4
+	"errors"
5
+	"fmt"
6
+	"io/ioutil"
7
+	"net/http"
8
+	"net/url"
9
+	"strings"
10
+
11
+	"github.com/docker/docker/pkg/log"
12
+)
13
+
14
+// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version.
15
+func scanForApiVersion(hostname string) (string, APIVersion) {
16
+	var (
17
+		chunks        []string
18
+		apiVersionStr string
19
+	)
20
+	if strings.HasSuffix(hostname, "/") {
21
+		chunks = strings.Split(hostname[:len(hostname)-1], "/")
22
+		apiVersionStr = chunks[len(chunks)-1]
23
+	} else {
24
+		chunks = strings.Split(hostname, "/")
25
+		apiVersionStr = chunks[len(chunks)-1]
26
+	}
27
+	for k, v := range apiVersions {
28
+		if apiVersionStr == v {
29
+			hostname = strings.Join(chunks[:len(chunks)-1], "/")
30
+			return hostname, k
31
+		}
32
+	}
33
+	return hostname, DefaultAPIVersion
34
+}
35
+
36
+func NewEndpoint(hostname string) (*Endpoint, error) {
37
+	var (
38
+		endpoint        Endpoint
39
+		trimmedHostname string
40
+		err             error
41
+	)
42
+	if !strings.HasPrefix(hostname, "http") {
43
+		hostname = "https://" + hostname
44
+	}
45
+	trimmedHostname, endpoint.Version = scanForApiVersion(hostname)
46
+	endpoint.URL, err = url.Parse(trimmedHostname)
47
+	if err != nil {
48
+		return nil, err
49
+	}
50
+
51
+	endpoint.URL.Scheme = "https"
52
+	if _, err := endpoint.Ping(); err != nil {
53
+		log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err)
54
+		// TODO: Check if http fallback is enabled
55
+		endpoint.URL.Scheme = "http"
56
+		if _, err = endpoint.Ping(); err != nil {
57
+			return nil, errors.New("Invalid Registry endpoint: " + err.Error())
58
+		}
59
+	}
60
+
61
+	return &endpoint, nil
62
+}
63
+
64
+type Endpoint struct {
65
+	URL     *url.URL
66
+	Version APIVersion
67
+}
68
+
69
+// Get the formated URL for the root of this registry Endpoint
70
+func (e Endpoint) String() string {
71
+	return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version)
72
+}
73
+
74
+func (e Endpoint) VersionString(version APIVersion) string {
75
+	return fmt.Sprintf("%s/v%d/", e.URL.String(), version)
76
+}
77
+
78
+func (e Endpoint) Ping() (RegistryInfo, error) {
79
+	if e.String() == IndexServerAddress() {
80
+		// Skip the check, we now this one is valid
81
+		// (and we never want to fallback to http in case of error)
82
+		return RegistryInfo{Standalone: false}, nil
83
+	}
84
+
85
+	req, err := http.NewRequest("GET", e.String()+"_ping", nil)
86
+	if err != nil {
87
+		return RegistryInfo{Standalone: false}, err
88
+	}
89
+
90
+	resp, _, err := doRequest(req, nil, ConnectTimeout)
91
+	if err != nil {
92
+		return RegistryInfo{Standalone: false}, err
93
+	}
94
+
95
+	defer resp.Body.Close()
96
+
97
+	jsonString, err := ioutil.ReadAll(resp.Body)
98
+	if err != nil {
99
+		return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err)
100
+	}
101
+
102
+	// If the header is absent, we assume true for compatibility with earlier
103
+	// versions of the registry. default to true
104
+	info := RegistryInfo{
105
+		Standalone: true,
106
+	}
107
+	if err := json.Unmarshal(jsonString, &info); err != nil {
108
+		log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err)
109
+		// don't stop here. Just assume sane defaults
110
+	}
111
+	if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" {
112
+		log.Debugf("Registry version header: '%s'", hdr)
113
+		info.Version = hdr
114
+	}
115
+	log.Debugf("RegistryInfo.Version: %q", info.Version)
116
+
117
+	standalone := resp.Header.Get("X-Docker-Registry-Standalone")
118
+	log.Debugf("Registry standalone header: '%s'", standalone)
119
+	// Accepted values are "true" (case-insensitive) and "1".
120
+	if strings.EqualFold(standalone, "true") || standalone == "1" {
121
+		info.Standalone = true
122
+	} else if len(standalone) > 0 {
123
+		// there is a header set, and it is not "true" or "1", so assume fails
124
+		info.Standalone = false
125
+	}
126
+	log.Debugf("RegistryInfo.Standalone: %t", info.Standalone)
127
+	return info, nil
128
+}
... ...
@@ -3,7 +3,6 @@ package registry
3 3
 import (
4 4
 	"crypto/tls"
5 5
 	"crypto/x509"
6
-	"encoding/json"
7 6
 	"errors"
8 7
 	"fmt"
9 8
 	"io/ioutil"
... ...
@@ -15,13 +14,13 @@ import (
15 15
 	"strings"
16 16
 	"time"
17 17
 
18
-	"github.com/docker/docker/pkg/log"
19 18
 	"github.com/docker/docker/utils"
20 19
 )
21 20
 
22 21
 var (
23 22
 	ErrAlreadyExists         = errors.New("Image already exists")
24 23
 	ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")")
24
+	ErrDoesNotExist          = errors.New("Image does not exist")
25 25
 	errLoginRequired         = errors.New("Authentication is required.")
26 26
 	validHex                 = regexp.MustCompile(`^([a-f0-9]{64})$`)
27 27
 	validNamespace           = regexp.MustCompile(`^([a-z0-9_]{4,30})$`)
... ...
@@ -152,55 +151,6 @@ func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*htt
152 152
 	return nil, nil, nil
153 153
 }
154 154
 
155
-func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) {
156
-	if endpoint == IndexServerAddress() {
157
-		// Skip the check, we now this one is valid
158
-		// (and we never want to fallback to http in case of error)
159
-		return RegistryInfo{Standalone: false}, nil
160
-	}
161
-
162
-	req, err := http.NewRequest("GET", endpoint+"_ping", nil)
163
-	if err != nil {
164
-		return RegistryInfo{Standalone: false}, err
165
-	}
166
-
167
-	resp, _, err := doRequest(req, nil, ConnectTimeout)
168
-	if err != nil {
169
-		return RegistryInfo{Standalone: false}, err
170
-	}
171
-
172
-	defer resp.Body.Close()
173
-
174
-	jsonString, err := ioutil.ReadAll(resp.Body)
175
-	if err != nil {
176
-		return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err)
177
-	}
178
-
179
-	// If the header is absent, we assume true for compatibility with earlier
180
-	// versions of the registry. default to true
181
-	info := RegistryInfo{
182
-		Standalone: true,
183
-	}
184
-	if err := json.Unmarshal(jsonString, &info); err != nil {
185
-		log.Debugf("Error unmarshalling the _ping RegistryInfo: %s", err)
186
-		// don't stop here. Just assume sane defaults
187
-	}
188
-	if hdr := resp.Header.Get("X-Docker-Registry-Version"); hdr != "" {
189
-		log.Debugf("Registry version header: '%s'", hdr)
190
-		info.Version = hdr
191
-	}
192
-	log.Debugf("RegistryInfo.Version: %q", info.Version)
193
-
194
-	standalone := resp.Header.Get("X-Docker-Registry-Standalone")
195
-	log.Debugf("Registry standalone header: '%s'", standalone)
196
-	if !strings.EqualFold(standalone, "true") && standalone != "1" && len(standalone) > 0 {
197
-		// there is a header set, and it is not "true" or "1", so assume fails
198
-		info.Standalone = false
199
-	}
200
-	log.Debugf("RegistryInfo.Standalone: %q", info.Standalone)
201
-	return info, nil
202
-}
203
-
204 155
 func validateRepositoryName(repositoryName string) error {
205 156
 	var (
206 157
 		namespace string
... ...
@@ -252,33 +202,6 @@ func ResolveRepositoryName(reposName string) (string, string, error) {
252 252
 	return hostname, reposName, nil
253 253
 }
254 254
 
255
-// this method expands the registry name as used in the prefix of a repo
256
-// to a full url. if it already is a url, there will be no change.
257
-// The registry is pinged to test if it http or https
258
-func ExpandAndVerifyRegistryUrl(hostname string) (string, error) {
259
-	if strings.HasPrefix(hostname, "http:") || strings.HasPrefix(hostname, "https:") {
260
-		// if there is no slash after https:// (8 characters) then we have no path in the url
261
-		if strings.LastIndex(hostname, "/") < 9 {
262
-			// there is no path given. Expand with default path
263
-			hostname = hostname + "/v1/"
264
-		}
265
-		if _, err := pingRegistryEndpoint(hostname); err != nil {
266
-			return "", errors.New("Invalid Registry endpoint: " + err.Error())
267
-		}
268
-		return hostname, nil
269
-	}
270
-	endpoint := fmt.Sprintf("https://%s/v1/", hostname)
271
-	if _, err := pingRegistryEndpoint(endpoint); err != nil {
272
-		log.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err)
273
-		endpoint = fmt.Sprintf("http://%s/v1/", hostname)
274
-		if _, err = pingRegistryEndpoint(endpoint); err != nil {
275
-			//TODO: triggering highland build can be done there without "failing"
276
-			return "", errors.New("Invalid Registry endpoint: " + err.Error())
277
-		}
278
-	}
279
-	return endpoint, nil
280
-}
281
-
282 255
 func trustedLocation(req *http.Request) bool {
283 256
 	var (
284 257
 		trusteds = []string{"docker.com", "docker.io"}
... ...
@@ -83,6 +83,8 @@ var (
83 83
 
84 84
 func init() {
85 85
 	r := mux.NewRouter()
86
+
87
+	// /v1/
86 88
 	r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET")
87 89
 	r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET")
88 90
 	r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT")
... ...
@@ -93,6 +95,10 @@ func init() {
93 93
 	r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE")
94 94
 	r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT")
95 95
 	r.HandleFunc("/v1/search", handlerSearch).Methods("GET")
96
+
97
+	// /v2/
98
+	r.HandleFunc("/v2/version", handlerGetPing).Methods("GET")
99
+
96 100
 	testHttpServer = httptest.NewServer(handlerAccessLog(r))
97 101
 }
98 102
 
... ...
@@ -18,7 +18,11 @@ var (
18 18
 
19 19
 func spawnTestRegistrySession(t *testing.T) *Session {
20 20
 	authConfig := &AuthConfig{}
21
-	r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"), true)
21
+	endpoint, err := NewEndpoint(makeURL("/v1/"))
22
+	if err != nil {
23
+		t.Fatal(err)
24
+	}
25
+	r, err := NewSession(authConfig, utils.NewHTTPRequestFactory(), endpoint, true)
22 26
 	if err != nil {
23 27
 		t.Fatal(err)
24 28
 	}
... ...
@@ -26,7 +30,11 @@ func spawnTestRegistrySession(t *testing.T) *Session {
26 26
 }
27 27
 
28 28
 func TestPingRegistryEndpoint(t *testing.T) {
29
-	regInfo, err := pingRegistryEndpoint(makeURL("/v1/"))
29
+	ep, err := NewEndpoint(makeURL("/v1/"))
30
+	if err != nil {
31
+		t.Fatal(err)
32
+	}
33
+	regInfo, err := ep.Ping()
30 34
 	if err != nil {
31 35
 		t.Fatal(err)
32 36
 	}
... ...
@@ -197,7 +205,7 @@ func TestPushImageJSONIndex(t *testing.T) {
197 197
 	if repoData == nil {
198 198
 		t.Fatal("Expected RepositoryData object")
199 199
 	}
200
-	repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint})
200
+	repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint.String()})
201 201
 	if err != nil {
202 202
 		t.Fatal(err)
203 203
 	}
... ...
@@ -40,11 +40,14 @@ func (s *Service) Auth(job *engine.Job) engine.Status {
40 40
 	job.GetenvJson("authConfig", authConfig)
41 41
 	// TODO: this is only done here because auth and registry need to be merged into one pkg
42 42
 	if addr := authConfig.ServerAddress; addr != "" && addr != IndexServerAddress() {
43
-		addr, err = ExpandAndVerifyRegistryUrl(addr)
43
+		endpoint, err := NewEndpoint(addr)
44 44
 		if err != nil {
45 45
 			return job.Error(err)
46 46
 		}
47
-		authConfig.ServerAddress = addr
47
+		if _, err := endpoint.Ping(); err != nil {
48
+			return job.Error(err)
49
+		}
50
+		authConfig.ServerAddress = endpoint.String()
48 51
 	}
49 52
 	status, err := Login(authConfig, HTTPRequestFactory(nil))
50 53
 	if err != nil {
... ...
@@ -86,11 +89,11 @@ func (s *Service) Search(job *engine.Job) engine.Status {
86 86
 	if err != nil {
87 87
 		return job.Error(err)
88 88
 	}
89
-	hostname, err = ExpandAndVerifyRegistryUrl(hostname)
89
+	endpoint, err := NewEndpoint(hostname)
90 90
 	if err != nil {
91 91
 		return job.Error(err)
92 92
 	}
93
-	r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), hostname, true)
93
+	r, err := NewSession(authConfig, HTTPRequestFactory(metaHeaders), endpoint, true)
94 94
 	if err != nil {
95 95
 		return job.Error(err)
96 96
 	}
... ...
@@ -25,15 +25,15 @@ import (
25 25
 type Session struct {
26 26
 	authConfig    *AuthConfig
27 27
 	reqFactory    *utils.HTTPRequestFactory
28
-	indexEndpoint string
28
+	indexEndpoint *Endpoint
29 29
 	jar           *cookiejar.Jar
30 30
 	timeout       TimeoutType
31 31
 }
32 32
 
33
-func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Session, err error) {
33
+func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpoint *Endpoint, timeout bool) (r *Session, err error) {
34 34
 	r = &Session{
35 35
 		authConfig:    authConfig,
36
-		indexEndpoint: indexEndpoint,
36
+		indexEndpoint: endpoint,
37 37
 	}
38 38
 
39 39
 	if timeout {
... ...
@@ -47,13 +47,13 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, index
47 47
 
48 48
 	// If we're working with a standalone private registry over HTTPS, send Basic Auth headers
49 49
 	// alongside our requests.
50
-	if indexEndpoint != IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") {
51
-		info, err := pingRegistryEndpoint(indexEndpoint)
50
+	if r.indexEndpoint.VersionString(1) != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" {
51
+		info, err := r.indexEndpoint.Ping()
52 52
 		if err != nil {
53 53
 			return nil, err
54 54
 		}
55 55
 		if info.Standalone {
56
-			log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", indexEndpoint)
56
+			log.Debugf("Endpoint %s is eligible for private registry registry. Enabling decorator.", r.indexEndpoint.String())
57 57
 			dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password)
58 58
 			factory.AddDecorator(dec)
59 59
 		}
... ...
@@ -261,8 +261,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) {
261 261
 }
262 262
 
263 263
 func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) {
264
-	indexEp := r.indexEndpoint
265
-	repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote)
264
+	repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote)
266 265
 
267 266
 	log.Debugf("[registry] Calling GET %s", repositoryTarget)
268 267
 
... ...
@@ -296,17 +295,13 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) {
296 296
 
297 297
 	var endpoints []string
298 298
 	if res.Header.Get("X-Docker-Endpoints") != "" {
299
-		endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp)
299
+		endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1))
300 300
 		if err != nil {
301 301
 			return nil, err
302 302
 		}
303 303
 	} else {
304 304
 		// Assume the endpoint is on the same host
305
-		u, err := url.Parse(indexEp)
306
-		if err != nil {
307
-			return nil, err
308
-		}
309
-		endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", u.Scheme, req.URL.Host))
305
+		endpoints = append(endpoints, fmt.Sprintf("%s://%s/v1/", r.indexEndpoint.URL.Scheme, req.URL.Host))
310 306
 	}
311 307
 
312 308
 	checksumsJSON, err := ioutil.ReadAll(res.Body)
... ...
@@ -474,7 +469,6 @@ func (r *Session) PushRegistryTag(remote, revision, tag, registry string, token
474 474
 
475 475
 func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) {
476 476
 	cleanImgList := []*ImgData{}
477
-	indexEp := r.indexEndpoint
478 477
 
479 478
 	if validate {
480 479
 		for _, elem := range imgList {
... ...
@@ -494,7 +488,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate
494 494
 	if validate {
495 495
 		suffix = "images"
496 496
 	}
497
-	u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix)
497
+	u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix)
498 498
 	log.Debugf("[registry] PUT %s", u)
499 499
 	log.Debugf("Image list pushed to index:\n%s", imgListJSON)
500 500
 	req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON))
... ...
@@ -552,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate
552 552
 		}
553 553
 
554 554
 		if res.Header.Get("X-Docker-Endpoints") != "" {
555
-			endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], indexEp)
555
+			endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1))
556 556
 			if err != nil {
557 557
 				return nil, err
558 558
 			}
... ...
@@ -578,7 +572,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate
578 578
 
579 579
 func (r *Session) SearchRepositories(term string) (*SearchResults, error) {
580 580
 	log.Debugf("Index server: %s", r.indexEndpoint)
581
-	u := r.indexEndpoint + "search?q=" + url.QueryEscape(term)
581
+	u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term)
582 582
 	req, err := r.reqFactory.NewRequest("GET", u, nil)
583 583
 	if err != nil {
584 584
 		return nil, err
585 585
new file mode 100644
... ...
@@ -0,0 +1,386 @@
0
+package registry
1
+
2
+import (
3
+	"encoding/json"
4
+	"fmt"
5
+	"io"
6
+	"io/ioutil"
7
+	"net/url"
8
+	"strconv"
9
+
10
+	"github.com/docker/docker/pkg/log"
11
+	"github.com/docker/docker/utils"
12
+	"github.com/gorilla/mux"
13
+)
14
+
15
+func newV2RegistryRouter() *mux.Router {
16
+	router := mux.NewRouter()
17
+
18
+	v2Router := router.PathPrefix("/v2/").Subrouter()
19
+
20
+	// Version Info
21
+	v2Router.Path("/version").Name("version")
22
+
23
+	// Image Manifests
24
+	v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests")
25
+
26
+	// List Image Tags
27
+	v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags")
28
+
29
+	// Download a blob
30
+	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")
31
+
32
+	// Upload a blob
33
+	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob")
34
+
35
+	// Mounting a blob in an image
36
+	v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")
37
+
38
+	return router
39
+}
40
+
41
+// APIVersion2 /v2/
42
+var v2HTTPRoutes = newV2RegistryRouter()
43
+
44
+func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) {
45
+	route := v2HTTPRoutes.Get(routeName)
46
+	if route == nil {
47
+		return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName)
48
+	}
49
+
50
+	varReplace := make([]string, 0, len(vars)*2)
51
+	for key, val := range vars {
52
+		varReplace = append(varReplace, key, val)
53
+	}
54
+
55
+	routePath, err := route.URLPath(varReplace...)
56
+	if err != nil {
57
+		return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err)
58
+	}
59
+
60
+	return &url.URL{
61
+		Scheme: e.URL.Scheme,
62
+		Host:   e.URL.Host,
63
+		Path:   routePath.Path,
64
+	}, nil
65
+}
66
+
67
+// V2 Provenance POC
68
+
69
+func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) {
70
+	routeURL, err := getV2URL(r.indexEndpoint, "version", nil)
71
+	if err != nil {
72
+		return nil, err
73
+	}
74
+
75
+	method := "GET"
76
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
77
+
78
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
79
+	if err != nil {
80
+		return nil, err
81
+	}
82
+	setTokenAuth(req, token)
83
+	res, _, err := r.doRequest(req)
84
+	if err != nil {
85
+		return nil, err
86
+	}
87
+	defer res.Body.Close()
88
+	if res.StatusCode != 200 {
89
+		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res)
90
+	}
91
+
92
+	decoder := json.NewDecoder(res.Body)
93
+	versionInfo := new(RegistryInfo)
94
+
95
+	err = decoder.Decode(versionInfo)
96
+	if err != nil {
97
+		return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err)
98
+	}
99
+
100
+	return versionInfo, nil
101
+}
102
+
103
+//
104
+// 1) Check if TarSum of each layer exists /v2/
105
+//  1.a) if 200, continue
106
+//  1.b) if 300, then push the
107
+//  1.c) if anything else, err
108
+// 2) PUT the created/signed manifest
109
+//
110
+func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) {
111
+	vars := map[string]string{
112
+		"imagename": imageName,
113
+		"tagname":   tagName,
114
+	}
115
+
116
+	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
117
+	if err != nil {
118
+		return nil, err
119
+	}
120
+
121
+	method := "GET"
122
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
123
+
124
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
125
+	if err != nil {
126
+		return nil, err
127
+	}
128
+	setTokenAuth(req, token)
129
+	res, _, err := r.doRequest(req)
130
+	if err != nil {
131
+		return nil, err
132
+	}
133
+	defer res.Body.Close()
134
+	if res.StatusCode != 200 {
135
+		if res.StatusCode == 401 {
136
+			return nil, errLoginRequired
137
+		} else if res.StatusCode == 404 {
138
+			return nil, ErrDoesNotExist
139
+		}
140
+		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res)
141
+	}
142
+
143
+	buf, err := ioutil.ReadAll(res.Body)
144
+	if err != nil {
145
+		return nil, fmt.Errorf("Error while reading the http response: %s", err)
146
+	}
147
+	return buf, nil
148
+}
149
+
150
+// - Succeeded to mount for this image scope
151
+// - Failed with no error (So continue to Push the Blob)
152
+// - Failed with error
153
+func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) {
154
+	vars := map[string]string{
155
+		"imagename": imageName,
156
+		"sumtype":   sumType,
157
+		"sum":       sum,
158
+	}
159
+
160
+	routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars)
161
+	if err != nil {
162
+		return false, err
163
+	}
164
+
165
+	method := "POST"
166
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
167
+
168
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
169
+	if err != nil {
170
+		return false, err
171
+	}
172
+	setTokenAuth(req, token)
173
+	res, _, err := r.doRequest(req)
174
+	if err != nil {
175
+		return false, err
176
+	}
177
+	res.Body.Close() // close early, since we're not needing a body on this call .. yet?
178
+	switch res.StatusCode {
179
+	case 200:
180
+		// return something indicating no push needed
181
+		return true, nil
182
+	case 300:
183
+		// return something indicating blob push needed
184
+		return false, nil
185
+	}
186
+	return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode)
187
+}
188
+
189
+func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error {
190
+	vars := map[string]string{
191
+		"imagename": imageName,
192
+		"sumtype":   sumType,
193
+		"sum":       sum,
194
+	}
195
+
196
+	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
197
+	if err != nil {
198
+		return err
199
+	}
200
+
201
+	method := "GET"
202
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
203
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
204
+	if err != nil {
205
+		return err
206
+	}
207
+	setTokenAuth(req, token)
208
+	res, _, err := r.doRequest(req)
209
+	if err != nil {
210
+		return err
211
+	}
212
+	defer res.Body.Close()
213
+	if res.StatusCode != 200 {
214
+		if res.StatusCode == 401 {
215
+			return errLoginRequired
216
+		}
217
+		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
218
+	}
219
+
220
+	_, err = io.Copy(blobWrtr, res.Body)
221
+	return err
222
+}
223
+
224
+func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) {
225
+	vars := map[string]string{
226
+		"imagename": imageName,
227
+		"sumtype":   sumType,
228
+		"sum":       sum,
229
+	}
230
+
231
+	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
232
+	if err != nil {
233
+		return nil, 0, err
234
+	}
235
+
236
+	method := "GET"
237
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
238
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
239
+	if err != nil {
240
+		return nil, 0, err
241
+	}
242
+	setTokenAuth(req, token)
243
+	res, _, err := r.doRequest(req)
244
+	if err != nil {
245
+		return nil, 0, err
246
+	}
247
+	if res.StatusCode != 200 {
248
+		if res.StatusCode == 401 {
249
+			return nil, 0, errLoginRequired
250
+		}
251
+		return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
252
+	}
253
+	lenStr := res.Header.Get("Content-Length")
254
+	l, err := strconv.ParseInt(lenStr, 10, 64)
255
+	if err != nil {
256
+		return nil, 0, err
257
+	}
258
+
259
+	return res.Body, l, err
260
+}
261
+
262
+// Push the image to the server for storage.
263
+// 'layer' is an uncompressed reader of the blob to be pushed.
264
+// The server will generate it's own checksum calculation.
265
+func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) {
266
+	vars := map[string]string{
267
+		"imagename": imageName,
268
+		"sumtype":   sumType,
269
+	}
270
+
271
+	routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars)
272
+	if err != nil {
273
+		return "", err
274
+	}
275
+
276
+	method := "PUT"
277
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
278
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr)
279
+	if err != nil {
280
+		return "", err
281
+	}
282
+	setTokenAuth(req, token)
283
+	res, _, err := r.doRequest(req)
284
+	if err != nil {
285
+		return "", err
286
+	}
287
+	defer res.Body.Close()
288
+	if res.StatusCode != 201 {
289
+		if res.StatusCode == 401 {
290
+			return "", errLoginRequired
291
+		}
292
+		return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res)
293
+	}
294
+
295
+	type sumReturn struct {
296
+		Checksum string `json:"checksum"`
297
+	}
298
+
299
+	decoder := json.NewDecoder(res.Body)
300
+	var sumInfo sumReturn
301
+
302
+	err = decoder.Decode(&sumInfo)
303
+	if err != nil {
304
+		return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err)
305
+	}
306
+
307
+	// XXX this is a json struct from the registry, with its checksum
308
+	return sumInfo.Checksum, nil
309
+}
310
+
311
+// Finally Push the (signed) manifest of the blobs we've just pushed
312
+func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error {
313
+	vars := map[string]string{
314
+		"imagename": imageName,
315
+		"tagname":   tagName,
316
+	}
317
+
318
+	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
319
+	if err != nil {
320
+		return err
321
+	}
322
+
323
+	method := "PUT"
324
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
325
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr)
326
+	if err != nil {
327
+		return err
328
+	}
329
+	setTokenAuth(req, token)
330
+	res, _, err := r.doRequest(req)
331
+	if err != nil {
332
+		return err
333
+	}
334
+	res.Body.Close()
335
+	if res.StatusCode != 201 {
336
+		if res.StatusCode == 401 {
337
+			return errLoginRequired
338
+		}
339
+		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res)
340
+	}
341
+
342
+	return nil
343
+}
344
+
345
+// Given a repository name, returns a json array of string tags
346
+func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) {
347
+	vars := map[string]string{
348
+		"imagename": imageName,
349
+	}
350
+
351
+	routeURL, err := getV2URL(r.indexEndpoint, "tags", vars)
352
+	if err != nil {
353
+		return nil, err
354
+	}
355
+
356
+	method := "GET"
357
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
358
+
359
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
360
+	if err != nil {
361
+		return nil, err
362
+	}
363
+	setTokenAuth(req, token)
364
+	res, _, err := r.doRequest(req)
365
+	if err != nil {
366
+		return nil, err
367
+	}
368
+	defer res.Body.Close()
369
+	if res.StatusCode != 200 {
370
+		if res.StatusCode == 401 {
371
+			return nil, errLoginRequired
372
+		} else if res.StatusCode == 404 {
373
+			return nil, ErrDoesNotExist
374
+		}
375
+		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res)
376
+	}
377
+
378
+	decoder := json.NewDecoder(res.Body)
379
+	var tags []string
380
+	err = decoder.Decode(&tags)
381
+	if err != nil {
382
+		return nil, fmt.Errorf("Error while decoding the http response: %s", err)
383
+	}
384
+	return tags, nil
385
+}
... ...
@@ -31,3 +31,29 @@ type RegistryInfo struct {
31 31
 	Version    string `json:"version"`
32 32
 	Standalone bool   `json:"standalone"`
33 33
 }
34
+
35
+type ManifestData struct {
36
+	Name          string   `json:"name"`
37
+	Tag           string   `json:"tag"`
38
+	Architecture  string   `json:"architecture"`
39
+	BlobSums      []string `json:"blobSums"`
40
+	History       []string `json:"history"`
41
+	SchemaVersion int      `json:"schemaVersion"`
42
+}
43
+
44
+type APIVersion int
45
+
46
+func (av APIVersion) String() string {
47
+	return apiVersions[av]
48
+}
49
+
50
+var DefaultAPIVersion APIVersion = APIVersion1
51
+var apiVersions = map[APIVersion]string{
52
+	1: "v1",
53
+	2: "v2",
54
+}
55
+
56
+const (
57
+	APIVersion1 = iota + 1
58
+	APIVersion2
59
+)
34 60
new file mode 100644
... ...
@@ -0,0 +1,74 @@
0
+package trust
1
+
2
+import (
3
+	"fmt"
4
+	"time"
5
+
6
+	"github.com/docker/docker/engine"
7
+	"github.com/docker/docker/pkg/log"
8
+	"github.com/docker/libtrust"
9
+)
10
+
11
+func (t *TrustStore) Install(eng *engine.Engine) error {
12
+	for name, handler := range map[string]engine.Handler{
13
+		"trust_key_check":   t.CmdCheckKey,
14
+		"trust_update_base": t.CmdUpdateBase,
15
+	} {
16
+		if err := eng.Register(name, handler); err != nil {
17
+			return fmt.Errorf("Could not register %q: %v", name, err)
18
+		}
19
+	}
20
+	return nil
21
+}
22
+
23
+func (t *TrustStore) CmdCheckKey(job *engine.Job) engine.Status {
24
+	if n := len(job.Args); n != 1 {
25
+		return job.Errorf("Usage: %s NAMESPACE", job.Name)
26
+	}
27
+	var (
28
+		namespace = job.Args[0]
29
+		keyBytes  = job.Getenv("PublicKey")
30
+	)
31
+
32
+	if keyBytes == "" {
33
+		return job.Errorf("Missing PublicKey")
34
+	}
35
+	pk, err := libtrust.UnmarshalPublicKeyJWK([]byte(keyBytes))
36
+	if err != nil {
37
+		return job.Errorf("Error unmarshalling public key: %s", err)
38
+	}
39
+
40
+	permission := uint16(job.GetenvInt("Permission"))
41
+	if permission == 0 {
42
+		permission = 0x03
43
+	}
44
+
45
+	t.RLock()
46
+	defer t.RUnlock()
47
+	if t.graph == nil {
48
+		job.Stdout.Write([]byte("no graph"))
49
+		return engine.StatusOK
50
+	}
51
+
52
+	// Check if any expired grants
53
+	verified, err := t.graph.Verify(pk, namespace, permission)
54
+	if err != nil {
55
+		return job.Errorf("Error verifying key to namespace: %s", namespace)
56
+	}
57
+	if !verified {
58
+		log.Debugf("Verification failed for %s using key %s", namespace, pk.KeyID())
59
+		job.Stdout.Write([]byte("not verified"))
60
+	} else if t.expiration.Before(time.Now()) {
61
+		job.Stdout.Write([]byte("expired"))
62
+	} else {
63
+		job.Stdout.Write([]byte("verified"))
64
+	}
65
+
66
+	return engine.StatusOK
67
+}
68
+
69
+func (t *TrustStore) CmdUpdateBase(job *engine.Job) engine.Status {
70
+	t.fetch()
71
+
72
+	return engine.StatusOK
73
+}
0 74
new file mode 100644
... ...
@@ -0,0 +1,199 @@
0
+package trust
1
+
2
+import (
3
+	"crypto/x509"
4
+	"errors"
5
+	"io/ioutil"
6
+	"net/http"
7
+	"net/url"
8
+	"os"
9
+	"path"
10
+	"path/filepath"
11
+	"sync"
12
+	"time"
13
+
14
+	"github.com/docker/docker/pkg/log"
15
+	"github.com/docker/libtrust/trustgraph"
16
+)
17
+
18
+type TrustStore struct {
19
+	path          string
20
+	caPool        *x509.CertPool
21
+	graph         trustgraph.TrustGraph
22
+	expiration    time.Time
23
+	fetcher       *time.Timer
24
+	fetchTime     time.Duration
25
+	autofetch     bool
26
+	httpClient    *http.Client
27
+	baseEndpoints map[string]*url.URL
28
+
29
+	sync.RWMutex
30
+}
31
+
32
+// defaultFetchtime represents the starting duration to wait between
33
+// fetching sections of the graph.  Unsuccessful fetches should
34
+// increase time between fetching.
35
+const defaultFetchtime = 45 * time.Second
36
+
37
+var baseEndpoints = map[string]string{"official": "https://dvjy3tqbc323p.cloudfront.net/trust/official.json"}
38
+
39
+func NewTrustStore(path string) (*TrustStore, error) {
40
+	abspath, err := filepath.Abs(path)
41
+	if err != nil {
42
+		return nil, err
43
+	}
44
+
45
+	// Create base graph url map
46
+	endpoints := map[string]*url.URL{}
47
+	for name, endpoint := range baseEndpoints {
48
+		u, err := url.Parse(endpoint)
49
+		if err != nil {
50
+			return nil, err
51
+		}
52
+		endpoints[name] = u
53
+	}
54
+
55
+	// Load grant files
56
+	t := &TrustStore{
57
+		path:          abspath,
58
+		caPool:        nil,
59
+		httpClient:    &http.Client{},
60
+		fetchTime:     time.Millisecond,
61
+		baseEndpoints: endpoints,
62
+	}
63
+
64
+	err = t.reload()
65
+	if err != nil {
66
+		return nil, err
67
+	}
68
+
69
+	return t, nil
70
+}
71
+
72
+func (t *TrustStore) reload() error {
73
+	t.Lock()
74
+	defer t.Unlock()
75
+
76
+	matches, err := filepath.Glob(filepath.Join(t.path, "*.json"))
77
+	if err != nil {
78
+		return err
79
+	}
80
+	statements := make([]*trustgraph.Statement, len(matches))
81
+	for i, match := range matches {
82
+		f, err := os.Open(match)
83
+		if err != nil {
84
+			return err
85
+		}
86
+		statements[i], err = trustgraph.LoadStatement(f, nil)
87
+		if err != nil {
88
+			f.Close()
89
+			return err
90
+		}
91
+		f.Close()
92
+	}
93
+	if len(statements) == 0 {
94
+		if t.autofetch {
95
+			log.Debugf("No grants, fetching")
96
+			t.fetcher = time.AfterFunc(t.fetchTime, t.fetch)
97
+		}
98
+		return nil
99
+	}
100
+
101
+	grants, expiration, err := trustgraph.CollapseStatements(statements, true)
102
+	if err != nil {
103
+		return err
104
+	}
105
+
106
+	t.expiration = expiration
107
+	t.graph = trustgraph.NewMemoryGraph(grants)
108
+	log.Debugf("Reloaded graph with %d grants expiring at %s", len(grants), expiration)
109
+
110
+	if t.autofetch {
111
+		nextFetch := expiration.Sub(time.Now())
112
+		if nextFetch < 0 {
113
+			nextFetch = defaultFetchtime
114
+		} else {
115
+			nextFetch = time.Duration(0.8 * (float64)(nextFetch))
116
+		}
117
+		t.fetcher = time.AfterFunc(nextFetch, t.fetch)
118
+	}
119
+
120
+	return nil
121
+}
122
+
123
+func (t *TrustStore) fetchBaseGraph(u *url.URL) (*trustgraph.Statement, error) {
124
+	req := &http.Request{
125
+		Method:     "GET",
126
+		URL:        u,
127
+		Proto:      "HTTP/1.1",
128
+		ProtoMajor: 1,
129
+		ProtoMinor: 1,
130
+		Header:     make(http.Header),
131
+		Body:       nil,
132
+		Host:       u.Host,
133
+	}
134
+
135
+	resp, err := t.httpClient.Do(req)
136
+	if err != nil {
137
+		return nil, err
138
+	}
139
+	if resp.StatusCode == 404 {
140
+		return nil, errors.New("base graph does not exist")
141
+	}
142
+
143
+	defer resp.Body.Close()
144
+
145
+	return trustgraph.LoadStatement(resp.Body, t.caPool)
146
+}
147
+
148
+// fetch retrieves updated base graphs.  This function cannot error, it
149
+// should only log errors
150
+func (t *TrustStore) fetch() {
151
+	t.Lock()
152
+	defer t.Unlock()
153
+
154
+	if t.autofetch && t.fetcher == nil {
155
+		// Do nothing ??
156
+		return
157
+	}
158
+
159
+	fetchCount := 0
160
+	for bg, ep := range t.baseEndpoints {
161
+		statement, err := t.fetchBaseGraph(ep)
162
+		if err != nil {
163
+			log.Infof("Trust graph fetch failed: %s", err)
164
+			continue
165
+		}
166
+		b, err := statement.Bytes()
167
+		if err != nil {
168
+			log.Infof("Bad trust graph statement: %s", err)
169
+			continue
170
+		}
171
+		// TODO check if value differs
172
+		err = ioutil.WriteFile(path.Join(t.path, bg+".json"), b, 0600)
173
+		if err != nil {
174
+			log.Infof("Error writing trust graph statement: %s", err)
175
+		}
176
+		fetchCount++
177
+	}
178
+	log.Debugf("Fetched %d base graphs at %s", fetchCount, time.Now())
179
+
180
+	if fetchCount > 0 {
181
+		go func() {
182
+			err := t.reload()
183
+			if err != nil {
184
+				// TODO log
185
+				log.Infof("Reload of trust graph failed: %s", err)
186
+			}
187
+		}()
188
+		t.fetchTime = defaultFetchtime
189
+		t.fetcher = nil
190
+	} else if t.autofetch {
191
+		maxTime := 10 * defaultFetchtime
192
+		t.fetchTime = time.Duration(1.5 * (float64)(t.fetchTime+time.Second))
193
+		if t.fetchTime > maxTime {
194
+			t.fetchTime = maxTime
195
+		}
196
+		t.fetcher = time.AfterFunc(t.fetchTime, t.fetch)
197
+	}
198
+}
0 199
new file mode 100644
... ...
@@ -0,0 +1,50 @@
0
+package trustgraph
1
+
2
+import "github.com/docker/libtrust"
3
+
4
+// TrustGraph represents a graph of authorization mapping
5
+// public keys to nodes and grants between nodes.
6
+type TrustGraph interface {
7
+	// Verifies that the given public key is allowed to perform
8
+	// the given action on the given node according to the trust
9
+	// graph.
10
+	Verify(libtrust.PublicKey, string, uint16) (bool, error)
11
+
12
+	// GetGrants returns an array of all grant chains which are used to
13
+	// allow the requested permission.
14
+	GetGrants(libtrust.PublicKey, string, uint16) ([][]*Grant, error)
15
+}
16
+
17
+// Grant represents a transfer of permission from one part of the
18
+// trust graph to another. This is the only way to delegate
19
+// permission between two different sub trees in the graph.
20
+type Grant struct {
21
+	// Subject is the namespace being granted
22
+	Subject string
23
+
24
+	// Permissions is a bit map of permissions
25
+	Permission uint16
26
+
27
+	// Grantee represents the node being granted
28
+	// a permission scope.  The grantee can be
29
+	// either a namespace item or a key id where namespace
30
+	// items will always start with a '/'.
31
+	Grantee string
32
+
33
+	// statement represents the statement used to create
34
+	// this object.
35
+	statement *Statement
36
+}
37
+
38
+// Permissions
39
+//  Read node 0x01 (can read node, no sub nodes)
40
+//  Write node 0x02 (can write to node object, cannot create subnodes)
41
+//  Read subtree 0x04 (delegates read to each sub node)
42
+//  Write subtree 0x08 (delegates write to each sub node, included create on the subject)
43
+//
44
+// Permission shortcuts
45
+// ReadItem = 0x01
46
+// WriteItem = 0x03
47
+// ReadAccess = 0x07
48
+// WriteAccess = 0x0F
49
+// Delegate = 0x0F
0 50
new file mode 100644
... ...
@@ -0,0 +1,133 @@
0
+package trustgraph
1
+
2
+import (
3
+	"strings"
4
+
5
+	"github.com/docker/libtrust"
6
+)
7
+
8
+type grantNode struct {
9
+	grants   []*Grant
10
+	children map[string]*grantNode
11
+}
12
+
13
+type memoryGraph struct {
14
+	roots map[string]*grantNode
15
+}
16
+
17
+func newGrantNode() *grantNode {
18
+	return &grantNode{
19
+		grants:   []*Grant{},
20
+		children: map[string]*grantNode{},
21
+	}
22
+}
23
+
24
+// NewMemoryGraph returns a new in memory trust graph created from
25
+// a static list of grants.  This graph is immutable after creation
26
+// and any alterations should create a new instance.
27
+func NewMemoryGraph(grants []*Grant) TrustGraph {
28
+	roots := map[string]*grantNode{}
29
+	for _, grant := range grants {
30
+		parts := strings.Split(grant.Grantee, "/")
31
+		nodes := roots
32
+		var node *grantNode
33
+		var nodeOk bool
34
+		for _, part := range parts {
35
+			node, nodeOk = nodes[part]
36
+			if !nodeOk {
37
+				node = newGrantNode()
38
+				nodes[part] = node
39
+			}
40
+			if part != "" {
41
+				node.grants = append(node.grants, grant)
42
+			}
43
+			nodes = node.children
44
+		}
45
+	}
46
+	return &memoryGraph{roots}
47
+}
48
+
49
+func (g *memoryGraph) getGrants(name string) []*Grant {
50
+	nameParts := strings.Split(name, "/")
51
+	nodes := g.roots
52
+	var node *grantNode
53
+	var nodeOk bool
54
+	for _, part := range nameParts {
55
+		node, nodeOk = nodes[part]
56
+		if !nodeOk {
57
+			return nil
58
+		}
59
+		nodes = node.children
60
+	}
61
+	return node.grants
62
+}
63
+
64
+func isSubName(name, sub string) bool {
65
+	if strings.HasPrefix(name, sub) {
66
+		if len(name) == len(sub) || name[len(sub)] == '/' {
67
+			return true
68
+		}
69
+	}
70
+	return false
71
+}
72
+
73
+type walkFunc func(*Grant, []*Grant) bool
74
+
75
+func foundWalkFunc(*Grant, []*Grant) bool {
76
+	return true
77
+}
78
+
79
+func (g *memoryGraph) walkGrants(start, target string, permission uint16, f walkFunc, chain []*Grant, visited map[*Grant]bool, collect bool) bool {
80
+	if visited == nil {
81
+		visited = map[*Grant]bool{}
82
+	}
83
+	grants := g.getGrants(start)
84
+	subGrants := make([]*Grant, 0, len(grants))
85
+	for _, grant := range grants {
86
+		if visited[grant] {
87
+			continue
88
+		}
89
+		visited[grant] = true
90
+		if grant.Permission&permission == permission {
91
+			if isSubName(target, grant.Subject) {
92
+				if f(grant, chain) {
93
+					return true
94
+				}
95
+			} else {
96
+				subGrants = append(subGrants, grant)
97
+			}
98
+		}
99
+	}
100
+	for _, grant := range subGrants {
101
+		var chainCopy []*Grant
102
+		if collect {
103
+			chainCopy = make([]*Grant, len(chain)+1)
104
+			copy(chainCopy, chain)
105
+			chainCopy[len(chainCopy)-1] = grant
106
+		} else {
107
+			chainCopy = nil
108
+		}
109
+
110
+		if g.walkGrants(grant.Subject, target, permission, f, chainCopy, visited, collect) {
111
+			return true
112
+		}
113
+	}
114
+	return false
115
+}
116
+
117
+func (g *memoryGraph) Verify(key libtrust.PublicKey, node string, permission uint16) (bool, error) {
118
+	return g.walkGrants(key.KeyID(), node, permission, foundWalkFunc, nil, nil, false), nil
119
+}
120
+
121
+func (g *memoryGraph) GetGrants(key libtrust.PublicKey, node string, permission uint16) ([][]*Grant, error) {
122
+	grants := [][]*Grant{}
123
+	collect := func(grant *Grant, chain []*Grant) bool {
124
+		grantChain := make([]*Grant, len(chain)+1)
125
+		copy(grantChain, chain)
126
+		grantChain[len(grantChain)-1] = grant
127
+		grants = append(grants, grantChain)
128
+		return false
129
+	}
130
+	g.walkGrants(key.KeyID(), node, permission, collect, nil, nil, true)
131
+	return grants, nil
132
+}
0 133
new file mode 100644
... ...
@@ -0,0 +1,174 @@
0
+package trustgraph
1
+
2
+import (
3
+	"fmt"
4
+	"testing"
5
+
6
+	"github.com/docker/libtrust"
7
+)
8
+
9
+func createTestKeysAndGrants(count int) ([]*Grant, []libtrust.PrivateKey) {
10
+	grants := make([]*Grant, count)
11
+	keys := make([]libtrust.PrivateKey, count)
12
+	for i := 0; i < count; i++ {
13
+		pk, err := libtrust.GenerateECP256PrivateKey()
14
+		if err != nil {
15
+			panic(err)
16
+		}
17
+		grant := &Grant{
18
+			Subject:    fmt.Sprintf("/user-%d", i+1),
19
+			Permission: 0x0f,
20
+			Grantee:    pk.KeyID(),
21
+		}
22
+		keys[i] = pk
23
+		grants[i] = grant
24
+	}
25
+	return grants, keys
26
+}
27
+
28
+func testVerified(t *testing.T, g TrustGraph, k libtrust.PublicKey, keyName, target string, permission uint16) {
29
+	if ok, err := g.Verify(k, target, permission); err != nil {
30
+		t.Fatalf("Unexpected error during verification: %s", err)
31
+	} else if !ok {
32
+		t.Errorf("key failed verification\n\tKey: %s(%s)\n\tNamespace: %s", keyName, k.KeyID(), target)
33
+	}
34
+}
35
+
36
+func testNotVerified(t *testing.T, g TrustGraph, k libtrust.PublicKey, keyName, target string, permission uint16) {
37
+	if ok, err := g.Verify(k, target, permission); err != nil {
38
+		t.Fatalf("Unexpected error during verification: %s", err)
39
+	} else if ok {
40
+		t.Errorf("key should have failed verification\n\tKey: %s(%s)\n\tNamespace: %s", keyName, k.KeyID(), target)
41
+	}
42
+}
43
+
44
+func TestVerify(t *testing.T) {
45
+	grants, keys := createTestKeysAndGrants(4)
46
+	extraGrants := make([]*Grant, 3)
47
+	extraGrants[0] = &Grant{
48
+		Subject:    "/user-3",
49
+		Permission: 0x0f,
50
+		Grantee:    "/user-2",
51
+	}
52
+	extraGrants[1] = &Grant{
53
+		Subject:    "/user-3/sub-project",
54
+		Permission: 0x0f,
55
+		Grantee:    "/user-4",
56
+	}
57
+	extraGrants[2] = &Grant{
58
+		Subject:    "/user-4",
59
+		Permission: 0x07,
60
+		Grantee:    "/user-1",
61
+	}
62
+	grants = append(grants, extraGrants...)
63
+
64
+	g := NewMemoryGraph(grants)
65
+
66
+	testVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-1", 0x0f)
67
+	testVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-1/some-project/sub-value", 0x0f)
68
+	testVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-4", 0x07)
69
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-2/", 0x0f)
70
+	testVerified(t, g, keys[2].PublicKey(), "user-key-3", "/user-3/sub-value", 0x0f)
71
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-3/sub-value", 0x0f)
72
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-3", 0x0f)
73
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-3/", 0x0f)
74
+	testVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-3/sub-project", 0x0f)
75
+	testVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-3/sub-project/app", 0x0f)
76
+	testVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-4", 0x0f)
77
+
78
+	testNotVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-2", 0x0f)
79
+	testNotVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-3/sub-value", 0x0f)
80
+	testNotVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-4", 0x0f)
81
+	testNotVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-1/", 0x0f)
82
+	testNotVerified(t, g, keys[2].PublicKey(), "user-key-3", "/user-2", 0x0f)
83
+	testNotVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-4", 0x0f)
84
+	testNotVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-3", 0x0f)
85
+}
86
+
87
+func TestCircularWalk(t *testing.T) {
88
+	grants, keys := createTestKeysAndGrants(3)
89
+	user1Grant := &Grant{
90
+		Subject:    "/user-2",
91
+		Permission: 0x0f,
92
+		Grantee:    "/user-1",
93
+	}
94
+	user2Grant := &Grant{
95
+		Subject:    "/user-1",
96
+		Permission: 0x0f,
97
+		Grantee:    "/user-2",
98
+	}
99
+	grants = append(grants, user1Grant, user2Grant)
100
+
101
+	g := NewMemoryGraph(grants)
102
+
103
+	testVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-1", 0x0f)
104
+	testVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-2", 0x0f)
105
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-2", 0x0f)
106
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-1", 0x0f)
107
+	testVerified(t, g, keys[2].PublicKey(), "user-key-3", "/user-3", 0x0f)
108
+
109
+	testNotVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-3", 0x0f)
110
+	testNotVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-3", 0x0f)
111
+}
112
+
113
+func assertGrantSame(t *testing.T, actual, expected *Grant) {
114
+	if actual != expected {
115
+		t.Fatalf("Unexpected grant retrieved\n\tExpected: %v\n\tActual: %v", expected, actual)
116
+	}
117
+}
118
+
119
+func TestGetGrants(t *testing.T) {
120
+	grants, keys := createTestKeysAndGrants(5)
121
+	extraGrants := make([]*Grant, 4)
122
+	extraGrants[0] = &Grant{
123
+		Subject:    "/user-3/friend-project",
124
+		Permission: 0x0f,
125
+		Grantee:    "/user-2/friends",
126
+	}
127
+	extraGrants[1] = &Grant{
128
+		Subject:    "/user-3/sub-project",
129
+		Permission: 0x0f,
130
+		Grantee:    "/user-4",
131
+	}
132
+	extraGrants[2] = &Grant{
133
+		Subject:    "/user-2/friends",
134
+		Permission: 0x0f,
135
+		Grantee:    "/user-5/fun-project",
136
+	}
137
+	extraGrants[3] = &Grant{
138
+		Subject:    "/user-5/fun-project",
139
+		Permission: 0x0f,
140
+		Grantee:    "/user-1",
141
+	}
142
+	grants = append(grants, extraGrants...)
143
+
144
+	g := NewMemoryGraph(grants)
145
+
146
+	grantChains, err := g.GetGrants(keys[3], "/user-3/sub-project/specific-app", 0x0f)
147
+	if err != nil {
148
+		t.Fatalf("Error getting grants: %s", err)
149
+	}
150
+	if len(grantChains) != 1 {
151
+		t.Fatalf("Expected number of grant chains returned, expected %d, received %d", 1, len(grantChains))
152
+	}
153
+	if len(grantChains[0]) != 2 {
154
+		t.Fatalf("Unexpected number of grants retrieved\n\tExpected: %d\n\tActual: %d", 2, len(grantChains[0]))
155
+	}
156
+	assertGrantSame(t, grantChains[0][0], grants[3])
157
+	assertGrantSame(t, grantChains[0][1], extraGrants[1])
158
+
159
+	grantChains, err = g.GetGrants(keys[0], "/user-3/friend-project/fun-app", 0x0f)
160
+	if err != nil {
161
+		t.Fatalf("Error getting grants: %s", err)
162
+	}
163
+	if len(grantChains) != 1 {
164
+		t.Fatalf("Expected number of grant chains returned, expected %d, received %d", 1, len(grantChains))
165
+	}
166
+	if len(grantChains[0]) != 4 {
167
+		t.Fatalf("Unexpected number of grants retrieved\n\tExpected: %d\n\tActual: %d", 2, len(grantChains[0]))
168
+	}
169
+	assertGrantSame(t, grantChains[0][0], grants[0])
170
+	assertGrantSame(t, grantChains[0][1], extraGrants[3])
171
+	assertGrantSame(t, grantChains[0][2], extraGrants[2])
172
+	assertGrantSame(t, grantChains[0][3], extraGrants[0])
173
+}
0 174
new file mode 100644
... ...
@@ -0,0 +1,227 @@
0
+package trustgraph
1
+
2
+import (
3
+	"crypto/x509"
4
+	"encoding/json"
5
+	"io"
6
+	"io/ioutil"
7
+	"sort"
8
+	"strings"
9
+	"time"
10
+
11
+	"github.com/docker/libtrust"
12
+)
13
+
14
+type jsonGrant struct {
15
+	Subject    string `json:"subject"`
16
+	Permission uint16 `json:"permission"`
17
+	Grantee    string `json:"grantee"`
18
+}
19
+
20
+type jsonRevocation struct {
21
+	Subject    string `json:"subject"`
22
+	Revocation uint16 `json:"revocation"`
23
+	Grantee    string `json:"grantee"`
24
+}
25
+
26
+type jsonStatement struct {
27
+	Revocations []*jsonRevocation `json:"revocations"`
28
+	Grants      []*jsonGrant      `json:"grants"`
29
+	Expiration  time.Time         `json:"expiration"`
30
+	IssuedAt    time.Time         `json:"issuedAt"`
31
+}
32
+
33
+func (g *jsonGrant) Grant(statement *Statement) *Grant {
34
+	return &Grant{
35
+		Subject:    g.Subject,
36
+		Permission: g.Permission,
37
+		Grantee:    g.Grantee,
38
+		statement:  statement,
39
+	}
40
+}
41
+
42
+// Statement represents a set of grants made from a verifiable
43
+// authority.  A statement has an expiration associated with it
44
+// set by the authority.
45
+type Statement struct {
46
+	jsonStatement
47
+
48
+	signature *libtrust.JSONSignature
49
+}
50
+
51
+// IsExpired returns whether the statement has expired
52
+func (s *Statement) IsExpired() bool {
53
+	return s.Expiration.Before(time.Now().Add(-10 * time.Second))
54
+}
55
+
56
+// Bytes returns an indented json representation of the statement
57
+// in a byte array.  This value can be written to a file or stream
58
+// without alteration.
59
+func (s *Statement) Bytes() ([]byte, error) {
60
+	return s.signature.PrettySignature("signatures")
61
+}
62
+
63
+// LoadStatement loads and verifies a statement from an input stream.
64
+func LoadStatement(r io.Reader, authority *x509.CertPool) (*Statement, error) {
65
+	b, err := ioutil.ReadAll(r)
66
+	if err != nil {
67
+		return nil, err
68
+	}
69
+	js, err := libtrust.ParsePrettySignature(b, "signatures")
70
+	if err != nil {
71
+		return nil, err
72
+	}
73
+	payload, err := js.Payload()
74
+	if err != nil {
75
+		return nil, err
76
+	}
77
+	var statement Statement
78
+	err = json.Unmarshal(payload, &statement.jsonStatement)
79
+	if err != nil {
80
+		return nil, err
81
+	}
82
+
83
+	if authority == nil {
84
+		_, err = js.Verify()
85
+		if err != nil {
86
+			return nil, err
87
+		}
88
+	} else {
89
+		_, err = js.VerifyChains(authority)
90
+		if err != nil {
91
+			return nil, err
92
+		}
93
+	}
94
+	statement.signature = js
95
+
96
+	return &statement, nil
97
+}
98
+
99
+// CreateStatements creates and signs a statement from a stream of grants
100
+// and revocations in a JSON array.
101
+func CreateStatement(grants, revocations io.Reader, expiration time.Duration, key libtrust.PrivateKey, chain []*x509.Certificate) (*Statement, error) {
102
+	var statement Statement
103
+	err := json.NewDecoder(grants).Decode(&statement.jsonStatement.Grants)
104
+	if err != nil {
105
+		return nil, err
106
+	}
107
+	err = json.NewDecoder(revocations).Decode(&statement.jsonStatement.Revocations)
108
+	if err != nil {
109
+		return nil, err
110
+	}
111
+	statement.jsonStatement.Expiration = time.Now().UTC().Add(expiration)
112
+	statement.jsonStatement.IssuedAt = time.Now().UTC()
113
+
114
+	b, err := json.MarshalIndent(&statement.jsonStatement, "", "   ")
115
+	if err != nil {
116
+		return nil, err
117
+	}
118
+
119
+	statement.signature, err = libtrust.NewJSONSignature(b)
120
+	if err != nil {
121
+		return nil, err
122
+	}
123
+	err = statement.signature.SignWithChain(key, chain)
124
+	if err != nil {
125
+		return nil, err
126
+	}
127
+
128
+	return &statement, nil
129
+}
130
+
131
+type statementList []*Statement
132
+
133
+func (s statementList) Len() int {
134
+	return len(s)
135
+}
136
+
137
+func (s statementList) Less(i, j int) bool {
138
+	return s[i].IssuedAt.Before(s[j].IssuedAt)
139
+}
140
+
141
+func (s statementList) Swap(i, j int) {
142
+	s[i], s[j] = s[j], s[i]
143
+}
144
+
145
+// CollapseStatements returns a single list of the valid statements as well as the
146
+// time when the next grant will expire.
147
+func CollapseStatements(statements []*Statement, useExpired bool) ([]*Grant, time.Time, error) {
148
+	sorted := make(statementList, 0, len(statements))
149
+	for _, statement := range statements {
150
+		if useExpired || !statement.IsExpired() {
151
+			sorted = append(sorted, statement)
152
+		}
153
+	}
154
+	sort.Sort(sorted)
155
+
156
+	var minExpired time.Time
157
+	var grantCount int
158
+	roots := map[string]*grantNode{}
159
+	for i, statement := range sorted {
160
+		if statement.Expiration.Before(minExpired) || i == 0 {
161
+			minExpired = statement.Expiration
162
+		}
163
+		for _, grant := range statement.Grants {
164
+			parts := strings.Split(grant.Grantee, "/")
165
+			nodes := roots
166
+			g := grant.Grant(statement)
167
+			grantCount = grantCount + 1
168
+
169
+			for _, part := range parts {
170
+				node, nodeOk := nodes[part]
171
+				if !nodeOk {
172
+					node = newGrantNode()
173
+					nodes[part] = node
174
+				}
175
+				node.grants = append(node.grants, g)
176
+				nodes = node.children
177
+			}
178
+		}
179
+
180
+		for _, revocation := range statement.Revocations {
181
+			parts := strings.Split(revocation.Grantee, "/")
182
+			nodes := roots
183
+
184
+			var node *grantNode
185
+			var nodeOk bool
186
+			for _, part := range parts {
187
+				node, nodeOk = nodes[part]
188
+				if !nodeOk {
189
+					break
190
+				}
191
+				nodes = node.children
192
+			}
193
+			if node != nil {
194
+				for _, grant := range node.grants {
195
+					if isSubName(grant.Subject, revocation.Subject) {
196
+						grant.Permission = grant.Permission &^ revocation.Revocation
197
+					}
198
+				}
199
+			}
200
+		}
201
+	}
202
+
203
+	retGrants := make([]*Grant, 0, grantCount)
204
+	for _, rootNodes := range roots {
205
+		retGrants = append(retGrants, rootNodes.grants...)
206
+	}
207
+
208
+	return retGrants, minExpired, nil
209
+}
210
+
211
+// FilterStatements filters the statements to statements including the given grants.
212
+func FilterStatements(grants []*Grant) ([]*Statement, error) {
213
+	statements := map[*Statement]bool{}
214
+	for _, grant := range grants {
215
+		if grant.statement != nil {
216
+			statements[grant.statement] = true
217
+		}
218
+	}
219
+	retStatements := make([]*Statement, len(statements))
220
+	var i int
221
+	for statement := range statements {
222
+		retStatements[i] = statement
223
+		i++
224
+	}
225
+	return retStatements, nil
226
+}
0 227
new file mode 100644
... ...
@@ -0,0 +1,417 @@
0
+package trustgraph
1
+
2
+import (
3
+	"bytes"
4
+	"crypto/x509"
5
+	"encoding/json"
6
+	"testing"
7
+	"time"
8
+
9
+	"github.com/docker/libtrust"
10
+	"github.com/docker/libtrust/testutil"
11
+)
12
+
13
+const testStatementExpiration = time.Hour * 5
14
+
15
+func generateStatement(grants []*Grant, key libtrust.PrivateKey, chain []*x509.Certificate) (*Statement, error) {
16
+	var statement Statement
17
+
18
+	statement.Grants = make([]*jsonGrant, len(grants))
19
+	for i, grant := range grants {
20
+		statement.Grants[i] = &jsonGrant{
21
+			Subject:    grant.Subject,
22
+			Permission: grant.Permission,
23
+			Grantee:    grant.Grantee,
24
+		}
25
+	}
26
+	statement.IssuedAt = time.Now()
27
+	statement.Expiration = time.Now().Add(testStatementExpiration)
28
+	statement.Revocations = make([]*jsonRevocation, 0)
29
+
30
+	marshalled, err := json.MarshalIndent(statement.jsonStatement, "", "   ")
31
+	if err != nil {
32
+		return nil, err
33
+	}
34
+
35
+	sig, err := libtrust.NewJSONSignature(marshalled)
36
+	if err != nil {
37
+		return nil, err
38
+	}
39
+	err = sig.SignWithChain(key, chain)
40
+	if err != nil {
41
+		return nil, err
42
+	}
43
+	statement.signature = sig
44
+
45
+	return &statement, nil
46
+}
47
+
48
+func generateTrustChain(t *testing.T, chainLen int) (libtrust.PrivateKey, *x509.CertPool, []*x509.Certificate) {
49
+	caKey, err := libtrust.GenerateECP256PrivateKey()
50
+	if err != nil {
51
+		t.Fatalf("Error generating key: %s", err)
52
+	}
53
+	ca, err := testutil.GenerateTrustCA(caKey.CryptoPublicKey(), caKey.CryptoPrivateKey())
54
+	if err != nil {
55
+		t.Fatalf("Error generating ca: %s", err)
56
+	}
57
+
58
+	parent := ca
59
+	parentKey := caKey
60
+	chain := make([]*x509.Certificate, chainLen)
61
+	for i := chainLen - 1; i > 0; i-- {
62
+		intermediatekey, err := libtrust.GenerateECP256PrivateKey()
63
+		if err != nil {
64
+			t.Fatalf("Error generate key: %s", err)
65
+		}
66
+		chain[i], err = testutil.GenerateIntermediate(intermediatekey.CryptoPublicKey(), parentKey.CryptoPrivateKey(), parent)
67
+		if err != nil {
68
+			t.Fatalf("Error generating intermdiate certificate: %s", err)
69
+		}
70
+		parent = chain[i]
71
+		parentKey = intermediatekey
72
+	}
73
+	trustKey, err := libtrust.GenerateECP256PrivateKey()
74
+	if err != nil {
75
+		t.Fatalf("Error generate key: %s", err)
76
+	}
77
+	chain[0], err = testutil.GenerateTrustCert(trustKey.CryptoPublicKey(), parentKey.CryptoPrivateKey(), parent)
78
+	if err != nil {
79
+		t.Fatalf("Error generate trust cert: %s", err)
80
+	}
81
+
82
+	caPool := x509.NewCertPool()
83
+	caPool.AddCert(ca)
84
+
85
+	return trustKey, caPool, chain
86
+}
87
+
88
+func TestLoadStatement(t *testing.T) {
89
+	grantCount := 4
90
+	grants, _ := createTestKeysAndGrants(grantCount)
91
+
92
+	trustKey, caPool, chain := generateTrustChain(t, 6)
93
+
94
+	statement, err := generateStatement(grants, trustKey, chain)
95
+	if err != nil {
96
+		t.Fatalf("Error generating statement: %s", err)
97
+	}
98
+
99
+	statementBytes, err := statement.Bytes()
100
+	if err != nil {
101
+		t.Fatalf("Error getting statement bytes: %s", err)
102
+	}
103
+
104
+	s2, err := LoadStatement(bytes.NewReader(statementBytes), caPool)
105
+	if err != nil {
106
+		t.Fatalf("Error loading statement: %s", err)
107
+	}
108
+	if len(s2.Grants) != grantCount {
109
+		t.Fatalf("Unexpected grant length\n\tExpected: %d\n\tActual: %d", grantCount, len(s2.Grants))
110
+	}
111
+
112
+	pool := x509.NewCertPool()
113
+	_, err = LoadStatement(bytes.NewReader(statementBytes), pool)
114
+	if err == nil {
115
+		t.Fatalf("No error thrown verifying without an authority")
116
+	} else if _, ok := err.(x509.UnknownAuthorityError); !ok {
117
+		t.Fatalf("Unexpected error verifying without authority: %s", err)
118
+	}
119
+
120
+	s2, err = LoadStatement(bytes.NewReader(statementBytes), nil)
121
+	if err != nil {
122
+		t.Fatalf("Error loading statement: %s", err)
123
+	}
124
+	if len(s2.Grants) != grantCount {
125
+		t.Fatalf("Unexpected grant length\n\tExpected: %d\n\tActual: %d", grantCount, len(s2.Grants))
126
+	}
127
+
128
+	badData := make([]byte, len(statementBytes))
129
+	copy(badData, statementBytes)
130
+	badData[0] = '['
131
+	_, err = LoadStatement(bytes.NewReader(badData), nil)
132
+	if err == nil {
133
+		t.Fatalf("No error thrown parsing bad json")
134
+	}
135
+
136
+	alteredData := make([]byte, len(statementBytes))
137
+	copy(alteredData, statementBytes)
138
+	alteredData[30] = '0'
139
+	_, err = LoadStatement(bytes.NewReader(alteredData), nil)
140
+	if err == nil {
141
+		t.Fatalf("No error thrown from bad data")
142
+	}
143
+}
144
+
145
+func TestCollapseGrants(t *testing.T) {
146
+	grantCount := 8
147
+	grants, keys := createTestKeysAndGrants(grantCount)
148
+	linkGrants := make([]*Grant, 4)
149
+	linkGrants[0] = &Grant{
150
+		Subject:    "/user-3",
151
+		Permission: 0x0f,
152
+		Grantee:    "/user-2",
153
+	}
154
+	linkGrants[1] = &Grant{
155
+		Subject:    "/user-3/sub-project",
156
+		Permission: 0x0f,
157
+		Grantee:    "/user-4",
158
+	}
159
+	linkGrants[2] = &Grant{
160
+		Subject:    "/user-6",
161
+		Permission: 0x0f,
162
+		Grantee:    "/user-7",
163
+	}
164
+	linkGrants[3] = &Grant{
165
+		Subject:    "/user-6/sub-project/specific-app",
166
+		Permission: 0x0f,
167
+		Grantee:    "/user-5",
168
+	}
169
+	trustKey, pool, chain := generateTrustChain(t, 3)
170
+
171
+	statements := make([]*Statement, 3)
172
+	var err error
173
+	statements[0], err = generateStatement(grants[0:4], trustKey, chain)
174
+	if err != nil {
175
+		t.Fatalf("Error generating statement: %s", err)
176
+	}
177
+	statements[1], err = generateStatement(grants[4:], trustKey, chain)
178
+	if err != nil {
179
+		t.Fatalf("Error generating statement: %s", err)
180
+	}
181
+	statements[2], err = generateStatement(linkGrants, trustKey, chain)
182
+	if err != nil {
183
+		t.Fatalf("Error generating statement: %s", err)
184
+	}
185
+
186
+	statementsCopy := make([]*Statement, len(statements))
187
+	for i, statement := range statements {
188
+		b, err := statement.Bytes()
189
+		if err != nil {
190
+			t.Fatalf("Error getting statement bytes: %s", err)
191
+		}
192
+		verifiedStatement, err := LoadStatement(bytes.NewReader(b), pool)
193
+		if err != nil {
194
+			t.Fatalf("Error loading statement: %s", err)
195
+		}
196
+		// Force sort by reversing order
197
+		statementsCopy[len(statementsCopy)-i-1] = verifiedStatement
198
+	}
199
+	statements = statementsCopy
200
+
201
+	collapsedGrants, expiration, err := CollapseStatements(statements, false)
202
+	if len(collapsedGrants) != 12 {
203
+		t.Fatalf("Unexpected number of grants\n\tExpected: %d\n\tActual: %s", 12, len(collapsedGrants))
204
+	}
205
+	if expiration.After(time.Now().Add(time.Hour*5)) || expiration.Before(time.Now()) {
206
+		t.Fatalf("Unexpected expiration time: %s", expiration.String())
207
+	}
208
+	g := NewMemoryGraph(collapsedGrants)
209
+
210
+	testVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-1", 0x0f)
211
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-2", 0x0f)
212
+	testVerified(t, g, keys[2].PublicKey(), "user-key-3", "/user-3", 0x0f)
213
+	testVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-4", 0x0f)
214
+	testVerified(t, g, keys[4].PublicKey(), "user-key-5", "/user-5", 0x0f)
215
+	testVerified(t, g, keys[5].PublicKey(), "user-key-6", "/user-6", 0x0f)
216
+	testVerified(t, g, keys[6].PublicKey(), "user-key-7", "/user-7", 0x0f)
217
+	testVerified(t, g, keys[7].PublicKey(), "user-key-8", "/user-8", 0x0f)
218
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-3", 0x0f)
219
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-3/sub-project/specific-app", 0x0f)
220
+	testVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-3/sub-project", 0x0f)
221
+	testVerified(t, g, keys[6].PublicKey(), "user-key-7", "/user-6", 0x0f)
222
+	testVerified(t, g, keys[6].PublicKey(), "user-key-7", "/user-6/sub-project/specific-app", 0x0f)
223
+	testVerified(t, g, keys[4].PublicKey(), "user-key-5", "/user-6/sub-project/specific-app", 0x0f)
224
+
225
+	testNotVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-3", 0x0f)
226
+	testNotVerified(t, g, keys[3].PublicKey(), "user-key-4", "/user-6/sub-project", 0x0f)
227
+	testNotVerified(t, g, keys[4].PublicKey(), "user-key-5", "/user-6/sub-project", 0x0f)
228
+
229
+	// Add revocation grant
230
+	statements = append(statements, &Statement{
231
+		jsonStatement{
232
+			IssuedAt:   time.Now(),
233
+			Expiration: time.Now().Add(testStatementExpiration),
234
+			Grants:     []*jsonGrant{},
235
+			Revocations: []*jsonRevocation{
236
+				&jsonRevocation{
237
+					Subject:    "/user-1",
238
+					Revocation: 0x0f,
239
+					Grantee:    keys[0].KeyID(),
240
+				},
241
+				&jsonRevocation{
242
+					Subject:    "/user-2",
243
+					Revocation: 0x08,
244
+					Grantee:    keys[1].KeyID(),
245
+				},
246
+				&jsonRevocation{
247
+					Subject:    "/user-6",
248
+					Revocation: 0x0f,
249
+					Grantee:    "/user-7",
250
+				},
251
+				&jsonRevocation{
252
+					Subject:    "/user-9",
253
+					Revocation: 0x0f,
254
+					Grantee:    "/user-10",
255
+				},
256
+			},
257
+		},
258
+		nil,
259
+	})
260
+
261
+	collapsedGrants, expiration, err = CollapseStatements(statements, false)
262
+	if len(collapsedGrants) != 12 {
263
+		t.Fatalf("Unexpected number of grants\n\tExpected: %d\n\tActual: %s", 12, len(collapsedGrants))
264
+	}
265
+	if expiration.After(time.Now().Add(time.Hour*5)) || expiration.Before(time.Now()) {
266
+		t.Fatalf("Unexpected expiration time: %s", expiration.String())
267
+	}
268
+	g = NewMemoryGraph(collapsedGrants)
269
+
270
+	testNotVerified(t, g, keys[0].PublicKey(), "user-key-1", "/user-1", 0x0f)
271
+	testNotVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-2", 0x0f)
272
+	testNotVerified(t, g, keys[6].PublicKey(), "user-key-7", "/user-6/sub-project/specific-app", 0x0f)
273
+
274
+	testVerified(t, g, keys[1].PublicKey(), "user-key-2", "/user-2", 0x07)
275
+}
276
+
277
+func TestFilterStatements(t *testing.T) {
278
+	grantCount := 8
279
+	grants, keys := createTestKeysAndGrants(grantCount)
280
+	linkGrants := make([]*Grant, 3)
281
+	linkGrants[0] = &Grant{
282
+		Subject:    "/user-3",
283
+		Permission: 0x0f,
284
+		Grantee:    "/user-2",
285
+	}
286
+	linkGrants[1] = &Grant{
287
+		Subject:    "/user-5",
288
+		Permission: 0x0f,
289
+		Grantee:    "/user-4",
290
+	}
291
+	linkGrants[2] = &Grant{
292
+		Subject:    "/user-7",
293
+		Permission: 0x0f,
294
+		Grantee:    "/user-6",
295
+	}
296
+
297
+	trustKey, _, chain := generateTrustChain(t, 3)
298
+
299
+	statements := make([]*Statement, 5)
300
+	var err error
301
+	statements[0], err = generateStatement(grants[0:2], trustKey, chain)
302
+	if err != nil {
303
+		t.Fatalf("Error generating statement: %s", err)
304
+	}
305
+	statements[1], err = generateStatement(grants[2:4], trustKey, chain)
306
+	if err != nil {
307
+		t.Fatalf("Error generating statement: %s", err)
308
+	}
309
+	statements[2], err = generateStatement(grants[4:6], trustKey, chain)
310
+	if err != nil {
311
+		t.Fatalf("Error generating statement: %s", err)
312
+	}
313
+	statements[3], err = generateStatement(grants[6:], trustKey, chain)
314
+	if err != nil {
315
+		t.Fatalf("Error generating statement: %s", err)
316
+	}
317
+	statements[4], err = generateStatement(linkGrants, trustKey, chain)
318
+	if err != nil {
319
+		t.Fatalf("Error generating statement: %s", err)
320
+	}
321
+	collapsed, _, err := CollapseStatements(statements, false)
322
+	if err != nil {
323
+		t.Fatalf("Error collapsing grants: %s", err)
324
+	}
325
+
326
+	// Filter 1, all 5 statements
327
+	filter1, err := FilterStatements(collapsed)
328
+	if err != nil {
329
+		t.Fatalf("Error filtering statements: %s", err)
330
+	}
331
+	if len(filter1) != 5 {
332
+		t.Fatalf("Wrong number of statements, expected %d, received %d", 5, len(filter1))
333
+	}
334
+
335
+	// Filter 2, one statement
336
+	filter2, err := FilterStatements([]*Grant{collapsed[0]})
337
+	if err != nil {
338
+		t.Fatalf("Error filtering statements: %s", err)
339
+	}
340
+	if len(filter2) != 1 {
341
+		t.Fatalf("Wrong number of statements, expected %d, received %d", 1, len(filter2))
342
+	}
343
+
344
+	// Filter 3, 2 statements, from graph lookup
345
+	g := NewMemoryGraph(collapsed)
346
+	lookupGrants, err := g.GetGrants(keys[1], "/user-3", 0x0f)
347
+	if err != nil {
348
+		t.Fatalf("Error looking up grants: %s", err)
349
+	}
350
+	if len(lookupGrants) != 1 {
351
+		t.Fatalf("Wrong numberof grant chains returned from lookup, expected %d, received %d", 1, len(lookupGrants))
352
+	}
353
+	if len(lookupGrants[0]) != 2 {
354
+		t.Fatalf("Wrong number of grants looked up, expected %d, received %d", 2, len(lookupGrants))
355
+	}
356
+	filter3, err := FilterStatements(lookupGrants[0])
357
+	if err != nil {
358
+		t.Fatalf("Error filtering statements: %s", err)
359
+	}
360
+	if len(filter3) != 2 {
361
+		t.Fatalf("Wrong number of statements, expected %d, received %d", 2, len(filter3))
362
+	}
363
+
364
+}
365
+
366
+func TestCreateStatement(t *testing.T) {
367
+	grantJSON := bytes.NewReader([]byte(`[
368
+   {
369
+      "subject": "/user-2",
370
+      "permission": 15,
371
+      "grantee": "/user-1"
372
+   },
373
+   {
374
+      "subject": "/user-7",
375
+      "permission": 1,
376
+      "grantee": "/user-9"
377
+   },
378
+   {
379
+      "subject": "/user-3",
380
+      "permission": 15,
381
+      "grantee": "/user-2"
382
+   }
383
+]`))
384
+	revocationJSON := bytes.NewReader([]byte(`[
385
+   {
386
+      "subject": "user-8",
387
+      "revocation": 12,
388
+      "grantee": "user-9"
389
+   }
390
+]`))
391
+
392
+	trustKey, pool, chain := generateTrustChain(t, 3)
393
+
394
+	statement, err := CreateStatement(grantJSON, revocationJSON, testStatementExpiration, trustKey, chain)
395
+	if err != nil {
396
+		t.Fatalf("Error creating statement: %s", err)
397
+	}
398
+
399
+	b, err := statement.Bytes()
400
+	if err != nil {
401
+		t.Fatalf("Error retrieving bytes: %s", err)
402
+	}
403
+
404
+	verified, err := LoadStatement(bytes.NewReader(b), pool)
405
+	if err != nil {
406
+		t.Fatalf("Error loading statement: %s", err)
407
+	}
408
+
409
+	if len(verified.Grants) != 3 {
410
+		t.Errorf("Unexpected number of grants, expected %d, received %d", 3, len(verified.Grants))
411
+	}
412
+
413
+	if len(verified.Revocations) != 1 {
414
+		t.Errorf("Unexpected number of revocations, expected %d, received %d", 1, len(verified.Revocations))
415
+	}
416
+}