Browse code

Add provenance pull flow for official images

Add support for pulling signed images from a version 2 registry.
Only official images within the library namespace will be pull from the
new registry and check the build signature.

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)

Derek McGowan authored on 2014/10/02 10:26:06
Showing 11 changed files
... ...
@@ -38,6 +38,7 @@ import (
38 38
 	"github.com/docker/docker/pkg/sysinfo"
39 39
 	"github.com/docker/docker/pkg/truncindex"
40 40
 	"github.com/docker/docker/runconfig"
41
+	"github.com/docker/docker/trust"
41 42
 	"github.com/docker/docker/utils"
42 43
 	"github.com/docker/docker/volumes"
43 44
 )
... ...
@@ -98,6 +99,7 @@ type Daemon struct {
98 98
 	containerGraph *graphdb.Database
99 99
 	driver         graphdriver.Driver
100 100
 	execDriver     execdriver.Driver
101
+	trustStore     *trust.TrustStore
101 102
 }
102 103
 
103 104
 // Install installs daemon capabilities to eng.
... ...
@@ -136,6 +138,9 @@ func (daemon *Daemon) Install(eng *engine.Engine) error {
136 136
 	if err := daemon.Repositories().Install(eng); err != nil {
137 137
 		return err
138 138
 	}
139
+	if err := daemon.trustStore.Install(eng); err != nil {
140
+		return err
141
+	}
139 142
 	// FIXME: this hack is necessary for legacy integration tests to access
140 143
 	// the daemon object.
141 144
 	eng.Hack_SetGlobalVar("httpapi.daemon", daemon)
... ...
@@ -813,6 +818,15 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
813 813
 		return nil, fmt.Errorf("Couldn't create Tag store: %s", err)
814 814
 	}
815 815
 
816
+	trustDir := path.Join(config.Root, "trust")
817
+	if err := os.MkdirAll(trustDir, 0700); err != nil && !os.IsExist(err) {
818
+		return nil, err
819
+	}
820
+	t, err := trust.NewTrustStore(trustDir)
821
+	if err != nil {
822
+		return nil, fmt.Errorf("could not create trust store: %s", err)
823
+	}
824
+
816 825
 	if !config.DisableNetwork {
817 826
 		job := eng.Job("init_networkdriver")
818 827
 
... ...
@@ -877,6 +891,7 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
877 877
 		sysInitPath:    sysInitPath,
878 878
 		execDriver:     ed,
879 879
 		eng:            eng,
880
+		trustStore:     t,
880 881
 	}
881 882
 	if err := daemon.checkLocaldns(); err != nil {
882 883
 		return nil, err
... ...
@@ -1,10 +1,14 @@
1 1
 package graph
2 2
 
3 3
 import (
4
+	"bytes"
5
+	"encoding/json"
4 6
 	"fmt"
5 7
 	"io"
8
+	"io/ioutil"
6 9
 	"net"
7 10
 	"net/url"
11
+	"os"
8 12
 	"strings"
9 13
 	"time"
10 14
 
... ...
@@ -13,8 +17,59 @@ import (
13 13
 	"github.com/docker/docker/pkg/log"
14 14
 	"github.com/docker/docker/registry"
15 15
 	"github.com/docker/docker/utils"
16
+	"github.com/docker/libtrust"
16 17
 )
17 18
 
19
+func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) {
20
+	sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures")
21
+	if err != nil {
22
+		return nil, false, fmt.Errorf("error parsing payload: %s", err)
23
+	}
24
+	keys, err := sig.Verify()
25
+	if err != nil {
26
+		return nil, false, fmt.Errorf("error verifying payload: %s", err)
27
+	}
28
+
29
+	payload, err := sig.Payload()
30
+	if err != nil {
31
+		return nil, false, fmt.Errorf("error retrieving payload: %s", err)
32
+	}
33
+
34
+	var manifest registry.ManifestData
35
+	if err := json.Unmarshal(payload, &manifest); err != nil {
36
+		return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err)
37
+	}
38
+
39
+	var verified bool
40
+	for _, key := range keys {
41
+		job := eng.Job("trust_key_check")
42
+		b, err := key.MarshalJSON()
43
+		if err != nil {
44
+			return nil, false, fmt.Errorf("error marshalling public key: %s", err)
45
+		}
46
+		namespace := manifest.Name
47
+		if namespace[0] != '/' {
48
+			namespace = "/" + namespace
49
+		}
50
+		stdoutBuffer := bytes.NewBuffer(nil)
51
+
52
+		job.Args = append(job.Args, namespace)
53
+		job.Setenv("PublicKey", string(b))
54
+		job.SetenvInt("Permission", 0x03)
55
+		job.Stdout.Add(stdoutBuffer)
56
+		if err = job.Run(); err != nil {
57
+			return nil, false, fmt.Errorf("error running key check: %s", err)
58
+		}
59
+		result := engine.Tail(stdoutBuffer, 1)
60
+		log.Debugf("Key check result: %q", result)
61
+		if result == "verified" {
62
+			verified = true
63
+		}
64
+	}
65
+
66
+	return &manifest, verified, nil
67
+}
68
+
18 69
 func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
19 70
 	if n := len(job.Args); n != 1 && n != 2 {
20 71
 		return job.Errorf("Usage: %s IMAGE [TAG]", job.Name)
... ...
@@ -62,14 +117,32 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status {
62 62
 		return job.Error(err)
63 63
 	}
64 64
 
65
-	if endpoint.String() == registry.IndexServerAddress() {
65
+	var isOfficial bool
66
+	if endpoint.VersionString(1) == registry.IndexServerAddress() {
66 67
 		// If pull "index.docker.io/foo/bar", it's stored locally under "foo/bar"
67 68
 		localName = remoteName
68 69
 
70
+		isOfficial = isOfficialName(remoteName)
71
+		if isOfficial && strings.IndexRune(remoteName, '/') == -1 {
72
+			remoteName = "library/" + remoteName
73
+		}
74
+
69 75
 		// Use provided mirrors, if any
70 76
 		mirrors = s.mirrors
71 77
 	}
72 78
 
79
+	if isOfficial || endpoint.Version == registry.APIVersion2 {
80
+		j := job.Eng.Job("trust_update_base")
81
+		if err = j.Run(); err != nil {
82
+			return job.Errorf("error updating trust base graph: %s", err)
83
+		}
84
+
85
+		if err := s.pullV2Repository(job.Eng, r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err == nil {
86
+			return engine.StatusOK
87
+		} else if err != registry.ErrDoesNotExist {
88
+			log.Errorf("Error from V2 registry: %s", err)
89
+		}
90
+	}
73 91
 	if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel"), mirrors); err != nil {
74 92
 		return job.Error(err)
75 93
 	}
... ...
@@ -317,3 +390,169 @@ func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint
317 317
 	}
318 318
 	return nil
319 319
 }
320
+
321
+// downloadInfo is used to pass information from download to extractor
322
+type downloadInfo struct {
323
+	imgJSON    []byte
324
+	img        *image.Image
325
+	tmpFile    *os.File
326
+	length     int64
327
+	downloaded bool
328
+	err        chan error
329
+}
330
+
331
+func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
332
+	if tag == "" {
333
+		log.Debugf("Pulling tag list from V2 registry for %s", remoteName)
334
+		tags, err := r.GetV2RemoteTags(remoteName, nil)
335
+		if err != nil {
336
+			return err
337
+		}
338
+		for _, t := range tags {
339
+			if err := s.pullV2Tag(eng, r, out, localName, remoteName, t, sf, parallel); err != nil {
340
+				return err
341
+			}
342
+		}
343
+	} else {
344
+		if err := s.pullV2Tag(eng, r, out, localName, remoteName, tag, sf, parallel); err != nil {
345
+			return err
346
+		}
347
+	}
348
+
349
+	return nil
350
+}
351
+
352
+func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, localName, remoteName, tag string, sf *utils.StreamFormatter, parallel bool) error {
353
+	log.Debugf("Pulling tag from V2 registry: %q", tag)
354
+	manifestBytes, err := r.GetV2ImageManifest(remoteName, tag, nil)
355
+	if err != nil {
356
+		return err
357
+	}
358
+
359
+	manifest, verified, err := s.verifyManifest(eng, manifestBytes)
360
+	if err != nil {
361
+		return fmt.Errorf("error verifying manifest: %s", err)
362
+	}
363
+
364
+	if len(manifest.BlobSums) != len(manifest.History) {
365
+		return fmt.Errorf("length of history not equal to number of layers")
366
+	}
367
+
368
+	if verified {
369
+		out.Write(sf.FormatStatus("", "The image you are pulling has been digitally signed by Docker, Inc."))
370
+	}
371
+	out.Write(sf.FormatStatus(tag, "Pulling from %s", localName))
372
+
373
+	downloads := make([]downloadInfo, len(manifest.BlobSums))
374
+
375
+	for i := len(manifest.BlobSums) - 1; i >= 0; i-- {
376
+		var (
377
+			sumStr  = manifest.BlobSums[i]
378
+			imgJSON = []byte(manifest.History[i])
379
+		)
380
+
381
+		img, err := image.NewImgJSON(imgJSON)
382
+		if err != nil {
383
+			return fmt.Errorf("failed to parse json: %s", err)
384
+		}
385
+		downloads[i].img = img
386
+
387
+		// Check if exists
388
+		if s.graph.Exists(img.ID) {
389
+			log.Debugf("Image already exists: %s", img.ID)
390
+			continue
391
+		}
392
+
393
+		chunks := strings.SplitN(sumStr, ":", 2)
394
+		if len(chunks) < 2 {
395
+			return fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)
396
+		}
397
+		sumType, checksum := chunks[0], chunks[1]
398
+		out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil))
399
+
400
+		downloadFunc := func(di *downloadInfo) error {
401
+			log.Infof("pulling blob %q to V1 img %s", sumStr, img.ID)
402
+
403
+			if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
404
+				if c != nil {
405
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil))
406
+					<-c
407
+					out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil))
408
+				} else {
409
+					log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
410
+				}
411
+			} else {
412
+				tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob")
413
+				if err != nil {
414
+					return err
415
+				}
416
+
417
+				r, l, err := r.GetV2ImageBlobReader(remoteName, sumType, checksum, nil)
418
+				if err != nil {
419
+					return err
420
+				}
421
+				defer r.Close()
422
+				io.Copy(tmpFile, utils.ProgressReader(r, int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading"))
423
+
424
+				out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil))
425
+
426
+				log.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name())
427
+				di.tmpFile = tmpFile
428
+				di.length = l
429
+				di.downloaded = true
430
+			}
431
+			di.imgJSON = imgJSON
432
+			defer s.poolRemove("pull", "img:"+img.ID)
433
+
434
+			return nil
435
+		}
436
+
437
+		if parallel {
438
+			downloads[i].err = make(chan error)
439
+			go func(di *downloadInfo) {
440
+				di.err <- downloadFunc(di)
441
+			}(&downloads[i])
442
+		} else {
443
+			err := downloadFunc(&downloads[i])
444
+			if err != nil {
445
+				return err
446
+			}
447
+		}
448
+	}
449
+
450
+	for i := len(downloads) - 1; i >= 0; i-- {
451
+		d := &downloads[i]
452
+		if d.err != nil {
453
+			err := <-d.err
454
+			if err != nil {
455
+				return err
456
+			}
457
+		}
458
+		if d.downloaded {
459
+			// if tmpFile is empty assume download and extracted elsewhere
460
+			defer os.Remove(d.tmpFile.Name())
461
+			defer d.tmpFile.Close()
462
+			d.tmpFile.Seek(0, 0)
463
+			if d.tmpFile != nil {
464
+				err = s.graph.Register(d.img, d.imgJSON,
465
+					utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting"))
466
+				if err != nil {
467
+					return err
468
+				}
469
+
470
+				// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
471
+			}
472
+			out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Pull complete", nil))
473
+
474
+		} else {
475
+			out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Already exists", nil))
476
+		}
477
+
478
+	}
479
+
480
+	if err = s.Set(localName, tag, downloads[0].img.ID, true); err != nil {
481
+		return err
482
+	}
483
+
484
+	return nil
485
+}
... ...
@@ -276,6 +276,20 @@ func (store *TagStore) GetRepoRefs() map[string][]string {
276 276
 	return reporefs
277 277
 }
278 278
 
279
+// isOfficialName returns whether a repo name is considered an official
280
+// repository.  Official repositories are repos with names within
281
+// the library namespace or which default to the library namespace
282
+// by not providing one.
283
+func isOfficialName(name string) bool {
284
+	if strings.HasPrefix(name, "library/") {
285
+		return true
286
+	}
287
+	if strings.IndexRune(name, '/') == -1 {
288
+		return true
289
+	}
290
+	return false
291
+}
292
+
279 293
 // Validate the name of a repository
280 294
 func validateRepoName(name string) error {
281 295
 	if name == "" {
... ...
@@ -2,15 +2,16 @@ package graph
2 2
 
3 3
 import (
4 4
 	"bytes"
5
+	"io"
6
+	"os"
7
+	"path"
8
+	"testing"
9
+
5 10
 	"github.com/docker/docker/daemon/graphdriver"
6 11
 	_ "github.com/docker/docker/daemon/graphdriver/vfs" // import the vfs driver so it is used in the tests
7 12
 	"github.com/docker/docker/image"
8 13
 	"github.com/docker/docker/utils"
9 14
 	"github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar"
10
-	"io"
11
-	"os"
12
-	"path"
13
-	"testing"
14 15
 )
15 16
 
16 17
 const (
... ...
@@ -132,3 +133,18 @@ func TestInvalidTagName(t *testing.T) {
132 132
 		}
133 133
 	}
134 134
 }
135
+
136
+func TestOfficialName(t *testing.T) {
137
+	names := map[string]bool{
138
+		"library/ubuntu":    true,
139
+		"nonlibrary/ubuntu": false,
140
+		"ubuntu":            true,
141
+		"other/library":     false,
142
+	}
143
+	for name, isOfficial := range names {
144
+		result := isOfficialName(name)
145
+		if result != isOfficial {
146
+			t.Errorf("Unexpected result for %s\n\tExpecting: %v\n\tActual: %v", name, isOfficial, result)
147
+		}
148
+	}
149
+}
... ...
@@ -20,6 +20,7 @@ import (
20 20
 var (
21 21
 	ErrAlreadyExists         = errors.New("Image already exists")
22 22
 	ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")")
23
+	ErrDoesNotExist          = errors.New("Image does not exist")
23 24
 	errLoginRequired         = errors.New("Authentication is required.")
24 25
 	validHex                 = regexp.MustCompile(`^([a-f0-9]{64})$`)
25 26
 	validNamespace           = regexp.MustCompile(`^([a-z0-9_]{4,30})$`)
... ...
@@ -83,6 +83,8 @@ var (
83 83
 
84 84
 func init() {
85 85
 	r := mux.NewRouter()
86
+
87
+	// /v1/
86 88
 	r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET")
87 89
 	r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET")
88 90
 	r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT")
... ...
@@ -93,6 +95,10 @@ func init() {
93 93
 	r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE")
94 94
 	r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT")
95 95
 	r.HandleFunc("/v1/search", handlerSearch).Methods("GET")
96
+
97
+	// /v2/
98
+	r.HandleFunc("/v2/version", handlerGetPing).Methods("GET")
99
+
96 100
 	testHttpServer = httptest.NewServer(handlerAccessLog(r))
97 101
 }
98 102
 
... ...
@@ -47,7 +47,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo
47 47
 
48 48
 	// If we're working with a standalone private registry over HTTPS, send Basic Auth headers
49 49
 	// alongside our requests.
50
-	if r.indexEndpoint.String() != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" {
50
+	if r.indexEndpoint.VersionString(1) != IndexServerAddress() && r.indexEndpoint.URL.Scheme == "https" {
51 51
 		info, err := r.indexEndpoint.Ping()
52 52
 		if err != nil {
53 53
 			return nil, err
... ...
@@ -261,7 +261,7 @@ func buildEndpointsList(headers []string, indexEp string) ([]string, error) {
261 261
 }
262 262
 
263 263
 func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) {
264
-	repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.String(), remote)
264
+	repositoryTarget := fmt.Sprintf("%srepositories/%s/images", r.indexEndpoint.VersionString(1), remote)
265 265
 
266 266
 	log.Debugf("[registry] Calling GET %s", repositoryTarget)
267 267
 
... ...
@@ -295,7 +295,7 @@ func (r *Session) GetRepositoryData(remote string) (*RepositoryData, error) {
295 295
 
296 296
 	var endpoints []string
297 297
 	if res.Header.Get("X-Docker-Endpoints") != "" {
298
-		endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String())
298
+		endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1))
299 299
 		if err != nil {
300 300
 			return nil, err
301 301
 		}
... ...
@@ -488,7 +488,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate
488 488
 	if validate {
489 489
 		suffix = "images"
490 490
 	}
491
-	u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.String(), remote, suffix)
491
+	u := fmt.Sprintf("%srepositories/%s/%s", r.indexEndpoint.VersionString(1), remote, suffix)
492 492
 	log.Debugf("[registry] PUT %s", u)
493 493
 	log.Debugf("Image list pushed to index:\n%s", imgListJSON)
494 494
 	req, err := r.reqFactory.NewRequest("PUT", u, bytes.NewReader(imgListJSON))
... ...
@@ -546,7 +546,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate
546 546
 		}
547 547
 
548 548
 		if res.Header.Get("X-Docker-Endpoints") != "" {
549
-			endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.String())
549
+			endpoints, err = buildEndpointsList(res.Header["X-Docker-Endpoints"], r.indexEndpoint.VersionString(1))
550 550
 			if err != nil {
551 551
 				return nil, err
552 552
 			}
... ...
@@ -572,7 +572,7 @@ func (r *Session) PushImageJSONIndex(remote string, imgList []*ImgData, validate
572 572
 
573 573
 func (r *Session) SearchRepositories(term string) (*SearchResults, error) {
574 574
 	log.Debugf("Index server: %s", r.indexEndpoint)
575
-	u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term)
575
+	u := r.indexEndpoint.VersionString(1) + "search?q=" + url.QueryEscape(term)
576 576
 	req, err := r.reqFactory.NewRequest("GET", u, nil)
577 577
 	if err != nil {
578 578
 		return nil, err
579 579
new file mode 100644
... ...
@@ -0,0 +1,386 @@
0
+package registry
1
+
2
+import (
3
+	"encoding/json"
4
+	"fmt"
5
+	"io"
6
+	"io/ioutil"
7
+	"net/url"
8
+	"strconv"
9
+
10
+	"github.com/docker/docker/pkg/log"
11
+	"github.com/docker/docker/utils"
12
+	"github.com/gorilla/mux"
13
+)
14
+
15
+func newV2RegistryRouter() *mux.Router {
16
+	router := mux.NewRouter()
17
+
18
+	v2Router := router.PathPrefix("/v2/").Subrouter()
19
+
20
+	// Version Info
21
+	v2Router.Path("/version").Name("version")
22
+
23
+	// Image Manifests
24
+	v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests")
25
+
26
+	// List Image Tags
27
+	v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags")
28
+
29
+	// Download a blob
30
+	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob")
31
+
32
+	// Upload a blob
33
+	v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name("uploadBlob")
34
+
35
+	// Mounting a blob in an image
36
+	v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob")
37
+
38
+	return router
39
+}
40
+
41
+// APIVersion2 /v2/
42
+var v2HTTPRoutes = newV2RegistryRouter()
43
+
44
+func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) {
45
+	route := v2HTTPRoutes.Get(routeName)
46
+	if route == nil {
47
+		return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName)
48
+	}
49
+
50
+	varReplace := make([]string, 0, len(vars)*2)
51
+	for key, val := range vars {
52
+		varReplace = append(varReplace, key, val)
53
+	}
54
+
55
+	routePath, err := route.URLPath(varReplace...)
56
+	if err != nil {
57
+		return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err)
58
+	}
59
+
60
+	return &url.URL{
61
+		Scheme: e.URL.Scheme,
62
+		Host:   e.URL.Host,
63
+		Path:   routePath.Path,
64
+	}, nil
65
+}
66
+
67
+// V2 Provenance POC
68
+
69
+func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) {
70
+	routeURL, err := getV2URL(r.indexEndpoint, "version", nil)
71
+	if err != nil {
72
+		return nil, err
73
+	}
74
+
75
+	method := "GET"
76
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
77
+
78
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
79
+	if err != nil {
80
+		return nil, err
81
+	}
82
+	setTokenAuth(req, token)
83
+	res, _, err := r.doRequest(req)
84
+	if err != nil {
85
+		return nil, err
86
+	}
87
+	defer res.Body.Close()
88
+	if res.StatusCode != 200 {
89
+		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res)
90
+	}
91
+
92
+	decoder := json.NewDecoder(res.Body)
93
+	versionInfo := new(RegistryInfo)
94
+
95
+	err = decoder.Decode(versionInfo)
96
+	if err != nil {
97
+		return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err)
98
+	}
99
+
100
+	return versionInfo, nil
101
+}
102
+
103
+//
104
+// 1) Check if TarSum of each layer exists /v2/
105
+//  1.a) if 200, continue
106
+//  1.b) if 300, then push the
107
+//  1.c) if anything else, err
108
+// 2) PUT the created/signed manifest
109
+//
110
+func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) {
111
+	vars := map[string]string{
112
+		"imagename": imageName,
113
+		"tagname":   tagName,
114
+	}
115
+
116
+	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
117
+	if err != nil {
118
+		return nil, err
119
+	}
120
+
121
+	method := "GET"
122
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
123
+
124
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
125
+	if err != nil {
126
+		return nil, err
127
+	}
128
+	setTokenAuth(req, token)
129
+	res, _, err := r.doRequest(req)
130
+	if err != nil {
131
+		return nil, err
132
+	}
133
+	defer res.Body.Close()
134
+	if res.StatusCode != 200 {
135
+		if res.StatusCode == 401 {
136
+			return nil, errLoginRequired
137
+		} else if res.StatusCode == 404 {
138
+			return nil, ErrDoesNotExist
139
+		}
140
+		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s:%s", res.StatusCode, imageName, tagName), res)
141
+	}
142
+
143
+	buf, err := ioutil.ReadAll(res.Body)
144
+	if err != nil {
145
+		return nil, fmt.Errorf("Error while reading the http response: %s", err)
146
+	}
147
+	return buf, nil
148
+}
149
+
150
+// - Succeeded to mount for this image scope
151
+// - Failed with no error (So continue to Push the Blob)
152
+// - Failed with error
153
+func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) {
154
+	vars := map[string]string{
155
+		"imagename": imageName,
156
+		"sumtype":   sumType,
157
+		"sum":       sum,
158
+	}
159
+
160
+	routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars)
161
+	if err != nil {
162
+		return false, err
163
+	}
164
+
165
+	method := "POST"
166
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
167
+
168
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
169
+	if err != nil {
170
+		return false, err
171
+	}
172
+	setTokenAuth(req, token)
173
+	res, _, err := r.doRequest(req)
174
+	if err != nil {
175
+		return false, err
176
+	}
177
+	res.Body.Close() // close early, since we're not needing a body on this call .. yet?
178
+	switch res.StatusCode {
179
+	case 200:
180
+		// return something indicating no push needed
181
+		return true, nil
182
+	case 300:
183
+		// return something indicating blob push needed
184
+		return false, nil
185
+	}
186
+	return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode)
187
+}
188
+
189
+func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error {
190
+	vars := map[string]string{
191
+		"imagename": imageName,
192
+		"sumtype":   sumType,
193
+		"sum":       sum,
194
+	}
195
+
196
+	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
197
+	if err != nil {
198
+		return err
199
+	}
200
+
201
+	method := "GET"
202
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
203
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
204
+	if err != nil {
205
+		return err
206
+	}
207
+	setTokenAuth(req, token)
208
+	res, _, err := r.doRequest(req)
209
+	if err != nil {
210
+		return err
211
+	}
212
+	defer res.Body.Close()
213
+	if res.StatusCode != 200 {
214
+		if res.StatusCode == 401 {
215
+			return errLoginRequired
216
+		}
217
+		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
218
+	}
219
+
220
+	_, err = io.Copy(blobWrtr, res.Body)
221
+	return err
222
+}
223
+
224
+func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) {
225
+	vars := map[string]string{
226
+		"imagename": imageName,
227
+		"sumtype":   sumType,
228
+		"sum":       sum,
229
+	}
230
+
231
+	routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars)
232
+	if err != nil {
233
+		return nil, 0, err
234
+	}
235
+
236
+	method := "GET"
237
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
238
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
239
+	if err != nil {
240
+		return nil, 0, err
241
+	}
242
+	setTokenAuth(req, token)
243
+	res, _, err := r.doRequest(req)
244
+	if err != nil {
245
+		return nil, 0, err
246
+	}
247
+	if res.StatusCode != 200 {
248
+		if res.StatusCode == 401 {
249
+			return nil, 0, errLoginRequired
250
+		}
251
+		return nil, 0, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res)
252
+	}
253
+	lenStr := res.Header.Get("Content-Length")
254
+	l, err := strconv.ParseInt(lenStr, 10, 64)
255
+	if err != nil {
256
+		return nil, 0, err
257
+	}
258
+
259
+	return res.Body, l, err
260
+}
261
+
262
+// Push the image to the server for storage.
263
+// 'layer' is an uncompressed reader of the blob to be pushed.
264
+// The server will generate it's own checksum calculation.
265
+func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) {
266
+	vars := map[string]string{
267
+		"imagename": imageName,
268
+		"sumtype":   sumType,
269
+	}
270
+
271
+	routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars)
272
+	if err != nil {
273
+		return "", err
274
+	}
275
+
276
+	method := "PUT"
277
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
278
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr)
279
+	if err != nil {
280
+		return "", err
281
+	}
282
+	setTokenAuth(req, token)
283
+	res, _, err := r.doRequest(req)
284
+	if err != nil {
285
+		return "", err
286
+	}
287
+	defer res.Body.Close()
288
+	if res.StatusCode != 201 {
289
+		if res.StatusCode == 401 {
290
+			return "", errLoginRequired
291
+		}
292
+		return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res)
293
+	}
294
+
295
+	type sumReturn struct {
296
+		Checksum string `json:"checksum"`
297
+	}
298
+
299
+	decoder := json.NewDecoder(res.Body)
300
+	var sumInfo sumReturn
301
+
302
+	err = decoder.Decode(&sumInfo)
303
+	if err != nil {
304
+		return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err)
305
+	}
306
+
307
+	// XXX this is a json struct from the registry, with its checksum
308
+	return sumInfo.Checksum, nil
309
+}
310
+
311
+// Finally Push the (signed) manifest of the blobs we've just pushed
312
+func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error {
313
+	vars := map[string]string{
314
+		"imagename": imageName,
315
+		"tagname":   tagName,
316
+	}
317
+
318
+	routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars)
319
+	if err != nil {
320
+		return err
321
+	}
322
+
323
+	method := "PUT"
324
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
325
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr)
326
+	if err != nil {
327
+		return err
328
+	}
329
+	setTokenAuth(req, token)
330
+	res, _, err := r.doRequest(req)
331
+	if err != nil {
332
+		return err
333
+	}
334
+	res.Body.Close()
335
+	if res.StatusCode != 201 {
336
+		if res.StatusCode == 401 {
337
+			return errLoginRequired
338
+		}
339
+		return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res)
340
+	}
341
+
342
+	return nil
343
+}
344
+
345
+// Given a repository name, returns a json array of string tags
346
+func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) {
347
+	vars := map[string]string{
348
+		"imagename": imageName,
349
+	}
350
+
351
+	routeURL, err := getV2URL(r.indexEndpoint, "tags", vars)
352
+	if err != nil {
353
+		return nil, err
354
+	}
355
+
356
+	method := "GET"
357
+	log.Debugf("[registry] Calling %q %s", method, routeURL.String())
358
+
359
+	req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil)
360
+	if err != nil {
361
+		return nil, err
362
+	}
363
+	setTokenAuth(req, token)
364
+	res, _, err := r.doRequest(req)
365
+	if err != nil {
366
+		return nil, err
367
+	}
368
+	defer res.Body.Close()
369
+	if res.StatusCode != 200 {
370
+		if res.StatusCode == 401 {
371
+			return nil, errLoginRequired
372
+		} else if res.StatusCode == 404 {
373
+			return nil, ErrDoesNotExist
374
+		}
375
+		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch for %s", res.StatusCode, imageName), res)
376
+	}
377
+
378
+	decoder := json.NewDecoder(res.Body)
379
+	var tags []string
380
+	err = decoder.Decode(&tags)
381
+	if err != nil {
382
+		return nil, fmt.Errorf("Error while decoding the http response: %s", err)
383
+	}
384
+	return tags, nil
385
+}
... ...
@@ -32,6 +32,15 @@ type RegistryInfo struct {
32 32
 	Standalone bool   `json:"standalone"`
33 33
 }
34 34
 
35
+type ManifestData struct {
36
+	Name          string   `json:"name"`
37
+	Tag           string   `json:"tag"`
38
+	Architecture  string   `json:"architecture"`
39
+	BlobSums      []string `json:"blobSums"`
40
+	History       []string `json:"history"`
41
+	SchemaVersion int      `json:"schemaVersion"`
42
+}
43
+
35 44
 type APIVersion int
36 45
 
37 46
 func (av APIVersion) String() string {
... ...
@@ -45,7 +54,6 @@ var apiVersions = map[APIVersion]string{
45 45
 }
46 46
 
47 47
 const (
48
-	_           = iota
49
-	APIVersion1 = iota
48
+	APIVersion1 = iota + 1
50 49
 	APIVersion2
51 50
 )
52 51
new file mode 100644
... ...
@@ -0,0 +1,74 @@
0
+package trust
1
+
2
+import (
3
+	"fmt"
4
+	"time"
5
+
6
+	"github.com/docker/docker/engine"
7
+	"github.com/docker/docker/pkg/log"
8
+	"github.com/docker/libtrust"
9
+)
10
+
11
+func (t *TrustStore) Install(eng *engine.Engine) error {
12
+	for name, handler := range map[string]engine.Handler{
13
+		"trust_key_check":   t.CmdCheckKey,
14
+		"trust_update_base": t.CmdUpdateBase,
15
+	} {
16
+		if err := eng.Register(name, handler); err != nil {
17
+			return fmt.Errorf("Could not register %q: %v", name, err)
18
+		}
19
+	}
20
+	return nil
21
+}
22
+
23
+func (t *TrustStore) CmdCheckKey(job *engine.Job) engine.Status {
24
+	if n := len(job.Args); n != 1 {
25
+		return job.Errorf("Usage: %s NAMESPACE", job.Name)
26
+	}
27
+	var (
28
+		namespace = job.Args[0]
29
+		keyBytes  = job.Getenv("PublicKey")
30
+	)
31
+
32
+	if keyBytes == "" {
33
+		return job.Errorf("Missing PublicKey")
34
+	}
35
+	pk, err := libtrust.UnmarshalPublicKeyJWK([]byte(keyBytes))
36
+	if err != nil {
37
+		return job.Errorf("Error unmarshalling public key: %s", err)
38
+	}
39
+
40
+	permission := uint16(job.GetenvInt("Permission"))
41
+	if permission == 0 {
42
+		permission = 0x03
43
+	}
44
+
45
+	t.RLock()
46
+	defer t.RUnlock()
47
+	if t.graph == nil {
48
+		job.Stdout.Write([]byte("no graph"))
49
+		return engine.StatusOK
50
+	}
51
+
52
+	// Check if any expired grants
53
+	verified, err := t.graph.Verify(pk, namespace, permission)
54
+	if err != nil {
55
+		return job.Errorf("Error verifying key to namespace: %s", namespace)
56
+	}
57
+	if !verified {
58
+		log.Debugf("Verification failed for %s using key %s", namespace, pk.KeyID())
59
+		job.Stdout.Write([]byte("not verified"))
60
+	} else if t.expiration.Before(time.Now()) {
61
+		job.Stdout.Write([]byte("expired"))
62
+	} else {
63
+		job.Stdout.Write([]byte("verified"))
64
+	}
65
+
66
+	return engine.StatusOK
67
+}
68
+
69
+func (t *TrustStore) CmdUpdateBase(job *engine.Job) engine.Status {
70
+	t.fetch()
71
+
72
+	return engine.StatusOK
73
+}
0 74
new file mode 100644
... ...
@@ -0,0 +1,199 @@
0
+package trust
1
+
2
+import (
3
+	"crypto/x509"
4
+	"errors"
5
+	"io/ioutil"
6
+	"net/http"
7
+	"net/url"
8
+	"os"
9
+	"path"
10
+	"path/filepath"
11
+	"sync"
12
+	"time"
13
+
14
+	"github.com/docker/docker/pkg/log"
15
+	"github.com/docker/libtrust/trustgraph"
16
+)
17
+
18
+type TrustStore struct {
19
+	path          string
20
+	caPool        *x509.CertPool
21
+	graph         trustgraph.TrustGraph
22
+	expiration    time.Time
23
+	fetcher       *time.Timer
24
+	fetchTime     time.Duration
25
+	autofetch     bool
26
+	httpClient    *http.Client
27
+	baseEndpoints map[string]*url.URL
28
+
29
+	sync.RWMutex
30
+}
31
+
32
+// defaultFetchtime represents the starting duration to wait between
33
+// fetching sections of the graph.  Unsuccessful fetches should
34
+// increase time between fetching.
35
+const defaultFetchtime = 45 * time.Second
36
+
37
+var baseEndpoints = map[string]string{"official": "https://dvjy3tqbc323p.cloudfront.net/trust/official.json"}
38
+
39
+func NewTrustStore(path string) (*TrustStore, error) {
40
+	abspath, err := filepath.Abs(path)
41
+	if err != nil {
42
+		return nil, err
43
+	}
44
+
45
+	// Create base graph url map
46
+	endpoints := map[string]*url.URL{}
47
+	for name, endpoint := range baseEndpoints {
48
+		u, err := url.Parse(endpoint)
49
+		if err != nil {
50
+			return nil, err
51
+		}
52
+		endpoints[name] = u
53
+	}
54
+
55
+	// Load grant files
56
+	t := &TrustStore{
57
+		path:          abspath,
58
+		caPool:        nil,
59
+		httpClient:    &http.Client{},
60
+		fetchTime:     time.Millisecond,
61
+		baseEndpoints: endpoints,
62
+	}
63
+
64
+	err = t.reload()
65
+	if err != nil {
66
+		return nil, err
67
+	}
68
+
69
+	return t, nil
70
+}
71
+
72
+func (t *TrustStore) reload() error {
73
+	t.Lock()
74
+	defer t.Unlock()
75
+
76
+	matches, err := filepath.Glob(filepath.Join(t.path, "*.json"))
77
+	if err != nil {
78
+		return err
79
+	}
80
+	statements := make([]*trustgraph.Statement, len(matches))
81
+	for i, match := range matches {
82
+		f, err := os.Open(match)
83
+		if err != nil {
84
+			return err
85
+		}
86
+		statements[i], err = trustgraph.LoadStatement(f, nil)
87
+		if err != nil {
88
+			f.Close()
89
+			return err
90
+		}
91
+		f.Close()
92
+	}
93
+	if len(statements) == 0 {
94
+		if t.autofetch {
95
+			log.Debugf("No grants, fetching")
96
+			t.fetcher = time.AfterFunc(t.fetchTime, t.fetch)
97
+		}
98
+		return nil
99
+	}
100
+
101
+	grants, expiration, err := trustgraph.CollapseStatements(statements, true)
102
+	if err != nil {
103
+		return err
104
+	}
105
+
106
+	t.expiration = expiration
107
+	t.graph = trustgraph.NewMemoryGraph(grants)
108
+	log.Debugf("Reloaded graph with %d grants expiring at %s", len(grants), expiration)
109
+
110
+	if t.autofetch {
111
+		nextFetch := expiration.Sub(time.Now())
112
+		if nextFetch < 0 {
113
+			nextFetch = defaultFetchtime
114
+		} else {
115
+			nextFetch = time.Duration(0.8 * (float64)(nextFetch))
116
+		}
117
+		t.fetcher = time.AfterFunc(nextFetch, t.fetch)
118
+	}
119
+
120
+	return nil
121
+}
122
+
123
+func (t *TrustStore) fetchBaseGraph(u *url.URL) (*trustgraph.Statement, error) {
124
+	req := &http.Request{
125
+		Method:     "GET",
126
+		URL:        u,
127
+		Proto:      "HTTP/1.1",
128
+		ProtoMajor: 1,
129
+		ProtoMinor: 1,
130
+		Header:     make(http.Header),
131
+		Body:       nil,
132
+		Host:       u.Host,
133
+	}
134
+
135
+	resp, err := t.httpClient.Do(req)
136
+	if err != nil {
137
+		return nil, err
138
+	}
139
+	if resp.StatusCode == 404 {
140
+		return nil, errors.New("base graph does not exist")
141
+	}
142
+
143
+	defer resp.Body.Close()
144
+
145
+	return trustgraph.LoadStatement(resp.Body, t.caPool)
146
+}
147
+
148
+// fetch retrieves updated base graphs.  This function cannot error, it
149
+// should only log errors
150
+func (t *TrustStore) fetch() {
151
+	t.Lock()
152
+	defer t.Unlock()
153
+
154
+	if t.autofetch && t.fetcher == nil {
155
+		// Do nothing ??
156
+		return
157
+	}
158
+
159
+	fetchCount := 0
160
+	for bg, ep := range t.baseEndpoints {
161
+		statement, err := t.fetchBaseGraph(ep)
162
+		if err != nil {
163
+			log.Infof("Trust graph fetch failed: %s", err)
164
+			continue
165
+		}
166
+		b, err := statement.Bytes()
167
+		if err != nil {
168
+			log.Infof("Bad trust graph statement: %s", err)
169
+			continue
170
+		}
171
+		// TODO check if value differs
172
+		err = ioutil.WriteFile(path.Join(t.path, bg+".json"), b, 0600)
173
+		if err != nil {
174
+			log.Infof("Error writing trust graph statement: %s", err)
175
+		}
176
+		fetchCount++
177
+	}
178
+	log.Debugf("Fetched %d base graphs at %s", fetchCount, time.Now())
179
+
180
+	if fetchCount > 0 {
181
+		go func() {
182
+			err := t.reload()
183
+			if err != nil {
184
+				// TODO log
185
+				log.Infof("Reload of trust graph failed: %s", err)
186
+			}
187
+		}()
188
+		t.fetchTime = defaultFetchtime
189
+		t.fetcher = nil
190
+	} else if t.autofetch {
191
+		maxTime := 10 * defaultFetchtime
192
+		t.fetchTime = time.Duration(1.5 * (float64)(t.fetchTime+time.Second))
193
+		if t.fetchTime > maxTime {
194
+			t.fetchTime = maxTime
195
+		}
196
+		t.fetcher = time.AfterFunc(t.fetchTime, t.fetch)
197
+	}
198
+}