Browse code

Push flow

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)

Derek McGowan authored on 2014/10/01 09:03:57
Showing 6 changed files
... ...
@@ -43,6 +43,7 @@ import (
43 43
 	"github.com/docker/docker/registry"
44 44
 	"github.com/docker/docker/runconfig"
45 45
 	"github.com/docker/docker/utils"
46
+	"github.com/docker/libtrust"
46 47
 )
47 48
 
48 49
 const (
... ...
@@ -1215,6 +1216,26 @@ func (cli *DockerCli) CmdPush(args ...string) error {
1215 1215
 
1216 1216
 	v := url.Values{}
1217 1217
 	v.Set("tag", tag)
1218
+
1219
+	body, _, err := readBody(cli.call("GET", "/images/"+remote+"/manifest?"+v.Encode(), nil, false))
1220
+	if err != nil {
1221
+		return err
1222
+	}
1223
+
1224
+	js, err := libtrust.NewJSONSignature(body)
1225
+	if err != nil {
1226
+		return err
1227
+	}
1228
+	err = js.Sign(cli.key)
1229
+	if err != nil {
1230
+		return err
1231
+	}
1232
+
1233
+	signedBody, err := js.PrettySignature("signatures")
1234
+	if err != nil {
1235
+		return err
1236
+	}
1237
+
1218 1238
 	push := func(authConfig registry.AuthConfig) error {
1219 1239
 		buf, err := json.Marshal(authConfig)
1220 1240
 		if err != nil {
... ...
@@ -1224,7 +1245,7 @@ func (cli *DockerCli) CmdPush(args ...string) error {
1224 1224
 			base64.URLEncoding.EncodeToString(buf),
1225 1225
 		}
1226 1226
 
1227
-		return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), nil, cli.out, map[string][]string{
1227
+		return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), bytes.NewReader(signedBody), cli.out, map[string][]string{
1228 1228
 			"X-Registry-Auth": registryAuthHeader,
1229 1229
 		})
1230 1230
 	}
... ...
@@ -608,6 +608,18 @@ func getImagesSearch(eng *engine.Engine, version version.Version, w http.Respons
608 608
 	return job.Run()
609 609
 }
610 610
 
611
+func getImageManifest(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
612
+	if err := parseForm(r); err != nil {
613
+		return err
614
+	}
615
+
616
+	job := eng.Job("image_manifest", vars["name"])
617
+	job.Setenv("tag", r.Form.Get("tag"))
618
+	job.Stdout.Add(utils.NewWriteFlusher(w))
619
+
620
+	return job.Run()
621
+}
622
+
611 623
 func postImagesPush(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
612 624
 	if vars == nil {
613 625
 		return fmt.Errorf("Missing parameter")
... ...
@@ -639,9 +651,15 @@ func postImagesPush(eng *engine.Engine, version version.Version, w http.Response
639 639
 		}
640 640
 	}
641 641
 
642
+	manifest, err := ioutil.ReadAll(r.Body)
643
+	if err != nil {
644
+		return err
645
+	}
646
+
642 647
 	job := eng.Job("push", vars["name"])
643 648
 	job.SetenvJson("metaHeaders", metaHeaders)
644 649
 	job.SetenvJson("authConfig", authConfig)
650
+	job.Setenv("manifest", string(manifest))
645 651
 	job.Setenv("tag", r.Form.Get("tag"))
646 652
 	if version.GreaterThan("1.0") {
647 653
 		job.SetenvBool("json", true)
... ...
@@ -1294,6 +1312,7 @@ func createRouter(eng *engine.Engine, logging, enableCors bool, dockerVersion st
1294 1294
 			"/images/viz":                     getImagesViz,
1295 1295
 			"/images/search":                  getImagesSearch,
1296 1296
 			"/images/get":                     getImagesGet,
1297
+			"/images/{name:.*}/manifest":      getImageManifest,
1297 1298
 			"/images/{name:.*}/get":           getImagesGet,
1298 1299
 			"/images/{name:.*}/history":       getImagesHistory,
1299 1300
 			"/images/{name:.*}/json":          getImagesByName,
1300 1301
new file mode 100644
... ...
@@ -0,0 +1,116 @@
0
+package graph
1
+
2
+import (
3
+	"encoding/json"
4
+	"fmt"
5
+	"io"
6
+	"io/ioutil"
7
+	"path"
8
+
9
+	"github.com/docker/docker/engine"
10
+	"github.com/docker/docker/pkg/tarsum"
11
+	"github.com/docker/docker/registry"
12
+	"github.com/docker/docker/runconfig"
13
+)
14
+
15
+func (s *TagStore) CmdManifest(job *engine.Job) engine.Status {
16
+	if len(job.Args) != 1 {
17
+		return job.Errorf("usage: %s NAME", job.Name)
18
+	}
19
+	name := job.Args[0]
20
+	tag := job.Getenv("tag")
21
+	if tag == "" {
22
+		tag = "latest"
23
+	}
24
+
25
+	// Resolve the Repository name from fqn to endpoint + name
26
+	_, remoteName, err := registry.ResolveRepositoryName(name)
27
+	if err != nil {
28
+		return job.Error(err)
29
+	}
30
+
31
+	manifest := &registry.ManifestData{
32
+		Name:          remoteName,
33
+		Tag:           tag,
34
+		SchemaVersion: 1,
35
+	}
36
+	localRepo, exists := s.Repositories[name]
37
+	if !exists {
38
+		return job.Errorf("Repo does not exist: %s", name)
39
+	}
40
+
41
+	layerId, exists := localRepo[tag]
42
+	if !exists {
43
+		return job.Errorf("Tag does not exist for %s: %s", name, tag)
44
+	}
45
+	tarsums := make([]string, 0, 4)
46
+	layersSeen := make(map[string]bool)
47
+
48
+	layer, err := s.graph.Get(layerId)
49
+	if err != nil {
50
+		return job.Error(err)
51
+	}
52
+	if layer.Config == nil {
53
+		return job.Errorf("Missing layer configuration")
54
+	}
55
+	manifest.Architecture = layer.Architecture
56
+	var metadata runconfig.Config
57
+	metadata = *layer.Config
58
+	history := make([]string, 0, cap(tarsums))
59
+
60
+	for ; layer != nil; layer, err = layer.GetParent() {
61
+		if err != nil {
62
+			return job.Error(err)
63
+		}
64
+
65
+		if layersSeen[layer.ID] {
66
+			break
67
+		}
68
+		if layer.Config != nil && metadata.Image != layer.ID {
69
+			err = runconfig.Merge(&metadata, layer.Config)
70
+			if err != nil {
71
+				return job.Error(err)
72
+			}
73
+		}
74
+
75
+		archive, err := layer.TarLayer()
76
+		if err != nil {
77
+			return job.Error(err)
78
+		}
79
+
80
+		tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version0)
81
+		if err != nil {
82
+			return job.Error(err)
83
+		}
84
+		if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
85
+			return job.Error(err)
86
+		}
87
+
88
+		tarId := tarSum.Sum(nil)
89
+		// Save tarsum to image json
90
+
91
+		tarsums = append(tarsums, tarId)
92
+
93
+		layersSeen[layer.ID] = true
94
+		jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json"))
95
+		if err != nil {
96
+			return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err))
97
+		}
98
+		history = append(history, string(jsonData))
99
+	}
100
+
101
+	manifest.BlobSums = tarsums
102
+	manifest.History = history
103
+
104
+	manifestBytes, err := json.MarshalIndent(manifest, "", "   ")
105
+	if err != nil {
106
+		return job.Error(err)
107
+	}
108
+
109
+	_, err = job.Stdout.Write(manifestBytes)
110
+	if err != nil {
111
+		return job.Error(err)
112
+	}
113
+
114
+	return engine.StatusOK
115
+}
... ...
@@ -1,15 +1,18 @@
1 1
 package graph
2 2
 
3 3
 import (
4
+	"bytes"
4 5
 	"fmt"
5 6
 	"io"
6 7
 	"io/ioutil"
7 8
 	"os"
8 9
 	"path"
10
+	"strings"
9 11
 	"sync"
10 12
 
11 13
 	log "github.com/Sirupsen/logrus"
12 14
 	"github.com/docker/docker/engine"
15
+	"github.com/docker/docker/image"
13 16
 	"github.com/docker/docker/pkg/archive"
14 17
 	"github.com/docker/docker/registry"
15 18
 	"github.com/docker/docker/utils"
... ...
@@ -267,6 +270,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status {
267 267
 	}
268 268
 
269 269
 	tag := job.Getenv("tag")
270
+	manifestBytes := job.Getenv("manifest")
270 271
 	job.GetenvJson("authConfig", authConfig)
271 272
 	job.GetenvJson("metaHeaders", &metaHeaders)
272 273
 
... ...
@@ -286,6 +290,92 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status {
286 286
 		return job.Error(err2)
287 287
 	}
288 288
 
289
+	var isOfficial bool
290
+	if endpoint.String() == registry.IndexServerAddress() {
291
+		isOfficial = isOfficialName(remoteName)
292
+		if isOfficial && strings.IndexRune(remoteName, '/') == -1 {
293
+			remoteName = "library/" + remoteName
294
+		}
295
+	}
296
+
297
+	if len(tag) == 0 {
298
+		tag = DEFAULTTAG
299
+	}
300
+	if isOfficial || endpoint.Version == registry.APIVersion2 {
301
+		j := job.Eng.Job("trust_update_base")
302
+		if err = j.Run(); err != nil {
303
+			return job.Errorf("error updating trust base graph: %s", err)
304
+		}
305
+
306
+		repoData, err := r.PushImageJSONIndex(remoteName, []*registry.ImgData{}, false, nil)
307
+		if err != nil {
308
+			return job.Error(err)
309
+		}
310
+
311
+		// try via manifest
312
+		manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes))
313
+		if err != nil {
314
+			return job.Errorf("error verifying manifest: %s", err)
315
+		}
316
+
317
+		if len(manifest.FSLayers) != len(manifest.History) {
318
+			return job.Errorf("length of history not equal to number of layers")
319
+		}
320
+
321
+		if !verified {
322
+			log.Debugf("Pushing unverified image")
323
+		}
324
+
325
+		for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
326
+			var (
327
+				sumStr  = manifest.FSLayers[i].BlobSum
328
+				imgJSON = []byte(manifest.History[i].V1Compatibility)
329
+			)
330
+
331
+			sumParts := strings.SplitN(sumStr, ":", 2)
332
+			if len(sumParts) < 2 {
333
+				return job.Errorf("Invalid checksum: %s", sumStr)
334
+			}
335
+			manifestSum := sumParts[1]
336
+
337
+			// for each layer, check if it exists ...
338
+			// XXX wait this requires having the TarSum of the layer.tar first
339
+			// skip this step for now. Just push the layer every time for this naive implementation
340
+			//shouldPush, err := r.PostV2ImageMountBlob(imageName, sumType, sum string, token []string)
341
+
342
+			img, err := image.NewImgJSON(imgJSON)
343
+			if err != nil {
344
+				return job.Errorf("Failed to parse json: %s", err)
345
+			}
346
+
347
+			img, err = s.graph.Get(img.ID)
348
+			if err != nil {
349
+				return job.Error(err)
350
+			}
351
+
352
+			arch, err := img.TarLayer()
353
+			if err != nil {
354
+				return job.Errorf("Could not get tar layer: %s", err)
355
+			}
356
+
357
+			_, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens)
358
+			if err != nil {
359
+				job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil))
360
+				return job.Error(err)
361
+			}
362
+			job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil))
363
+		}
364
+
365
+		// push the manifest
366
+		err = r.PutV2ImageManifest(remoteName, tag, bytes.NewReader([]byte(manifestBytes)), repoData.Tokens)
367
+		if err != nil {
368
+			return job.Error(err)
369
+		}
370
+
371
+		// done, no fallback to V1
372
+		return engine.StatusOK
373
+	}
374
+
289 375
 	if err != nil {
290 376
 		reposLen := 1
291 377
 		if tag == "" {
... ...
@@ -25,6 +25,7 @@ func (s *TagStore) Install(eng *engine.Engine) error {
25 25
 		"import":         s.CmdImport,
26 26
 		"pull":           s.CmdPull,
27 27
 		"push":           s.CmdPush,
28
+		"image_manifest": s.CmdManifest,
28 29
 	} {
29 30
 		if err := eng.Register(name, handler); err != nil {
30 31
 			return fmt.Errorf("Could not register %q: %v", name, err)
... ...
@@ -267,7 +267,7 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s
267 267
 // Push the image to the server for storage.
268 268
 // 'layer' is an uncompressed reader of the blob to be pushed.
269 269
 // The server will generate it's own checksum calculation.
270
-func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) {
270
+func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) {
271 271
 	vars := map[string]string{
272 272
 		"imagename": imageName,
273 273
 		"sumtype":   sumType,
... ...
@@ -285,6 +285,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t
285 285
 		return "", err
286 286
 	}
287 287
 	setTokenAuth(req, token)
288
+	req.Header.Set("X-Tarsum", sumStr)
288 289
 	res, _, err := r.doRequest(req)
289 290
 	if err != nil {
290 291
 		return "", err
... ...
@@ -309,6 +310,10 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t
309 309
 		return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err)
310 310
 	}
311 311
 
312
+	if sumInfo.Checksum != sumStr {
313
+		return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr)
314
+	}
315
+
312 316
 	// XXX this is a json struct from the registry, with its checksum
313 317
 	return sumInfo.Checksum, nil
314 318
 }