Browse code

builder-next: support for inline cache from local images

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2019/03/15 06:27:18
Showing 2 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,163 @@
0
+package localinlinecache
1
+
2
+import (
3
+	"context"
4
+	"encoding/json"
5
+	"time"
6
+
7
+	"github.com/containerd/containerd/content"
8
+	"github.com/containerd/containerd/images"
9
+	distreference "github.com/docker/distribution/reference"
10
+	imagestore "github.com/docker/docker/image"
11
+	"github.com/docker/docker/reference"
12
+	"github.com/moby/buildkit/cache/remotecache"
13
+	registryremotecache "github.com/moby/buildkit/cache/remotecache/registry"
14
+	v1 "github.com/moby/buildkit/cache/remotecache/v1"
15
+	"github.com/moby/buildkit/session"
16
+	"github.com/moby/buildkit/solver"
17
+	"github.com/moby/buildkit/util/resolver"
18
+	"github.com/moby/buildkit/worker"
19
+	digest "github.com/opencontainers/go-digest"
20
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
21
+	"github.com/pkg/errors"
22
+)
23
+
24
+// ResolveCacheImporterFunc returns a resolver function for local inline cache
25
+func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
26
+
27
+	upstream := registryremotecache.ResolveCacheImporterFunc(sm, resolverOpt)
28
+
29
+	return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
30
+		if dt, err := tryImportLocal(rs, is, attrs["ref"]); err == nil {
31
+			return newLocalImporter(dt), specs.Descriptor{}, nil
32
+		}
33
+		return upstream(ctx, attrs)
34
+	}
35
+}
36
+
37
+func tryImportLocal(rs reference.Store, is imagestore.Store, refStr string) ([]byte, error) {
38
+	ref, err := distreference.ParseNormalizedNamed(refStr)
39
+	if err != nil {
40
+		return nil, err
41
+	}
42
+	dgst, err := rs.Get(ref)
43
+	if err != nil {
44
+		return nil, err
45
+	}
46
+	img, err := is.Get(imagestore.ID(dgst))
47
+	if err != nil {
48
+		return nil, err
49
+	}
50
+
51
+	return img.RawJSON(), nil
52
+}
53
+
54
+func newLocalImporter(dt []byte) remotecache.Importer {
55
+	return &localImporter{dt: dt}
56
+}
57
+
58
+type localImporter struct {
59
+	dt []byte
60
+}
61
+
62
+func (li *localImporter) Resolve(ctx context.Context, _ specs.Descriptor, id string, w worker.Worker) (solver.CacheManager, error) {
63
+	cc := v1.NewCacheChains()
64
+	if err := li.importInlineCache(ctx, li.dt, cc); err != nil {
65
+		return nil, err
66
+	}
67
+
68
+	keysStorage, resultStorage, err := v1.NewCacheKeyStorage(cc, w)
69
+	if err != nil {
70
+		return nil, err
71
+	}
72
+	return solver.NewCacheManager(id, keysStorage, resultStorage), nil
73
+}
74
+
75
+func (li *localImporter) importInlineCache(ctx context.Context, dt []byte, cc solver.CacheExporterTarget) error {
76
+	var img image
77
+
78
+	if err := json.Unmarshal(dt, &img); err != nil {
79
+		return err
80
+	}
81
+
82
+	if img.Cache == nil {
83
+		return nil
84
+	}
85
+
86
+	var config v1.CacheConfig
87
+	if err := json.Unmarshal(img.Cache, &config.Records); err != nil {
88
+		return err
89
+	}
90
+
91
+	createdDates, createdMsg, err := parseCreatedLayerInfo(img)
92
+	if err != nil {
93
+		return err
94
+	}
95
+
96
+	layers := v1.DescriptorProvider{}
97
+	for i, diffID := range img.Rootfs.DiffIDs {
98
+		dgst := digest.Digest(diffID.String())
99
+		desc := specs.Descriptor{
100
+			Digest:      dgst,
101
+			Size:        -1,
102
+			MediaType:   images.MediaTypeDockerSchema2Layer,
103
+			Annotations: map[string]string{},
104
+		}
105
+		if createdAt := createdDates[i]; createdAt != "" {
106
+			desc.Annotations["buildkit/createdat"] = createdAt
107
+		}
108
+		if createdBy := createdMsg[i]; createdBy != "" {
109
+			desc.Annotations["buildkit/description"] = createdBy
110
+		}
111
+		desc.Annotations["containerd.io/uncompressed"] = img.Rootfs.DiffIDs[i].String()
112
+		layers[dgst] = v1.DescriptorProviderPair{
113
+			Descriptor: desc,
114
+			Provider:   &emptyProvider{},
115
+		}
116
+		config.Layers = append(config.Layers, v1.CacheLayer{
117
+			Blob:        dgst,
118
+			ParentIndex: i - 1,
119
+		})
120
+	}
121
+
122
+	return v1.ParseConfig(config, layers, cc)
123
+}
124
+
125
+type image struct {
126
+	Rootfs struct {
127
+		DiffIDs []digest.Digest `json:"diff_ids"`
128
+	} `json:"rootfs"`
129
+	Cache   []byte `json:"moby.buildkit.cache.v0"`
130
+	History []struct {
131
+		Created    *time.Time `json:"created,omitempty"`
132
+		CreatedBy  string     `json:"created_by,omitempty"`
133
+		EmptyLayer bool       `json:"empty_layer,omitempty"`
134
+	} `json:"history,omitempty"`
135
+}
136
+
137
+func parseCreatedLayerInfo(img image) ([]string, []string, error) {
138
+	dates := make([]string, 0, len(img.Rootfs.DiffIDs))
139
+	createdBy := make([]string, 0, len(img.Rootfs.DiffIDs))
140
+	for _, h := range img.History {
141
+		if !h.EmptyLayer {
142
+			str := ""
143
+			if h.Created != nil {
144
+				dt, err := h.Created.MarshalText()
145
+				if err != nil {
146
+					return nil, nil, err
147
+				}
148
+				str = string(dt)
149
+			}
150
+			dates = append(dates, str)
151
+			createdBy = append(createdBy, h.CreatedBy)
152
+		}
153
+	}
154
+	return dates, createdBy, nil
155
+}
156
+
157
+type emptyProvider struct {
158
+}
159
+
160
+func (p *emptyProvider) ReaderAt(ctx context.Context, dec specs.Descriptor) (content.ReaderAt, error) {
161
+	return nil, errors.Errorf("ReaderAt not implemented for empty provider")
162
+}
... ...
@@ -8,6 +8,7 @@ import (
8 8
 	"github.com/containerd/containerd/content/local"
9 9
 	"github.com/docker/docker/api/types"
10 10
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
11
+	"github.com/docker/docker/builder/builder-next/adapters/localinlinecache"
11 12
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
12 13
 	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
13 14
 	"github.com/docker/docker/builder/builder-next/imagerefchecker"
... ...
@@ -19,7 +20,6 @@ import (
19 19
 	"github.com/moby/buildkit/cache/metadata"
20 20
 	"github.com/moby/buildkit/cache/remotecache"
21 21
 	inlineremotecache "github.com/moby/buildkit/cache/remotecache/inline"
22
-	registryremotecache "github.com/moby/buildkit/cache/remotecache/registry"
23 22
 	"github.com/moby/buildkit/client"
24 23
 	"github.com/moby/buildkit/control"
25 24
 	"github.com/moby/buildkit/frontend"
... ...
@@ -175,7 +175,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
175 175
 		Frontends:        frontends,
176 176
 		CacheKeyStorage:  cacheStorage,
177 177
 		ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{
178
-			"registry": registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt),
178
+			"registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt, dist.ReferenceStore, dist.ImageStore),
179 179
 		},
180 180
 		ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
181 181
 			"inline": inlineremotecache.ResolveCacheExporterFunc(),