Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
| ... | ... |
@@ -45,7 +45,6 @@ import ( |
| 45 | 45 |
|
| 46 | 46 |
// SourceOpt is options for creating the image source |
| 47 | 47 |
type SourceOpt struct {
|
| 48 |
- SessionManager *session.Manager |
|
| 49 | 48 |
ContentStore content.Store |
| 50 | 49 |
CacheAccessor cache.Accessor |
| 51 | 50 |
ReferenceStore reference.Store |
| ... | ... |
@@ -73,19 +72,19 @@ func (is *imageSource) ID() string {
|
| 73 | 73 |
return source.DockerImageScheme |
| 74 | 74 |
} |
| 75 | 75 |
|
| 76 |
-func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string) remotes.Resolver {
|
|
| 76 |
+func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string, sm *session.Manager) remotes.Resolver {
|
|
| 77 | 77 |
opt := docker.ResolverOptions{
|
| 78 | 78 |
Client: tracing.DefaultClient, |
| 79 | 79 |
} |
| 80 | 80 |
if rfn != nil {
|
| 81 | 81 |
opt = rfn(ref) |
| 82 | 82 |
} |
| 83 |
- opt.Credentials = is.getCredentialsFromSession(ctx) |
|
| 83 |
+ opt.Credentials = is.getCredentialsFromSession(ctx, sm) |
|
| 84 | 84 |
r := docker.NewResolver(opt) |
| 85 | 85 |
return r |
| 86 | 86 |
} |
| 87 | 87 |
|
| 88 |
-func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
|
|
| 88 |
+func (is *imageSource) getCredentialsFromSession(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {
|
|
| 89 | 89 |
id := session.FromContext(ctx) |
| 90 | 90 |
if id == "" {
|
| 91 | 91 |
// can be removed after containerd/containerd#2812 |
| ... | ... |
@@ -97,7 +96,7 @@ func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(strin |
| 97 | 97 |
timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| 98 | 98 |
defer cancel() |
| 99 | 99 |
|
| 100 |
- caller, err := is.SessionManager.Get(timeoutCtx, id) |
|
| 100 |
+ caller, err := sm.Get(timeoutCtx, id) |
|
| 101 | 101 |
if err != nil {
|
| 102 | 102 |
return "", "", err |
| 103 | 103 |
} |
| ... | ... |
@@ -122,13 +121,13 @@ func (is *imageSource) resolveLocal(refStr string) ([]byte, error) {
|
| 122 | 122 |
return img.RawJSON(), nil |
| 123 | 123 |
} |
| 124 | 124 |
|
| 125 |
-func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *ocispec.Platform) (digest.Digest, []byte, error) {
|
|
| 125 |
+func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *ocispec.Platform, sm *session.Manager) (digest.Digest, []byte, error) {
|
|
| 126 | 126 |
type t struct {
|
| 127 | 127 |
dgst digest.Digest |
| 128 | 128 |
dt []byte |
| 129 | 129 |
} |
| 130 | 130 |
res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
|
| 131 |
- dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref), is.ContentStore, platform) |
|
| 131 |
+ dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, platform) |
|
| 132 | 132 |
if err != nil {
|
| 133 | 133 |
return nil, err |
| 134 | 134 |
} |
| ... | ... |
@@ -142,14 +141,14 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform * |
| 142 | 142 |
return typed.dgst, typed.dt, nil |
| 143 | 143 |
} |
| 144 | 144 |
|
| 145 |
-func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt) (digest.Digest, []byte, error) {
|
|
| 145 |
+func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error) {
|
|
| 146 | 146 |
resolveMode, err := source.ParseImageResolveMode(opt.ResolveMode) |
| 147 | 147 |
if err != nil {
|
| 148 | 148 |
return "", nil, err |
| 149 | 149 |
} |
| 150 | 150 |
switch resolveMode {
|
| 151 | 151 |
case source.ResolveModeForcePull: |
| 152 |
- dgst, dt, err := is.resolveRemote(ctx, ref, opt.Platform) |
|
| 152 |
+ dgst, dt, err := is.resolveRemote(ctx, ref, opt.Platform, sm) |
|
| 153 | 153 |
// TODO: pull should fallback to local in case of failure to allow offline behavior |
| 154 | 154 |
// the fallback doesn't work currently |
| 155 | 155 |
return dgst, dt, err |
| ... | ... |
@@ -171,13 +170,13 @@ func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt g |
| 171 | 171 |
return "", dt, err |
| 172 | 172 |
} |
| 173 | 173 |
// fallback to remote |
| 174 |
- return is.resolveRemote(ctx, ref, opt.Platform) |
|
| 174 |
+ return is.resolveRemote(ctx, ref, opt.Platform, sm) |
|
| 175 | 175 |
} |
| 176 | 176 |
// should never happen |
| 177 | 177 |
return "", nil, fmt.Errorf("builder cannot resolve image %s: invalid mode %q", ref, opt.ResolveMode)
|
| 178 | 178 |
} |
| 179 | 179 |
|
| 180 |
-func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
|
|
| 180 |
+func (is *imageSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) {
|
|
| 181 | 181 |
imageIdentifier, ok := id.(*source.ImageIdentifier) |
| 182 | 182 |
if !ok {
|
| 183 | 183 |
return nil, errors.Errorf("invalid image identifier %v", id)
|
| ... | ... |
@@ -191,8 +190,9 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc |
| 191 | 191 |
p := &puller{
|
| 192 | 192 |
src: imageIdentifier, |
| 193 | 193 |
is: is, |
| 194 |
- resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String()), |
|
| 194 |
+ resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String(), sm), |
|
| 195 | 195 |
platform: platform, |
| 196 |
+ sm: sm, |
|
| 196 | 197 |
} |
| 197 | 198 |
return p, nil |
| 198 | 199 |
} |
| ... | ... |
@@ -208,6 +208,7 @@ type puller struct {
|
| 208 | 208 |
resolver remotes.Resolver |
| 209 | 209 |
config []byte |
| 210 | 210 |
platform ocispec.Platform |
| 211 |
+ sm *session.Manager |
|
| 211 | 212 |
} |
| 212 | 213 |
|
| 213 | 214 |
func (p *puller) mainManifestKey(dgst digest.Digest, platform ocispec.Platform) (digest.Digest, error) {
|
| ... | ... |
@@ -294,7 +295,7 @@ func (p *puller) resolve(ctx context.Context) error {
|
| 294 | 294 |
resolveProgressDone(err) |
| 295 | 295 |
return |
| 296 | 296 |
} |
| 297 |
- _, dt, err := p.is.ResolveImageConfig(ctx, ref.String(), gw.ResolveImageConfigOpt{Platform: &p.platform, ResolveMode: resolveModeToString(p.src.ResolveMode)})
|
|
| 297 |
+ _, dt, err := p.is.ResolveImageConfig(ctx, ref.String(), gw.ResolveImageConfigOpt{Platform: &p.platform, ResolveMode: resolveModeToString(p.src.ResolveMode)}, p.sm)
|
|
| 298 | 298 |
if err != nil {
|
| 299 | 299 |
p.resolveErr = err |
| 300 | 300 |
resolveProgressDone(err) |
| ... | ... |
@@ -420,7 +421,7 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
|
| 420 | 420 |
) |
| 421 | 421 |
} |
| 422 | 422 |
|
| 423 |
- if err := images.Dispatch(ctx, images.Handlers(handlers...), p.desc); err != nil {
|
|
| 423 |
+ if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, p.desc); err != nil {
|
|
| 424 | 424 |
stopProgress() |
| 425 | 425 |
return nil, err |
| 426 | 426 |
} |
| ... | ... |
@@ -5,6 +5,7 @@ import ( |
| 5 | 5 |
"fmt" |
| 6 | 6 |
"io" |
| 7 | 7 |
"net" |
| 8 |
+ "strconv" |
|
| 8 | 9 |
"strings" |
| 9 | 10 |
"sync" |
| 10 | 11 |
"time" |
| ... | ... |
@@ -318,6 +319,16 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. |
| 318 | 318 |
exporterAttrs["name"] = strings.Join(opt.Options.Tags, ",") |
| 319 | 319 |
} |
| 320 | 320 |
|
| 321 |
+ cache := controlapi.CacheOptions{}
|
|
| 322 |
+ |
|
| 323 |
+ if inlineCache := opt.Options.BuildArgs["BUILDKIT_INLINE_CACHE"]; inlineCache != nil {
|
|
| 324 |
+ if b, err := strconv.ParseBool(*inlineCache); err == nil && b {
|
|
| 325 |
+ cache.Exports = append(cache.Exports, &controlapi.CacheOptionsEntry{
|
|
| 326 |
+ Type: "inline", |
|
| 327 |
+ }) |
|
| 328 |
+ } |
|
| 329 |
+ } |
|
| 330 |
+ |
|
| 321 | 331 |
req := &controlapi.SolveRequest{
|
| 322 | 332 |
Ref: id, |
| 323 | 333 |
Exporter: "moby", |
| ... | ... |
@@ -325,6 +336,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. |
| 325 | 325 |
Frontend: "dockerfile.v0", |
| 326 | 326 |
FrontendAttrs: frontendAttrs, |
| 327 | 327 |
Session: opt.Options.SessionID, |
| 328 |
+ Cache: cache, |
|
| 328 | 329 |
} |
| 329 | 330 |
|
| 330 | 331 |
if opt.Options.NetworkMode == "host" {
|
| ... | ... |
@@ -18,10 +18,10 @@ import ( |
| 18 | 18 |
"github.com/moby/buildkit/cache" |
| 19 | 19 |
"github.com/moby/buildkit/cache/metadata" |
| 20 | 20 |
"github.com/moby/buildkit/cache/remotecache" |
| 21 |
+ inlineremotecache "github.com/moby/buildkit/cache/remotecache/inline" |
|
| 21 | 22 |
registryremotecache "github.com/moby/buildkit/cache/remotecache/registry" |
| 22 | 23 |
"github.com/moby/buildkit/client" |
| 23 | 24 |
"github.com/moby/buildkit/control" |
| 24 |
- "github.com/moby/buildkit/exporter" |
|
| 25 | 25 |
"github.com/moby/buildkit/frontend" |
| 26 | 26 |
dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder" |
| 27 | 27 |
"github.com/moby/buildkit/frontend/gateway" |
| ... | ... |
@@ -95,7 +95,6 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 95 | 95 |
} |
| 96 | 96 |
|
| 97 | 97 |
src, err := containerimage.NewSource(containerimage.SourceOpt{
|
| 98 |
- SessionManager: opt.SessionManager, |
|
| 99 | 98 |
CacheAccessor: cm, |
| 100 | 99 |
ContentStore: store, |
| 101 | 100 |
DownloadManager: dist.DownloadManager, |
| ... | ... |
@@ -139,7 +138,6 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 139 | 139 |
|
| 140 | 140 |
wopt := mobyworker.Opt{
|
| 141 | 141 |
ID: "moby", |
| 142 |
- SessionManager: opt.SessionManager, |
|
| 143 | 142 |
MetadataStore: md, |
| 144 | 143 |
ContentStore: store, |
| 145 | 144 |
CacheManager: cm, |
| ... | ... |
@@ -149,10 +147,8 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 149 | 149 |
ImageSource: src, |
| 150 | 150 |
DownloadManager: dist.DownloadManager, |
| 151 | 151 |
V2MetadataService: dist.V2MetadataService, |
| 152 |
- Exporters: map[string]exporter.Exporter{
|
|
| 153 |
- "moby": exp, |
|
| 154 |
- }, |
|
| 155 |
- Transport: rt, |
|
| 152 |
+ Exporter: exp, |
|
| 153 |
+ Transport: rt, |
|
| 156 | 154 |
} |
| 157 | 155 |
|
| 158 | 156 |
wc := &worker.Controller{}
|
| ... | ... |
@@ -175,6 +171,9 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 175 | 175 |
ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{
|
| 176 | 176 |
"registry": registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt), |
| 177 | 177 |
}, |
| 178 |
+ ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
|
|
| 179 |
+ "inline": inlineremotecache.ResolveCacheExporterFunc(), |
|
| 180 |
+ }, |
|
| 178 | 181 |
// TODO: set ResolveCacheExporterFunc for exporting cache |
| 179 | 182 |
}) |
| 180 | 183 |
} |
| ... | ... |
@@ -148,7 +148,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source) |
| 148 | 148 |
|
| 149 | 149 |
diffs, history = normalizeLayersAndHistory(diffs, history, ref) |
| 150 | 150 |
|
| 151 |
- config, err = patchImageConfig(config, diffs, history) |
|
| 151 |
+ config, err = patchImageConfig(config, diffs, history, inp.Metadata[exptypes.ExporterInlineCache]) |
|
| 152 | 152 |
if err != nil {
|
| 153 | 153 |
return nil, err |
| 154 | 154 |
} |
| ... | ... |
@@ -41,7 +41,7 @@ func parseHistoryFromConfig(dt []byte) ([]ocispec.History, error) {
|
| 41 | 41 |
return config.History, nil |
| 42 | 42 |
} |
| 43 | 43 |
|
| 44 |
-func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History) ([]byte, error) {
|
|
| 44 |
+func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History, cache []byte) ([]byte, error) {
|
|
| 45 | 45 |
m := map[string]json.RawMessage{}
|
| 46 | 46 |
if err := json.Unmarshal(dt, &m); err != nil {
|
| 47 | 47 |
return nil, errors.Wrap(err, "failed to parse image config for patch") |
| ... | ... |
@@ -77,6 +77,14 @@ func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History) |
| 77 | 77 |
m["created"] = dt |
| 78 | 78 |
} |
| 79 | 79 |
|
| 80 |
+ if cache != nil {
|
|
| 81 |
+ dt, err := json.Marshal(cache) |
|
| 82 |
+ if err != nil {
|
|
| 83 |
+ return nil, err |
|
| 84 |
+ } |
|
| 85 |
+ m["moby.buildkit.cache.v0"] = dt |
|
| 86 |
+ } |
|
| 87 |
+ |
|
| 80 | 88 |
dt, err = json.Marshal(m) |
| 81 | 89 |
return dt, errors.Wrap(err, "failed to marshal config after patch") |
| 82 | 90 |
} |
| ... | ... |
@@ -23,6 +23,7 @@ import ( |
| 23 | 23 |
"github.com/moby/buildkit/client" |
| 24 | 24 |
"github.com/moby/buildkit/executor" |
| 25 | 25 |
"github.com/moby/buildkit/exporter" |
| 26 |
+ localexporter "github.com/moby/buildkit/exporter/local" |
|
| 26 | 27 |
"github.com/moby/buildkit/frontend" |
| 27 | 28 |
gw "github.com/moby/buildkit/frontend/gateway/client" |
| 28 | 29 |
"github.com/moby/buildkit/session" |
| ... | ... |
@@ -47,17 +48,16 @@ type Opt struct {
|
| 47 | 47 |
ID string |
| 48 | 48 |
Labels map[string]string |
| 49 | 49 |
GCPolicy []client.PruneInfo |
| 50 |
- SessionManager *session.Manager |
|
| 51 | 50 |
MetadataStore *metadata.Store |
| 52 | 51 |
Executor executor.Executor |
| 53 | 52 |
Snapshotter snapshot.Snapshotter |
| 54 | 53 |
ContentStore content.Store |
| 55 | 54 |
CacheManager cache.Manager |
| 56 | 55 |
ImageSource source.Source |
| 57 |
- Exporters map[string]exporter.Exporter |
|
| 58 | 56 |
DownloadManager distribution.RootFSDownloadManager |
| 59 | 57 |
V2MetadataService distmetadata.V2MetadataService |
| 60 | 58 |
Transport nethttp.RoundTripper |
| 59 |
+ Exporter exporter.Exporter |
|
| 61 | 60 |
} |
| 62 | 61 |
|
| 63 | 62 |
// Worker is a local worker instance with dedicated snapshotter, cache, and so on. |
| ... | ... |
@@ -99,9 +99,8 @@ func NewWorker(opt Opt) (*Worker, error) {
|
| 99 | 99 |
} |
| 100 | 100 |
|
| 101 | 101 |
ss, err := local.NewSource(local.Opt{
|
| 102 |
- SessionManager: opt.SessionManager, |
|
| 103 |
- CacheAccessor: cm, |
|
| 104 |
- MetadataStore: opt.MetadataStore, |
|
| 102 |
+ CacheAccessor: cm, |
|
| 103 |
+ MetadataStore: opt.MetadataStore, |
|
| 105 | 104 |
}) |
| 106 | 105 |
if err == nil {
|
| 107 | 106 |
sm.Register(ss) |
| ... | ... |
@@ -146,13 +145,13 @@ func (w *Worker) LoadRef(id string, hidden bool) (cache.ImmutableRef, error) {
|
| 146 | 146 |
} |
| 147 | 147 |
|
| 148 | 148 |
// ResolveOp converts a LLB vertex into a LLB operation |
| 149 |
-func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solver.Op, error) {
|
|
| 149 |
+func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *session.Manager) (solver.Op, error) {
|
|
| 150 | 150 |
if baseOp, ok := v.Sys().(*pb.Op); ok {
|
| 151 | 151 |
switch op := baseOp.Op.(type) {
|
| 152 | 152 |
case *pb.Op_Source: |
| 153 |
- return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, w) |
|
| 153 |
+ return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, sm, w) |
|
| 154 | 154 |
case *pb.Op_Exec: |
| 155 |
- return ops.NewExecOp(v, op, w.CacheManager, w.Opt.SessionManager, w.MetadataStore, w.Executor, w) |
|
| 155 |
+ return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager, sm, w.MetadataStore, w.Executor, w) |
|
| 156 | 156 |
case *pb.Op_Build: |
| 157 | 157 |
return ops.NewBuildOp(v, op, s, w) |
| 158 | 158 |
} |
| ... | ... |
@@ -161,13 +160,13 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solve |
| 161 | 161 |
} |
| 162 | 162 |
|
| 163 | 163 |
// ResolveImageConfig returns image config for an image |
| 164 |
-func (w *Worker) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt) (digest.Digest, []byte, error) {
|
|
| 164 |
+func (w *Worker) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error) {
|
|
| 165 | 165 |
// ImageSource is typically source/containerimage |
| 166 | 166 |
resolveImageConfig, ok := w.ImageSource.(resolveImageConfig) |
| 167 | 167 |
if !ok {
|
| 168 | 168 |
return "", nil, errors.Errorf("worker %q does not implement ResolveImageConfig", w.ID())
|
| 169 | 169 |
} |
| 170 |
- return resolveImageConfig.ResolveImageConfig(ctx, ref, opt) |
|
| 170 |
+ return resolveImageConfig.ResolveImageConfig(ctx, ref, opt, sm) |
|
| 171 | 171 |
} |
| 172 | 172 |
|
| 173 | 173 |
// Exec executes a process directly on a worker |
| ... | ... |
@@ -191,12 +190,17 @@ func (w *Worker) Prune(ctx context.Context, ch chan client.UsageInfo, info ...cl |
| 191 | 191 |
} |
| 192 | 192 |
|
| 193 | 193 |
// Exporter returns exporter by name |
| 194 |
-func (w *Worker) Exporter(name string) (exporter.Exporter, error) {
|
|
| 195 |
- exp, ok := w.Exporters[name] |
|
| 196 |
- if !ok {
|
|
| 194 |
+func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter, error) {
|
|
| 195 |
+ switch name {
|
|
| 196 |
+ case "moby": |
|
| 197 |
+ return w.Opt.Exporter, nil |
|
| 198 |
+ case client.ExporterLocal: |
|
| 199 |
+ return localexporter.New(localexporter.Opt{
|
|
| 200 |
+ SessionManager: sm, |
|
| 201 |
+ }) |
|
| 202 |
+ default: |
|
| 197 | 203 |
return nil, errors.Errorf("exporter %q could not be found", name)
|
| 198 | 204 |
} |
| 199 |
- return exp, nil |
|
| 200 | 205 |
} |
| 201 | 206 |
|
| 202 | 207 |
// GetRemote returns a remote snapshot reference for a local one |
| ... | ... |
@@ -338,5 +342,5 @@ func oneOffProgress(ctx context.Context, id string) func(err error) error {
|
| 338 | 338 |
} |
| 339 | 339 |
|
| 340 | 340 |
type resolveImageConfig interface {
|
| 341 |
- ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt) (digest.Digest, []byte, error) |
|
| 341 |
+ ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error) |
|
| 342 | 342 |
} |