package remotecache
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/core/images"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/compression"
"github.com/moby/buildkit/util/contentutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/progress/logs"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
type ResolveCacheExporterFunc func(ctx context.Context, g session.Group, attrs map[string]string) (Exporter, error)
type Exporter interface {
solver.CacheExporterTarget
// Name uniquely identifies the exporter
Name() string
// Finalize finalizes and return metadata that are returned to the client
// e.g. ExporterResponseManifestDesc
Finalize(ctx context.Context) (map[string]string, error)
Config() Config
}
type Config struct {
Compression compression.Config
}
type CacheType int
const (
// ExportResponseManifestDesc is a key for the map returned from Exporter.Finalize.
// The map value is a JSON string of an OCI desciptor of a manifest.
ExporterResponseManifestDesc = "cache.manifest"
)
const (
NotSet CacheType = iota
ManifestList
ImageManifest
)
func (data CacheType) String() string {
switch data {
case ManifestList:
return "Manifest List"
case ImageManifest:
return "Image Manifest"
default:
return "Not Set"
}
}
func NewExporter(ingester content.Ingester, ref string, oci bool, imageManifest bool, compressionConfig compression.Config) Exporter {
cc := v1.NewCacheChains()
return &contentCacheExporter{CacheExporterTarget: cc, chains: cc, ingester: ingester, oci: oci, imageManifest: imageManifest, ref: ref, comp: compressionConfig}
}
type ExportableCache struct {
// This cache describes two distinct styles of exportable cache, one is an Index (or Manifest List) of blobs,
// or as an artifact using the OCI image manifest format.
ExportedManifest ocispecs.Manifest
ExportedIndex ocispecs.Index
CacheType CacheType
OCI bool
}
func NewExportableCache(oci bool, imageManifest bool) (*ExportableCache, error) {
var mediaType string
if imageManifest {
mediaType = ocispecs.MediaTypeImageManifest
if !oci {
return nil, errors.Errorf("invalid configuration for remote cache, OCI mediatypes are required for image-manifest cache format")
}
} else {
if oci {
mediaType = ocispecs.MediaTypeImageIndex
} else {
mediaType = images.MediaTypeDockerSchema2ManifestList
}
}
cacheType := ManifestList
if imageManifest {
cacheType = ImageManifest
}
schemaVersion := specs.Versioned{SchemaVersion: 2}
switch cacheType {
case ManifestList:
return &ExportableCache{ExportedIndex: ocispecs.Index{
MediaType: mediaType,
Versioned: schemaVersion,
},
CacheType: cacheType,
OCI: oci,
}, nil
case ImageManifest:
return &ExportableCache{ExportedManifest: ocispecs.Manifest{
MediaType: mediaType,
Versioned: schemaVersion,
},
CacheType: cacheType,
OCI: oci,
}, nil
default:
return nil, errors.Errorf("exportable cache type not set")
}
}
func (ec *ExportableCache) MediaType() string {
if ec.CacheType == ManifestList {
return ec.ExportedIndex.MediaType
}
return ec.ExportedManifest.MediaType
}
func (ec *ExportableCache) AddCacheBlob(blob ocispecs.Descriptor) {
if ec.CacheType == ManifestList {
ec.ExportedIndex.Manifests = append(ec.ExportedIndex.Manifests, blob)
} else {
ec.ExportedManifest.Layers = append(ec.ExportedManifest.Layers, blob)
}
}
func (ec *ExportableCache) FinalizeCache(ctx context.Context) {
if ec.CacheType == ManifestList {
ec.ExportedIndex.Manifests = compression.ConvertAllLayerMediaTypes(ctx, ec.OCI, ec.ExportedIndex.Manifests...)
} else {
ec.ExportedManifest.Layers = compression.ConvertAllLayerMediaTypes(ctx, ec.OCI, ec.ExportedManifest.Layers...)
}
}
func (ec *ExportableCache) SetConfig(config ocispecs.Descriptor) {
if ec.CacheType == ManifestList {
ec.ExportedIndex.Manifests = append(ec.ExportedIndex.Manifests, config)
} else {
ec.ExportedManifest.Config = config
}
}
func (ec *ExportableCache) MarshalJSON() ([]byte, error) {
if ec.CacheType == ManifestList {
return json.Marshal(ec.ExportedIndex)
}
return json.Marshal(ec.ExportedManifest)
}
type contentCacheExporter struct {
solver.CacheExporterTarget
chains *v1.CacheChains
ingester content.Ingester
oci bool
imageManifest bool
ref string
comp compression.Config
}
func (ce *contentCacheExporter) Name() string {
return "exporting content cache"
}
func (ce *contentCacheExporter) Config() Config {
return Config{
Compression: ce.comp,
}
}
func (ce *contentCacheExporter) Finalize(ctx context.Context) (map[string]string, error) {
res := make(map[string]string)
config, descs, err := ce.chains.Marshal(ctx)
if err != nil {
return nil, err
}
if len(config.Layers) == 0 {
bklog.G(ctx).Warn("failed to match any cache with layers")
return nil, progress.OneOff(ctx, "skipping cache export for empty result")(nil)
}
cache, err := NewExportableCache(ce.oci, ce.imageManifest)
if err != nil {
return nil, err
}
for _, l := range config.Layers {
dgstPair, ok := descs[l.Blob]
if !ok {
return nil, errors.Errorf("missing blob %s", l.Blob)
}
layerDone := progress.OneOff(ctx, fmt.Sprintf("writing layer %s", l.Blob))
if err := contentutil.Copy(ctx, ce.ingester, dgstPair.Provider, dgstPair.Descriptor, ce.ref, logs.LoggerFromContext(ctx)); err != nil {
return nil, layerDone(errors.Wrap(err, "error writing layer blob"))
}
layerDone(nil)
cache.AddCacheBlob(dgstPair.Descriptor)
}
cache.FinalizeCache(ctx)
dt, err := json.Marshal(config)
if err != nil {
return nil, err
}
dgst := digest.FromBytes(dt)
desc := ocispecs.Descriptor{
Digest: dgst,
Size: int64(len(dt)),
MediaType: v1.CacheConfigMediaTypeV0,
}
configDone := progress.OneOff(ctx, fmt.Sprintf("writing config %s", dgst))
if err := content.WriteBlob(ctx, ce.ingester, dgst.String(), bytes.NewReader(dt), desc); err != nil {
return nil, configDone(errors.Wrap(err, "error writing config blob"))
}
configDone(nil)
cache.SetConfig(desc)
dt, err = cache.MarshalJSON()
if err != nil {
return nil, errors.Wrap(err, "failed to marshal manifest")
}
dgst = digest.FromBytes(dt)
desc = ocispecs.Descriptor{
Digest: dgst,
Size: int64(len(dt)),
MediaType: cache.MediaType(),
}
mfstLog := fmt.Sprintf("writing cache manifest %s", dgst)
if ce.imageManifest {
mfstLog = fmt.Sprintf("writing cache image manifest %s", dgst)
}
mfstDone := progress.OneOff(ctx, mfstLog)
if err := content.WriteBlob(ctx, ce.ingester, dgst.String(), bytes.NewReader(dt), desc); err != nil {
return nil, mfstDone(errors.Wrap(err, "error writing manifest blob"))
}
descJSON, err := json.Marshal(desc)
if err != nil {
return nil, err
}
res[ExporterResponseManifestDesc] = string(descJSON)
mfstDone(nil)
return res, nil
}