Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
| ... | ... |
@@ -35,9 +35,9 @@ import ( |
| 35 | 35 |
"github.com/moby/buildkit/util/progress" |
| 36 | 36 |
"github.com/moby/buildkit/util/tracing" |
| 37 | 37 |
digest "github.com/opencontainers/go-digest" |
| 38 |
+ "github.com/opencontainers/image-spec/identity" |
|
| 38 | 39 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 39 | 40 |
"github.com/pkg/errors" |
| 40 |
- netcontext "golang.org/x/net/context" |
|
| 41 | 41 |
"golang.org/x/time/rate" |
| 42 | 42 |
) |
| 43 | 43 |
|
| ... | ... |
@@ -152,60 +152,66 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc |
| 152 | 152 |
} |
| 153 | 153 |
|
| 154 | 154 |
type puller struct {
|
| 155 |
- is *imageSource |
|
| 156 |
- resolveOnce sync.Once |
|
| 157 |
- src *source.ImageIdentifier |
|
| 158 |
- desc ocispec.Descriptor |
|
| 159 |
- ref string |
|
| 160 |
- resolveErr error |
|
| 161 |
- resolver remotes.Resolver |
|
| 162 |
- imageID image.ID |
|
| 163 |
- cacheKey digest.Digest |
|
| 155 |
+ is *imageSource |
|
| 156 |
+ resolveOnce sync.Once |
|
| 157 |
+ resolveLocalOnce sync.Once |
|
| 158 |
+ src *source.ImageIdentifier |
|
| 159 |
+ desc ocispec.Descriptor |
|
| 160 |
+ ref string |
|
| 161 |
+ resolveErr error |
|
| 162 |
+ resolver remotes.Resolver |
|
| 163 |
+ config []byte |
|
| 164 | 164 |
} |
| 165 | 165 |
|
| 166 |
-func (p *puller) resolve(ctx context.Context) error {
|
|
| 167 |
- p.resolveOnce.Do(func() {
|
|
| 168 |
- resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String()) |
|
| 166 |
+func (p *puller) mainManifestKey(dgst digest.Digest) (digest.Digest, error) {
|
|
| 167 |
+ dt, err := json.Marshal(struct {
|
|
| 168 |
+ Digest digest.Digest |
|
| 169 |
+ OS string |
|
| 170 |
+ Arch string |
|
| 171 |
+ }{
|
|
| 172 |
+ Digest: p.desc.Digest, |
|
| 173 |
+ OS: runtime.GOOS, |
|
| 174 |
+ Arch: runtime.GOARCH, |
|
| 175 |
+ }) |
|
| 176 |
+ if err != nil {
|
|
| 177 |
+ return "", err |
|
| 178 |
+ } |
|
| 179 |
+ return digest.FromBytes(dt), nil |
|
| 180 |
+} |
|
| 169 | 181 |
|
| 170 |
- // dgst := p.src.Reference.Digest() |
|
| 171 |
- // if dgst != "" {
|
|
| 172 |
- // info, err := p.is.ContentStore.Info(ctx, dgst) |
|
| 173 |
- // if err == nil {
|
|
| 174 |
- // p.ref = p.src.Reference.String() |
|
| 175 |
- // ra, err := p.is.ContentStore.ReaderAt(ctx, dgst) |
|
| 176 |
- // if err == nil {
|
|
| 177 |
- // mt, err := imageutil.DetectManifestMediaType(ra) |
|
| 178 |
- // if err == nil {
|
|
| 179 |
- // p.desc = ocispec.Descriptor{
|
|
| 180 |
- // Size: info.Size, |
|
| 181 |
- // Digest: dgst, |
|
| 182 |
- // MediaType: mt, |
|
| 183 |
- // } |
|
| 184 |
- // resolveProgressDone(nil) |
|
| 185 |
- // return |
|
| 186 |
- // } |
|
| 187 |
- // } |
|
| 188 |
- // } |
|
| 189 |
- // } |
|
| 190 |
- |
|
| 191 |
- // ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String()) |
|
| 192 |
- // if err != nil {
|
|
| 193 |
- // p.resolveErr = err |
|
| 194 |
- // resolveProgressDone(err) |
|
| 195 |
- // return |
|
| 196 |
- // } |
|
| 182 |
+func (p *puller) resolveLocal() {
|
|
| 183 |
+ p.resolveLocalOnce.Do(func() {
|
|
| 184 |
+ dgst := p.src.Reference.Digest() |
|
| 185 |
+ if dgst != "" {
|
|
| 186 |
+ info, err := p.is.ContentStore.Info(context.TODO(), dgst) |
|
| 187 |
+ if err == nil {
|
|
| 188 |
+ p.ref = p.src.Reference.String() |
|
| 189 |
+ ra, err := p.is.ContentStore.ReaderAt(context.TODO(), dgst) |
|
| 190 |
+ if err == nil {
|
|
| 191 |
+ mt, err := imageutil.DetectManifestMediaType(ra) |
|
| 192 |
+ if err == nil {
|
|
| 193 |
+ p.desc = ocispec.Descriptor{
|
|
| 194 |
+ Size: info.Size, |
|
| 195 |
+ Digest: dgst, |
|
| 196 |
+ MediaType: mt, |
|
| 197 |
+ } |
|
| 198 |
+ } |
|
| 199 |
+ } |
|
| 200 |
+ } |
|
| 201 |
+ } |
|
| 197 | 202 |
|
| 198 | 203 |
if preferLocal {
|
| 199 | 204 |
dt, err := p.is.resolveLocal(p.src.Reference.String()) |
| 200 | 205 |
if err == nil {
|
| 201 |
- dgst := digest.FromBytes(dt) |
|
| 202 |
- p.imageID = image.ID(dgst) |
|
| 203 |
- p.cacheKey = dgst |
|
| 204 |
- resolveProgressDone(nil) |
|
| 205 |
- return |
|
| 206 |
+ p.config = dt |
|
| 206 | 207 |
} |
| 207 |
- |
|
| 208 | 208 |
} |
| 209 |
+ }) |
|
| 210 |
+} |
|
| 211 |
+ |
|
| 212 |
+func (p *puller) resolve(ctx context.Context) error {
|
|
| 213 |
+ p.resolveOnce.Do(func() {
|
|
| 214 |
+ resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String()) |
|
| 209 | 215 |
|
| 210 | 216 |
ref, err := distreference.ParseNormalizedNamed(p.src.Reference.String()) |
| 211 | 217 |
if err != nil {
|
| ... | ... |
@@ -214,52 +220,82 @@ func (p *puller) resolve(ctx context.Context) error {
|
| 214 | 214 |
return |
| 215 | 215 |
} |
| 216 | 216 |
|
| 217 |
- outRef, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String()) |
|
| 218 |
- if err != nil {
|
|
| 219 |
- p.resolveErr = err |
|
| 220 |
- resolveProgressDone(err) |
|
| 221 |
- return |
|
| 222 |
- } |
|
| 217 |
+ if p.desc.Digest == "" && p.config == nil {
|
|
| 218 |
+ origRef, desc, err := p.resolver.Resolve(ctx, ref.String()) |
|
| 219 |
+ if err != nil {
|
|
| 220 |
+ p.resolveErr = err |
|
| 221 |
+ resolveProgressDone(err) |
|
| 222 |
+ return |
|
| 223 |
+ } |
|
| 223 | 224 |
|
| 224 |
- ref, err = distreference.WithDigest(ref, desc.Digest) |
|
| 225 |
- if err != nil {
|
|
| 226 |
- p.resolveErr = err |
|
| 227 |
- resolveProgressDone(err) |
|
| 228 |
- return |
|
| 225 |
+ p.desc = desc |
|
| 226 |
+ p.ref = origRef |
|
| 229 | 227 |
} |
| 230 | 228 |
|
| 231 |
- _, dt, err := p.is.ResolveImageConfig(ctx, ref.String()) |
|
| 232 |
- if err != nil {
|
|
| 233 |
- p.resolveErr = err |
|
| 234 |
- resolveProgressDone(err) |
|
| 235 |
- return |
|
| 229 |
+ if p.config == nil {
|
|
| 230 |
+ ref, err := distreference.WithDigest(ref, p.desc.Digest) |
|
| 231 |
+ if err != nil {
|
|
| 232 |
+ p.resolveErr = err |
|
| 233 |
+ resolveProgressDone(err) |
|
| 234 |
+ return |
|
| 235 |
+ } |
|
| 236 |
+ |
|
| 237 |
+ _, dt, err := p.is.ResolveImageConfig(ctx, ref.String()) |
|
| 238 |
+ if err != nil {
|
|
| 239 |
+ p.resolveErr = err |
|
| 240 |
+ resolveProgressDone(err) |
|
| 241 |
+ return |
|
| 242 |
+ } |
|
| 243 |
+ |
|
| 244 |
+ p.config = dt |
|
| 236 | 245 |
} |
| 237 |
- p.desc = desc |
|
| 238 |
- p.cacheKey = digest.FromBytes(dt) |
|
| 239 |
- p.ref = outRef |
|
| 240 | 246 |
resolveProgressDone(nil) |
| 241 | 247 |
}) |
| 242 | 248 |
return p.resolveErr |
| 243 | 249 |
} |
| 244 | 250 |
|
| 245 | 251 |
func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error) {
|
| 252 |
+ p.resolveLocal() |
|
| 253 |
+ |
|
| 254 |
+ if p.desc.Digest != "" && index == 0 {
|
|
| 255 |
+ dgst, err := p.mainManifestKey(p.desc.Digest) |
|
| 256 |
+ if err != nil {
|
|
| 257 |
+ return "", false, err |
|
| 258 |
+ } |
|
| 259 |
+ return dgst.String(), false, nil |
|
| 260 |
+ } |
|
| 261 |
+ |
|
| 262 |
+ if p.config != nil {
|
|
| 263 |
+ return cacheKeyFromConfig(p.config).String(), true, nil |
|
| 264 |
+ } |
|
| 265 |
+ |
|
| 246 | 266 |
if err := p.resolve(ctx); err != nil {
|
| 247 | 267 |
return "", false, err |
| 248 | 268 |
} |
| 249 |
- return p.cacheKey.String(), true, nil |
|
| 269 |
+ |
|
| 270 |
+ if p.desc.Digest != "" && index == 0 {
|
|
| 271 |
+ dgst, err := p.mainManifestKey(p.desc.Digest) |
|
| 272 |
+ if err != nil {
|
|
| 273 |
+ return "", false, err |
|
| 274 |
+ } |
|
| 275 |
+ return dgst.String(), false, nil |
|
| 276 |
+ } |
|
| 277 |
+ |
|
| 278 |
+ return cacheKeyFromConfig(p.config).String(), true, nil |
|
| 250 | 279 |
} |
| 251 | 280 |
|
| 252 | 281 |
func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
|
| 282 |
+ p.resolveLocal() |
|
| 253 | 283 |
if err := p.resolve(ctx); err != nil {
|
| 254 | 284 |
return nil, err |
| 255 | 285 |
} |
| 256 | 286 |
|
| 257 |
- if p.imageID != "" {
|
|
| 258 |
- img, err := p.is.ImageStore.Get(p.imageID) |
|
| 287 |
+ if p.config != nil {
|
|
| 288 |
+ img, err := p.is.ImageStore.Get(image.ID(digest.Digest(p.config))) |
|
| 259 | 289 |
if err != nil {
|
| 260 | 290 |
return nil, err |
| 261 | 291 |
} |
| 262 |
- ref, err := p.is.CacheAccessor.Get(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
|
|
| 292 |
+ ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
|
|
| 263 | 293 |
if err != nil {
|
| 264 | 294 |
return nil, err |
| 265 | 295 |
} |
| ... | ... |
@@ -423,7 +459,7 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
|
| 423 | 423 |
} |
| 424 | 424 |
stopProgress() |
| 425 | 425 |
|
| 426 |
- ref, err := p.is.CacheAccessor.Get(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
|
|
| 426 |
+ ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
|
|
| 427 | 427 |
release() |
| 428 | 428 |
if err != nil {
|
| 429 | 429 |
return nil, err |
| ... | ... |
@@ -453,7 +489,7 @@ func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
|
| 453 | 453 |
return ld.diffID, nil |
| 454 | 454 |
} |
| 455 | 455 |
|
| 456 |
-func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
|
|
| 456 |
+func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
|
|
| 457 | 457 |
rc, err := ld.fetcher.Fetch(ctx, ld.desc) |
| 458 | 458 |
if err != nil {
|
| 459 | 459 |
return nil, 0, err |
| ... | ... |
@@ -654,3 +690,17 @@ func oneOffProgress(ctx context.Context, id string) func(err error) error {
|
| 654 | 654 |
return err |
| 655 | 655 |
} |
| 656 | 656 |
} |
| 657 |
+ |
|
| 658 |
+// cacheKeyFromConfig returns a stable digest from image config. If image config |
|
| 659 |
+// is a known oci image we will use chainID of layers. |
|
| 660 |
+func cacheKeyFromConfig(dt []byte) digest.Digest {
|
|
| 661 |
+ var img ocispec.Image |
|
| 662 |
+ err := json.Unmarshal(dt, &img) |
|
| 663 |
+ if err != nil {
|
|
| 664 |
+ return digest.FromBytes(dt) |
|
| 665 |
+ } |
|
| 666 |
+ if img.RootFS.Type != "layers" {
|
|
| 667 |
+ return digest.FromBytes(dt) |
|
| 668 |
+ } |
|
| 669 |
+ return identity.ChainID(img.RootFS.DiffIDs) |
|
| 670 |
+} |
| ... | ... |
@@ -17,7 +17,6 @@ import ( |
| 17 | 17 |
"github.com/moby/buildkit/identity" |
| 18 | 18 |
"github.com/moby/buildkit/session" |
| 19 | 19 |
"github.com/pkg/errors" |
| 20 |
- netcontext "golang.org/x/net/context" |
|
| 21 | 20 |
"golang.org/x/sync/errgroup" |
| 22 | 21 |
grpcmetadata "google.golang.org/grpc/metadata" |
| 23 | 22 |
) |
| ... | ... |
@@ -184,7 +183,7 @@ func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
|
| 184 | 184 |
return sp.SendMsg(resp) |
| 185 | 185 |
} |
| 186 | 186 |
|
| 187 |
-func (sp *statusProxy) Context() netcontext.Context {
|
|
| 187 |
+func (sp *statusProxy) Context() context.Context {
|
|
| 188 | 188 |
return sp.ctx |
| 189 | 189 |
} |
| 190 | 190 |
func (sp *statusProxy) SendMsg(m interface{}) error {
|
| ... | ... |
@@ -36,7 +36,6 @@ import ( |
| 36 | 36 |
digest "github.com/opencontainers/go-digest" |
| 37 | 37 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 38 | 38 |
"github.com/pkg/errors" |
| 39 |
- netcontext "golang.org/x/net/context" |
|
| 40 | 39 |
) |
| 41 | 40 |
|
| 42 | 41 |
// WorkerOpt is specific to a worker. |
| ... | ... |
@@ -243,7 +242,7 @@ func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
|
| 243 | 243 |
return ld.diffID, nil |
| 244 | 244 |
} |
| 245 | 245 |
|
| 246 |
-func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
|
|
| 246 |
+func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
|
|
| 247 | 247 |
done := oneOffProgress(ld.pctx, fmt.Sprintf("pulling %s", ld.desc.Digest))
|
| 248 | 248 |
if err := contentutil.Copy(ctx, ld.w.ContentStore, ld.provider, ld.desc); err != nil {
|
| 249 | 249 |
return nil, 0, done(err) |