4f0d95fa |
package distribution // import "github.com/docker/docker/distribution" |
694df3ff |
import ( |
7d62e40f |
"context" |
694df3ff |
"encoding/json"
"fmt"
"io"
"io/ioutil" |
7b81bc14 |
"net/url" |
694df3ff |
"os"
"runtime" |
0380fbff |
"strings" |
694df3ff |
|
337ba71f |
"github.com/containerd/containerd/platforms" |
694df3ff |
"github.com/docker/distribution" |
2bb8c85b |
"github.com/docker/distribution/manifest/manifestlist" |
e037bade |
"github.com/docker/distribution/manifest/ocischema" |
694df3ff |
"github.com/docker/distribution/manifest/schema1" |
94726f7c |
"github.com/docker/distribution/manifest/schema2" |
3a127939 |
"github.com/docker/distribution/reference" |
9d6acbee |
"github.com/docker/distribution/registry/api/errcode" |
7b81bc14 |
"github.com/docker/distribution/registry/client/auth" |
056bf9f2 |
"github.com/docker/distribution/registry/client/transport" |
694df3ff |
"github.com/docker/docker/distribution/metadata" |
572ce802 |
"github.com/docker/docker/distribution/xfer" |
694df3ff |
"github.com/docker/docker/image" |
023166b5 |
v1 "github.com/docker/docker/image/v1" |
694df3ff |
"github.com/docker/docker/layer" |
572ce802 |
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progress" |
694df3ff |
"github.com/docker/docker/pkg/stringid" |
b21d9ab5 |
"github.com/docker/docker/pkg/system" |
3a127939 |
refstore "github.com/docker/docker/reference" |
694df3ff |
"github.com/docker/docker/registry" |
f23c00d8 |
"github.com/opencontainers/go-digest" |
83908836 |
specs "github.com/opencontainers/image-spec/specs-go/v1" |
ebcb7d6b |
"github.com/pkg/errors" |
1009e6a4 |
"github.com/sirupsen/logrus" |
694df3ff |
)
|
9b6dcc8b |
var ( |
3c7676a0 |
errRootFSMismatch = errors.New("layers from manifest don't match image configuration")
errRootFSInvalid = errors.New("invalid rootfs in image configuration") |
9b6dcc8b |
) |
94726f7c |
|
8f26fe4f |
// ImageConfigPullError is an error pulling the image config blob
// (only applies to schema2).
type ImageConfigPullError struct {
Err error
}
// Error returns the error string for ImageConfigPullError.
func (e ImageConfigPullError) Error() string {
return "error pulling image configuration: " + e.Err.Error()
}
|
694df3ff |
type v2Puller struct { |
d3bd14a4 |
V2MetadataService metadata.V2MetadataService |
63099477 |
endpoint registry.APIEndpoint
config *ImagePullConfig
repoInfo *registry.RepositoryInfo
repo distribution.Repository |
a57478d6 |
// confirmedV2 is set to true if we confirm we're talking to a v2
// registry. This is used to limit fallbacks to the v1 protocol.
confirmedV2 bool |
694df3ff |
}
|
337ba71f |
func (p *v2Puller) Pull(ctx context.Context, ref reference.Named, platform *specs.Platform) (err error) { |
694df3ff |
// TODO(tiborvass): was ReceiveTimeout |
a57478d6 |
p.repo, p.confirmedV2, err = NewV2Repository(ctx, p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull") |
694df3ff |
if err != nil { |
572ce802 |
logrus.Warnf("Error getting v2 registry: %v", err) |
5e8af46f |
return err |
694df3ff |
}
|
337ba71f |
if err = p.pullV2Repository(ctx, ref, platform); err != nil { |
589a5226 |
if _, ok := err.(fallbackError); ok {
return err
} |
305801f5 |
if continueOnError(err, p.endpoint.Mirror) { |
5e8af46f |
return fallbackError{
err: err,
confirmedV2: p.confirmedV2,
transportOK: true,
} |
694df3ff |
}
} |
a57478d6 |
return err |
694df3ff |
}
|
337ba71f |
func (p *v2Puller) pullV2Repository(ctx context.Context, ref reference.Named, platform *specs.Platform) (err error) { |
589a5226 |
var layersDownloaded bool |
eeb2d4c1 |
if !reference.IsNameOnly(ref) { |
337ba71f |
layersDownloaded, err = p.pullV2Tag(ctx, ref, platform) |
589a5226 |
if err != nil {
return err
} |
694df3ff |
} else { |
c168a005 |
tags, err := p.repo.Tags(ctx).All(ctx) |
694df3ff |
if err != nil { |
589a5226 |
// If this repository doesn't exist on V2, we should
// permit a fallback to V1.
return allowV1Fallback(err) |
694df3ff |
}
|
589a5226 |
// The v2 registry knows about this repository, so we will not
// allow fallback to the v1 protocol even if we encounter an
// error later on. |
a57478d6 |
p.confirmedV2 = true
|
694df3ff |
for _, tag := range tags { |
ffded61d |
tagRef, err := reference.WithTag(ref, tag) |
694df3ff |
if err != nil {
return err
} |
337ba71f |
pulledNew, err := p.pullV2Tag(ctx, tagRef, platform) |
589a5226 |
if err != nil {
// Since this is the pull-all-tags case, don't
// allow an error pulling a particular tag to
// make the whole pull fall back to v1.
if fallbackErr, ok := err.(fallbackError); ok {
return fallbackErr.err
}
return err
}
// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
layersDownloaded = layersDownloaded || pulledNew |
694df3ff |
}
}
|
3a127939 |
writeStatus(reference.FamiliarString(ref), p.config.ProgressOutput, layersDownloaded) |
694df3ff |
return nil
}
|
572ce802 |
type v2LayerDescriptor struct { |
63099477 |
digest digest.Digest |
633f9252 |
diffID layer.DiffID |
63099477 |
repoInfo *registry.RepositoryInfo
repo distribution.Repository |
d3bd14a4 |
V2MetadataService metadata.V2MetadataService |
f425529e |
tmpFile *os.File |
056bf9f2 |
verifier digest.Verifier |
2c60430a |
src distribution.Descriptor |
694df3ff |
}
|
572ce802 |
func (ld *v2LayerDescriptor) Key() string {
return "v2:" + ld.digest.String()
} |
694df3ff |
|
572ce802 |
func (ld *v2LayerDescriptor) ID() string {
return stringid.TruncateID(ld.digest.String())
} |
694df3ff |
|
572ce802 |
func (ld *v2LayerDescriptor) DiffID() (layer.DiffID, error) { |
633f9252 |
if ld.diffID != "" {
return ld.diffID, nil
} |
63099477 |
return ld.V2MetadataService.GetDiffID(ld.digest) |
572ce802 |
}
func (ld *v2LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
logrus.Debugf("pulling blob %q", ld.digest) |
694df3ff |
|
056bf9f2 |
var (
err error
offset int64
) |
f425529e |
if ld.tmpFile == nil {
ld.tmpFile, err = createDownloadFile() |
056bf9f2 |
if err != nil {
return nil, 0, xfer.DoNotRetry{Err: err}
} |
f425529e |
} else { |
056bf9f2 |
offset, err = ld.tmpFile.Seek(0, os.SEEK_END)
if err != nil {
logrus.Debugf("error seeking to end of download file: %v", err)
offset = 0
ld.tmpFile.Close()
if err := os.Remove(ld.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
}
ld.tmpFile, err = createDownloadFile()
if err != nil {
return nil, 0, xfer.DoNotRetry{Err: err}
}
} else if offset != 0 {
logrus.Debugf("attempting to resume download of %q from %d bytes", ld.digest, offset)
} |
f425529e |
}
tmpFile := ld.tmpFile |
694df3ff |
|
05bd0435 |
layerDownload, err := ld.open(ctx) |
694df3ff |
if err != nil { |
8f26fe4f |
logrus.Errorf("Error initiating layer download: %v", err) |
572ce802 |
return nil, 0, retryOnError(err) |
694df3ff |
}
|
056bf9f2 |
if offset != 0 {
_, err := layerDownload.Seek(offset, os.SEEK_SET)
if err != nil {
if err := ld.truncateDownloadFile(); err != nil {
return nil, 0, xfer.DoNotRetry{Err: err}
}
return nil, 0, err
}
} |
572ce802 |
size, err := layerDownload.Seek(0, os.SEEK_END) |
39589800 |
if err != nil {
// Seek failed, perhaps because there was no Content-Length
// header. This shouldn't fail the download, because we can
// still continue without a progress bar. |
572ce802 |
size = 0 |
39589800 |
} else { |
056bf9f2 |
if size != 0 && offset > size { |
a72b45db |
logrus.Debug("Partial download is larger than full blob. Starting over") |
056bf9f2 |
offset = 0
if err := ld.truncateDownloadFile(); err != nil {
return nil, 0, xfer.DoNotRetry{Err: err}
}
}
// Restore the seek offset either at the beginning of the
// stream, or just after the last byte we have from previous
// attempts.
_, err = layerDownload.Seek(offset, os.SEEK_SET) |
39589800 |
if err != nil { |
572ce802 |
return nil, 0, err |
39589800 |
}
}
|
056bf9f2 |
reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerDownload), progressOutput, size-offset, ld.ID(), "Downloading") |
572ce802 |
defer reader.Close()
|
056bf9f2 |
if ld.verifier == nil { |
7a855799 |
ld.verifier = ld.digest.Verifier() |
694df3ff |
}
|
056bf9f2 |
_, err = io.Copy(tmpFile, io.TeeReader(reader, ld.verifier)) |
572ce802 |
if err != nil { |
056bf9f2 |
if err == transport.ErrWrongCodeForByteRange {
if err := ld.truncateDownloadFile(); err != nil {
return nil, 0, xfer.DoNotRetry{Err: err}
}
return nil, 0, err |
5a363ce6 |
} |
572ce802 |
return nil, 0, retryOnError(err)
} |
694df3ff |
|
572ce802 |
progress.Update(progressOutput, ld.ID(), "Verifying Checksum") |
694df3ff |
|
056bf9f2 |
if !ld.verifier.Verified() { |
572ce802 |
err = fmt.Errorf("filesystem layer verification failed for digest %s", ld.digest) |
694df3ff |
logrus.Error(err) |
5a363ce6 |
|
056bf9f2 |
// Allow a retry if this digest verification error happened
// after a resumed download.
if offset != 0 {
if err := ld.truncateDownloadFile(); err != nil {
return nil, 0, xfer.DoNotRetry{Err: err}
} |
572ce802 |
|
056bf9f2 |
return nil, 0, err
} |
572ce802 |
return nil, 0, xfer.DoNotRetry{Err: err} |
694df3ff |
}
|
572ce802 |
progress.Update(progressOutput, ld.ID(), "Download complete") |
694df3ff |
|
572ce802 |
logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), tmpFile.Name())
|
5a363ce6 |
_, err = tmpFile.Seek(0, os.SEEK_SET)
if err != nil {
tmpFile.Close()
if err := os.Remove(tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
} |
f425529e |
ld.tmpFile = nil |
056bf9f2 |
ld.verifier = nil |
5a363ce6 |
return nil, 0, xfer.DoNotRetry{Err: err}
} |
930ae3db |
// hand off the temporary file to the download manager, so it will only
// be closed once
ld.tmpFile = nil
return ioutils.NewReadCloserWrapper(tmpFile, func() error {
tmpFile.Close()
err := os.RemoveAll(tmpFile.Name())
if err != nil {
logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
}
return err
}), size, nil |
f425529e |
}
func (ld *v2LayerDescriptor) Close() {
if ld.tmpFile != nil {
ld.tmpFile.Close()
if err := os.RemoveAll(ld.tmpFile.Name()); err != nil {
logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
}
} |
572ce802 |
} |
694df3ff |
|
056bf9f2 |
func (ld *v2LayerDescriptor) truncateDownloadFile() error {
// Need a new hash context since we will be redoing the download
ld.verifier = nil
if _, err := ld.tmpFile.Seek(0, os.SEEK_SET); err != nil { |
8f26fe4f |
logrus.Errorf("error seeking to beginning of download file: %v", err) |
056bf9f2 |
return err
}
if err := ld.tmpFile.Truncate(0); err != nil { |
8f26fe4f |
logrus.Errorf("error truncating download file: %v", err) |
056bf9f2 |
return err
}
return nil
}
|
572ce802 |
func (ld *v2LayerDescriptor) Registered(diffID layer.DiffID) {
// Cache mapping from this layer's DiffID to the blobsum |
3a127939 |
ld.V2MetadataService.Add(diffID, metadata.V2Metadata{Digest: ld.digest, SourceRepository: ld.repoInfo.Name.Name()}) |
694df3ff |
}
|
337ba71f |
func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform *specs.Platform) (tagUpdated bool, err error) { |
c168a005 |
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return false, err
}
var (
manifest distribution.Manifest
tagOrDigest string // Used for logging/progress only
) |
0bff591b |
if digested, isDigested := ref.(reference.Canonical); isDigested { |
c168a005 |
manifest, err = manSvc.Get(ctx, digested.Digest())
if err != nil {
return false, err
} |
694df3ff |
tagOrDigest = digested.Digest().String() |
0bff591b |
} else if tagged, isTagged := ref.(reference.NamedTagged); isTagged {
manifest, err = manSvc.Get(ctx, "", distribution.WithTag(tagged.Tag()))
if err != nil {
return false, allowV1Fallback(err)
}
tagOrDigest = tagged.Tag() |
694df3ff |
} else { |
3a127939 |
return false, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", reference.FamiliarString(ref)) |
694df3ff |
}
|
c168a005 |
if manifest == nil {
return false, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest)
} |
694df3ff |
|
9b6dcc8b |
if m, ok := manifest.(*schema2.DeserializedManifest); ok { |
3c7676a0 |
var allowedMediatype bool
for _, t := range p.config.Schema2Types {
if m.Manifest.Config.MediaType == t {
allowedMediatype = true
break
}
}
if !allowedMediatype {
configClass := mediaTypeClasses[m.Manifest.Config.MediaType]
if configClass == "" {
configClass = "unknown"
} |
ebcb7d6b |
return false, invalidManifestClassError{m.Manifest.Config.MediaType, configClass} |
9b6dcc8b |
}
}
|
c168a005 |
// If manSvc.Get succeeded, we can be confident that the registry on
// the other side speaks the v2 protocol.
p.confirmedV2 = true
|
3a127939 |
logrus.Debugf("Pulling ref from V2 registry: %s", reference.FamiliarString(ref))
progress.Message(p.config.ProgressOutput, tagOrDigest, "Pulling from "+reference.FamiliarName(p.repo.Named())) |
c168a005 |
|
94726f7c |
var ( |
80522398 |
id digest.Digest |
94726f7c |
manifestDigest digest.Digest
) |
c168a005 |
switch v := manifest.(type) {
case *schema1.SignedManifest: |
3c7676a0 |
if p.config.RequireSchema2 {
return false, fmt.Errorf("invalid manifest: not schema2")
} |
f9232e3f |
// give registries time to upgrade to schema2 and only warn if we know a registry has been upgraded long time ago
// TODO: condition to be removed
if reference.Domain(ref) == "docker.io" {
msg := fmt.Sprintf("Image %s uses outdated schema1 manifest format. Please upgrade to a schema2 image for better future compatibility. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", ref)
logrus.Warn(msg)
progress.Message(p.config.ProgressOutput, "", msg)
} |
023166b5 |
|
337ba71f |
id, manifestDigest, err = p.pullSchema1(ctx, ref, v, platform) |
94726f7c |
if err != nil {
return false, err
}
case *schema2.DeserializedManifest: |
337ba71f |
id, manifestDigest, err = p.pullSchema2(ctx, ref, v, platform) |
c168a005 |
if err != nil {
return false, err
} |
e037bade |
case *ocischema.DeserializedManifest:
id, manifestDigest, err = p.pullOCI(ctx, ref, v, platform)
if err != nil {
return false, err
} |
2bb8c85b |
case *manifestlist.DeserializedManifestList: |
337ba71f |
id, manifestDigest, err = p.pullManifestList(ctx, ref, v, platform) |
2bb8c85b |
if err != nil {
return false, err
} |
c168a005 |
default: |
ebcb7d6b |
return false, invalidManifestFormatError{} |
694df3ff |
}
|
94726f7c |
progress.Message(p.config.ProgressOutput, "", "Digest: "+manifestDigest.String())
|
3c7676a0 |
if p.config.ReferenceStore != nil {
oldTagID, err := p.config.ReferenceStore.Get(ref)
if err == nil {
if oldTagID == id {
return false, addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id)
} |
3a127939 |
} else if err != refstore.ErrDoesNotExist { |
33984f25 |
return false, err
} |
3c7676a0 |
if canonical, ok := ref.(reference.Canonical); ok {
if err = p.config.ReferenceStore.AddDigest(canonical, id, true); err != nil {
return false, err
}
} else {
if err = addDigestReference(p.config.ReferenceStore, ref, manifestDigest, id); err != nil {
return false, err
}
if err = p.config.ReferenceStore.AddTag(ref, id, true); err != nil {
return false, err
} |
33984f25 |
} |
694df3ff |
} |
c168a005 |
return true, nil
} |
a57478d6 |
|
337ba71f |
func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unverifiedManifest *schema1.SignedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) { |
694df3ff |
var verifiedManifest *schema1.Manifest |
94726f7c |
verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref) |
694df3ff |
if err != nil { |
94726f7c |
return "", "", err |
694df3ff |
}
rootFS := image.NewRootFS()
// remove duplicate layers and check parent chain validity
err = fixManifestLayers(verifiedManifest)
if err != nil { |
94726f7c |
return "", "", err |
694df3ff |
}
|
572ce802 |
var descriptors []xfer.DownloadDescriptor |
694df3ff |
// Image history converted to the new format
var history []image.History
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly.
for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
blobSum := verifiedManifest.FSLayers[i].BlobSum
var throwAway struct {
ThrowAway bool `json:"throwaway,omitempty"`
}
if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil { |
94726f7c |
return "", "", err |
694df3ff |
}
h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway)
if err != nil { |
94726f7c |
return "", "", err |
694df3ff |
}
history = append(history, h)
if throwAway.ThrowAway {
continue
}
|
572ce802 |
layerDescriptor := &v2LayerDescriptor{ |
63099477 |
digest: blobSum,
repoInfo: p.repoInfo,
repo: p.repo,
V2MetadataService: p.V2MetadataService, |
694df3ff |
}
|
572ce802 |
descriptors = append(descriptors, layerDescriptor) |
694df3ff |
}
|
ce8e529e |
// The v1 manifest itself doesn't directly contain an OS. However, |
b21d9ab5 |
// the history does, but unfortunately that's a string, so search through |
ce8e529e |
// all the history until hopefully we find one which indicates the OS. |
0380fbff |
// supertest2014/nyan is an example of a registry image with schemav1.
configOS := runtime.GOOS |
4ec9766a |
if system.LCOWSupported() { |
b21d9ab5 |
type config struct {
Os string `json:"os,omitempty"`
}
for _, v := range verifiedManifest.History {
var c config
if err := json.Unmarshal([]byte(v.V1Compatibility), &c); err == nil {
if c.Os != "" { |
0380fbff |
configOS = c.Os |
b21d9ab5 |
break
}
}
}
}
|
35193c0e |
// In the situation that the API call didn't specify an OS explicitly, but
// we support the operating system, switch to that operating system.
// eg FROM supertest2014/nyan with no platform specifier, and docker build
// with no --platform= flag under LCOW. |
337ba71f |
requestedOS := ""
if platform != nil {
requestedOS = platform.OS
} else if system.IsOSSupported(configOS) { |
35193c0e |
requestedOS = configOS
}
|
0380fbff |
// Early bath if the requested OS doesn't match that of the configuration.
// This avoids doing the download, only to potentially fail later. |
d98ecf2d |
if !strings.EqualFold(configOS, requestedOS) { |
0380fbff |
return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configOS, requestedOS)
}
|
ce8e529e |
resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, configOS, descriptors, p.config.ProgressOutput) |
572ce802 |
if err != nil { |
94726f7c |
return "", "", err |
694df3ff |
} |
572ce802 |
defer release() |
694df3ff |
|
572ce802 |
config, err := v1.MakeConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), &resultRootFS, history) |
694df3ff |
if err != nil { |
94726f7c |
return "", "", err |
694df3ff |
}
|
3c7676a0 |
imageID, err := p.config.ImageStore.Put(config) |
694df3ff |
if err != nil { |
94726f7c |
return "", "", err |
694df3ff |
}
|
94726f7c |
manifestDigest = digest.FromBytes(unverifiedManifest.Canonical)
|
3c7676a0 |
return imageID, manifestDigest, nil |
94726f7c |
}
|
e037bade |
func (p *v2Puller) pullSchema2Layers(ctx context.Context, target distribution.Descriptor, layers []distribution.Descriptor, platform *specs.Platform) (id digest.Digest, err error) { |
3c7676a0 |
if _, err := p.config.ImageStore.Get(target.Digest); err == nil { |
94726f7c |
// If the image already exists locally, no need to pull
// anything. |
e037bade |
return target.Digest, nil |
94726f7c |
}
|
05bd0435 |
var descriptors []xfer.DownloadDescriptor
// Note that the order of this loop is in the direction of bottom-most
// to top-most, so that the downloads slice gets ordered correctly. |
e037bade |
for _, d := range layers { |
05bd0435 |
layerDescriptor := &v2LayerDescriptor{
digest: d.Digest,
repo: p.repo,
repoInfo: p.repoInfo,
V2MetadataService: p.V2MetadataService, |
2c60430a |
src: d, |
05bd0435 |
}
descriptors = append(descriptors, layerDescriptor)
}
|
94726f7c |
configChan := make(chan []byte, 1) |
d5530406 |
configErrChan := make(chan error, 1)
layerErrChan := make(chan error, 1)
downloadsDone := make(chan struct{}) |
94726f7c |
var cancel func()
ctx, cancel = context.WithCancel(ctx) |
d5530406 |
defer cancel() |
94726f7c |
// Pull the image config
go func() { |
80522398 |
configJSON, err := p.pullSchema2Config(ctx, target.Digest) |
94726f7c |
if err != nil { |
d5530406 |
configErrChan <- ImageConfigPullError{Err: err} |
94726f7c |
cancel()
return
}
configChan <- configJSON
}()
|
7450c258 |
var ( |
83908836 |
configJSON []byte // raw serialized image config
downloadedRootFS *image.RootFS // rootFS from registered layers
configRootFS *image.RootFS // rootFS from configuration
release func() // release resources from rootFS download
configPlatform *specs.Platform // for LCOW when registering downloaded layers |
7450c258 |
) |
f342b271 |
|
337ba71f |
layerStoreOS := runtime.GOOS
if platform != nil {
layerStoreOS = platform.OS
}
|
f342b271 |
// https://github.com/docker/docker/issues/24766 - Err on the side of caution, |
8437d0a3 |
// explicitly blocking images intended for linux from the Windows daemon. On
// Windows, we do this before the attempt to download, effectively serialising
// the download slightly slowing it down. We have to do it this way, as
// chances are the download of layers itself would fail due to file names
// which aren't suitable for NTFS. At some point in the future, if a similar
// check to block Windows images being pulled on Linux is implemented, it
// may be necessary to perform the same type of serialisation.
if runtime.GOOS == "windows" { |
83908836 |
configJSON, configRootFS, configPlatform, err = receiveConfig(p.config.ImageStore, configChan, configErrChan) |
8437d0a3 |
if err != nil { |
e037bade |
return "", err |
8437d0a3 |
} |
3c7676a0 |
if configRootFS == nil { |
e037bade |
return "", errRootFSInvalid |
8437d0a3 |
} |
83908836 |
if err := checkImageCompatibility(configPlatform.OS, configPlatform.OSVersion); err != nil { |
e037bade |
return "", err |
83908836 |
} |
633f9252 |
if len(descriptors) != len(configRootFS.DiffIDs) { |
e037bade |
return "", errRootFSMismatch |
633f9252 |
} |
337ba71f |
if platform == nil {
// Early bath if the requested OS doesn't match that of the configuration.
// This avoids doing the download, only to potentially fail later.
if !system.IsOSSupported(configPlatform.OS) { |
e037bade |
return "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, layerStoreOS) |
337ba71f |
}
layerStoreOS = configPlatform.OS |
0380fbff |
}
|
633f9252 |
// Populate diff ids in descriptors to avoid downloading foreign layers
// which have been side loaded
for i := range descriptors {
descriptors[i].(*v2LayerDescriptor).diffID = configRootFS.DiffIDs[i]
} |
7450c258 |
}
|
3c7676a0 |
if p.config.DownloadManager != nil { |
d5530406 |
go func() {
var (
err error
rootFS image.RootFS
)
downloadRootFS := *image.NewRootFS() |
337ba71f |
rootFS, release, err = p.config.DownloadManager.Download(ctx, downloadRootFS, layerStoreOS, descriptors, p.config.ProgressOutput) |
d5530406 |
if err != nil {
// Intentionally do not cancel the config download here
// as the error from config download (if there is one)
// is more interesting than the layer download error
layerErrChan <- err
return |
94726f7c |
} |
3c7676a0 |
|
d5530406 |
downloadedRootFS = &rootFS
close(downloadsDone)
}()
} else {
// We have nothing to download
close(downloadsDone) |
94726f7c |
}
|
8437d0a3 |
if configJSON == nil { |
d98ecf2d |
configJSON, configRootFS, _, err = receiveConfig(p.config.ImageStore, configChan, configErrChan) |
d5530406 |
if err == nil && configRootFS == nil {
err = errRootFSInvalid
} |
8437d0a3 |
if err != nil { |
d5530406 |
cancel()
select {
case <-downloadsDone:
case <-layerErrChan:
} |
e037bade |
return "", err |
8437d0a3 |
} |
d5530406 |
} |
94726f7c |
|
d5530406 |
select {
case <-downloadsDone:
case err = <-layerErrChan: |
e037bade |
return "", err |
d5530406 |
}
if release != nil {
defer release() |
9b6dcc8b |
}
|
3c7676a0 |
if downloadedRootFS != nil {
// The DiffIDs returned in rootFS MUST match those in the config.
// Otherwise the image config could be referencing layers that aren't
// included in the manifest.
if len(downloadedRootFS.DiffIDs) != len(configRootFS.DiffIDs) { |
e037bade |
return "", errRootFSMismatch |
94726f7c |
} |
3c7676a0 |
for i := range downloadedRootFS.DiffIDs {
if downloadedRootFS.DiffIDs[i] != configRootFS.DiffIDs[i] { |
e037bade |
return "", errRootFSMismatch |
3c7676a0 |
}
} |
94726f7c |
}
|
3c7676a0 |
imageID, err := p.config.ImageStore.Put(configJSON) |
94726f7c |
if err != nil { |
e037bade |
return "", err
}
return imageID, nil
}
func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil { |
94726f7c |
return "", "", err
} |
e037bade |
id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
return id, manifestDigest, err
} |
94726f7c |
|
e037bade |
func (p *v2Puller) pullOCI(ctx context.Context, ref reference.Named, mfst *ocischema.DeserializedManifest, platform *specs.Platform) (id digest.Digest, manifestDigest digest.Digest, err error) {
manifestDigest, err = schema2ManifestDigest(ref, mfst)
if err != nil {
return "", "", err
}
id, err = p.pullSchema2Layers(ctx, mfst.Target(), mfst.Layers, platform)
return id, manifestDigest, err |
94726f7c |
}
|
83908836 |
func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan error) ([]byte, *image.RootFS, *specs.Platform, error) { |
7450c258 |
select {
case configJSON := <-configChan: |
83908836 |
rootfs, err := s.RootFSFromConfig(configJSON)
if err != nil {
return nil, nil, nil, err
}
platform, err := s.PlatformFromConfig(configJSON) |
3c7676a0 |
if err != nil { |
83908836 |
return nil, nil, nil, err |
7450c258 |
} |
83908836 |
return configJSON, rootfs, platform, nil |
7450c258 |
case err := <-errChan: |
83908836 |
return nil, nil, nil, err |
7450c258 |
// Don't need a case for ctx.Done in the select because cancellation
// will trigger an error in p.pullSchema2ImageConfig.
}
}
|
2bb8c85b |
// pullManifestList handles "manifest lists" which point to various |
40af5691 |
// platform-specific manifests. |
337ba71f |
func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, pp *specs.Platform) (id digest.Digest, manifestListDigest digest.Digest, err error) { |
2bb8c85b |
manifestListDigest, err = schema2ManifestDigest(ref, mfstList)
if err != nil {
return "", "", err
}
|
337ba71f |
var platform specs.Platform
if pp != nil {
platform = *pp |
35193c0e |
} |
337ba71f |
logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), platforms.Format(platform), runtime.GOARCH) |
2bb8c85b |
|
337ba71f |
manifestMatches := filterManifests(mfstList.Manifests, platform) |
38aef56e |
if len(manifestMatches) == 0 { |
4b9db209 |
errMsg := fmt.Sprintf("no matching manifest for %s in the manifest list entries", formatPlatform(platform)) |
9a8cb931 |
logrus.Debugf(errMsg)
return "", "", errors.New(errMsg) |
2bb8c85b |
}
|
38aef56e |
if len(manifestMatches) > 1 {
logrus.Debugf("found multiple matches in manifest list, choosing best match %s", manifestMatches[0].Digest.String())
}
manifestDigest := manifestMatches[0].Digest
|
83908836 |
if err := checkImageCompatibility(manifestMatches[0].Platform.OS, manifestMatches[0].Platform.OSVersion); err != nil {
return "", "", err
}
|
2bb8c85b |
manSvc, err := p.repo.Manifests(ctx)
if err != nil {
return "", "", err
}
manifest, err := manSvc.Get(ctx, manifestDigest)
if err != nil {
return "", "", err
}
|
c85eb008 |
manifestRef, err := reference.WithDigest(reference.TrimNamed(ref), manifestDigest) |
2bb8c85b |
if err != nil {
return "", "", err
}
switch v := manifest.(type) {
case *schema1.SignedManifest: |
f9232e3f |
msg := fmt.Sprintf("[DEPRECATION NOTICE] v2 schema1 manifests in manifest lists are not supported and will break in a future release. Suggest author of %s to upgrade to v2 schema2. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/", ref) |
023166b5 |
logrus.Warn(msg)
progress.Message(p.config.ProgressOutput, "", msg)
|
337ba71f |
platform := toOCIPlatform(manifestMatches[0].Platform)
id, _, err = p.pullSchema1(ctx, manifestRef, v, &platform) |
2bb8c85b |
if err != nil {
return "", "", err
}
case *schema2.DeserializedManifest: |
337ba71f |
platform := toOCIPlatform(manifestMatches[0].Platform)
id, _, err = p.pullSchema2(ctx, manifestRef, v, &platform) |
2bb8c85b |
if err != nil {
return "", "", err
} |
e037bade |
case *ocischema.DeserializedManifest:
platform := toOCIPlatform(manifestMatches[0].Platform)
id, _, err = p.pullOCI(ctx, manifestRef, v, &platform)
if err != nil {
return "", "", err
} |
2bb8c85b |
default:
return "", "", errors.New("unsupported manifest format")
}
|
80522398 |
return id, manifestListDigest, err |
2bb8c85b |
}
|
80522398 |
func (p *v2Puller) pullSchema2Config(ctx context.Context, dgst digest.Digest) (configJSON []byte, err error) { |
94726f7c |
blobs := p.repo.Blobs(ctx)
configJSON, err = blobs.Get(ctx, dgst)
if err != nil {
return nil, err
}
// Verify image config digest |
7a855799 |
verifier := dgst.Verifier() |
94726f7c |
if _, err := verifier.Write(configJSON); err != nil {
return nil, err
}
if !verifier.Verified() {
err := fmt.Errorf("image config verification failed for digest %s", dgst)
logrus.Error(err)
return nil, err |
694df3ff |
}
|
94726f7c |
return configJSON, nil |
694df3ff |
}
|
2bb8c85b |
// schema2ManifestDigest computes the manifest digest, and, if pulling by
// digest, ensures that it matches the requested digest.
func schema2ManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) {
_, canonical, err := mfst.Payload()
if err != nil {
return "", err
}
// If pull by digest, then verify the manifest digest.
if digested, isDigested := ref.(reference.Canonical); isDigested { |
7a855799 |
verifier := digested.Digest().Verifier() |
2bb8c85b |
if _, err := verifier.Write(canonical); err != nil {
return "", err
}
if !verifier.Verified() {
err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest())
logrus.Error(err)
return "", err
}
return digested.Digest(), nil
}
return digest.FromBytes(canonical), nil
}
|
589a5226 |
// allowV1Fallback checks if the error is a possible reason to fallback to v1
// (even if confirmedV2 has been set already), and if so, wraps the error in
// a fallbackError with confirmedV2 set to false. Otherwise, it returns the
// error unmodified.
func allowV1Fallback(err error) error {
switch v := err.(type) {
case errcode.Errors:
if len(v) != 0 { |
8f26fe4f |
if v0, ok := v[0].(errcode.Error); ok && shouldV2Fallback(v0) { |
5e8af46f |
return fallbackError{
err: err,
confirmedV2: false,
transportOK: true,
} |
589a5226 |
}
}
case errcode.Error: |
8f26fe4f |
if shouldV2Fallback(v) { |
5e8af46f |
return fallbackError{
err: err,
confirmedV2: false,
transportOK: true,
} |
589a5226 |
} |
7b81bc14 |
case *url.Error:
if v.Err == auth.ErrNoBasicAuthCredentials {
return fallbackError{err: err, confirmedV2: false}
} |
589a5226 |
}
return err
}
|
709bf8b7 |
func verifySchema1Manifest(signedManifest *schema1.SignedManifest, ref reference.Reference) (m *schema1.Manifest, err error) { |
694df3ff |
// If pull by digest, then verify the manifest digest. NOTE: It is
// important to do this first, before any other content validation. If the
// digest cannot be verified, don't even bother with those other things. |
2655954c |
if digested, isCanonical := ref.(reference.Canonical); isCanonical { |
7a855799 |
verifier := digested.Digest().Verifier() |
c168a005 |
if _, err := verifier.Write(signedManifest.Canonical); err != nil { |
694df3ff |
return nil, err
}
if !verifier.Verified() {
err := fmt.Errorf("image verification failed for digest %s", digested.Digest())
logrus.Error(err)
return nil, err
}
} |
c168a005 |
m = &signedManifest.Manifest |
694df3ff |
if m.SchemaVersion != 1 { |
3a127939 |
return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, reference.FamiliarString(ref)) |
694df3ff |
}
if len(m.FSLayers) != len(m.History) { |
3a127939 |
return nil, fmt.Errorf("length of history not equal to number of layers for %q", reference.FamiliarString(ref)) |
694df3ff |
}
if len(m.FSLayers) == 0 { |
3a127939 |
return nil, fmt.Errorf("no FSLayers in manifest for %q", reference.FamiliarString(ref)) |
694df3ff |
}
return m, nil
}
// fixManifestLayers removes repeated layers from the manifest and checks the
// correctness of the parent chain.
func fixManifestLayers(m *schema1.Manifest) error {
imgs := make([]*image.V1Image, len(m.FSLayers))
for i := range m.FSLayers {
img := &image.V1Image{}
if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil {
return err
}
imgs[i] = img
if err := v1.ValidateID(img.ID); err != nil {
return err
}
}
if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" {
// Windows base layer can point to a base layer parent that is not in manifest. |
4dbc78a5 |
return errors.New("invalid parent ID in the base layer of the image") |
694df3ff |
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range imgs {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(imgs) - 2; i >= 0; i-- {
if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
m.History = append(m.History[:i], m.History[i+1:]...)
} else if imgs[i].Parent != imgs[i+1].ID { |
9b47b7b1 |
return fmt.Errorf("invalid parent ID. Expected %v, got %v", imgs[i+1].ID, imgs[i].Parent) |
694df3ff |
}
}
return nil
} |
f425529e |
func createDownloadFile() (*os.File, error) {
return ioutil.TempFile("", "GetImageBlob")
} |
337ba71f |
func toOCIPlatform(p manifestlist.PlatformSpec) specs.Platform {
return specs.Platform{
OS: p.OS,
Architecture: p.Architecture,
Variant: p.Variant,
OSFeatures: p.OSFeatures,
OSVersion: p.OSVersion,
}
} |