Co-authored-by: Djordje Lukic <djordje.lukic@docker.com>
Signed-off-by: Laura Brehm <laurabrehm@hey.com>
| ... | ... |
@@ -14,6 +14,7 @@ import ( |
| 14 | 14 |
containerpkg "github.com/docker/docker/container" |
| 15 | 15 |
"github.com/docker/docker/image" |
| 16 | 16 |
"github.com/docker/docker/layer" |
| 17 |
+ "github.com/opencontainers/go-digest" |
|
| 17 | 18 |
) |
| 18 | 19 |
|
| 19 | 20 |
const ( |
| ... | ... |
@@ -45,7 +46,7 @@ type Backend interface {
|
| 45 | 45 |
// ContainerCreateWorkdir creates the workdir |
| 46 | 46 |
ContainerCreateWorkdir(containerID string) error |
| 47 | 47 |
|
| 48 |
- CreateImage(config []byte, parent string) (Image, error) |
|
| 48 |
+ CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (Image, error) |
|
| 49 | 49 |
|
| 50 | 50 |
ImageCacheBuilder |
| 51 | 51 |
} |
| ... | ... |
@@ -104,6 +105,7 @@ type ROLayer interface {
|
| 104 | 104 |
Release() error |
| 105 | 105 |
NewRWLayer() (RWLayer, error) |
| 106 | 106 |
DiffID() layer.DiffID |
| 107 |
+ ContentStoreDigest() digest.Digest |
|
| 107 | 108 |
} |
| 108 | 109 |
|
| 109 | 110 |
// RWLayer is active layer that can be read/modified |
| ... | ... |
@@ -63,7 +63,7 @@ func (b *Builder) commitContainer(ctx context.Context, dispatchState *dispatchSt |
| 63 | 63 |
return err |
| 64 | 64 |
} |
| 65 | 65 |
|
| 66 |
-func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error {
|
|
| 66 |
+func (b *Builder) exportImage(ctx context.Context, state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error {
|
|
| 67 | 67 |
newLayer, err := layer.Commit() |
| 68 | 68 |
if err != nil {
|
| 69 | 69 |
return err |
| ... | ... |
@@ -98,7 +98,15 @@ func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, paren |
| 98 | 98 |
return errors.Wrap(err, "failed to encode image config") |
| 99 | 99 |
} |
| 100 | 100 |
|
| 101 |
- exportedImage, err := b.docker.CreateImage(config, state.imageID) |
|
| 101 |
+ // when writing the new image's manifest, we now need to pass in the new layer's digest. |
|
| 102 |
+ // before the containerd store work this was unnecessary since we get the layer id |
|
| 103 |
+ // from the image's RootFS ChainID -- see: |
|
| 104 |
+ // https://github.com/moby/moby/blob/8cf66ed7322fa885ef99c4c044fa23e1727301dc/image/store.go#L162 |
|
| 105 |
+ // however, with the containerd store we can't do this. An alternative implementation here |
|
| 106 |
+ // without changing the signature would be to get the layer digest by walking the content store |
|
| 107 |
+ // and filtering the objects to find the layer with the DiffID we want, but that has performance |
|
| 108 |
+ // implications that should be called out/investigated |
|
| 109 |
+ exportedImage, err := b.docker.CreateImage(ctx, config, state.imageID, newLayer.ContentStoreDigest()) |
|
| 102 | 110 |
if err != nil {
|
| 103 | 111 |
return errors.Wrapf(err, "failed to export image") |
| 104 | 112 |
} |
| ... | ... |
@@ -170,7 +178,7 @@ func (b *Builder) performCopy(ctx context.Context, req dispatchRequest, inst cop |
| 170 | 170 |
return errors.Wrapf(err, "failed to copy files") |
| 171 | 171 |
} |
| 172 | 172 |
} |
| 173 |
- return b.exportImage(state, rwLayer, imageMount.Image(), runConfigWithCommentCmd) |
|
| 173 |
+ return b.exportImage(ctx, state, rwLayer, imageMount.Image(), runConfigWithCommentCmd) |
|
| 174 | 174 |
} |
| 175 | 175 |
|
| 176 | 176 |
func createDestInfo(workingDir string, inst copyInstruction, rwLayer builder.RWLayer, platform string) (copyInfo, error) {
|
| ... | ... |
@@ -1,6 +1,7 @@ |
| 1 | 1 |
package dockerfile // import "github.com/docker/docker/builder/dockerfile" |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "context" |
|
| 4 | 5 |
"fmt" |
| 5 | 6 |
"os" |
| 6 | 7 |
"runtime" |
| ... | ... |
@@ -193,6 +194,7 @@ type MockROLayer struct {
|
| 193 | 193 |
diffID layer.DiffID |
| 194 | 194 |
} |
| 195 | 195 |
|
| 196 |
+func (l *MockROLayer) ContentStoreDigest() digest.Digest { return "" }
|
|
| 196 | 197 |
func (l *MockROLayer) Release() error { return nil }
|
| 197 | 198 |
func (l *MockROLayer) NewRWLayer() (builder.RWLayer, error) { return nil, nil }
|
| 198 | 199 |
func (l *MockROLayer) DiffID() layer.DiffID { return l.diffID }
|
| ... | ... |
@@ -217,6 +219,6 @@ func TestExportImage(t *testing.T) {
|
| 217 | 217 |
imageSources: getMockImageSource(nil, nil, nil), |
| 218 | 218 |
docker: getMockBuildBackend(), |
| 219 | 219 |
} |
| 220 |
- err := b.exportImage(ds, layer, parentImage, runConfig) |
|
| 220 |
+ err := b.exportImage(context.TODO(), ds, layer, parentImage, runConfig) |
|
| 221 | 221 |
assert.NilError(t, err) |
| 222 | 222 |
} |
| ... | ... |
@@ -13,6 +13,7 @@ import ( |
| 13 | 13 |
containerpkg "github.com/docker/docker/container" |
| 14 | 14 |
"github.com/docker/docker/image" |
| 15 | 15 |
"github.com/docker/docker/layer" |
| 16 |
+ "github.com/opencontainers/go-digest" |
|
| 16 | 17 |
) |
| 17 | 18 |
|
| 18 | 19 |
// MockBackend implements the builder.Backend interface for unit testing |
| ... | ... |
@@ -80,7 +81,7 @@ func (m *MockBackend) MakeImageCache(ctx context.Context, cacheFrom []string) (b |
| 80 | 80 |
return nil, nil |
| 81 | 81 |
} |
| 82 | 82 |
|
| 83 |
-func (m *MockBackend) CreateImage(config []byte, parent string) (builder.Image, error) {
|
|
| 83 |
+func (m *MockBackend) CreateImage(ctx context.Context, config []byte, parent string, layerDigest digest.Digest) (builder.Image, error) {
|
|
| 84 | 84 |
return &mockImage{id: "test"}, nil
|
| 85 | 85 |
} |
| 86 | 86 |
|
| ... | ... |
@@ -119,6 +120,10 @@ func (mic *mockImageCache) GetCache(parentID string, cfg *container.Config) (str |
| 119 | 119 |
|
| 120 | 120 |
type mockLayer struct{}
|
| 121 | 121 |
|
| 122 |
+func (l *mockLayer) ContentStoreDigest() digest.Digest {
|
|
| 123 |
+ return "" |
|
| 124 |
+} |
|
| 125 |
+ |
|
| 122 | 126 |
func (l *mockLayer) Release() error {
|
| 123 | 127 |
return nil |
| 124 | 128 |
} |
| ... | ... |
@@ -2,11 +2,82 @@ package containerd |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 |
+ "reflect" |
|
| 5 | 6 |
|
| 7 |
+ "github.com/docker/docker/api/types/container" |
|
| 8 |
+ imagetype "github.com/docker/docker/api/types/image" |
|
| 6 | 9 |
"github.com/docker/docker/builder" |
| 10 |
+ "github.com/docker/docker/image" |
|
| 7 | 11 |
) |
| 8 | 12 |
|
| 9 | 13 |
// MakeImageCache creates a stateful image cache. |
| 10 | 14 |
func (i *ImageService) MakeImageCache(ctx context.Context, cacheFrom []string) (builder.ImageCache, error) {
|
| 11 |
- panic("not implemented")
|
|
| 15 |
+ images := []*image.Image{}
|
|
| 16 |
+ for _, c := range cacheFrom {
|
|
| 17 |
+ im, err := i.GetImage(ctx, c, imagetype.GetImageOpts{})
|
|
| 18 |
+ if err != nil {
|
|
| 19 |
+ return nil, err |
|
| 20 |
+ } |
|
| 21 |
+ images = append(images, im) |
|
| 22 |
+ } |
|
| 23 |
+ return &imageCache{images: images, c: i}, nil
|
|
| 24 |
+} |
|
| 25 |
+ |
|
| 26 |
+type imageCache struct {
|
|
| 27 |
+ images []*image.Image |
|
| 28 |
+ c *ImageService |
|
| 29 |
+} |
|
| 30 |
+ |
|
| 31 |
+func (ic *imageCache) GetCache(parentID string, cfg *container.Config) (imageID string, err error) {
|
|
| 32 |
+ ctx := context.TODO() |
|
| 33 |
+ cfgCpy := *cfg |
|
| 34 |
+ i, err := ic.c.GetImage(ctx, parentID, imagetype.GetImageOpts{})
|
|
| 35 |
+ if err != nil {
|
|
| 36 |
+ for _, ii := range ic.images {
|
|
| 37 |
+ if ii.ID().String() == parentID {
|
|
| 38 |
+ if compare(ii.RunConfig(), &cfgCpy) {
|
|
| 39 |
+ return ii.ID().String(), nil |
|
| 40 |
+ } |
|
| 41 |
+ } |
|
| 42 |
+ } |
|
| 43 |
+ } else {
|
|
| 44 |
+ children, err := ic.c.Children(ctx, i.ID()) |
|
| 45 |
+ if err != nil {
|
|
| 46 |
+ return "", err |
|
| 47 |
+ } |
|
| 48 |
+ for _, ch := range children {
|
|
| 49 |
+ childImage, err := ic.c.GetImage(context.TODO(), ch.String(), imagetype.GetImageOpts{})
|
|
| 50 |
+ if err != nil {
|
|
| 51 |
+ return "", err |
|
| 52 |
+ } |
|
| 53 |
+ // this implementation looks correct but it's currently not working |
|
| 54 |
+ // with the containerd store as we're not storing the image ContainerConfig |
|
| 55 |
+ // and so intermediate images with ContainerConfigs such as |
|
| 56 |
+ // #(nop) COPY file:c6ab44934e83eeb07289a211582c6faa25dea7d06dae077b6ef76029e92400ce in ... |
|
| 57 |
+ // are not getting a hit |
|
| 58 |
+ if compare(&childImage.ContainerConfig, &cfgCpy) {
|
|
| 59 |
+ return ch.String(), nil |
|
| 60 |
+ } |
|
| 61 |
+ } |
|
| 62 |
+ } |
|
| 63 |
+ return "", nil |
|
| 64 |
+} |
|
| 65 |
+ |
|
| 66 |
+// compare two Config structs. Do not consider the "Hostname" field as it |
|
| 67 |
+// defaults to the randomly generated short container ID or "Image" as it |
|
| 68 |
+// represents the name of the image as it was passed in (symbolic) and does |
|
| 69 |
+// not provide any meaningful information about whether the image is usable |
|
| 70 |
+// as cache. |
|
| 71 |
+// If OpenStdin is set, then it differs |
|
| 72 |
+func compare(a, b *container.Config) bool {
|
|
| 73 |
+ if a == nil || b == nil || |
|
| 74 |
+ a.OpenStdin || b.OpenStdin {
|
|
| 75 |
+ return false |
|
| 76 |
+ } |
|
| 77 |
+ |
|
| 78 |
+ a.Image = "" |
|
| 79 |
+ a.Hostname = "" |
|
| 80 |
+ b.Image = "" |
|
| 81 |
+ b.Hostname = "" |
|
| 82 |
+ return reflect.DeepEqual(a, b) |
|
| 12 | 83 |
} |
| ... | ... |
@@ -2,23 +2,490 @@ package containerd |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 |
- "errors" |
|
| 5 |
+ "fmt" |
|
| 6 |
+ "io" |
|
| 7 |
+ "os" |
|
| 8 |
+ "runtime" |
|
| 9 |
+ "time" |
|
| 6 | 10 |
|
| 11 |
+ "github.com/containerd/containerd" |
|
| 12 |
+ cerrdefs "github.com/containerd/containerd/errdefs" |
|
| 13 |
+ "github.com/containerd/containerd/leases" |
|
| 14 |
+ "github.com/containerd/containerd/mount" |
|
| 15 |
+ "github.com/containerd/containerd/platforms" |
|
| 16 |
+ "github.com/containerd/containerd/rootfs" |
|
| 17 |
+ "github.com/docker/distribution/reference" |
|
| 7 | 18 |
"github.com/docker/docker/api/types/backend" |
| 19 |
+ imagetypes "github.com/docker/docker/api/types/image" |
|
| 20 |
+ "github.com/docker/docker/api/types/registry" |
|
| 21 |
+ registrypkg "github.com/docker/docker/registry" |
|
| 22 |
+ |
|
| 23 |
+ // "github.com/docker/docker/api/types/container" |
|
| 24 |
+ containerdimages "github.com/containerd/containerd/images" |
|
| 25 |
+ "github.com/docker/docker/api/types/image" |
|
| 8 | 26 |
"github.com/docker/docker/builder" |
| 9 | 27 |
"github.com/docker/docker/errdefs" |
| 28 |
+ dimage "github.com/docker/docker/image" |
|
| 29 |
+ "github.com/docker/docker/layer" |
|
| 30 |
+ "github.com/docker/docker/pkg/progress" |
|
| 31 |
+ "github.com/docker/docker/pkg/streamformatter" |
|
| 32 |
+ "github.com/docker/docker/pkg/stringid" |
|
| 33 |
+ "github.com/docker/docker/pkg/system" |
|
| 34 |
+ "github.com/opencontainers/go-digest" |
|
| 35 |
+ "github.com/opencontainers/image-spec/identity" |
|
| 36 |
+ ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
|
| 37 |
+ "github.com/sirupsen/logrus" |
|
| 10 | 38 |
) |
| 11 | 39 |
|
| 12 | 40 |
// GetImageAndReleasableLayer returns an image and releaseable layer for a |
| 13 | 41 |
// reference or ID. Every call to GetImageAndReleasableLayer MUST call |
| 14 | 42 |
// releasableLayer.Release() to prevent leaking of layers. |
| 15 | 43 |
func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) {
|
| 16 |
- return nil, nil, errdefs.NotImplemented(errors.New("not implemented"))
|
|
| 44 |
+ if refOrID == "" { // from SCRATCH
|
|
| 45 |
+ os := runtime.GOOS |
|
| 46 |
+ if runtime.GOOS == "windows" {
|
|
| 47 |
+ os = "linux" |
|
| 48 |
+ } |
|
| 49 |
+ if opts.Platform != nil {
|
|
| 50 |
+ os = opts.Platform.OS |
|
| 51 |
+ } |
|
| 52 |
+ if !system.IsOSSupported(os) {
|
|
| 53 |
+ return nil, nil, system.ErrNotSupportedOperatingSystem |
|
| 54 |
+ } |
|
| 55 |
+ return nil, &rolayer{
|
|
| 56 |
+ key: "", |
|
| 57 |
+ c: i.client, |
|
| 58 |
+ snapshotter: i.snapshotter, |
|
| 59 |
+ diffID: "", |
|
| 60 |
+ root: "", |
|
| 61 |
+ }, nil |
|
| 62 |
+ } |
|
| 63 |
+ |
|
| 64 |
+ if opts.PullOption != backend.PullOptionForcePull {
|
|
| 65 |
+ // TODO(laurazard): same as below |
|
| 66 |
+ img, err := i.GetImage(ctx, refOrID, image.GetImageOpts{Platform: opts.Platform})
|
|
| 67 |
+ if err != nil && opts.PullOption == backend.PullOptionNoPull {
|
|
| 68 |
+ return nil, nil, err |
|
| 69 |
+ } |
|
| 70 |
+ imgDesc, err := i.resolveDescriptor(ctx, refOrID) |
|
| 71 |
+ if err != nil && !errdefs.IsNotFound(err) {
|
|
| 72 |
+ return nil, nil, err |
|
| 73 |
+ } |
|
| 74 |
+ if img != nil {
|
|
| 75 |
+ if !system.IsOSSupported(img.OperatingSystem()) {
|
|
| 76 |
+ return nil, nil, system.ErrNotSupportedOperatingSystem |
|
| 77 |
+ } |
|
| 78 |
+ |
|
| 79 |
+ layer, err := newROLayerForImage(ctx, &imgDesc, i, opts, refOrID, opts.Platform) |
|
| 80 |
+ if err != nil {
|
|
| 81 |
+ return nil, nil, err |
|
| 82 |
+ } |
|
| 83 |
+ |
|
| 84 |
+ return img, layer, nil |
|
| 85 |
+ } |
|
| 86 |
+ } |
|
| 87 |
+ |
|
| 88 |
+ ctx, _, err := i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) |
|
| 89 |
+ if err != nil {
|
|
| 90 |
+ return nil, nil, fmt.Errorf("failed to create lease for commit: %w", err)
|
|
| 91 |
+ } |
|
| 92 |
+ |
|
| 93 |
+ // TODO(laurazard): do we really need a new method here to pull the image? |
|
| 94 |
+ imgDesc, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform) |
|
| 95 |
+ if err != nil {
|
|
| 96 |
+ return nil, nil, err |
|
| 97 |
+ } |
|
| 98 |
+ |
|
| 99 |
+ // TODO(laurazard): pullForBuilder should return whatever we |
|
| 100 |
+ // need here instead of having to go and get it again |
|
| 101 |
+ img, err := i.GetImage(ctx, refOrID, imagetypes.GetImageOpts{
|
|
| 102 |
+ Platform: opts.Platform, |
|
| 103 |
+ }) |
|
| 104 |
+ if err != nil {
|
|
| 105 |
+ return nil, nil, err |
|
| 106 |
+ } |
|
| 107 |
+ |
|
| 108 |
+ layer, err := newROLayerForImage(ctx, imgDesc, i, opts, refOrID, opts.Platform) |
|
| 109 |
+ if err != nil {
|
|
| 110 |
+ return nil, nil, err |
|
| 111 |
+ } |
|
| 112 |
+ |
|
| 113 |
+ return img, layer, nil |
|
| 114 |
+} |
|
| 115 |
+ |
|
| 116 |
+func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]registry.AuthConfig, output io.Writer, platform *ocispec.Platform) (*ocispec.Descriptor, error) {
|
|
| 117 |
+ ref, err := reference.ParseNormalizedNamed(name) |
|
| 118 |
+ if err != nil {
|
|
| 119 |
+ return nil, err |
|
| 120 |
+ } |
|
| 121 |
+ taggedRef := reference.TagNameOnly(ref) |
|
| 122 |
+ |
|
| 123 |
+ pullRegistryAuth := ®istry.AuthConfig{}
|
|
| 124 |
+ if len(authConfigs) > 0 {
|
|
| 125 |
+ // The request came with a full auth config, use it |
|
| 126 |
+ repoInfo, err := i.registryService.ResolveRepository(ref) |
|
| 127 |
+ if err != nil {
|
|
| 128 |
+ return nil, err |
|
| 129 |
+ } |
|
| 130 |
+ |
|
| 131 |
+ resolvedConfig := registrypkg.ResolveAuthConfig(authConfigs, repoInfo.Index) |
|
| 132 |
+ pullRegistryAuth = &resolvedConfig |
|
| 133 |
+ } |
|
| 134 |
+ |
|
| 135 |
+ if err := i.PullImage(ctx, ref.Name(), taggedRef.(reference.NamedTagged).Tag(), platform, nil, pullRegistryAuth, output); err != nil {
|
|
| 136 |
+ return nil, err |
|
| 137 |
+ } |
|
| 138 |
+ |
|
| 139 |
+ img, err := i.GetImage(ctx, name, imagetypes.GetImageOpts{Platform: platform})
|
|
| 140 |
+ if err != nil {
|
|
| 141 |
+ if errdefs.IsNotFound(err) && img != nil && platform != nil {
|
|
| 142 |
+ imgPlat := ocispec.Platform{
|
|
| 143 |
+ OS: img.OS, |
|
| 144 |
+ Architecture: img.BaseImgArch(), |
|
| 145 |
+ Variant: img.BaseImgVariant(), |
|
| 146 |
+ } |
|
| 147 |
+ |
|
| 148 |
+ p := *platform |
|
| 149 |
+ if !platforms.Only(p).Match(imgPlat) {
|
|
| 150 |
+ po := streamformatter.NewJSONProgressOutput(output, false) |
|
| 151 |
+ progress.Messagef(po, "", ` |
|
| 152 |
+WARNING: Pulled image with specified platform (%s), but the resulting image's configured platform (%s) does not match. |
|
| 153 |
+This is most likely caused by a bug in the build system that created the fetched image (%s). |
|
| 154 |
+Please notify the image author to correct the configuration.`, |
|
| 155 |
+ platforms.Format(p), platforms.Format(imgPlat), name, |
|
| 156 |
+ ) |
|
| 157 |
+ logrus.WithError(err).WithField("image", name).Warn("Ignoring error about platform mismatch where the manifest list points to an image whose configuration does not match the platform in the manifest.")
|
|
| 158 |
+ } |
|
| 159 |
+ } else {
|
|
| 160 |
+ return nil, err |
|
| 161 |
+ } |
|
| 162 |
+ } |
|
| 163 |
+ |
|
| 164 |
+ if !system.IsOSSupported(img.OperatingSystem()) {
|
|
| 165 |
+ return nil, system.ErrNotSupportedOperatingSystem |
|
| 166 |
+ } |
|
| 167 |
+ |
|
| 168 |
+ imgDesc, err := i.resolveDescriptor(ctx, name) |
|
| 169 |
+ if err != nil {
|
|
| 170 |
+ return nil, err |
|
| 171 |
+ } |
|
| 172 |
+ |
|
| 173 |
+ return &imgDesc, err |
|
| 174 |
+} |
|
| 175 |
+ |
|
| 176 |
+func newROLayerForImage(ctx context.Context, imgDesc *ocispec.Descriptor, i *ImageService, opts backend.GetImageAndLayerOptions, refOrID string, platform *ocispec.Platform) (builder.ROLayer, error) {
|
|
| 177 |
+ if imgDesc == nil {
|
|
| 178 |
+ return nil, fmt.Errorf("can't make an RO layer for a nil image :'(")
|
|
| 179 |
+ } |
|
| 180 |
+ |
|
| 181 |
+ platMatcher := platforms.Default() |
|
| 182 |
+ if platform != nil {
|
|
| 183 |
+ platMatcher = platforms.Only(*platform) |
|
| 184 |
+ } |
|
| 185 |
+ |
|
| 186 |
+ // this needs it's own context + lease so that it doesn't get cleaned before we're ready |
|
| 187 |
+ confDesc, err := containerdimages.Config(ctx, i.client.ContentStore(), *imgDesc, platMatcher) |
|
| 188 |
+ if err != nil {
|
|
| 189 |
+ return nil, err |
|
| 190 |
+ } |
|
| 191 |
+ |
|
| 192 |
+ diffIDs, err := containerdimages.RootFS(ctx, i.client.ContentStore(), confDesc) |
|
| 193 |
+ if err != nil {
|
|
| 194 |
+ return nil, err |
|
| 195 |
+ } |
|
| 196 |
+ parent := identity.ChainID(diffIDs).String() |
|
| 197 |
+ |
|
| 198 |
+ s := i.client.SnapshotService(i.snapshotter) |
|
| 199 |
+ key := stringid.GenerateRandomID() |
|
| 200 |
+ ctx, _, err = i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) |
|
| 201 |
+ if err != nil {
|
|
| 202 |
+ return nil, fmt.Errorf("failed to create lease for commit: %w", err)
|
|
| 203 |
+ } |
|
| 204 |
+ mounts, err := s.View(ctx, key, parent) |
|
| 205 |
+ if err != nil {
|
|
| 206 |
+ return nil, err |
|
| 207 |
+ } |
|
| 208 |
+ |
|
| 209 |
+ tempMountLocation := os.TempDir() |
|
| 210 |
+ root, err := os.MkdirTemp(tempMountLocation, "rootfs-mount") |
|
| 211 |
+ if err != nil {
|
|
| 212 |
+ return nil, err |
|
| 213 |
+ } |
|
| 214 |
+ |
|
| 215 |
+ if err := mount.All(mounts, root); err != nil {
|
|
| 216 |
+ return nil, err |
|
| 217 |
+ } |
|
| 218 |
+ |
|
| 219 |
+ return &rolayer{
|
|
| 220 |
+ key: key, |
|
| 221 |
+ c: i.client, |
|
| 222 |
+ snapshotter: i.snapshotter, |
|
| 223 |
+ diffID: digest.Digest(parent), |
|
| 224 |
+ root: root, |
|
| 225 |
+ contentStoreDigest: "", |
|
| 226 |
+ }, nil |
|
| 227 |
+} |
|
| 228 |
+ |
|
| 229 |
+type rolayer struct {
|
|
| 230 |
+ key string |
|
| 231 |
+ c *containerd.Client |
|
| 232 |
+ snapshotter string |
|
| 233 |
+ diffID digest.Digest |
|
| 234 |
+ root string |
|
| 235 |
+ contentStoreDigest digest.Digest |
|
| 236 |
+} |
|
| 237 |
+ |
|
| 238 |
+func (rl *rolayer) ContentStoreDigest() digest.Digest {
|
|
| 239 |
+ return rl.contentStoreDigest |
|
| 240 |
+} |
|
| 241 |
+ |
|
| 242 |
+func (rl *rolayer) DiffID() layer.DiffID {
|
|
| 243 |
+ if rl.diffID == "" {
|
|
| 244 |
+ return layer.DigestSHA256EmptyTar |
|
| 245 |
+ } |
|
| 246 |
+ return layer.DiffID(rl.diffID) |
|
| 247 |
+} |
|
| 248 |
+ |
|
| 249 |
+func (rl *rolayer) Release() error {
|
|
| 250 |
+ snapshotter := rl.c.SnapshotService(rl.snapshotter) |
|
| 251 |
+ err := snapshotter.Remove(context.TODO(), rl.key) |
|
| 252 |
+ if err != nil && !cerrdefs.IsNotFound(err) {
|
|
| 253 |
+ return err |
|
| 254 |
+ } |
|
| 255 |
+ |
|
| 256 |
+ if rl.root == "" { // nothing to release
|
|
| 257 |
+ return nil |
|
| 258 |
+ } |
|
| 259 |
+ if err := mount.UnmountAll(rl.root, 0); err != nil {
|
|
| 260 |
+ logrus.WithError(err).WithField("root", rl.root).Error("failed to unmount ROLayer")
|
|
| 261 |
+ return err |
|
| 262 |
+ } |
|
| 263 |
+ if err := os.Remove(rl.root); err != nil {
|
|
| 264 |
+ logrus.WithError(err).WithField("dir", rl.root).Error("failed to remove mount temp dir")
|
|
| 265 |
+ return err |
|
| 266 |
+ } |
|
| 267 |
+ rl.root = "" |
|
| 268 |
+ return nil |
|
| 269 |
+} |
|
| 270 |
+ |
|
| 271 |
+// NewRWLayer creates a new read-write layer for the builder |
|
| 272 |
+func (rl *rolayer) NewRWLayer() (builder.RWLayer, error) {
|
|
| 273 |
+ snapshotter := rl.c.SnapshotService(rl.snapshotter) |
|
| 274 |
+ |
|
| 275 |
+ // we need this here for the prepared snapshots or |
|
| 276 |
+ // we'll have racy behaviour where sometimes they |
|
| 277 |
+ // will get GC'd before we commit/use them |
|
| 278 |
+ ctx, _, err := rl.c.WithLease(context.TODO(), leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) |
|
| 279 |
+ if err != nil {
|
|
| 280 |
+ return nil, fmt.Errorf("failed to create lease for commit: %w", err)
|
|
| 281 |
+ } |
|
| 282 |
+ |
|
| 283 |
+ key := stringid.GenerateRandomID() |
|
| 284 |
+ mounts, err := snapshotter.Prepare(ctx, key, rl.diffID.String()) |
|
| 285 |
+ if err != nil {
|
|
| 286 |
+ return nil, err |
|
| 287 |
+ } |
|
| 288 |
+ |
|
| 289 |
+ root, err := os.MkdirTemp(os.TempDir(), "rootfs-mount") |
|
| 290 |
+ if err != nil {
|
|
| 291 |
+ return nil, err |
|
| 292 |
+ } |
|
| 293 |
+ if err := mount.All(mounts, root); err != nil {
|
|
| 294 |
+ return nil, err |
|
| 295 |
+ } |
|
| 296 |
+ |
|
| 297 |
+ return &rwlayer{
|
|
| 298 |
+ key: key, |
|
| 299 |
+ parent: rl.key, |
|
| 300 |
+ c: rl.c, |
|
| 301 |
+ snapshotter: rl.snapshotter, |
|
| 302 |
+ root: root, |
|
| 303 |
+ }, nil |
|
| 304 |
+} |
|
| 305 |
+ |
|
| 306 |
+type rwlayer struct {
|
|
| 307 |
+ key string |
|
| 308 |
+ parent string |
|
| 309 |
+ c *containerd.Client |
|
| 310 |
+ snapshotter string |
|
| 311 |
+ root string |
|
| 312 |
+} |
|
| 313 |
+ |
|
| 314 |
+func (rw *rwlayer) Root() string {
|
|
| 315 |
+ return rw.root |
|
| 316 |
+} |
|
| 317 |
+ |
|
| 318 |
+func (rw *rwlayer) Commit() (builder.ROLayer, error) {
|
|
| 319 |
+ // we need this here for the prepared snapshots or |
|
| 320 |
+ // we'll have racy behaviour where sometimes they |
|
| 321 |
+ // will get GC'd before we commit/use them |
|
| 322 |
+ ctx, _, err := rw.c.WithLease(context.TODO(), leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) |
|
| 323 |
+ if err != nil {
|
|
| 324 |
+ return nil, fmt.Errorf("failed to create lease for commit: %w", err)
|
|
| 325 |
+ } |
|
| 326 |
+ snapshotter := rw.c.SnapshotService(rw.snapshotter) |
|
| 327 |
+ |
|
| 328 |
+ key := stringid.GenerateRandomID() |
|
| 329 |
+ err = snapshotter.Commit(ctx, key, rw.key) |
|
| 330 |
+ if err != nil && !cerrdefs.IsAlreadyExists(err) {
|
|
| 331 |
+ return nil, err |
|
| 332 |
+ } |
|
| 333 |
+ |
|
| 334 |
+ differ := rw.c.DiffService() |
|
| 335 |
+ desc, err := rootfs.CreateDiff(ctx, key, snapshotter, differ) |
|
| 336 |
+ if err != nil {
|
|
| 337 |
+ return nil, err |
|
| 338 |
+ } |
|
| 339 |
+ info, err := rw.c.ContentStore().Info(ctx, desc.Digest) |
|
| 340 |
+ if err != nil {
|
|
| 341 |
+ return nil, err |
|
| 342 |
+ } |
|
| 343 |
+ diffIDStr, ok := info.Labels["containerd.io/uncompressed"] |
|
| 344 |
+ if !ok {
|
|
| 345 |
+ return nil, fmt.Errorf("invalid differ response with no diffID")
|
|
| 346 |
+ } |
|
| 347 |
+ diffID, err := digest.Parse(diffIDStr) |
|
| 348 |
+ if err != nil {
|
|
| 349 |
+ return nil, err |
|
| 350 |
+ } |
|
| 351 |
+ |
|
| 352 |
+ return &rolayer{
|
|
| 353 |
+ key: key, |
|
| 354 |
+ c: rw.c, |
|
| 355 |
+ snapshotter: rw.snapshotter, |
|
| 356 |
+ diffID: diffID, |
|
| 357 |
+ root: "", |
|
| 358 |
+ contentStoreDigest: desc.Digest, |
|
| 359 |
+ }, nil |
|
| 360 |
+} |
|
| 361 |
+ |
|
| 362 |
+func (rw *rwlayer) Release() error {
|
|
| 363 |
+ snapshotter := rw.c.SnapshotService(rw.snapshotter) |
|
| 364 |
+ err := snapshotter.Remove(context.TODO(), rw.key) |
|
| 365 |
+ if err != nil && !cerrdefs.IsNotFound(err) {
|
|
| 366 |
+ return err |
|
| 367 |
+ } |
|
| 368 |
+ |
|
| 369 |
+ if rw.root == "" { // nothing to release
|
|
| 370 |
+ return nil |
|
| 371 |
+ } |
|
| 372 |
+ if err := mount.UnmountAll(rw.root, 0); err != nil {
|
|
| 373 |
+ logrus.WithError(err).WithField("root", rw.root).Error("failed to unmount ROLayer")
|
|
| 374 |
+ return err |
|
| 375 |
+ } |
|
| 376 |
+ if err := os.Remove(rw.root); err != nil {
|
|
| 377 |
+ logrus.WithError(err).WithField("dir", rw.root).Error("failed to remove mount temp dir")
|
|
| 378 |
+ return err |
|
| 379 |
+ } |
|
| 380 |
+ rw.root = "" |
|
| 381 |
+ return nil |
|
| 17 | 382 |
} |
| 18 | 383 |
|
| 19 | 384 |
// CreateImage creates a new image by adding a config and ID to the image store. |
| 20 | 385 |
// This is similar to LoadImage() except that it receives JSON encoded bytes of |
| 21 | 386 |
// an image instead of a tar archive. |
| 22 |
-func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) {
|
|
| 23 |
- return nil, errdefs.NotImplemented(errors.New("not implemented"))
|
|
| 387 |
+func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent string, layerDigest digest.Digest) (builder.Image, error) {
|
|
| 388 |
+ imgToCreate, err := dimage.NewFromJSON(config) |
|
| 389 |
+ if err != nil {
|
|
| 390 |
+ return nil, err |
|
| 391 |
+ } |
|
| 392 |
+ |
|
| 393 |
+ rootfs := ocispec.RootFS{
|
|
| 394 |
+ Type: imgToCreate.RootFS.Type, |
|
| 395 |
+ DiffIDs: []digest.Digest{},
|
|
| 396 |
+ } |
|
| 397 |
+ for _, diffId := range imgToCreate.RootFS.DiffIDs {
|
|
| 398 |
+ rootfs.DiffIDs = append(rootfs.DiffIDs, digest.Digest(diffId)) |
|
| 399 |
+ } |
|
| 400 |
+ exposedPorts := make(map[string]struct{}, len(imgToCreate.Config.ExposedPorts))
|
|
| 401 |
+ for k, v := range imgToCreate.Config.ExposedPorts {
|
|
| 402 |
+ exposedPorts[string(k)] = v |
|
| 403 |
+ } |
|
| 404 |
+ |
|
| 405 |
+ // make an ocispec.Image from the docker/image.Image |
|
| 406 |
+ ociImgToCreate := ocispec.Image{
|
|
| 407 |
+ Created: &imgToCreate.Created, |
|
| 408 |
+ Author: imgToCreate.Author, |
|
| 409 |
+ Architecture: imgToCreate.Architecture, |
|
| 410 |
+ Variant: imgToCreate.Variant, |
|
| 411 |
+ OS: imgToCreate.OS, |
|
| 412 |
+ OSVersion: imgToCreate.OSVersion, |
|
| 413 |
+ OSFeatures: imgToCreate.OSFeatures, |
|
| 414 |
+ Config: ocispec.ImageConfig{
|
|
| 415 |
+ User: imgToCreate.Config.User, |
|
| 416 |
+ ExposedPorts: exposedPorts, |
|
| 417 |
+ Env: imgToCreate.Config.Env, |
|
| 418 |
+ Entrypoint: imgToCreate.Config.Entrypoint, |
|
| 419 |
+ Cmd: imgToCreate.Config.Cmd, |
|
| 420 |
+ Volumes: imgToCreate.Config.Volumes, |
|
| 421 |
+ WorkingDir: imgToCreate.Config.WorkingDir, |
|
| 422 |
+ Labels: imgToCreate.Config.Labels, |
|
| 423 |
+ StopSignal: imgToCreate.Config.StopSignal, |
|
| 424 |
+ }, |
|
| 425 |
+ RootFS: rootfs, |
|
| 426 |
+ // TODO(laurazard) |
|
| 427 |
+ History: []ocispec.History{},
|
|
| 428 |
+ } |
|
| 429 |
+ |
|
| 430 |
+ var layers []ocispec.Descriptor |
|
| 431 |
+ // if the image has a parent, we need to start with the parents layers descriptors |
|
| 432 |
+ if parent != "" {
|
|
| 433 |
+ parentDesc, err := i.resolveDescriptor(ctx, parent) |
|
| 434 |
+ if err != nil {
|
|
| 435 |
+ return nil, err |
|
| 436 |
+ } |
|
| 437 |
+ parentImageManifest, err := containerdimages.Manifest(ctx, i.client.ContentStore(), parentDesc, platforms.Default()) |
|
| 438 |
+ if err != nil {
|
|
| 439 |
+ return nil, err |
|
| 440 |
+ } |
|
| 441 |
+ |
|
| 442 |
+ layers = parentImageManifest.Layers |
|
| 443 |
+ } |
|
| 444 |
+ |
|
| 445 |
+ // get the info for the new layers |
|
| 446 |
+ info, err := i.client.ContentStore().Info(ctx, layerDigest) |
|
| 447 |
+ if err != nil {
|
|
| 448 |
+ return nil, err |
|
| 449 |
+ } |
|
| 450 |
+ |
|
| 451 |
+ // append the new layer descriptor |
|
| 452 |
+ layers = append(layers, |
|
| 453 |
+ ocispec.Descriptor{
|
|
| 454 |
+ MediaType: containerdimages.MediaTypeDockerSchema2LayerGzip, |
|
| 455 |
+ Digest: layerDigest, |
|
| 456 |
+ Size: info.Size, |
|
| 457 |
+ }, |
|
| 458 |
+ ) |
|
| 459 |
+ |
|
| 460 |
+ commitManifestDesc, err := writeContentsForImage(ctx, i.snapshotter, i.client.ContentStore(), ociImgToCreate, layers) |
|
| 461 |
+ if err != nil {
|
|
| 462 |
+ return nil, err |
|
| 463 |
+ } |
|
| 464 |
+ |
|
| 465 |
+ // image create |
|
| 466 |
+ img := containerdimages.Image{
|
|
| 467 |
+ Name: danglingImageName(commitManifestDesc.Digest), |
|
| 468 |
+ Target: commitManifestDesc, |
|
| 469 |
+ CreatedAt: time.Now(), |
|
| 470 |
+ } |
|
| 471 |
+ |
|
| 472 |
+ createdImage, err := i.client.ImageService().Update(ctx, img) |
|
| 473 |
+ if err != nil {
|
|
| 474 |
+ if !cerrdefs.IsNotFound(err) {
|
|
| 475 |
+ return nil, err |
|
| 476 |
+ } |
|
| 477 |
+ |
|
| 478 |
+ if createdImage, err = i.client.ImageService().Create(ctx, img); err != nil {
|
|
| 479 |
+ return nil, fmt.Errorf("failed to create new image: %w", err)
|
|
| 480 |
+ } |
|
| 481 |
+ } |
|
| 482 |
+ |
|
| 483 |
+ if err := i.unpackImage(ctx, createdImage, platforms.DefaultSpec()); err != nil {
|
|
| 484 |
+ return nil, err |
|
| 485 |
+ } |
|
| 486 |
+ |
|
| 487 |
+ newImage := dimage.NewImage(dimage.ID(createdImage.Target.Digest)) |
|
| 488 |
+ newImage.V1Image = imgToCreate.V1Image |
|
| 489 |
+ newImage.V1Image.ID = string(createdImage.Target.Digest) |
|
| 490 |
+ return newImage, nil |
|
| 24 | 491 |
} |
| ... | ... |
@@ -6,7 +6,6 @@ import ( |
| 6 | 6 |
"crypto/rand" |
| 7 | 7 |
"encoding/base64" |
| 8 | 8 |
"encoding/json" |
| 9 |
- "errors" |
|
| 10 | 9 |
"fmt" |
| 11 | 10 |
"runtime" |
| 12 | 11 |
"strings" |
| ... | ... |
@@ -20,7 +19,6 @@ import ( |
| 20 | 20 |
"github.com/containerd/containerd/rootfs" |
| 21 | 21 |
"github.com/containerd/containerd/snapshots" |
| 22 | 22 |
"github.com/docker/docker/api/types/backend" |
| 23 |
- "github.com/docker/docker/errdefs" |
|
| 24 | 23 |
"github.com/docker/docker/image" |
| 25 | 24 |
"github.com/opencontainers/go-digest" |
| 26 | 25 |
"github.com/opencontainers/image-spec/identity" |
| ... | ... |
@@ -298,5 +296,13 @@ func uniquePart() string {
|
| 298 | 298 |
// |
| 299 | 299 |
// This is a temporary shim. Should be removed when builder stops using commit. |
| 300 | 300 |
func (i *ImageService) CommitBuildStep(ctx context.Context, c backend.CommitConfig) (image.ID, error) {
|
| 301 |
- return "", errdefs.NotImplemented(errors.New("not implemented"))
|
|
| 301 |
+ ctr := i.containers.Get(c.ContainerID) |
|
| 302 |
+ if ctr == nil {
|
|
| 303 |
+ // TODO: use typed error |
|
| 304 |
+ return "", fmt.Errorf("container not found: %s", c.ContainerID)
|
|
| 305 |
+ } |
|
| 306 |
+ c.ContainerMountLabel = ctr.MountLabel |
|
| 307 |
+ c.ContainerOS = ctr.OS |
|
| 308 |
+ c.ParentImageID = string(ctr.ImageID) |
|
| 309 |
+ return i.CommitImage(ctx, c) |
|
| 302 | 310 |
} |
| ... | ... |
@@ -10,6 +10,7 @@ import ( |
| 10 | 10 |
"github.com/containerd/containerd/plugin" |
| 11 | 11 |
"github.com/containerd/containerd/remotes/docker" |
| 12 | 12 |
"github.com/containerd/containerd/snapshots" |
| 13 |
+ "github.com/docker/distribution/reference" |
|
| 13 | 14 |
imagetypes "github.com/docker/docker/api/types/image" |
| 14 | 15 |
"github.com/docker/docker/container" |
| 15 | 16 |
daemonevents "github.com/docker/docker/daemon/events" |
| ... | ... |
@@ -17,6 +18,7 @@ import ( |
| 17 | 17 |
"github.com/docker/docker/errdefs" |
| 18 | 18 |
"github.com/docker/docker/image" |
| 19 | 19 |
"github.com/docker/docker/layer" |
| 20 |
+ "github.com/docker/docker/registry" |
|
| 20 | 21 |
"github.com/opencontainers/go-digest" |
| 21 | 22 |
"github.com/opencontainers/image-spec/identity" |
| 22 | 23 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -41,6 +43,7 @@ type RegistryHostsProvider interface {
|
| 41 | 41 |
|
| 42 | 42 |
type RegistryConfigProvider interface {
|
| 43 | 43 |
IsInsecureRegistry(host string) bool |
| 44 |
+ ResolveRepository(name reference.Named) (*registry.RepositoryInfo, error) |
|
| 44 | 45 |
} |
| 45 | 46 |
|
| 46 | 47 |
type ImageServiceConfig struct {
|
| ... | ... |
@@ -16,6 +16,7 @@ import ( |
| 16 | 16 |
"github.com/docker/docker/image" |
| 17 | 17 |
"github.com/docker/docker/layer" |
| 18 | 18 |
"github.com/docker/docker/pkg/archive" |
| 19 |
+ "github.com/opencontainers/go-digest" |
|
| 19 | 20 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 20 | 21 |
) |
| 21 | 22 |
|
| ... | ... |
@@ -27,7 +28,7 @@ type ImageService interface {
|
| 27 | 27 |
|
| 28 | 28 |
PullImage(ctx context.Context, name, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error |
| 29 | 29 |
PushImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error |
| 30 |
- CreateImage(config []byte, parent string) (builder.Image, error) |
|
| 30 |
+ CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (builder.Image, error) |
|
| 31 | 31 |
ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) |
| 32 | 32 |
ExportImage(ctx context.Context, names []string, outStream io.Writer) error |
| 33 | 33 |
PerformWithBaseFS(ctx context.Context, c *container.Container, fn func(string) error) error |
| ... | ... |
@@ -19,6 +19,7 @@ import ( |
| 19 | 19 |
"github.com/docker/docker/pkg/stringid" |
| 20 | 20 |
"github.com/docker/docker/pkg/system" |
| 21 | 21 |
registrypkg "github.com/docker/docker/registry" |
| 22 |
+ "github.com/opencontainers/go-digest" |
|
| 22 | 23 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| 23 | 24 |
"github.com/pkg/errors" |
| 24 | 25 |
"github.com/sirupsen/logrus" |
| ... | ... |
@@ -30,6 +31,10 @@ type roLayer struct {
|
| 30 | 30 |
roLayer layer.Layer |
| 31 | 31 |
} |
| 32 | 32 |
|
| 33 |
+func (l *roLayer) ContentStoreDigest() digest.Digest {
|
|
| 34 |
+ return "" |
|
| 35 |
+} |
|
| 36 |
+ |
|
| 33 | 37 |
func (l *roLayer) DiffID() layer.DiffID {
|
| 34 | 38 |
if l.roLayer == nil {
|
| 35 | 39 |
return layer.DigestSHA256EmptyTar |
| ... | ... |
@@ -241,7 +246,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s |
| 241 | 241 |
// CreateImage creates a new image by adding a config and ID to the image store. |
| 242 | 242 |
// This is similar to LoadImage() except that it receives JSON encoded bytes of |
| 243 | 243 |
// an image instead of a tar archive. |
| 244 |
-func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) {
|
|
| 244 |
+func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent string, _ digest.Digest) (builder.Image, error) {
|
|
| 245 | 245 |
id, err := i.imageStore.Create(config) |
| 246 | 246 |
if err != nil {
|
| 247 | 247 |
return nil, errors.Wrapf(err, "failed to create image") |