Browse code

daemon: Emit Image Create event when image is built

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

Paweł Gronowski authored on 2024/06/07 21:40:50
Showing 12 changed files
... ...
@@ -273,7 +273,18 @@ func (s *systemRouter) getEvents(ctx context.Context, w http.ResponseWriter, r *
273 273
 	buffered, l := s.backend.SubscribeToEvents(since, until, ef)
274 274
 	defer s.backend.UnsubscribeFromEvents(l)
275 275
 
276
+	shouldSkip := func(ev events.Message) bool { return false }
277
+	if versions.LessThan(httputils.VersionFromContext(ctx), "1.46") {
278
+		// Image create events were added in API 1.46
279
+		shouldSkip = func(ev events.Message) bool {
280
+			return ev.Type == "image" && ev.Action == "create"
281
+		}
282
+	}
283
+
276 284
 	for _, ev := range buffered {
285
+		if shouldSkip(ev) {
286
+			continue
287
+		}
277 288
 		if err := enc.Encode(ev); err != nil {
278 289
 			return err
279 290
 		}
... ...
@@ -291,6 +302,9 @@ func (s *systemRouter) getEvents(ctx context.Context, w http.ResponseWriter, r *
291 291
 				log.G(ctx).Warnf("unexpected event message: %q", ev)
292 292
 				continue
293 293
 			}
294
+			if shouldSkip(jev) {
295
+				continue
296
+			}
294 297
 			if err := enc.Encode(jev); err != nil {
295 298
 				return err
296 299
 			}
... ...
@@ -9507,7 +9507,7 @@ paths:
9507 9507
 
9508 9508
         Containers report these events: `attach`, `commit`, `copy`, `create`, `destroy`, `detach`, `die`, `exec_create`, `exec_detach`, `exec_start`, `exec_die`, `export`, `health_status`, `kill`, `oom`, `pause`, `rename`, `resize`, `restart`, `start`, `stop`, `top`, `unpause`, `update`, and `prune`
9509 9509
 
9510
-        Images report these events: `delete`, `import`, `load`, `pull`, `push`, `save`, `tag`, `untag`, and `prune`
9510
+        Images report these events: `create, `delete`, `import`, `load`, `pull`, `push`, `save`, `tag`, `untag`, and `prune`
9511 9511
 
9512 9512
         Volumes report these events: `create`, `mount`, `unmount`, `destroy`, and `prune`
9513 9513
 
... ...
@@ -430,23 +430,24 @@ func newRouterOptions(ctx context.Context, config *config.Config, d *daemon.Daem
430 430
 	cgroupParent := newCgroupParent(config)
431 431
 
432 432
 	bk, err := buildkit.New(ctx, buildkit.Opt{
433
-		SessionManager:      sm,
434
-		Root:                filepath.Join(config.Root, "buildkit"),
435
-		EngineID:            d.ID(),
436
-		Dist:                d.DistributionServices(),
437
-		ImageTagger:         d.ImageService(),
438
-		NetworkController:   d.NetworkController(),
439
-		DefaultCgroupParent: cgroupParent,
440
-		RegistryHosts:       d.RegistryHosts,
441
-		BuilderConfig:       config.Builder,
442
-		Rootless:            daemon.Rootless(config),
443
-		IdentityMapping:     d.IdentityMapping(),
444
-		DNSConfig:           config.DNSConfig,
445
-		ApparmorProfile:     daemon.DefaultApparmorProfile(),
446
-		UseSnapshotter:      d.UsesSnapshotter(),
447
-		Snapshotter:         d.ImageService().StorageDriver(),
448
-		ContainerdAddress:   config.ContainerdAddr,
449
-		ContainerdNamespace: config.ContainerdNamespace,
433
+		SessionManager:        sm,
434
+		Root:                  filepath.Join(config.Root, "buildkit"),
435
+		EngineID:              d.ID(),
436
+		Dist:                  d.DistributionServices(),
437
+		ImageTagger:           d.ImageService(),
438
+		NetworkController:     d.NetworkController(),
439
+		DefaultCgroupParent:   cgroupParent,
440
+		RegistryHosts:         d.RegistryHosts,
441
+		BuilderConfig:         config.Builder,
442
+		Rootless:              daemon.Rootless(config),
443
+		IdentityMapping:       d.IdentityMapping(),
444
+		DNSConfig:             config.DNSConfig,
445
+		ApparmorProfile:       daemon.DefaultApparmorProfile(),
446
+		UseSnapshotter:        d.UsesSnapshotter(),
447
+		Snapshotter:           d.ImageService().StorageDriver(),
448
+		ContainerdAddress:     config.ContainerdAddr,
449
+		ContainerdNamespace:   config.ContainerdNamespace,
450
+		ImageExportedCallback: d.ImageExportedByBuildkit,
450 451
 	})
451 452
 	if err != nil {
452 453
 		return routerOptions{}, err
453 454
new file mode 100644
... ...
@@ -0,0 +1,16 @@
0
+package daemon
1
+
2
+import (
3
+	"context"
4
+
5
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
6
+)
7
+
8
+// ImageExportedByBuildkit is a callback that is called when an image is exported by buildkit.
9
+// This is used to log the image creation event for untagged images.
10
+// When no tag is given, buildkit doesn't call the image service so it has no
11
+// way of knowing the image was created.
12
+func (daemon *Daemon) ImageExportedByBuildkit(ctx context.Context, id string, desc ocispec.Descriptor) error {
13
+	daemon.imageService.LogImageEvent(id, id, "create")
14
+	return nil
15
+}
... ...
@@ -23,9 +23,11 @@ import (
23 23
 	"github.com/distribution/reference"
24 24
 	"github.com/docker/docker/api/types/backend"
25 25
 	"github.com/docker/docker/api/types/container"
26
+	"github.com/docker/docker/api/types/events"
26 27
 	"github.com/docker/docker/api/types/registry"
27 28
 	"github.com/docker/docker/builder"
28 29
 	"github.com/docker/docker/errdefs"
30
+	"github.com/docker/docker/image"
29 31
 	dimage "github.com/docker/docker/image"
30 32
 	"github.com/docker/docker/layer"
31 33
 	"github.com/docker/docker/pkg/archive"
... ...
@@ -533,11 +535,14 @@ func (i *ImageService) createImageOCI(ctx context.Context, imgToCreate imagespec
533 533
 		}
534 534
 	}
535 535
 
536
+	id := image.ID(createdImage.Target.Digest)
537
+	i.LogImageEvent(id.String(), id.String(), events.ActionCreate)
538
+
536 539
 	if err := i.unpackImage(ctx, i.StorageDriver(), img, manifestDesc); err != nil {
537 540
 		return "", err
538 541
 	}
539 542
 
540
-	return dimage.ID(createdImage.Target.Digest), nil
543
+	return id, nil
541 544
 }
542 545
 
543 546
 // writeContentsForImage will commit oci image config and manifest into containerd's content store.
... ...
@@ -6,6 +6,7 @@ import (
6 6
 	"io"
7 7
 
8 8
 	"github.com/docker/docker/api/types/backend"
9
+	"github.com/docker/docker/api/types/events"
9 10
 	"github.com/docker/docker/image"
10 11
 	"github.com/docker/docker/layer"
11 12
 	"github.com/docker/docker/pkg/ioutils"
... ...
@@ -62,6 +63,9 @@ func (i *ImageService) CommitImage(ctx context.Context, c backend.CommitConfig)
62 62
 	if err != nil {
63 63
 		return "", err
64 64
 	}
65
+
66
+	i.LogImageEvent(id.String(), id.String(), events.ActionCreate)
67
+
65 68
 	if err := i.imageStore.SetBuiltLocally(id); err != nil {
66 69
 		return "", err
67 70
 	}
... ...
@@ -32,6 +32,8 @@ keywords: "API, Docker, rcli, REST, documentation"
32 32
   the multi-platform image.
33 33
 * `POST /containers/create` now takes `Options` as part of `HostConfig.Mounts.TmpfsOptions` to set options for tmpfs mounts.
34 34
 * `POST /services/create` now takes `Options` as part of `ContainerSpec.Mounts.TmpfsOptions`, to set options for tmpfs mounts.
35
+* `GET /events` now supports image `create` event that is emitted when a new
36
+  image is built regardless if it was tagged or not.
35 37
 
36 38
 ### Deprecated Config fields in `GET /images/{name}/json` response
37 39
 
... ...
@@ -2,6 +2,7 @@ package build // import "github.com/docker/docker/integration-cli/cli/build"
2 2
 
3 3
 import (
4 4
 	"io"
5
+	"os"
5 6
 	"strings"
6 7
 	"testing"
7 8
 
... ...
@@ -30,6 +31,21 @@ func WithDockerfile(dockerfile string) func(*icmd.Cmd) func() {
30 30
 	}
31 31
 }
32 32
 
33
+// WithBuildkit sets an DOCKER_BUILDKIT environment variable to make the build use buildkit (or not)
34
+func WithBuildkit(useBuildkit bool) func(*icmd.Cmd) func() {
35
+	return func(cmd *icmd.Cmd) func() {
36
+		val := "0"
37
+		if useBuildkit {
38
+			val = "1"
39
+		}
40
+		if cmd.Env == nil {
41
+			cmd.Env = os.Environ()
42
+		}
43
+		cmd.Env = append(cmd.Env, "DOCKER_BUILDKIT="+val)
44
+		return nil
45
+	}
46
+}
47
+
33 48
 // WithoutCache makes the build ignore cache
34 49
 func WithoutCache(cmd *icmd.Cmd) func() {
35 50
 	cmd.Command = append(cmd.Command, "--no-cache")
... ...
@@ -3,6 +3,7 @@ package cli // import "github.com/docker/docker/integration-cli/cli"
3 3
 import (
4 4
 	"fmt"
5 5
 	"io"
6
+	"os"
6 7
 	"strings"
7 8
 	"testing"
8 9
 	"time"
... ...
@@ -161,7 +162,10 @@ func WithTimeout(timeout time.Duration) func(cmd *icmd.Cmd) func() {
161 161
 // WithEnvironmentVariables sets the specified environment variables for the command to run
162 162
 func WithEnvironmentVariables(envs ...string) func(cmd *icmd.Cmd) func() {
163 163
 	return func(cmd *icmd.Cmd) func() {
164
-		cmd.Env = envs
164
+		if cmd.Env == nil {
165
+			cmd.Env = os.Environ()
166
+		}
167
+		cmd.Env = append(cmd.Env, envs...)
165 168
 		return nil
166 169
 	}
167 170
 }
... ...
@@ -6192,3 +6192,41 @@ func (s *DockerCLIBuildSuite) TestBuildIidFileCleanupOnFail(c *testing.T) {
6192 6192
 	assert.ErrorContains(c, err, "")
6193 6193
 	assert.Equal(c, os.IsNotExist(err), true)
6194 6194
 }
6195
+
6196
+func (s *DockerCLIBuildSuite) TestBuildEmitsImageCreateEvent(t *testing.T) {
6197
+	for _, tc := range []struct {
6198
+		buildkit bool
6199
+	}{
6200
+		{buildkit: false},
6201
+		{buildkit: true},
6202
+	} {
6203
+		tc := tc
6204
+		t.Run(fmt.Sprintf("buildkit=%v", tc.buildkit), func(t *testing.T) {
6205
+			skip.If(t, DaemonIsWindows, "Buildkit is not supported on Windows")
6206
+
6207
+			before := time.Now()
6208
+
6209
+			b := cli.Docker(cli.Args("build"),
6210
+				build.WithoutCache,
6211
+				build.WithDockerfile("FROM busybox\nRUN echo hi >/hello"),
6212
+				build.WithBuildkit(tc.buildkit),
6213
+			)
6214
+			b.Assert(t, icmd.Success)
6215
+			t.Log(b.Stdout())
6216
+			t.Log(b.Stderr())
6217
+
6218
+			cmd := cli.Docker(
6219
+				cli.Args("events",
6220
+					"--filter", "action=create,type=image",
6221
+					"--since", before.Format(time.RFC3339),
6222
+				),
6223
+				cli.WithTimeout(time.Millisecond*300),
6224
+				cli.WithEnvironmentVariables("DOCKER_API_VERSION=v1.46"), // FIXME(thaJeztah): integration-cli runs docker CLI 17.06; we're "upgrading" the API version to a version it doesn't support here ;)
6225
+			)
6226
+
6227
+			t.Log(cmd.Stdout())
6228
+
6229
+			assert.Check(t, is.Contains(cmd.Stdout(), "image create"))
6230
+		})
6231
+	}
6232
+}
... ...
@@ -338,9 +338,10 @@ func (s *DockerCLIEventSuite) TestEventsFilterImageLabels(c *testing.T) {
338 338
 	label := "io.docker.testing=image"
339 339
 
340 340
 	// Build a test image.
341
-	buildImageSuccessfully(c, name, build.WithDockerfile(fmt.Sprintf(`
342
-		FROM busybox:latest
343
-		LABEL %s`, label)))
341
+	buildImageSuccessfully(c, name,
342
+		build.WithDockerfile("FROM busybox:latest\nLABEL "+label),
343
+		build.WithoutCache, // Make sure image is actually built
344
+	)
344 345
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag1")
345 346
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag2")
346 347
 	cli.DockerCmd(c, "tag", "busybox:latest", "labelfiltertest:tag3")
... ...
@@ -560,9 +561,10 @@ func (s *DockerCLIEventSuite) TestEventsFilterType(c *testing.T) {
560 560
 	label := "io.docker.testing=image"
561 561
 
562 562
 	// Build a test image.
563
-	buildImageSuccessfully(c, name, build.WithDockerfile(fmt.Sprintf(`
564
-		FROM busybox:latest
565
-		LABEL %s`, label)))
563
+	buildImageSuccessfully(c, name,
564
+		build.WithDockerfile("FROM busybox:latest\nLABEL "+label),
565
+		build.WithoutCache, // Make sure image is actually built
566
+	)
566 567
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag1")
567 568
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag2")
568 569
 	cli.DockerCmd(c, "tag", "busybox:latest", "labelfiltertest:tag3")
... ...
@@ -571,7 +573,7 @@ func (s *DockerCLIEventSuite) TestEventsFilterType(c *testing.T) {
571 571
 		"events",
572 572
 		"--since", since,
573 573
 		"--until", daemonUnixTime(c),
574
-		"--filter", fmt.Sprintf("label=%s", label),
574
+		"--filter", "label="+label,
575 575
 		"--filter", "type=image",
576 576
 	).Stdout()
577 577
 
... ...
@@ -3,14 +3,17 @@ package build // import "github.com/docker/docker/integration/build"
3 3
 import (
4 4
 	"archive/tar"
5 5
 	"bytes"
6
+	"context"
6 7
 	"encoding/json"
7 8
 	"io"
8 9
 	"os"
9 10
 	"strings"
10 11
 	"testing"
12
+	"time"
11 13
 
12 14
 	"github.com/docker/docker/api/types"
13 15
 	"github.com/docker/docker/api/types/container"
16
+	"github.com/docker/docker/api/types/events"
14 17
 	"github.com/docker/docker/api/types/filters"
15 18
 	"github.com/docker/docker/errdefs"
16 19
 	"github.com/docker/docker/pkg/jsonmessage"
... ...
@@ -727,6 +730,68 @@ func TestBuildWorkdirNoCacheMiss(t *testing.T) {
727 727
 	}
728 728
 }
729 729
 
730
+func TestBuildEmitsImageCreateEvent(t *testing.T) {
731
+	ctx := setupTest(t)
732
+
733
+	dockerfile := "FROM busybox\nRUN echo hello > /hello"
734
+	source := fakecontext.New(t, "", fakecontext.WithDockerfile(dockerfile))
735
+	defer source.Close()
736
+
737
+	apiClient := testEnv.APIClient()
738
+
739
+	for _, builderVersion := range []types.BuilderVersion{types.BuilderV1, types.BuilderBuildKit} {
740
+		builderVersion := builderVersion
741
+		t.Run("v"+string(builderVersion), func(t *testing.T) {
742
+			if builderVersion == types.BuilderBuildKit {
743
+				skip.If(t, testEnv.UsingSnapshotter(),
744
+					"FIXME: Passing a context via a tarball is not supported with the containerd image store. See: https://github.com/moby/moby/issues/47717")
745
+				skip.If(t, testEnv.DaemonInfo.OSType == "windows", "Buildkit is not supported on Windows")
746
+			}
747
+
748
+			ctx, cancel := context.WithCancel(ctx)
749
+			defer cancel()
750
+
751
+			since := time.Now()
752
+
753
+			resp, err := apiClient.ImageBuild(ctx, source.AsTarReader(t), types.ImageBuildOptions{
754
+				Version: builderVersion,
755
+				NoCache: true,
756
+			})
757
+			assert.NilError(t, err)
758
+
759
+			defer resp.Body.Close()
760
+
761
+			out := bytes.NewBuffer(nil)
762
+			_, err = io.Copy(out, resp.Body)
763
+			assert.NilError(t, err)
764
+
765
+			t.Log(out.String())
766
+
767
+			eventsChan, errs := apiClient.Events(ctx, events.ListOptions{
768
+				Since: since.Format(time.RFC3339Nano),
769
+				Until: time.Now().Format(time.RFC3339Nano),
770
+			})
771
+			imageCreateEvts := 0
772
+			finished := false
773
+			for !finished {
774
+				select {
775
+				case evt := <-eventsChan:
776
+					t.Log("Got event type:", evt.Type, "action:", evt.Action)
777
+
778
+					if evt.Type == events.ImageEventType && evt.Action == events.ActionCreate {
779
+						imageCreateEvts++
780
+					}
781
+				case err := <-errs:
782
+					assert.Check(t, err == nil || err == io.EOF)
783
+					finished = true
784
+				}
785
+			}
786
+
787
+			assert.Check(t, is.Equal(1, imageCreateEvts))
788
+		})
789
+	}
790
+}
791
+
730 792
 func readBuildImageIDs(t *testing.T, rd io.Reader) string {
731 793
 	t.Helper()
732 794
 	decoder := json.NewDecoder(rd)