@@ -64,3 +64,4 @@ Flags for debugging IT on Swarm itself:

  - `-rand-seed N`: the random seed. This flag is useful for deterministic replaying. By default(0), the timestamp is used.

  - `-filters-file FILE`: the file contains `-check.f` strings. By default, the file is automatically generated.

  - `-dry-run`: skip the actual workload

+ - `keep-executor`: do not auto-remove executor containers, which is used for running privileged programs on Swarm

@@ -19,69 +19,76 @@ import (

 // image needs to be the worker image itself. testFlags are OR-set of regexp for filtering tests.

 type testChunkExecutor func(image string, tests []string) (int64, string, error)

-func dryTestChunkExecutor(image string, tests []string) (int64, string, error) {

-	return 0, fmt.Sprintf("DRY RUN (image=%q, tests=%v)", image, tests), nil

+func dryTestChunkExecutor() testChunkExecutor {

+	return func(image string, tests []string) (int64, string, error) {

+		return 0, fmt.Sprintf("DRY RUN (image=%q, tests=%v)", image, tests), nil

+	}

 }

 // privilegedTestChunkExecutor invokes a privileged container from the worker

 // service via bind-mounted API socket so as to execute the test chunk

-func privilegedTestChunkExecutor(image string, tests []string) (int64, string, error) {

-	cli, err := client.NewEnvClient()

-	if err != nil {

-		return 0, "", err

-	}

-	// propagate variables from the host (needs to be defined in the compose file)

-	experimental := os.Getenv("DOCKER_EXPERIMENTAL")

-	graphdriver := os.Getenv("DOCKER_GRAPHDRIVER")

-	if graphdriver == "" {

-		info, err := cli.Info(context.Background())

+func privilegedTestChunkExecutor(autoRemove bool) testChunkExecutor {

+	return func(image string, tests []string) (int64, string, error) {

+		cli, err := client.NewEnvClient()

 		if err != nil {

 			return 0, "", err

 		}

-		graphdriver = info.Driver

-	}

-	// `daemon_dest` is similar to `$DEST` (e.g. `bundles/VERSION/test-integration-cli`)

-	// but it exists outside of `bundles` so as to make `$DOCKER_GRAPHDRIVER` work.

-	//

-	// Without this hack, `$DOCKER_GRAPHDRIVER` fails because of (e.g.) `overlay2 is not supported over overlayfs`

-	//

-	// see integration-cli/daemon/daemon.go

-	daemonDest := "/daemon_dest"

-	config := container.Config{

-		Image: image,

-		Env: []string{

-			"TESTFLAGS=-check.f " + strings.Join(tests, "|"),

-			"KEEPBUNDLE=1",

-			"DOCKER_INTEGRATION_TESTS_VERIFIED=1", // for avoiding rebuilding integration-cli

-			"DOCKER_EXPERIMENTAL=" + experimental,

-			"DOCKER_GRAPHDRIVER=" + graphdriver,

-			"DOCKER_INTEGRATION_DAEMON_DEST=" + daemonDest,

-		},

-		// TODO: set label?

-		Entrypoint: []string{"hack/dind"},

-		Cmd:        []string{"hack/make.sh", "test-integration-cli"},

-	}

-	hostConfig := container.HostConfig{

-		AutoRemove: true,

-		Privileged: true,

-		Mounts: []mount.Mount{

-			{

-				Type:   mount.TypeVolume,

-				Target: daemonDest,

+		// propagate variables from the host (needs to be defined in the compose file)

+		experimental := os.Getenv("DOCKER_EXPERIMENTAL")

+		graphdriver := os.Getenv("DOCKER_GRAPHDRIVER")

+		if graphdriver == "" {

+			info, err := cli.Info(context.Background())

+			if err != nil {

+				return 0, "", err

+			}

+			graphdriver = info.Driver

+		}

+		// `daemon_dest` is similar to `$DEST` (e.g. `bundles/VERSION/test-integration-cli`)

+		// but it exists outside of `bundles` so as to make `$DOCKER_GRAPHDRIVER` work.

+		//

+		// Without this hack, `$DOCKER_GRAPHDRIVER` fails because of (e.g.) `overlay2 is not supported over overlayfs`

+		//

+		// see integration-cli/daemon/daemon.go

+		daemonDest := "/daemon_dest"

+		config := container.Config{

+			Image: image,

+			Env: []string{

+				"TESTFLAGS=-check.f " + strings.Join(tests, "|"),

+				"KEEPBUNDLE=1",

+				"DOCKER_INTEGRATION_TESTS_VERIFIED=1", // for avoiding rebuilding integration-cli

+				"DOCKER_EXPERIMENTAL=" + experimental,

+				"DOCKER_GRAPHDRIVER=" + graphdriver,

+				"DOCKER_INTEGRATION_DAEMON_DEST=" + daemonDest,

 			},

-		},

-	}

-	id, stream, err := runContainer(context.Background(), cli, config, hostConfig)

-	if err != nil {

-		return 0, "", err

-	}

-	var b bytes.Buffer

-	teeContainerStream(&b, os.Stdout, os.Stderr, stream)

-	rc, err := cli.ContainerWait(context.Background(), id)

-	if err != nil {

-		return 0, "", err

+			Labels: map[string]string{

+				"org.dockerproject.integration-cli-on-swarm":         "",

+				"org.dockerproject.integration-cli-on-swarm.comment": "this non-service container is created for running privileged programs on Swarm. you can remove this container manually if the corresponding service is already stopped.",

+			},

+			Entrypoint: []string{"hack/dind"},

+			Cmd:        []string{"hack/make.sh", "test-integration-cli"},

+		}

+		hostConfig := container.HostConfig{

+			AutoRemove: autoRemove,

+			Privileged: true,

+			Mounts: []mount.Mount{

+				{

+					Type:   mount.TypeVolume,

+					Target: daemonDest,

+				},

+			},

+		}

+		id, stream, err := runContainer(context.Background(), cli, config, hostConfig)

+		if err != nil {

+			return 0, "", err

+		}

+		var b bytes.Buffer

+		teeContainerStream(&b, os.Stdout, os.Stderr, stream)

+		rc, err := cli.ContainerWait(context.Background(), id)

+		if err != nil {

+			return 0, "", err

+		}

+		return rc, b.String(), nil

 	}

-	return rc, b.String(), nil

 }

 func runContainer(ctx context.Context, cli *client.Client, config container.Config, hostConfig container.HostConfig) (string, io.ReadCloser, error) {

@@ -24,6 +24,7 @@ func validImageDigest(s string) bool {

 func xmain() error {

 	workerImageDigest := flag.String("worker-image-digest", "", "Needs to be the digest of this worker image itself")

 	dryRun := flag.Bool("dry-run", false, "Dry run")

+	keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm")

 	flag.Parse()

 	if !validImageDigest(*workerImageDigest) {

 		// Because of issue #29582.

@@ -31,9 +32,9 @@ func xmain() error {

 		// So, `docker run localregistry.example.com/blahblah:latest` fails: `Unable to find image 'localregistry.example.com/blahblah:latest' locally`

 		return fmt.Errorf("worker-image-digest must be a digest, got %q", *workerImageDigest)

 	}

-	executor := privilegedTestChunkExecutor

+	executor := privilegedTestChunkExecutor(!*keepExecutor)

 	if *dryRun {

-		executor = dryTestChunkExecutor

+		executor = dryTestChunkExecutor()

 	}

 	return handle(*workerImageDigest, executor)

 }

@@ -16,7 +16,7 @@ version: "3"

 services:

   worker:

     image: "{{.WorkerImage}}"

-    command: ["-worker-image-digest={{.WorkerImageDigest}}", "-dry-run={{.DryRun}}"]

+    command: ["-worker-image-digest={{.WorkerImageDigest}}", "-dry-run={{.DryRun}}", "-keep-executor={{.KeepExecutor}}"]

     networks:

       - net

     volumes:

@@ -57,14 +57,15 @@ volumes:

 `

 type composeOptions struct {

-	Replicas    int

-	Chunks      int

-	MasterImage string

-	WorkerImage string

-	Volume      string

-	Shuffle     bool

-	RandSeed    int64

-	DryRun      bool

+	Replicas     int

+	Chunks       int

+	MasterImage  string

+	WorkerImage  string

+	Volume       string

+	Shuffle      bool

+	RandSeed     int64

+	DryRun       bool

+	KeepExecutor bool

 }

 type composeTemplateOptions struct {

@@ -41,6 +41,7 @@ func xmain() error {

 	randSeed := flag.Int64("rand-seed", int64(0), "Random seed used for shuffling (0 == curent time)")

 	filtersFile := flag.String("filters-file", "", "Path to optional file composed of `-check.f` filter strings")

 	dryRun := flag.Bool("dry-run", false, "Dry run")

+	keepExecutor := flag.Bool("keep-executor", false, "Do not auto-remove executor containers, which is used for running privileged programs on Swarm")

 	flag.Parse()

 	if *chunks == 0 {

 		*chunks = *replicas

@@ -72,14 +73,15 @@ func xmain() error {

 		workerImageForStack = *pushWorkerImage

 	}

 	compose, err := createCompose("", cli, composeOptions{

-		Replicas:    *replicas,

-		Chunks:      *chunks,

-		MasterImage: defaultMasterImageName,

-		WorkerImage: workerImageForStack,

-		Volume:      defaultVolumeName,

-		Shuffle:     *shuffle,

-		RandSeed:    *randSeed,

-		DryRun:      *dryRun,

+		Replicas:     *replicas,

+		Chunks:       *chunks,

+		MasterImage:  defaultMasterImageName,

+		WorkerImage:  workerImageForStack,

+		Volume:       defaultVolumeName,

+		Shuffle:      *shuffle,

+		RandSeed:     *randSeed,

+		DryRun:       *dryRun,

+		KeepExecutor: *keepExecutor,

 	})

 	if err != nil {

 		return err