@@ -4,19 +4,15 @@ package main

 import (

 	"fmt"

-	"path"

 	"strconv"

 	"strings"

 	"time"

 	"github.com/docker/docker/api/types"

 	"github.com/docker/docker/api/types/swarm"

-	"github.com/docker/docker/api/types/swarm/runtime"

 	"github.com/docker/docker/integration-cli/checker"

 	"github.com/docker/docker/integration-cli/daemon"

 	testdaemon "github.com/docker/docker/internal/test/daemon"

-	"github.com/docker/docker/internal/test/fixtures/plugin"

-	"github.com/docker/docker/internal/test/registry"

 	"github.com/go-check/check"

 	"golang.org/x/net/context"

 	"golang.org/x/sys/unix"

@@ -611,78 +607,3 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) {

 		}

 	}

 }

-

-// Test plugins deployed via swarm services

-func (s *DockerSwarmSuite) TestAPISwarmServicesPlugin(c *check.C) {

-	testRequires(c, ExperimentalDaemon, DaemonIsLinux, IsAmd64)

-

-	reg := registry.NewV2(c)

-	defer reg.Close()

-

-	repo := path.Join(privateRegistryURL, "swarm", "test:v1")

-	repo2 := path.Join(privateRegistryURL, "swarm", "test:v2")

-	name := "test"

-

-	err := plugin.CreateInRegistry(context.Background(), repo, nil)

-	c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin"))

-	err = plugin.CreateInRegistry(context.Background(), repo2, nil)

-	c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin"))

-

-	d1 := s.AddDaemon(c, true, true)

-	d2 := s.AddDaemon(c, true, true)

-	d3 := s.AddDaemon(c, true, false)

-

-	makePlugin := func(repo, name string, constraints []string) func(*swarm.Service) {

-		return func(s *swarm.Service) {

-			s.Spec.TaskTemplate.Runtime = "plugin"

-			s.Spec.TaskTemplate.PluginSpec = &runtime.PluginSpec{

-				Name:   name,

-				Remote: repo,

-			}

-			if constraints != nil {

-				s.Spec.TaskTemplate.Placement = &swarm.Placement{

-					Constraints: constraints,

-				}

-			}

-		}

-	}

-

-	id := d1.CreateService(c, makePlugin(repo, name, nil))

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True)

-

-	service := d1.GetService(c, id)

-	d1.UpdateService(c, service, makePlugin(repo2, name, nil))

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginImage(name), checker.Equals, repo2)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginImage(name), checker.Equals, repo2)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginImage(name), checker.Equals, repo2)

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True)

-

-	d1.RemoveService(c, id)

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False)

-

-	// constrain to managers only

-	id = d1.CreateService(c, makePlugin(repo, name, []string{"node.role==manager"}))

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) // Not a manager, not running it

-	d1.RemoveService(c, id)

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False)

-

-	// with no name

-	id = d1.CreateService(c, makePlugin(repo, "", nil))

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.True)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.True)

-	d1.RemoveService(c, id)

-	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.False)

-	waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.False)

-	waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.False)

-}

@@ -19,10 +19,10 @@ import (

 // ServicePoll tweaks the pollSettings for `service`

 func ServicePoll(config *poll.Settings) {

 	// Override the default pollSettings for `service` resource here ...

-

+	config.Timeout = 30 * time.Second

+	config.Delay = 100 * time.Millisecond

 	if runtime.GOARCH == "arm64" || runtime.GOARCH == "arm" {

 		config.Timeout = 1 * time.Minute

-		config.Delay = 100 * time.Millisecond

 	}

 }

new file mode 100644

@@ -0,0 +1,120 @@

+package service

+

+import (

+	"context"

+	"io"

+	"io/ioutil"

+	"os"

+	"path"

+	"testing"

+

+	"github.com/docker/docker/api/types"

+	swarmtypes "github.com/docker/docker/api/types/swarm"

+	"github.com/docker/docker/api/types/swarm/runtime"

+	"github.com/docker/docker/integration/internal/swarm"

+	"github.com/docker/docker/internal/test/daemon"

+	"github.com/docker/docker/internal/test/fixtures/plugin"

+	"github.com/docker/docker/internal/test/registry"

+	"github.com/gotestyourself/gotestyourself/assert"

+	"github.com/gotestyourself/gotestyourself/poll"

+	"github.com/gotestyourself/gotestyourself/skip"

+)

+

+func TestServicePlugin(t *testing.T) {

+	skip.If(t, testEnv.DaemonInfo.OSType == "windows")

+	skip.If(t, os.Getenv("DOCKER_ENGINE_GOARCH") != "amd64")

+	defer setupTest(t)()

+

+	reg := registry.NewV2(t)

+	defer reg.Close()

+

+	repo := path.Join(registry.DefaultURL, "swarm", "test:v1")

+	repo2 := path.Join(registry.DefaultURL, "swarm", "test:v2")

+	name := "test"

+

+	d := daemon.New(t)

+	d.StartWithBusybox(t)

+	apiclient := d.NewClientT(t)

+	err := plugin.Create(context.Background(), apiclient, repo)

+	assert.NilError(t, err)

+	r, err := apiclient.PluginPush(context.Background(), repo, "")

+	assert.NilError(t, err)

+	_, err = io.Copy(ioutil.Discard, r)

+	assert.NilError(t, err)

+	err = apiclient.PluginRemove(context.Background(), repo, types.PluginRemoveOptions{})

+	assert.NilError(t, err)

+	err = plugin.Create(context.Background(), apiclient, repo2)

+	assert.NilError(t, err)

+	r, err = apiclient.PluginPush(context.Background(), repo2, "")

+	assert.NilError(t, err)

+	_, err = io.Copy(ioutil.Discard, r)

+	assert.NilError(t, err)

+	err = apiclient.PluginRemove(context.Background(), repo2, types.PluginRemoveOptions{})

+	assert.NilError(t, err)

+	d.Stop(t)

+

+	d1 := swarm.NewSwarm(t, testEnv, daemon.WithExperimental)

+	defer d1.Stop(t)

+	d2 := daemon.New(t, daemon.WithExperimental, daemon.WithSwarmPort(daemon.DefaultSwarmPort+1))

+	d2.StartAndSwarmJoin(t, d1, true)

+	defer d2.Stop(t)

+	d3 := daemon.New(t, daemon.WithExperimental, daemon.WithSwarmPort(daemon.DefaultSwarmPort+2))

+	d3.StartAndSwarmJoin(t, d1, false)

+	defer d3.Stop(t)

+

+	id := d1.CreateService(t, makePlugin(repo, name, nil))

+	poll.WaitOn(t, d1.PluginIsRunning(name), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginIsRunning(name), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginIsRunning(name), swarm.ServicePoll)

+

+	service := d1.GetService(t, id)

+	d1.UpdateService(t, service, makePlugin(repo2, name, nil))

+	poll.WaitOn(t, d1.PluginReferenceIs(name, repo2), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginReferenceIs(name, repo2), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginReferenceIs(name, repo2), swarm.ServicePoll)

+	poll.WaitOn(t, d1.PluginIsRunning(name), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginIsRunning(name), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginIsRunning(name), swarm.ServicePoll)

+

+	d1.RemoveService(t, id)

+	poll.WaitOn(t, d1.PluginIsNotPresent(name), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginIsNotPresent(name), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginIsNotPresent(name), swarm.ServicePoll)

+

+	// constrain to managers only

+	id = d1.CreateService(t, makePlugin(repo, name, []string{"node.role==manager"}))

+	poll.WaitOn(t, d1.PluginIsRunning(name), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginIsRunning(name), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginIsNotPresent(name), swarm.ServicePoll)

+

+	d1.RemoveService(t, id)

+	poll.WaitOn(t, d1.PluginIsNotPresent(name), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginIsNotPresent(name), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginIsNotPresent(name), swarm.ServicePoll)

+

+	// with no name

+	id = d1.CreateService(t, makePlugin(repo, "", nil))

+	poll.WaitOn(t, d1.PluginIsRunning(repo), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginIsRunning(repo), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginIsRunning(repo), swarm.ServicePoll)

+

+	d1.RemoveService(t, id)

+	poll.WaitOn(t, d1.PluginIsNotPresent(repo), swarm.ServicePoll)

+	poll.WaitOn(t, d2.PluginIsNotPresent(repo), swarm.ServicePoll)

+	poll.WaitOn(t, d3.PluginIsNotPresent(repo), swarm.ServicePoll)

+}

+

+func makePlugin(repo, name string, constraints []string) func(*swarmtypes.Service) {

+	return func(s *swarmtypes.Service) {

+		s.Spec.TaskTemplate.Runtime = "plugin"

+		s.Spec.TaskTemplate.PluginSpec = &runtime.PluginSpec{

+			Name:   name,

+			Remote: repo,

+		}

+		if constraints != nil {

+			s.Spec.TaskTemplate.Placement = &swarmtypes.Placement{

+				Constraints: constraints,

+			}

+		}

+	}

+}

@@ -113,7 +113,7 @@ func New(t testingT, ops ...func(*Daemon)) *Daemon {

 		execRoot:        filepath.Join(os.TempDir(), "docker-execroot", id),

 		dockerdBinary:   defaultDockerdBinary,

 		swarmListenAddr: defaultSwarmListenAddr,

-		SwarmPort:       defaultSwarmPort,

+		SwarmPort:       DefaultSwarmPort,

 		log:             t,

 	}

new file mode 100644

@@ -0,0 +1,77 @@

+package daemon

+

+import (

+	"context"

+

+	"github.com/docker/docker/api/types"

+	"github.com/docker/docker/client"

+	"github.com/gotestyourself/gotestyourself/poll"

+)

+

+// PluginIsRunning provides a poller to check if the specified plugin is running

+func (d *Daemon) PluginIsRunning(name string) func(poll.LogT) poll.Result {

+	return withClient(d, withPluginInspect(name, func(plugin *types.Plugin, t poll.LogT) poll.Result {

+		if plugin.Enabled {

+			return poll.Success()

+		}

+		return poll.Continue("plugin %q is not enabled", name)

+	}))

+}

+

+// PluginIsNotRunning provides a poller to check if the specified plugin is not running

+func (d *Daemon) PluginIsNotRunning(name string) func(poll.LogT) poll.Result {

+	return withClient(d, withPluginInspect(name, func(plugin *types.Plugin, t poll.LogT) poll.Result {

+		if !plugin.Enabled {

+			return poll.Success()

+		}

+		return poll.Continue("plugin %q is enabled", name)

+	}))

+}

+

+// PluginIsNotPresent provides a poller to check if the specified plugin is not present

+func (d *Daemon) PluginIsNotPresent(name string) func(poll.LogT) poll.Result {

+	return withClient(d, func(c client.APIClient, t poll.LogT) poll.Result {

+		_, _, err := c.PluginInspectWithRaw(context.Background(), name)

+		if client.IsErrNotFound(err) {

+			return poll.Success()

+		}

+		if err != nil {

+			return poll.Error(err)

+		}

+		return poll.Continue("plugin %q exists")

+	})

+}

+

+// PluginReferenceIs provides a poller to check if the specified plugin has the specified reference

+func (d *Daemon) PluginReferenceIs(name, expectedRef string) func(poll.LogT) poll.Result {

+	return withClient(d, withPluginInspect(name, func(plugin *types.Plugin, t poll.LogT) poll.Result {

+		if plugin.PluginReference == expectedRef {

+			return poll.Success()

+		}

+		return poll.Continue("plugin %q reference is not %q", name, expectedRef)

+	}))

+}

+

+func withPluginInspect(name string, f func(*types.Plugin, poll.LogT) poll.Result) func(client.APIClient, poll.LogT) poll.Result {

+	return func(c client.APIClient, t poll.LogT) poll.Result {

+		plugin, _, err := c.PluginInspectWithRaw(context.Background(), name)

+		if client.IsErrNotFound(err) {

+			return poll.Continue("plugin %q not found", name)

+		}

+		if err != nil {

+			return poll.Error(err)

+		}

+		return f(plugin, t)

+	}

+

+}

+

+func withClient(d *Daemon, f func(client.APIClient, poll.LogT) poll.Result) func(poll.LogT) poll.Result {

+	return func(t poll.LogT) poll.Result {

+		c, err := d.NewClient()

+		if err != nil {

+			poll.Error(err)

+		}

+		return f(c, t)

+	}

+}

@@ -10,7 +10,8 @@ import (

 )

 const (

-	defaultSwarmPort       = 2477

+	// DefaultSwarmPort is the default port use for swarm in the tests

+	DefaultSwarmPort       = 2477

 	defaultSwarmListenAddr = "0.0.0.0"

 )