@@ -16,7 +16,6 @@ func NewPluginCommand(dockerCli *command.DockerCli) *cobra.Command {

 			cmd.SetOutput(dockerCli.Err())

 			cmd.HelpFunc()(cmd, args)

 		},

-		Tags: map[string]string{"experimental": ""},

 	}

 	cmd.AddCommand(

@@ -21,6 +21,7 @@ type CommonAPIClient interface {

 	ImageAPIClient

 	NodeAPIClient

 	NetworkAPIClient

+	PluginAPIClient

 	ServiceAPIClient

 	SwarmAPIClient

 	SecretAPIClient

@@ -104,6 +105,19 @@ type NodeAPIClient interface {

 	NodeUpdate(ctx context.Context, nodeID string, version swarm.Version, node swarm.NodeSpec) error

 }

+// PluginAPIClient defines API client methods for the plugins

+type PluginAPIClient interface {

+	PluginList(ctx context.Context) (types.PluginsListResponse, error)

+	PluginRemove(ctx context.Context, name string, options types.PluginRemoveOptions) error

+	PluginEnable(ctx context.Context, name string) error

+	PluginDisable(ctx context.Context, name string) error

+	PluginInstall(ctx context.Context, name string, options types.PluginInstallOptions) error

+	PluginPush(ctx context.Context, name string, registryAuth string) error

+	PluginSet(ctx context.Context, name string, args []string) error

+	PluginInspectWithRaw(ctx context.Context, name string) (*types.Plugin, []byte, error)

+	PluginCreate(ctx context.Context, createContext io.Reader, options types.PluginCreateOptions) error

+}

+

 // ServiceAPIClient defines API client methods for the services

 type ServiceAPIClient interface {

 	ServiceCreate(ctx context.Context, service swarm.ServiceSpec, options types.ServiceCreateOptions) (types.ServiceCreateResponse, error)

@@ -1,15 +1,12 @@

 package client

 import (

-	"io"

-

 	"github.com/docker/docker/api/types"

 	"golang.org/x/net/context"

 )

 type apiClientExperimental interface {

 	CheckpointAPIClient

-	PluginAPIClient

 }

 // CheckpointAPIClient defines API client methods for the checkpoints

@@ -18,16 +15,3 @@ type CheckpointAPIClient interface {

 	CheckpointDelete(ctx context.Context, container string, options types.CheckpointDeleteOptions) error

 	CheckpointList(ctx context.Context, container string, options types.CheckpointListOptions) ([]types.Checkpoint, error)

 }

-

-// PluginAPIClient defines API client methods for the plugins

-type PluginAPIClient interface {

-	PluginList(ctx context.Context) (types.PluginsListResponse, error)

-	PluginRemove(ctx context.Context, name string, options types.PluginRemoveOptions) error

-	PluginEnable(ctx context.Context, name string) error

-	PluginDisable(ctx context.Context, name string) error

-	PluginInstall(ctx context.Context, name string, options types.PluginInstallOptions) error

-	PluginPush(ctx context.Context, name string, registryAuth string) error

-	PluginSet(ctx context.Context, name string, args []string) error

-	PluginInspectWithRaw(ctx context.Context, name string) (*types.Plugin, []byte, error)

-	PluginCreate(ctx context.Context, createContext io.Reader, options types.PluginCreateOptions) error

-}

@@ -19,6 +19,7 @@ import (

 	"github.com/docker/docker/api/server/router/container"

 	"github.com/docker/docker/api/server/router/image"

 	"github.com/docker/docker/api/server/router/network"

+	pluginrouter "github.com/docker/docker/api/server/router/plugin"

 	swarmrouter "github.com/docker/docker/api/server/router/swarm"

 	systemrouter "github.com/docker/docker/api/server/router/system"

 	"github.com/docker/docker/api/server/router/volume"

@@ -38,6 +39,7 @@ import (

 	"github.com/docker/docker/pkg/plugingetter"

 	"github.com/docker/docker/pkg/signal"

 	"github.com/docker/docker/pkg/system"

+	"github.com/docker/docker/plugin"

 	"github.com/docker/docker/registry"

 	"github.com/docker/docker/runconfig"

 	"github.com/docker/docker/utils"

@@ -457,6 +459,7 @@ func initRouter(s *apiserver.Server, d *daemon.Daemon, c *cluster.Cluster) {

 		volume.NewRouter(d),

 		build.NewRouter(dockerfile.NewBuildManager(d)),

 		swarmrouter.NewRouter(d, c),

+		pluginrouter.NewRouter(plugin.GetManager()),

 	}...)

 	if d.NetworkControllerEnabled() {

@@ -4,14 +4,12 @@ import (

 	"github.com/docker/docker/api/server/httputils"

 	"github.com/docker/docker/api/server/router"

 	checkpointrouter "github.com/docker/docker/api/server/router/checkpoint"

-	pluginrouter "github.com/docker/docker/api/server/router/plugin"

 	"github.com/docker/docker/daemon"

-	"github.com/docker/docker/plugin"

 )

 func addExperimentalRouters(routers []router.Router, d *daemon.Daemon, decoder httputils.ContainerDecoder) []router.Router {

 	if !d.HasExperimental() {

 		return []router.Router{}

 	}

-	return append(routers, checkpointrouter.NewRouter(d, decoder), pluginrouter.NewRouter(plugin.GetManager()))

+	return append(routers, checkpointrouter.NewRouter(d, decoder))

 }

@@ -29,6 +29,7 @@ import (

 	"github.com/docker/docker/daemon/events"

 	"github.com/docker/docker/daemon/exec"

 	"github.com/docker/docker/dockerversion"

+	"github.com/docker/docker/plugin"

 	"github.com/docker/libnetwork/cluster"

 	// register graph drivers

 	_ "github.com/docker/docker/daemon/graphdriver/register"

@@ -1267,3 +1268,16 @@ func (daemon *Daemon) GetCluster() Cluster {

 func (daemon *Daemon) SetCluster(cluster Cluster) {

 	daemon.cluster = cluster

 }

+

+func (daemon *Daemon) pluginInit(cfg *Config, remote libcontainerd.Remote) error {

+	return plugin.Init(cfg.Root, daemon.PluginStore, remote, daemon.RegistryService, cfg.LiveRestoreEnabled, daemon.LogPluginEvent)

+}

+

+func (daemon *Daemon) pluginShutdown() {

+	manager := plugin.GetManager()

+	// Check for a valid manager object. In error conditions, daemon init can fail

+	// and shutdown called, before plugin manager is initialized.

+	if manager != nil {

+		manager.Shutdown()

+	}

+}

@@ -1,30 +1,7 @@

 package daemon

-import (

-	"github.com/docker/docker/api/types/container"

-	"github.com/docker/docker/libcontainerd"

-	"github.com/docker/docker/plugin"

-)

+import "github.com/docker/docker/api/types/container"

 func (daemon *Daemon) verifyExperimentalContainerSettings(hostConfig *container.HostConfig, config *container.Config) ([]string, error) {

 	return nil, nil

 }

-

-func (daemon *Daemon) pluginInit(cfg *Config, remote libcontainerd.Remote) error {

-	if !daemon.HasExperimental() {

-		return nil

-	}

-	return plugin.Init(cfg.Root, daemon.PluginStore, remote, daemon.RegistryService, cfg.LiveRestoreEnabled, daemon.LogPluginEvent)

-}

-

-func (daemon *Daemon) pluginShutdown() {

-	if !daemon.HasExperimental() {

-		return

-	}

-	manager := plugin.GetManager()

-	// Check for a valid manager object. In error conditions, daemon init can fail

-	// and shutdown called, before plugin manager is initialized.

-	if manager != nil {

-		manager.Shutdown()

-	}

-}

@@ -5,7 +5,6 @@ aliases: [

 title: "Plugin config"

 description: "How develop and use a plugin with the managed plugin system"

 keywords: "API, Usage, plugins, documentation, developer"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -19,8 +18,8 @@ advisory: "experimental"

 # Plugin Config Version 0 of Plugin V2

-This document outlines the format of the V0 plugin config. The plugin

-config described herein was introduced in the Docker daemon (experimental version) in the [v1.12.0

+This document outlines the format of the V0 plugin configuration. The plugin

+config described herein was introduced in the Docker daemon in the [v1.12.0

 release](https://github.com/docker/docker/commit/f37117045c5398fd3dca8016ea8ca0cb47e7312b).

 Plugin configs describe the various constituents of a docker plugin. Plugin

@@ -171,7 +170,6 @@ Config provides the base accessible fields for working with V0 plugin format

 ```

 {

-       	"configVersion": "v0",

        	"description": "A test plugin for Docker",

        	"documentation": "https://docs.docker.com/engine/extend/plugins/",

        	"entrypoint": ["plugin-no-remove", "/data"],

@@ -1,5 +1,4 @@

 ---

-advisory: experimental

 aliases:

 - /engine/extend/

 description: Develop and use a plugin with the managed plugin system

@@ -18,9 +17,6 @@ title: Managed plugin system

 # Docker Engine managed plugin system

-This document describes the plugin system available today in the **experimental

-build** of Docker 1.12:

-

 * [Installing and using a plugin](index.md#installing-and-using-a-plugin)

 * [Developing a plugin](index.md#developing-a-plugin)

@@ -17,8 +17,8 @@ keywords: "Examples, Usage, plugins, docker, documentation, user guide"

 # Use Docker Engine plugins

 This document describes the Docker Engine plugins generally available in Docker

-Engine. To view information on plugins managed by Docker Engine currently in

-experimental status, refer to [Docker Engine plugin system](index.md).

+Engine. To view information on plugins managed by Docker,

+refer to [Docker Engine plugin system](index.md).

 You can extend the capabilities of the Docker Engine by loading third-party

 plugins. This page explains the types of plugins and provides links to several

@@ -19,8 +19,7 @@ Docker plugins are out-of-process extensions which add capabilities to the

 Docker Engine.

 This document describes the Docker Engine plugin API. To view information on

-plugins managed by Docker Engine currently in experimental status, refer to

-[Docker Engine plugin system](index.md).

+plugins managed by Docker Engine, refer to [Docker Engine plugin system](index.md).

 This page is intended for people who want to develop their own Docker plugin.

 If you just want to learn about or use Docker plugins, look

@@ -17,8 +17,8 @@ aliases: ["/engine/extend/authorization/"]

 # Create an authorization plugin

 This document describes the Docker Engine plugins generally available in Docker

-Engine. To view information on plugins managed by Docker Engine currently in

-experimental status, refer to [Docker Engine plugin system](index.md).

+Engine. To view information on plugins managed by Docker Engine,

+refer to [Docker Engine plugin system](index.md).

 Docker's out-of-the-box authorization model is all or nothing. Any user with

 permission to access the Docker daemon can run any Docker client command. The

@@ -184,6 +184,16 @@ This section lists each version from latest to oldest.  Each listing includes a

 * `POST /services/create` and `POST /services/(id or name)/update` now accept the `TTY` parameter, which allocate a pseudo-TTY in container.

 * `POST /services/create` and `POST /services/(id or name)/update` now accept the `DNSConfig` parameter, which specifies DNS related configurations in resolver configuration file (resolv.conf) through `Nameservers`, `Search`, and `Options`.

 * `GET /networks/(id or name)` now includes IP and name of all peers nodes for swarm mode overlay networks.

+* `GET /plugins` list plugins.

+* `POST /plugins/pull?name=<plugin name>` pulls a plugin.

+* `GET /plugins/(plugin name)` inspect a plugin.

+* `POST /plugins/(plugin name)/set` configure a plugin.

+* `POST /plugins/(plugin name)/enable` enable a plugin.

+* `POST /plugins/(plugin name)/disable` disable a plugin.

+* `POST /plugins/(plugin name)/push` push a pluging.

+* `POST /plugins/create?name=(plugin name)` create a plugin.

+* `DELETE /plugins/(plugin name)` delete a plugin.

+

 ### v1.24 API changes

@@ -4291,7 +4291,7 @@ Content-Type: application/json

 ### Configure a plugin

-POST /plugins/(plugin name)/set`

+`POST /plugins/(plugin name)/set`

 **Example request**:

@@ -4438,11 +4438,9 @@ Content-Type: text/plain; charset=utf-8

 -   **204** - no error

 -   **500** - server error

-<!-- TODO Document "docker plugin push" endpoint once we have "plugin build"

-

 ### Push a plugin

-`POST /v1.25/plugins/tiborvass/(plugin name)/push HTTP/1.1`

+`POST /v1.25/plugins/(plugin name)/push`

 Pushes a plugin to the registry.

@@ -4464,7 +4462,6 @@ an image](#create-an-image) section for more details.

 -   **200** - no error

 -   **404** - plugin not installed

 ## 3.7 Nodes

@@ -1,8 +1,7 @@

 ---

-title: "plugin create (experimental)"

+title: "plugin create"

 description: "the plugin create command description and usage"

 keywords: "plugin, create"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,10 +13,12 @@ advisory: "experimental"

      will be rejected.

 -->

+# plugin create

+

 ```markdown

 Usage:  docker plugin create [OPTIONS] reponame[:tag] PATH-TO-ROOTFS

-create a plugin from the given PATH-TO-ROOTFS, which contains the plugin's root filesystem and the config file, config.json

+Create a plugin from a rootfs and configuration

 Options:

       --compress   Compress the context using gzip 

@@ -2,7 +2,6 @@

 title: "plugin disable"

 description: "the plugin disable command description and usage"

 keywords: "plugin, disable"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,7 +13,7 @@ advisory: "experimental"

      will be rejected.

 -->

-# plugin disable (experimental)

+# plugin disable

 ```markdown

 Usage:  docker plugin disable PLUGIN

@@ -2,7 +2,6 @@

 title: "plugin enable"

 description: "the plugin enable command description and usage"

 keywords: "plugin, enable"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,7 +13,7 @@ advisory: "experimental"

      will be rejected.

 -->

-# plugin enable (experimental)

+# plugin enable

 ```markdown

 Usage:  docker plugin enable PLUGIN

@@ -2,7 +2,6 @@

 title: "plugin inspect"

 description: "The plugin inspect command description and usage"

 keywords: "plugin, inspect"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,7 +13,7 @@ advisory: "experimental"

      will be rejected.

 -->

-# plugin inspect (experimental)

+# plugin inspect

 ```markdown

 Usage:  docker plugin inspect [OPTIONS] PLUGIN [PLUGIN...]

@@ -2,7 +2,6 @@

 title: "plugin install"

 description: "the plugin install command description and usage"

 keywords: "plugin, install"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,7 +13,7 @@ advisory: "experimental"

      will be rejected.

 -->

-# plugin install (experimental)

+# plugin install

 ```markdown

 Usage:  docker plugin install [OPTIONS] PLUGIN [KEY=VALUE...]

@@ -2,7 +2,6 @@

 title: "plugin ls"

 description: "The plugin ls command description and usage"

 keywords: "plugin, list"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,7 +13,7 @@ advisory: "experimental"

      will be rejected.

 -->

-# plugin ls (experimental)

+# plugin ls

 ```markdown

 Usage:  docker plugin ls [OPTIONS]

@@ -2,7 +2,6 @@

 title: "plugin rm"

 description: "the plugin rm command description and usage"

 keywords: "plugin, rm"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,7 +13,7 @@ advisory: "experimental"

      will be rejected.

 -->

-# plugin rm (experimental)

+# plugin rm

 ```markdown

 Usage:  docker plugin rm [OPTIONS] PLUGIN [PLUGIN...]

@@ -2,7 +2,6 @@

 title: "plugin set"

 description: "the plugin set command description and usage"

 keywords: "plugin, set"

-advisory: "experimental"

 ---

 <!-- This file is maintained within the docker/docker Github

@@ -14,7 +13,7 @@ advisory: "experimental"

      will be rejected.

 -->

-# plugin set (experimental)

+# plugin set

 ```markdown

 Usage:  docker plugin set PLUGIN KEY=VALUE [KEY=VALUE...]

@@ -30,7 +30,7 @@ type DockerAuthzV2Suite struct {

 }

 func (s *DockerAuthzV2Suite) SetUpTest(c *check.C) {

-	testRequires(c, DaemonIsLinux, ExperimentalDaemon, Network)

+	testRequires(c, DaemonIsLinux, Network)

 	s.d = NewDaemon(c)

 	c.Assert(s.d.Start(), check.IsNil)

 }

deleted file mode 100644

@@ -1,243 +0,0 @@

-// +build linux

-

-package main

-

-import (

-	"os"

-	"os/exec"

-	"path/filepath"

-	"strings"

-	"syscall"

-

-	"github.com/docker/docker/pkg/integration/checker"

-	"github.com/go-check/check"

-)

-

-var pluginName = "tiborvass/no-remove"

-

-// TestDaemonRestartWithPluginEnabled tests state restore for an enabled plugin

-func (s *DockerDaemonSuite) TestDaemonRestartWithPluginEnabled(c *check.C) {

-	testRequires(c, Network, ExperimentalDaemon)

-

-	if err := s.d.Start(); err != nil {

-		c.Fatalf("Could not start daemon: %v", err)

-	}

-

-	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

-		c.Fatalf("Could not install plugin: %v %s", err, out)

-	}

-

-	defer func() {

-		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

-			c.Fatalf("Could not disable plugin: %v %s", err, out)

-		}

-		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

-			c.Fatalf("Could not remove plugin: %v %s", err, out)

-		}

-	}()

-

-	if err := s.d.Restart(); err != nil {

-		c.Fatalf("Could not restart daemon: %v", err)

-	}

-

-	out, err := s.d.Cmd("plugin", "ls")

-	if err != nil {

-		c.Fatalf("Could not list plugins: %v %s", err, out)

-	}

-	c.Assert(out, checker.Contains, pluginName)

-	c.Assert(out, checker.Contains, "true")

-}

-

-// TestDaemonRestartWithPluginDisabled tests state restore for a disabled plugin

-func (s *DockerDaemonSuite) TestDaemonRestartWithPluginDisabled(c *check.C) {

-	testRequires(c, Network, ExperimentalDaemon)

-

-	if err := s.d.Start(); err != nil {

-		c.Fatalf("Could not start daemon: %v", err)

-	}

-

-	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName, "--disable"); err != nil {

-		c.Fatalf("Could not install plugin: %v %s", err, out)

-	}

-

-	defer func() {

-		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

-			c.Fatalf("Could not remove plugin: %v %s", err, out)

-		}

-	}()

-

-	if err := s.d.Restart(); err != nil {

-		c.Fatalf("Could not restart daemon: %v", err)

-	}

-

-	out, err := s.d.Cmd("plugin", "ls")

-	if err != nil {

-		c.Fatalf("Could not list plugins: %v %s", err, out)

-	}

-	c.Assert(out, checker.Contains, pluginName)

-	c.Assert(out, checker.Contains, "false")

-}

-

-// TestDaemonKillLiveRestoreWithPlugins SIGKILLs daemon started with --live-restore.

-// Plugins should continue to run.

-func (s *DockerDaemonSuite) TestDaemonKillLiveRestoreWithPlugins(c *check.C) {

-	testRequires(c, Network, ExperimentalDaemon)

-

-	if err := s.d.Start("--live-restore"); err != nil {

-		c.Fatalf("Could not start daemon: %v", err)

-	}

-	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

-		c.Fatalf("Could not install plugin: %v %s", err, out)

-	}

-	defer func() {

-		if err := s.d.Restart("--live-restore"); err != nil {

-			c.Fatalf("Could not restart daemon: %v", err)

-		}

-		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

-			c.Fatalf("Could not disable plugin: %v %s", err, out)

-		}

-		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

-			c.Fatalf("Could not remove plugin: %v %s", err, out)

-		}

-	}()

-

-	if err := s.d.Kill(); err != nil {

-		c.Fatalf("Could not kill daemon: %v", err)

-	}

-

-	cmd := exec.Command("pgrep", "-f", "plugin-no-remove")

-	if out, ec, err := runCommandWithOutput(cmd); ec != 0 {

-		c.Fatalf("Expected exit code '0', got %d err: %v output: %s ", ec, err, out)

-	}

-}

-

-// TestDaemonShutdownLiveRestoreWithPlugins SIGTERMs daemon started with --live-restore.

-// Plugins should continue to run.

-func (s *DockerDaemonSuite) TestDaemonShutdownLiveRestoreWithPlugins(c *check.C) {

-	testRequires(c, Network, ExperimentalDaemon)

-

-	if err := s.d.Start("--live-restore"); err != nil {

-		c.Fatalf("Could not start daemon: %v", err)

-	}

-	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

-		c.Fatalf("Could not install plugin: %v %s", err, out)

-	}

-	defer func() {

-		if err := s.d.Restart("--live-restore"); err != nil {

-			c.Fatalf("Could not restart daemon: %v", err)

-		}

-		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

-			c.Fatalf("Could not disable plugin: %v %s", err, out)

-		}

-		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

-			c.Fatalf("Could not remove plugin: %v %s", err, out)

-		}

-	}()

-

-	if err := s.d.cmd.Process.Signal(os.Interrupt); err != nil {

-		c.Fatalf("Could not kill daemon: %v", err)

-	}

-

-	cmd := exec.Command("pgrep", "-f", "plugin-no-remove")

-	if out, ec, err := runCommandWithOutput(cmd); ec != 0 {

-		c.Fatalf("Expected exit code '0', got %d err: %v output: %s ", ec, err, out)

-	}

-}

-

-// TestDaemonShutdownWithPlugins shuts down running plugins.

-func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) {

-	testRequires(c, Network, ExperimentalDaemon)

-

-	if err := s.d.Start(); err != nil {

-		c.Fatalf("Could not start daemon: %v", err)

-	}

-	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

-		c.Fatalf("Could not install plugin: %v %s", err, out)

-	}

-

-	defer func() {

-		if err := s.d.Restart(); err != nil {

-			c.Fatalf("Could not restart daemon: %v", err)

-		}

-		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

-			c.Fatalf("Could not disable plugin: %v %s", err, out)

-		}

-		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

-			c.Fatalf("Could not remove plugin: %v %s", err, out)

-		}

-	}()

-

-	if err := s.d.cmd.Process.Signal(os.Interrupt); err != nil {

-		c.Fatalf("Could not kill daemon: %v", err)

-	}

-

-	for {

-		if err := syscall.Kill(s.d.cmd.Process.Pid, 0); err == syscall.ESRCH {

-			break

-		}

-	}

-

-	cmd := exec.Command("pgrep", "-f", "plugin-no-remove")

-	if out, ec, err := runCommandWithOutput(cmd); ec != 1 {

-		c.Fatalf("Expected exit code '1', got %d err: %v output: %s ", ec, err, out)

-	}

-}

-

-// TestVolumePlugin tests volume creation using a plugin.

-func (s *DockerDaemonSuite) TestVolumePlugin(c *check.C) {

-	testRequires(c, Network, ExperimentalDaemon)

-

-	volName := "plugin-volume"

-	volRoot := "/data"

-	destDir := "/tmp/data/"

-	destFile := "foo"

-

-	if err := s.d.Start(); err != nil {

-		c.Fatalf("Could not start daemon: %v", err)

-	}

-	out, err := s.d.Cmd("plugin", "install", pluginName, "--grant-all-permissions")

-	if err != nil {

-		c.Fatalf("Could not install plugin: %v %s", err, out)

-	}

-	defer func() {

-		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

-			c.Fatalf("Could not disable plugin: %v %s", err, out)

-		}

-		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

-			c.Fatalf("Could not remove plugin: %v %s", err, out)

-		}

-	}()

-

-	out, err = s.d.Cmd("volume", "create", "-d", pluginName, volName)

-	if err != nil {

-		c.Fatalf("Could not create volume: %v %s", err, out)

-	}

-	defer func() {

-		if out, err := s.d.Cmd("volume", "remove", volName); err != nil {

-			c.Fatalf("Could not remove volume: %v %s", err, out)

-		}

-	}()

-

-	out, err = s.d.Cmd("volume", "ls")

-	if err != nil {

-		c.Fatalf("Could not list volume: %v %s", err, out)

-	}

-	c.Assert(out, checker.Contains, volName)

-	c.Assert(out, checker.Contains, pluginName)

-

-	mountPoint, err := s.d.Cmd("volume", "inspect", volName, "--format", "{{.Mountpoint}}")

-	if err != nil {

-		c.Fatalf("Could not inspect volume: %v %s", err, mountPoint)

-	}

-	mountPoint = strings.TrimSpace(mountPoint)

-

-	out, err = s.d.Cmd("run", "--rm", "-v", volName+":"+destDir, "busybox", "touch", destDir+destFile)

-	c.Assert(err, checker.IsNil, check.Commentf(out))

-	path := filepath.Join(mountPoint, destFile)

-	_, err = os.Lstat(path)

-	c.Assert(err, checker.IsNil)

-

-	// tiborvass/no-remove is a volume plugin that persists data on disk at /data,

-	// even after the volume is removed. So perform an explicit filesystem cleanup.

-	os.RemoveAll(volRoot)

-}

new file mode 100644

@@ -0,0 +1,243 @@

+// +build linux

+

+package main

+

+import (

+	"os"

+	"os/exec"

+	"path/filepath"

+	"strings"

+	"syscall"

+

+	"github.com/docker/docker/pkg/integration/checker"

+	"github.com/go-check/check"

+)

+

+var pluginName = "tiborvass/no-remove"

+

+// TestDaemonRestartWithPluginEnabled tests state restore for an enabled plugin

+func (s *DockerDaemonSuite) TestDaemonRestartWithPluginEnabled(c *check.C) {

+	testRequires(c, Network)

+

+	if err := s.d.Start(); err != nil {

+		c.Fatalf("Could not start daemon: %v", err)

+	}

+

+	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

+		c.Fatalf("Could not install plugin: %v %s", err, out)

+	}

+

+	defer func() {

+		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

+			c.Fatalf("Could not disable plugin: %v %s", err, out)

+		}

+		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

+			c.Fatalf("Could not remove plugin: %v %s", err, out)

+		}

+	}()

+

+	if err := s.d.Restart(); err != nil {

+		c.Fatalf("Could not restart daemon: %v", err)

+	}

+

+	out, err := s.d.Cmd("plugin", "ls")

+	if err != nil {

+		c.Fatalf("Could not list plugins: %v %s", err, out)

+	}

+	c.Assert(out, checker.Contains, pluginName)

+	c.Assert(out, checker.Contains, "true")

+}

+

+// TestDaemonRestartWithPluginDisabled tests state restore for a disabled plugin

+func (s *DockerDaemonSuite) TestDaemonRestartWithPluginDisabled(c *check.C) {

+	testRequires(c, Network)

+

+	if err := s.d.Start(); err != nil {

+		c.Fatalf("Could not start daemon: %v", err)

+	}

+

+	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName, "--disable"); err != nil {

+		c.Fatalf("Could not install plugin: %v %s", err, out)

+	}

+

+	defer func() {

+		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

+			c.Fatalf("Could not remove plugin: %v %s", err, out)

+		}

+	}()

+

+	if err := s.d.Restart(); err != nil {

+		c.Fatalf("Could not restart daemon: %v", err)

+	}

+

+	out, err := s.d.Cmd("plugin", "ls")

+	if err != nil {

+		c.Fatalf("Could not list plugins: %v %s", err, out)

+	}

+	c.Assert(out, checker.Contains, pluginName)

+	c.Assert(out, checker.Contains, "false")

+}

+

+// TestDaemonKillLiveRestoreWithPlugins SIGKILLs daemon started with --live-restore.

+// Plugins should continue to run.

+func (s *DockerDaemonSuite) TestDaemonKillLiveRestoreWithPlugins(c *check.C) {

+	testRequires(c, Network)

+

+	if err := s.d.Start("--live-restore"); err != nil {

+		c.Fatalf("Could not start daemon: %v", err)

+	}

+	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

+		c.Fatalf("Could not install plugin: %v %s", err, out)

+	}

+	defer func() {

+		if err := s.d.Restart("--live-restore"); err != nil {

+			c.Fatalf("Could not restart daemon: %v", err)

+		}

+		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

+			c.Fatalf("Could not disable plugin: %v %s", err, out)

+		}

+		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

+			c.Fatalf("Could not remove plugin: %v %s", err, out)

+		}

+	}()

+

+	if err := s.d.Kill(); err != nil {

+		c.Fatalf("Could not kill daemon: %v", err)

+	}

+

+	cmd := exec.Command("pgrep", "-f", "plugin-no-remove")

+	if out, ec, err := runCommandWithOutput(cmd); ec != 0 {

+		c.Fatalf("Expected exit code '0', got %d err: %v output: %s ", ec, err, out)

+	}

+}

+

+// TestDaemonShutdownLiveRestoreWithPlugins SIGTERMs daemon started with --live-restore.

+// Plugins should continue to run.

+func (s *DockerDaemonSuite) TestDaemonShutdownLiveRestoreWithPlugins(c *check.C) {

+	testRequires(c, Network)

+

+	if err := s.d.Start("--live-restore"); err != nil {

+		c.Fatalf("Could not start daemon: %v", err)

+	}

+	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

+		c.Fatalf("Could not install plugin: %v %s", err, out)

+	}

+	defer func() {

+		if err := s.d.Restart("--live-restore"); err != nil {

+			c.Fatalf("Could not restart daemon: %v", err)

+		}

+		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

+			c.Fatalf("Could not disable plugin: %v %s", err, out)

+		}

+		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

+			c.Fatalf("Could not remove plugin: %v %s", err, out)

+		}

+	}()

+

+	if err := s.d.cmd.Process.Signal(os.Interrupt); err != nil {

+		c.Fatalf("Could not kill daemon: %v", err)

+	}

+

+	cmd := exec.Command("pgrep", "-f", "plugin-no-remove")

+	if out, ec, err := runCommandWithOutput(cmd); ec != 0 {

+		c.Fatalf("Expected exit code '0', got %d err: %v output: %s ", ec, err, out)

+	}

+}

+

+// TestDaemonShutdownWithPlugins shuts down running plugins.

+func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) {

+	testRequires(c, Network)

+

+	if err := s.d.Start(); err != nil {

+		c.Fatalf("Could not start daemon: %v", err)

+	}

+	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {

+		c.Fatalf("Could not install plugin: %v %s", err, out)

+	}

+

+	defer func() {

+		if err := s.d.Restart(); err != nil {

+			c.Fatalf("Could not restart daemon: %v", err)

+		}

+		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {

+			c.Fatalf("Could not disable plugin: %v %s", err, out)

+		}

+		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {

+			c.Fatalf("Could not remove plugin: %v %s", err, out)

+		}