@@ -126,7 +126,7 @@ func (s *containerRouter) postContainerExecStart(ctx context.Context, w http.Res

 			return err

 		}

 		stdout.Write([]byte(err.Error() + "\r\n"))

-		logrus.Errorf("Error running exec in container: %v", err)

+		logrus.Errorf("Error running exec %s in container: %v", execName, err)

 	}

 	return nil

 }

@@ -102,7 +102,7 @@ func (c *containerManager) Run(ctx context.Context, cID string, stdout, stderr i

 func logCancellationError(cancelErrCh chan error, msg string) {

 	if cancelErr := <-cancelErrCh; cancelErr != nil {

-		logrus.Debugf("Build cancelled (%v): ", cancelErr, msg)

+		logrus.Debugf("Build cancelled (%v): %s", cancelErr, msg)

 	}

 }

@@ -27,6 +27,8 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) {

 	flags.Var(opts.NewNamedListOptsRef("exec-opts", &conf.ExecOptions, nil), "exec-opt", "Runtime execution options")

 	flags.StringVarP(&conf.Pidfile, "pidfile", "p", defaultPidFile, "Path to use for daemon PID file")

 	flags.StringVarP(&conf.Root, "graph", "g", defaultDataRoot, "Root of the Docker runtime")

+	flags.StringVar(&conf.ExecRoot, "exec-root", defaultExecRoot, "Root directory for execution state files")

+	flags.StringVar(&conf.ContainerdAddr, "containerd", "", "containerd grpc address")

 	// "--graph" is "soft-deprecated" in favor of "data-root". This flag was added

 	// before Docker 1.0, so won't be removed, only hidden, to discourage its usage.

@@ -29,13 +29,11 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) {

 	flags.BoolVar(&conf.BridgeConfig.EnableIPForward, "ip-forward", true, "Enable net.ipv4.ip_forward")

 	flags.BoolVar(&conf.BridgeConfig.EnableIPMasq, "ip-masq", true, "Enable IP masquerading")

 	flags.BoolVar(&conf.BridgeConfig.EnableIPv6, "ipv6", false, "Enable IPv6 networking")

-	flags.StringVar(&conf.ExecRoot, "exec-root", defaultExecRoot, "Root directory for execution state files")

 	flags.StringVar(&conf.BridgeConfig.FixedCIDRv6, "fixed-cidr-v6", "", "IPv6 subnet for fixed IPs")

 	flags.BoolVar(&conf.BridgeConfig.EnableUserlandProxy, "userland-proxy", true, "Use userland proxy for loopback traffic")

 	flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", "", "Path to the userland proxy binary")

 	flags.StringVar(&conf.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers")

 	flags.StringVar(&conf.RemappedRoot, "userns-remap", "", "User/Group setting for user namespaces")

-	flags.StringVar(&conf.ContainerdAddr, "containerd", "", "Path to containerd socket")

 	flags.BoolVar(&conf.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running")

 	flags.IntVar(&conf.OOMScoreAdjust, "oom-score-adjust", -500, "Set the oom_score_adj for the daemon")

 	flags.BoolVar(&conf.Init, "init", false, "Run an init in the container to forward signals and reap processes")

@@ -11,6 +11,7 @@ import (

 var (

 	defaultPidFile  string

 	defaultDataRoot = filepath.Join(os.Getenv("programdata"), "docker")

+	defaultExecRoot = filepath.Join(os.Getenv("programdata"), "docker", "exec-root")

 )

 // installConfigFlags adds flags to the pflag.FlagSet to configure the daemon

@@ -204,7 +204,11 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {

 		return err

 	}

-	containerdRemote, err := libcontainerd.New(cli.getLibcontainerdRoot(), cli.getPlatformRemoteOptions()...)

+	rOpts, err := cli.getRemoteOptions()

+	if err != nil {

+		return fmt.Errorf("Failed to generate containerd options: %s", err)

+	}

+	containerdRemote, err := libcontainerd.New(filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), rOpts...)

 	if err != nil {

 		return err

 	}

@@ -560,6 +564,17 @@ func (cli *DaemonCli) initMiddlewares(s *apiserver.Server, cfg *apiserver.Config

 	return nil

 }

+func (cli *DaemonCli) getRemoteOptions() ([]libcontainerd.RemoteOption, error) {

+	opts := []libcontainerd.RemoteOption{}

+

+	pOpts, err := cli.getPlatformRemoteOptions()

+	if err != nil {

+		return nil, err

+	}

+	opts = append(opts, pOpts...)

+	return opts, nil

+}

+

 // validates that the plugins requested with the --authorization-plugin flag are valid AuthzDriver

 // plugins present on the host and available to the daemon

 func validateAuthzPlugins(requestedPlugins []string, pg plugingetter.PluginGetter) error {

@@ -11,5 +11,5 @@ func preNotifySystem() {

 // notifySystem sends a message to the host when the server is ready to be used

 func notifySystem() {

 	// Tell the init daemon we are accepting requests

-	go systemdDaemon.SdNotify("READY=1")

+	go systemdDaemon.SdNotify(false, "READY=1")

 }

@@ -41,20 +41,8 @@ func preNotifySystem() {

 func notifySystem() {

 }

-func (cli *DaemonCli) getPlatformRemoteOptions() []libcontainerd.RemoteOption {

-	opts := []libcontainerd.RemoteOption{}

-	if cli.Config.ContainerdAddr != "" {

-		opts = append(opts, libcontainerd.WithRemoteAddr(cli.Config.ContainerdAddr))

-	} else {

-		opts = append(opts, libcontainerd.WithStartDaemon(true))

-	}

-	return opts

-}

-

-// getLibcontainerdRoot gets the root directory for libcontainerd/containerd to

-// store their state.

-func (cli *DaemonCli) getLibcontainerdRoot() string {

-	return filepath.Join(cli.Config.ExecRoot, "libcontainerd")

+func (cli *DaemonCli) getPlatformRemoteOptions() ([]libcontainerd.RemoteOption, error) {

+	return nil, nil

 }

 // getSwarmRunRoot gets the root directory for swarm to store runtime state

@@ -10,9 +10,11 @@ import (

 	"path/filepath"

 	"strconv"

+	"github.com/containerd/containerd/linux"

 	"github.com/docker/docker/cmd/dockerd/hack"

 	"github.com/docker/docker/daemon"

 	"github.com/docker/docker/libcontainerd"

+	"github.com/docker/docker/pkg/parsers/kernel"

 	"github.com/docker/libnetwork/portallocator"

 	"golang.org/x/sys/unix"

 )

@@ -35,42 +37,48 @@ func getDaemonConfDir(_ string) string {

 	return "/etc/docker"

 }

-// setupConfigReloadTrap configures the USR2 signal to reload the configuration.

-func (cli *DaemonCli) setupConfigReloadTrap() {

-	c := make(chan os.Signal, 1)

-	signal.Notify(c, unix.SIGHUP)

-	go func() {

-		for range c {

-			cli.reloadConfig()

-		}

-	}()

-}

+func (cli *DaemonCli) getPlatformRemoteOptions() ([]libcontainerd.RemoteOption, error) {

+	// On older kernel, letting putting the containerd-shim in its own

+	// namespace will effectively prevent operations such as unlink, rename

+	// and remove on mountpoints that were present at the time the shim

+	// namespace was created. This would led to a famous EBUSY will trying to

+	// remove shm mounts.

+	var noNewNS bool

+	if !kernel.CheckKernelVersion(3, 18, 0) {

+		noNewNS = true

+	}

-func (cli *DaemonCli) getPlatformRemoteOptions() []libcontainerd.RemoteOption {

 	opts := []libcontainerd.RemoteOption{

-		libcontainerd.WithDebugLog(cli.Config.Debug),

 		libcontainerd.WithOOMScore(cli.Config.OOMScoreAdjust),

+		libcontainerd.WithPlugin("linux", &linux.Config{

+			Shim:          daemon.DefaultShimBinary,

+			Runtime:       daemon.DefaultRuntimeBinary,

+			RuntimeRoot:   filepath.Join(cli.Config.Root, "runc"),

+			ShimDebug:     cli.Config.Debug,

+			ShimNoMountNS: noNewNS,

+		}),

+	}

+	if cli.Config.Debug {

+		opts = append(opts, libcontainerd.WithLogLevel("debug"))

 	}

 	if cli.Config.ContainerdAddr != "" {

 		opts = append(opts, libcontainerd.WithRemoteAddr(cli.Config.ContainerdAddr))

 	} else {

 		opts = append(opts, libcontainerd.WithStartDaemon(true))

 	}

-	if daemon.UsingSystemd(cli.Config) {

-		args := []string{"--systemd-cgroup=true"}

-		opts = append(opts, libcontainerd.WithRuntimeArgs(args))

-	}

-	if cli.Config.LiveRestoreEnabled {

-		opts = append(opts, libcontainerd.WithLiveRestore(true))

-	}

-	opts = append(opts, libcontainerd.WithRuntimePath(daemon.DefaultRuntimeBinary))

-	return opts

+

+	return opts, nil

 }

-// getLibcontainerdRoot gets the root directory for libcontainerd/containerd to

-// store their state.

-func (cli *DaemonCli) getLibcontainerdRoot() string {

-	return filepath.Join(cli.Config.ExecRoot, "libcontainerd")

+// setupConfigReloadTrap configures the USR2 signal to reload the configuration.

+func (cli *DaemonCli) setupConfigReloadTrap() {

+	c := make(chan os.Signal, 1)

+	signal.Notify(c, unix.SIGHUP)

+	go func() {

+		for range c {

+			cli.reloadConfig()

+		}

+	}()

 }

 // getSwarmRunRoot gets the root directory for swarm to store runtime state

@@ -48,6 +48,10 @@ func notifyShutdown(err error) {

 	}

 }

+func (cli *DaemonCli) getPlatformRemoteOptions() ([]libcontainerd.RemoteOption, error) {

+	return nil, nil

+}

+

 // setupConfigReloadTrap configures a Win32 event to reload the configuration.

 func (cli *DaemonCli) setupConfigReloadTrap() {

 	go func() {

@@ -65,17 +69,6 @@ func (cli *DaemonCli) setupConfigReloadTrap() {

 	}()

 }

-func (cli *DaemonCli) getPlatformRemoteOptions() []libcontainerd.RemoteOption {

-	return nil

-}

-

-// getLibcontainerdRoot gets the root directory for libcontainerd to store its

-// state. The Windows libcontainerd implementation does not need to write a spec

-// or state to disk, so this is a no-op.

-func (cli *DaemonCli) getLibcontainerdRoot() string {

-	return ""

-}

-

 // getSwarmRunRoot gets the root directory for swarm to store runtime state

 // For example, the control socket

 func (cli *DaemonCli) getSwarmRunRoot() string {

@@ -15,6 +15,7 @@ import (

 	"syscall"

 	"time"

+	"github.com/containerd/containerd"

 	containertypes "github.com/docker/docker/api/types/container"

 	mounttypes "github.com/docker/docker/api/types/mount"

 	networktypes "github.com/docker/docker/api/types/network"

@@ -61,6 +62,18 @@ var (

 	errInvalidNetwork  = errors.New("invalid network settings while building port map info")

 )

+// ExitStatus provides exit reasons for a container.

+type ExitStatus struct {

+	// The exit code with which the container exited.

+	ExitCode int

+

+	// Whether the container encountered an OOM.

+	OOMKilled bool

+

+	// Time at which the container died

+	ExitedAt time.Time

+}

+

 // Container holds the structure defining a container object.

 type Container struct {

 	StreamConfig *stream.Config

@@ -996,10 +1009,10 @@ func (container *Container) CloseStreams() error {

 }

 // InitializeStdio is called by libcontainerd to connect the stdio.

-func (container *Container) InitializeStdio(iop libcontainerd.IOPipe) error {

+func (container *Container) InitializeStdio(iop *libcontainerd.IOPipe) (containerd.IO, error) {

 	if err := container.startLogging(); err != nil {

 		container.Reset(false)

-		return err

+		return nil, err

 	}

 	container.StreamConfig.CopyToPipe(iop)

@@ -1012,7 +1025,7 @@ func (container *Container) InitializeStdio(iop libcontainerd.IOPipe) error {

 		}

 	}

-	return nil

+	return &cio{IO: iop, sc: container.StreamConfig}, nil

 }

 // SecretMountPath returns the path of the secret mount for the container

@@ -1069,3 +1082,21 @@ func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string

 	env = ReplaceOrAppendEnvValues(env, container.Config.Env)

 	return env

 }

+

+type cio struct {

+	containerd.IO

+

+	sc *stream.Config

+}

+

+func (i *cio) Close() error {

+	i.IO.Close()

+

+	return i.sc.CloseStreams()

+}

+

+func (i *cio) Wait() {

+	i.sc.Wait()

+

+	i.IO.Wait()

+}

@@ -24,15 +24,6 @@ const (

 	containerSecretMountPath = "/run/secrets"

 )

-// ExitStatus provides exit reasons for a container.

-type ExitStatus struct {

-	// The exit code with which the container exited.

-	ExitCode int

-

-	// Whether the container encountered an OOM.

-	OOMKilled bool

-}

-

 // TrySetNetworkMount attempts to set the network mounts given a provided destination and

 // the path to use for it; return true if the given destination was a network mount file

 func (container *Container) TrySetNetworkMount(destination string, path string) bool {

@@ -18,12 +18,6 @@ const (

 	containerInternalConfigsDirPath  = `C:\ProgramData\Docker\internal\configs`

 )

-// ExitStatus provides exit reasons for a container.

-type ExitStatus struct {

-	// The exit code with which the container exited.

-	ExitCode int

-}

-

 // UnmountIpcMount unmounts Ipc related mounts.

 // This is a NOOP on windows.

 func (container *Container) UnmountIpcMount(unmount func(pth string) error) error {

@@ -276,6 +276,7 @@ func (s *State) SetExitCode(ec int) {

 // SetRunning sets the state of the container to "running".

 func (s *State) SetRunning(pid int, initial bool) {

 	s.ErrorMsg = ""

+	s.Paused = false

 	s.Running = true

 	s.Restarting = false

 	if initial {

@@ -294,9 +295,14 @@ func (s *State) SetStopped(exitStatus *ExitStatus) {

 	s.Paused = false

 	s.Restarting = false

 	s.Pid = 0

-	s.FinishedAt = time.Now().UTC()

-	s.setFromExitStatus(exitStatus)

-	close(s.waitStop) // Fire waiters for stop

+	if exitStatus.ExitedAt.IsZero() {

+		s.FinishedAt = time.Now().UTC()

+	} else {

+		s.FinishedAt = exitStatus.ExitedAt

+	}

+	s.ExitCodeValue = exitStatus.ExitCode

+	s.OOMKilled = exitStatus.OOMKilled

+	close(s.waitStop) // fire waiters for stop

 	s.waitStop = make(chan struct{})

 }

@@ -310,8 +316,9 @@ func (s *State) SetRestarting(exitStatus *ExitStatus) {

 	s.Paused = false

 	s.Pid = 0

 	s.FinishedAt = time.Now().UTC()

-	s.setFromExitStatus(exitStatus)

-	close(s.waitStop) // Fire waiters for stop

+	s.ExitCodeValue = exitStatus.ExitCode

+	s.OOMKilled = exitStatus.OOMKilled

+	close(s.waitStop) // fire waiters for stop

 	s.waitStop = make(chan struct{})

 }

deleted file mode 100644

@@ -1,10 +0,0 @@

-// +build linux freebsd

-

-package container

-

-// setFromExitStatus is a platform specific helper function to set the state

-// based on the ExitStatus structure.

-func (s *State) setFromExitStatus(exitStatus *ExitStatus) {

-	s.ExitCodeValue = exitStatus.ExitCode

-	s.OOMKilled = exitStatus.OOMKilled

-}

deleted file mode 100644

@@ -1,7 +0,0 @@

-package container

-

-// setFromExitStatus is a platform specific helper function to set the state

-// based on the ExitStatus structure.

-func (s *State) setFromExitStatus(exitStatus *ExitStatus) {

-	s.ExitCodeValue = exitStatus.ExitCode

-}

@@ -114,12 +114,12 @@ func (c *Config) CloseStreams() error {

 }

 // CopyToPipe connects streamconfig with a libcontainerd.IOPipe

-func (c *Config) CopyToPipe(iop libcontainerd.IOPipe) {

+func (c *Config) CopyToPipe(iop *libcontainerd.IOPipe) {

 	copyFunc := func(w io.Writer, r io.ReadCloser) {

 		c.Add(1)

 		go func() {

 			if _, err := pools.Copy(w, r); err != nil {

-				logrus.Errorf("stream copy error: %+v", err)

+				logrus.Errorf("stream copy error: %v", err)

 			}

 			r.Close()

 			c.Done()

@@ -138,7 +138,7 @@ func (c *Config) CopyToPipe(iop libcontainerd.IOPipe) {

 			go func() {

 				pools.Copy(iop.Stdin, stdin)

 				if err := iop.Stdin.Close(); err != nil {

-					logrus.Warnf("failed to close stdin: %+v", err)

+					logrus.Warnf("failed to close stdin: %v", err)

 				}

 			}()

 		}

@@ -1,6 +1,7 @@

 package daemon

 import (

+	"context"

 	"encoding/json"

 	"fmt"

 	"io/ioutil"

@@ -17,7 +18,7 @@ var (

 )

 // getCheckpointDir verifies checkpoint directory for create,remove, list options and checks if checkpoint already exists

-func getCheckpointDir(checkDir, checkpointID string, ctrName string, ctrID string, ctrCheckpointDir string, create bool) (string, error) {

+func getCheckpointDir(checkDir, checkpointID, ctrName, ctrID, ctrCheckpointDir string, create bool) (string, error) {

 	var checkpointDir string

 	var err2 error

 	if checkDir != "" {

@@ -32,7 +33,10 @@ func getCheckpointDir(checkDir, checkpointID string, ctrName string, ctrID strin

 		case err == nil && stat.IsDir():

 			err2 = fmt.Errorf("checkpoint with name %s already exists for container %s", checkpointID, ctrName)

 		case err != nil && os.IsNotExist(err):

-			err2 = nil

+			err2 = os.MkdirAll(checkpointAbsDir, 0700)

+			if os.IsExist(err2) {

+				err2 = nil

+			}

 		case err != nil:

 			err2 = err

 		case err == nil:

@@ -48,7 +52,7 @@ func getCheckpointDir(checkDir, checkpointID string, ctrName string, ctrID strin

 			err2 = fmt.Errorf("%s exists and is not a directory", checkpointAbsDir)

 		}

 	}

-	return checkpointDir, err2

+	return checkpointAbsDir, err2

 }

 // CheckpointCreate checkpoints the process running in a container with CRIU

@@ -62,6 +66,10 @@ func (daemon *Daemon) CheckpointCreate(name string, config types.CheckpointCreat

 		return fmt.Errorf("Container %s not running", name)

 	}

+	if container.Config.Tty {

+		return fmt.Errorf("checkpoint not support on containers with tty")

+	}

+

 	if !validCheckpointNamePattern.MatchString(config.CheckpointID) {

 		return fmt.Errorf("Invalid checkpoint ID (%s), only %s are allowed", config.CheckpointID, validCheckpointNameChars)

 	}

@@ -71,8 +79,9 @@ func (daemon *Daemon) CheckpointCreate(name string, config types.CheckpointCreat

 		return fmt.Errorf("cannot checkpoint container %s: %s", name, err)

 	}

-	err = daemon.containerd.CreateCheckpoint(container.ID, config.CheckpointID, checkpointDir, config.Exit)

+	err = daemon.containerd.CreateCheckpoint(context.Background(), container.ID, checkpointDir, config.Exit)

 	if err != nil {

+		os.RemoveAll(checkpointDir)

 		return fmt.Errorf("Cannot checkpoint container %s: %s", name, err)

 	}

@@ -101,6 +101,7 @@ type CommonConfig struct {

 	RawLogs              bool                      `json:"raw-logs,omitempty"`

 	RootDeprecated       string                    `json:"graph,omitempty"`

 	Root                 string                    `json:"data-root,omitempty"`

+	ExecRoot             string                    `json:"exec-root,omitempty"`

 	SocketGroup          string                    `json:"group,omitempty"`

 	CorsHeaders          string                    `json:"api-cors-header,omitempty"`

@@ -172,6 +173,10 @@ type CommonConfig struct {

 	NodeGenericResources string `json:"node-generic-resources,omitempty"`

 	// NetworkControlPlaneMTU allows to specify the control plane MTU, this will allow to optimize the network use in some components

 	NetworkControlPlaneMTU int `json:"network-control-plane-mtu,omitempty"`

+

+	// ContainerAddr is the address used to connect to containerd if we're

+	// not starting it ourselves

+	ContainerdAddr string `json:"containerd,omitempty"`

 }

 // IsValueSet returns true if a configuration value

@@ -11,8 +11,6 @@ import (

 // CommonUnixConfig defines configuration of a docker daemon that is

 // common across Unix platforms.

 type CommonUnixConfig struct {

-	ExecRoot          string                   `json:"exec-root,omitempty"`

-	ContainerdAddr    string                   `json:"containerd,omitempty"`

 	Runtimes          map[string]types.Runtime `json:"runtimes,omitempty"`

 	DefaultRuntime    string                   `json:"default-runtime,omitempty"`

 	DefaultInitBinary string                   `json:"default-init,omitempty"`

@@ -18,7 +18,7 @@ import (

 	"sync"

 	"time"

-	containerd "github.com/containerd/containerd/api/grpc/types"

+	"github.com/docker/docker/api/errdefs"

 	"github.com/docker/docker/api/types"

 	containertypes "github.com/docker/docker/api/types/container"

 	"github.com/docker/docker/api/types/swarm"

@@ -62,11 +62,10 @@ import (

 	"github.com/pkg/errors"

 )

-var (

-	// DefaultRuntimeBinary is the default runtime to be used by

-	// containerd if none is specified

-	DefaultRuntimeBinary = "docker-runc"

+// MainNamespace is the name of the namespace used for users containers

+const MainNamespace = "moby"

+var (

 	errSystemNotSupported = errors.New("the Docker daemon is not supported on this platform")

 )

@@ -170,7 +169,7 @@ func (daemon *Daemon) restore() error {

 				continue

 			}

 			container.RWLayer = rwlayer

-			logrus.Debugf("Loaded container %v", container.ID)

+			logrus.Debugf("Loaded container %v, isRunning: %v", container.ID, container.IsRunning())

 			containers[container.ID] = container

 		} else {

@@ -209,8 +208,10 @@ func (daemon *Daemon) restore() error {

 		}

 	}

-	var wg sync.WaitGroup

-	var mapLock sync.Mutex

+	var (

+		wg      sync.WaitGroup

+		mapLock sync.Mutex

+	)

 	for _, c := range containers {

 		wg.Add(1)

 		go func(c *container.Container) {

@@ -221,11 +222,74 @@ func (daemon *Daemon) restore() error {

 			}

 			daemon.setStateCounter(c)

+

+			logrus.WithFields(logrus.Fields{

+				"container": c.ID,

+				"running":   c.IsRunning(),

+				"paused":    c.IsPaused(),

+			}).Debug("restoring container")

+

+			var (

+				err      error

+				alive    bool

+				ec       uint32

+				exitedAt time.Time

+			)

+

+			alive, _, err = daemon.containerd.Restore(context.Background(), c.ID, c.InitializeStdio)

+			if err != nil && !errdefs.IsNotFound(err) {

+				logrus.Errorf("Failed to restore container %s with containerd: %s", c.ID, err)

+				return

+			}

+			if !alive {

+				ec, exitedAt, err = daemon.containerd.DeleteTask(context.Background(), c.ID)

+				if err != nil && !errdefs.IsNotFound(err) {

+					logrus.WithError(err).Errorf("Failed to delete container %s from containerd", c.ID)

+					return

+				}

+			}

+

 			if c.IsRunning() || c.IsPaused() {

 				c.RestartManager().Cancel() // manually start containers because some need to wait for swarm networking

-				if err := daemon.containerd.Restore(c.ID, c.InitializeStdio); err != nil {

-					logrus.Errorf("Failed to restore %s with containerd: %s", c.ID, err)

-					return

+

+				if c.IsPaused() && alive {

+					s, err := daemon.containerd.Status(context.Background(), c.ID)

+					if err != nil {

+						logrus.WithError(err).WithField("container", c.ID).

+							Errorf("Failed to get container status")

+					} else {

+						logrus.WithField("container", c.ID).WithField("state", s).

+							Info("restored container paused")

+						switch s {

+						case libcontainerd.StatusPaused, libcontainerd.StatusPausing:

+							// nothing to do

+						case libcontainerd.StatusStopped:

+							alive = false

+						case libcontainerd.StatusUnknown:

+							logrus.WithField("container", c.ID).

+								Error("Unknown status for container during restore")

+						default:

+							// running

+							c.Lock()

+							c.Paused = false

+							daemon.setStateCounter(c)

+							if err := c.CheckpointTo(daemon.containersReplica); err != nil {

+								logrus.WithError(err).WithField("container", c.ID).

+									Error("Failed to update stopped container state")

+							}

+							c.Unlock()

+						}

+					}

+				}

+

+				if !alive {

+					c.Lock()

+					c.SetStopped(&container.ExitStatus{ExitCode: int(ec), ExitedAt: exitedAt})

+					daemon.Cleanup(c)

+					if err := c.CheckpointTo(daemon.containersReplica); err != nil {

+						logrus.Errorf("Failed to update stopped container %s state: %v", c.ID, err)

+					}

+					c.Unlock()

 				}

 				// we call Mount and then Unmount to get BaseFs of the container

@@ -253,11 +317,9 @@ func (daemon *Daemon) restore() error {

 					activeSandboxes[c.NetworkSettings.SandboxID] = options

 					mapLock.Unlock()

 				}

+			} else {

+				// get list of containers we need to restart

-			}

-			// fixme: only if not running

-			// get list of containers we need to restart

-			if !c.IsRunning() && !c.IsPaused() {

 				// Do not autostart containers which

 				// has endpoints in a swarm scope

 				// network yet since the cluster is

@@ -289,7 +351,7 @@ func (daemon *Daemon) restore() error {

 				c.RemovalInProgress = false

 				c.Dead = true

 				if err := c.CheckpointTo(daemon.containersReplica); err != nil {

-					logrus.Errorf("Failed to update container %s state: %v", c.ID, err)

+					logrus.Errorf("Failed to update RemovalInProgress container %s state: %v", c.ID, err)

 				}

 			}

 			c.Unlock()

@@ -559,6 +621,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe

 	d := &Daemon{

 		configStore: config,

+		PluginStore: pluginStore,

 		startupDone: make(chan struct{}),

 	}

 	// Ensure the daemon is properly shutdown if there is a failure during

@@ -606,6 +669,16 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe

 		return nil, err

 	}

+	// Create the directory where we'll store the runtime scripts (i.e. in

+	// order to support runtimeArgs)

+	daemonRuntimes := filepath.Join(config.Root, "runtimes")

+	if err := system.MkdirAll(daemonRuntimes, 0700, ""); err != nil && !os.IsExist(err) {

+		return nil, err

+	}

+	if err := d.loadRuntimes(); err != nil {

+		return nil, err

+	}

+

 	if runtime.GOOS == "windows" {

 		if err := system.MkdirAll(filepath.Join(config.Root, "credentialspecs"), 0, ""); err != nil && !os.IsExist(err) {

 			return nil, err

@@ -635,7 +708,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe

 	}

 	d.RegistryService = registryService

-	d.PluginStore = pluginStore

 	logger.RegisterPluginGetter(d.PluginStore)

 	metricsSockPath, err := d.listenMetricsSock()

@@ -645,7 +717,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe

 	registerMetricsPluginCallback(d.PluginStore, metricsSockPath)

 	createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) {

-		return pluginexec.New(containerdRemote, m)

+		return pluginexec.New(getPluginExecRoot(config.Root), containerdRemote, m)

 	}

 	// Plugin system initialization should happen before restore. Do not change order.

@@ -802,13 +874,13 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe

 	d.idMappings = idMappings

 	d.seccompEnabled = sysInfo.Seccomp

 	d.apparmorEnabled = sysInfo.AppArmor

+	d.containerdRemote = containerdRemote

 	d.linkIndex = newLinkIndex()

-	d.containerdRemote = containerdRemote

 	go d.execCommandGC()

-	d.containerd, err = containerdRemote.Client(d)

+	d.containerd, err = containerdRemote.NewClient(MainNamespace, d)

 	if err != nil {

 		return nil, err

 	}

@@ -1171,19 +1243,6 @@ func (daemon *Daemon) networkOptions(dconfig *config.Config, pg plugingetter.Plu

 	return options, nil

 }

-func copyBlkioEntry(entries []*containerd.BlkioStatsEntry) []types.BlkioStatEntry {

-	out := make([]types.BlkioStatEntry, len(entries))

-	for i, re := range entries {

-		out[i] = types.BlkioStatEntry{

-			Major: re.Major,

-			Minor: re.Minor,

-			Op:    re.Op,

-			Value: re.Value,

-		}

-	}

-	return out

-}

-

 // GetCluster returns the cluster

 func (daemon *Daemon) GetCluster() Cluster {

 	return daemon.cluster

@@ -5,6 +5,7 @@ package daemon

 import (

 	"bufio"

 	"bytes"

+	"context"

 	"fmt"

 	"io/ioutil"

 	"net"

@@ -16,6 +17,7 @@ import (

 	"strings"

 	"time"

+	containerd_cgroups "github.com/containerd/cgroups"

 	"github.com/docker/docker/api/types"

 	"github.com/docker/docker/api/types/blkiodev"

 	pblkiodev "github.com/docker/docker/api/types/blkiodev"

@@ -26,6 +28,7 @@ import (

 	"github.com/docker/docker/opts"

 	"github.com/docker/docker/pkg/containerfs"

 	"github.com/docker/docker/pkg/idtools"

+	"github.com/docker/docker/pkg/ioutils"

 	"github.com/docker/docker/pkg/parsers"

 	"github.com/docker/docker/pkg/parsers/kernel"

 	"github.com/docker/docker/pkg/sysinfo"

@@ -38,7 +41,6 @@ import (

 	"github.com/docker/libnetwork/netutils"

 	"github.com/docker/libnetwork/options"

 	lntypes "github.com/docker/libnetwork/types"

-	"github.com/golang/protobuf/ptypes"

 	"github.com/opencontainers/runc/libcontainer/cgroups"

 	rsystem "github.com/opencontainers/runc/libcontainer/system"

 	specs "github.com/opencontainers/runtime-spec/specs-go"

@@ -50,6 +52,14 @@ import (

 )

 const (

+	// DefaultShimBinary is the default shim to be used by containerd if none

+	// is specified

+	DefaultShimBinary = "docker-containerd-shim"

+

+	// DefaultRuntimeBinary is the default runtime to be used by

+	// containerd if none is specified

+	DefaultRuntimeBinary = "docker-runc"

+

 	// See https://git.kernel.org/cgit/linux/kernel/git/tip/tip.git/tree/kernel/sched/sched.h?id=8cd9234c64c584432f6992fe944ca9e46ca8ea76#n269

 	linuxMinCPUShares = 2

 	linuxMaxCPUShares = 262144

@@ -63,6 +73,10 @@ const (

 	// constant for cgroup drivers

 	cgroupFsDriver      = "cgroupfs"

 	cgroupSystemdDriver = "systemd"

+

+	// DefaultRuntimeName is the default runtime to be used by

+	// containerd if none is specified

+	DefaultRuntimeName = "docker-runc"

 )

 type containerGetter interface {

@@ -623,6 +637,54 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.

 	return warnings, nil

 }

+func (daemon *Daemon) loadRuntimes() error {

+	return daemon.initRuntimes(daemon.configStore.Runtimes)

+}

+

+func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error) {

+	runtimeDir := filepath.Join(daemon.configStore.Root, "runtimes")

+	// Remove old temp directory if any

+	os.RemoveAll(runtimeDir + "-old")

+	tmpDir, err := ioutils.TempDir(daemon.configStore.Root, "gen-runtimes")

+	if err != nil {

+		return errors.Wrapf(err, "failed to get temp dir to generate runtime scripts")

+	}

+	defer func() {

+		if err != nil {

+			if err1 := os.RemoveAll(tmpDir); err1 != nil {

+				logrus.WithError(err1).WithField("dir", tmpDir).

+					Warnf("failed to remove tmp dir")

+			}

+			return

+		}

+

+		if err = os.Rename(runtimeDir, runtimeDir+"-old"); err != nil {

+			return

+		}

+		if err = os.Rename(tmpDir, runtimeDir); err != nil {

+			err = errors.Wrapf(err, "failed to setup runtimes dir, new containers may not start")

+			return

+		}

+		if err = os.RemoveAll(runtimeDir + "-old"); err != nil {

+			logrus.WithError(err).WithField("dir", tmpDir).

+				Warnf("failed to remove old runtimes dir")

+		}

+	}()

+

+	for name, rt := range runtimes {

+		if len(rt.Args) == 0 {

+			continue

+		}

+

+		script := filepath.Join(tmpDir, name)

+		content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))