Browse code

Windows libcontainerd implementation

Signed-off-by: John Howard <jhoward@microsoft.com>
Signed-off-by: John Starks <jostarks@microsoft.com>
Signed-off-by: Darren Stahl <darst@microsoft.com>
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

John Howard authored on 2016/03/19 03:53:27
Showing 22 changed files
... ...
@@ -7,7 +7,6 @@ import (
7 7
 	"os"
8 8
 	"path/filepath"
9 9
 
10
-	"github.com/docker/docker/daemon/execdriver"
11 10
 	"github.com/docker/docker/volume"
12 11
 	containertypes "github.com/docker/engine-api/types/container"
13 12
 )
... ...
@@ -23,6 +22,12 @@ type Container struct {
23 23
 	// Fields below here are platform specific.
24 24
 }
25 25
 
26
+// ExitStatus provides exit reasons for a container.
27
+type ExitStatus struct {
28
+	// The exit code with which the container exited.
29
+	ExitCode int
30
+}
31
+
26 32
 // CreateDaemonEnvironment creates a new environment variable slice for this container.
27 33
 func (container *Container) CreateDaemonEnvironment(linkedEnv []string) []string {
28 34
 	// On Windows, nothing to link. Just return the container environment.
... ...
@@ -35,7 +40,7 @@ func (container *Container) UnmountIpcMounts(unmount func(pth string) error) {
35 35
 }
36 36
 
37 37
 // IpcMounts returns the list of Ipc related mounts.
38
-func (container *Container) IpcMounts() []execdriver.Mount {
38
+func (container *Container) IpcMounts() []Mount {
39 39
 	return nil
40 40
 }
41 41
 
... ...
@@ -45,7 +50,7 @@ func (container *Container) UnmountVolumes(forceSyscall bool, volumeEventLog fun
45 45
 }
46 46
 
47 47
 // TmpfsMounts returns the list of tmpfs mounts
48
-func (container *Container) TmpfsMounts() []execdriver.Mount {
48
+func (container *Container) TmpfsMounts() []Mount {
49 49
 	return nil
50 50
 }
51 51
 
52 52
new file mode 100644
... ...
@@ -0,0 +1,8 @@
0
+package container
1
+
2
+// Mount contains information for a mount operation.
3
+type Mount struct {
4
+	Source      string `json:"source"`
5
+	Destination string `json:"destination"`
6
+	Writable    bool   `json:"writable"`
7
+}
... ...
@@ -1,9 +1,7 @@
1 1
 package container
2 2
 
3
-import "github.com/docker/docker/daemon/execdriver"
4
-
5 3
 // setFromExitStatus is a platform specific helper function to set the state
6 4
 // based on the ExitStatus structure.
7
-func (s *State) setFromExitStatus(exitStatus *execdriver.ExitStatus) {
5
+func (s *State) setFromExitStatus(exitStatus *ExitStatus) {
8 6
 	s.ExitCode = exitStatus.ExitCode
9 7
 }
... ...
@@ -7,8 +7,9 @@ import (
7 7
 )
8 8
 
9 9
 var (
10
-	defaultPidFile = os.Getenv("programdata") + string(os.PathSeparator) + "docker.pid"
11
-	defaultGraph   = os.Getenv("programdata") + string(os.PathSeparator) + "docker"
10
+	defaultPidFile  = os.Getenv("programdata") + string(os.PathSeparator) + "docker.pid"
11
+	defaultGraph    = os.Getenv("programdata") + string(os.PathSeparator) + "docker"
12
+	defaultExecRoot = defaultGraph
12 13
 )
13 14
 
14 15
 // bridgeConfig stores all the bridge driver specific
... ...
@@ -4,14 +4,9 @@ package daemon
4 4
 
5 5
 import (
6 6
 	"fmt"
7
-	"strings"
8
-
9
-	networktypes "github.com/docker/engine-api/types/network"
10 7
 
11 8
 	"github.com/docker/docker/container"
12
-	"github.com/docker/docker/daemon/execdriver"
13
-	"github.com/docker/docker/daemon/execdriver/windows"
14
-	"github.com/docker/docker/layer"
9
+	networktypes "github.com/docker/engine-api/types/network"
15 10
 	"github.com/docker/libnetwork"
16 11
 )
17 12
 
... ...
@@ -29,135 +24,6 @@ func (daemon *Daemon) DisconnectFromNetwork(container *container.Container, n li
29 29
 	return fmt.Errorf("Windows does not support disconnecting a running container from a network")
30 30
 }
31 31
 
32
-func (daemon *Daemon) populateCommand(c *container.Container, env []string) error {
33
-	en := &execdriver.Network{
34
-		Interface: nil,
35
-	}
36
-
37
-	var epList []string
38
-
39
-	// Connect all the libnetwork allocated networks to the container
40
-	if c.NetworkSettings != nil {
41
-		for n := range c.NetworkSettings.Networks {
42
-			sn, err := daemon.FindNetwork(n)
43
-			if err != nil {
44
-				continue
45
-			}
46
-
47
-			ep, err := c.GetEndpointInNetwork(sn)
48
-			if err != nil {
49
-				continue
50
-			}
51
-
52
-			data, err := ep.DriverInfo()
53
-			if err != nil {
54
-				continue
55
-			}
56
-			if data["hnsid"] != nil {
57
-				epList = append(epList, data["hnsid"].(string))
58
-			}
59
-		}
60
-	}
61
-
62
-	if daemon.netController == nil {
63
-		parts := strings.SplitN(string(c.HostConfig.NetworkMode), ":", 2)
64
-		switch parts[0] {
65
-		case "none":
66
-		case "default", "": // empty string to support existing containers
67
-			if !c.Config.NetworkDisabled {
68
-				en.Interface = &execdriver.NetworkInterface{
69
-					MacAddress:   c.Config.MacAddress,
70
-					Bridge:       daemon.configStore.bridgeConfig.Iface,
71
-					PortBindings: c.HostConfig.PortBindings,
72
-
73
-					// TODO Windows. Include IPAddress. There already is a
74
-					// property IPAddress on execDrive.CommonNetworkInterface,
75
-					// but there is no CLI option in docker to pass through
76
-					// an IPAddress on docker run.
77
-				}
78
-			}
79
-		default:
80
-			return fmt.Errorf("invalid network mode: %s", c.HostConfig.NetworkMode)
81
-		}
82
-	}
83
-
84
-	// TODO Windows. More resource controls to be implemented later.
85
-	resources := &execdriver.Resources{
86
-		CommonResources: execdriver.CommonResources{
87
-			CPUShares: c.HostConfig.CPUShares,
88
-		},
89
-	}
90
-
91
-	processConfig := execdriver.ProcessConfig{
92
-		CommonProcessConfig: execdriver.CommonProcessConfig{
93
-			Entrypoint: c.Path,
94
-			Arguments:  c.Args,
95
-			Tty:        c.Config.Tty,
96
-		},
97
-		ConsoleSize: c.HostConfig.ConsoleSize,
98
-	}
99
-
100
-	processConfig.Env = env
101
-
102
-	var layerPaths []string
103
-	img, err := daemon.imageStore.Get(c.ImageID)
104
-	if err != nil {
105
-		return fmt.Errorf("Failed to graph.Get on ImageID %s - %s", c.ImageID, err)
106
-	}
107
-
108
-	if img.RootFS != nil && img.RootFS.Type == "layers+base" {
109
-		max := len(img.RootFS.DiffIDs)
110
-		for i := 0; i <= max; i++ {
111
-			img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
112
-			path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
113
-			if err != nil {
114
-				return fmt.Errorf("Failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStore, img.RootFS.ChainID(), err)
115
-			}
116
-			// Reverse order, expecting parent most first
117
-			layerPaths = append([]string{path}, layerPaths...)
118
-		}
119
-	}
120
-
121
-	m, err := c.RWLayer.Metadata()
122
-	if err != nil {
123
-		return fmt.Errorf("Failed to get layer metadata - %s", err)
124
-	}
125
-	layerFolder := m["dir"]
126
-
127
-	var hvPartition bool
128
-	// Work out the isolation (whether it is a hypervisor partition)
129
-	if c.HostConfig.Isolation.IsDefault() {
130
-		// Not specified by caller. Take daemon default
131
-		hvPartition = windows.DefaultIsolation.IsHyperV()
132
-	} else {
133
-		// Take value specified by caller
134
-		hvPartition = c.HostConfig.Isolation.IsHyperV()
135
-	}
136
-
137
-	c.Command = &execdriver.Command{
138
-		CommonCommand: execdriver.CommonCommand{
139
-			ID:            c.ID,
140
-			Rootfs:        c.BaseFS,
141
-			WorkingDir:    c.Config.WorkingDir,
142
-			Network:       en,
143
-			MountLabel:    c.GetMountLabel(),
144
-			Resources:     resources,
145
-			ProcessConfig: processConfig,
146
-			ProcessLabel:  c.GetProcessLabel(),
147
-		},
148
-		FirstStart:  !c.HasBeenStartedBefore,
149
-		LayerFolder: layerFolder,
150
-		LayerPaths:  layerPaths,
151
-		Hostname:    c.Config.Hostname,
152
-		Isolation:   string(c.HostConfig.Isolation),
153
-		ArgsEscaped: c.Config.ArgsEscaped,
154
-		HvPartition: hvPartition,
155
-		EpList:      epList,
156
-	}
157
-
158
-	return nil
159
-}
160
-
161 32
 // getSize returns real size & virtual size
162 33
 func (daemon *Daemon) getSize(container *container.Container) (int64, int64) {
163 34
 	// TODO Windows
... ...
@@ -18,11 +18,13 @@ import (
18 18
 	"github.com/docker/docker/layer"
19 19
 	"github.com/docker/docker/reference"
20 20
 	"github.com/docker/docker/runconfig"
21
-	containertypes "github.com/docker/engine-api/types/container"
22 21
 	// register the windows graph driver
23 22
 	"github.com/docker/docker/daemon/graphdriver/windows"
24 23
 	"github.com/docker/docker/pkg/idtools"
24
+	"github.com/docker/docker/pkg/parsers"
25 25
 	"github.com/docker/docker/pkg/system"
26
+	"github.com/docker/engine-api/types"
27
+	containertypes "github.com/docker/engine-api/types/container"
26 28
 	"github.com/docker/libnetwork"
27 29
 	nwconfig "github.com/docker/libnetwork/config"
28 30
 	winlibnetwork "github.com/docker/libnetwork/drivers/windows"
... ...
@@ -39,7 +41,7 @@ const (
39 39
 	windowsMaxCPUShares  = 10000
40 40
 )
41 41
 
42
-func getBlkioWeightDevices(config *containertypes.HostConfig) ([]*blkiodev.WeightDevice, error) {
42
+func getBlkioWeightDevices(config *containertypes.HostConfig) ([]blkiodev.WeightDevice, error) {
43 43
 	return nil, nil
44 44
 }
45 45
 
... ...
@@ -47,19 +49,19 @@ func parseSecurityOpt(container *container.Container, config *containertypes.Hos
47 47
 	return nil
48 48
 }
49 49
 
50
-func getBlkioReadIOpsDevices(config *containertypes.HostConfig) ([]*blkiodev.ThrottleDevice, error) {
50
+func getBlkioReadIOpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) {
51 51
 	return nil, nil
52 52
 }
53 53
 
54
-func getBlkioWriteIOpsDevices(config *containertypes.HostConfig) ([]*blkiodev.ThrottleDevice, error) {
54
+func getBlkioWriteIOpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) {
55 55
 	return nil, nil
56 56
 }
57 57
 
58
-func getBlkioReadBpsDevices(config *containertypes.HostConfig) ([]*blkiodev.ThrottleDevice, error) {
58
+func getBlkioReadBpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) {
59 59
 	return nil, nil
60 60
 }
61 61
 
62
-func getBlkioWriteBpsDevices(config *containertypes.HostConfig) ([]*blkiodev.ThrottleDevice, error) {
62
+func getBlkioWriteBpsDevices(config *containertypes.HostConfig) ([]blkiodev.ThrottleDevice, error) {
63 63
 	return nil, nil
64 64
 }
65 65
 
... ...
@@ -287,6 +289,10 @@ func (daemon *Daemon) registerLinks(container *container.Container, hostConfig *
287 287
 	return nil
288 288
 }
289 289
 
290
+func (daemon *Daemon) cleanupMountsByID(in string) error {
291
+	return nil
292
+}
293
+
290 294
 func (daemon *Daemon) cleanupMounts() error {
291 295
 	return nil
292 296
 }
... ...
@@ -307,8 +313,19 @@ func setupDaemonRoot(config *Config, rootDir string, rootUID, rootGID int) error
307 307
 // conditionalMountOnStart is a platform specific helper function during the
308 308
 // container start to call mount.
309 309
 func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error {
310
+
311
+	// Are we going to run as a Hyper-V container?
312
+	hv := false
313
+	if container.HostConfig.Isolation.IsDefault() {
314
+		// Container is set to use the default, so take the default from the daemon configuration
315
+		hv = daemon.defaultIsolation.IsHyperV()
316
+	} else {
317
+		// Container is requesting an isolation mode. Honour it.
318
+		hv = container.HostConfig.Isolation.IsHyperV()
319
+	}
320
+
310 321
 	// We do not mount if a Hyper-V container
311
-	if !container.HostConfig.Isolation.IsHyperV() {
322
+	if !hv {
312 323
 		if err := daemon.Mount(container); err != nil {
313 324
 			return err
314 325
 		}
... ...
@@ -318,11 +335,12 @@ func (daemon *Daemon) conditionalMountOnStart(container *container.Container) er
318 318
 
319 319
 // conditionalUnmountOnCleanup is a platform specific helper function called
320 320
 // during the cleanup of a container to unmount.
321
-func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) {
321
+func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error {
322 322
 	// We do not unmount if a Hyper-V container
323 323
 	if !container.HostConfig.Isolation.IsHyperV() {
324
-		daemon.Unmount(container)
324
+		return daemon.Unmount(container)
325 325
 	}
326
+	return nil
326 327
 }
327 328
 
328 329
 func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) error {
... ...
@@ -404,3 +422,35 @@ func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) erro
404 404
 func driverOptions(config *Config) []nwconfig.Option {
405 405
 	return []nwconfig.Option{}
406 406
 }
407
+
408
+func (daemon *Daemon) stats(c *container.Container) (*types.StatsJSON, error) {
409
+	return nil, nil
410
+}
411
+
412
+// setDefaultIsolation determine the default isolation mode for the
413
+// daemon to run in. This is only applicable on Windows
414
+func (daemon *Daemon) setDefaultIsolation() error {
415
+	daemon.defaultIsolation = containertypes.Isolation("process")
416
+	for _, option := range daemon.configStore.ExecOptions {
417
+		key, val, err := parsers.ParseKeyValueOpt(option)
418
+		if err != nil {
419
+			return err
420
+		}
421
+		key = strings.ToLower(key)
422
+		switch key {
423
+
424
+		case "isolation":
425
+			if !containertypes.Isolation(val).IsValid() {
426
+				return fmt.Errorf("Invalid exec-opt value for 'isolation':'%s'", val)
427
+			}
428
+			if containertypes.Isolation(val).IsHyperV() {
429
+				daemon.defaultIsolation = containertypes.Isolation("hyperv")
430
+			}
431
+		default:
432
+			return fmt.Errorf("Unrecognised exec-opt '%s'\n", key)
433
+		}
434
+	}
435
+
436
+	logrus.Infof("Windows default isolation mode: %s", daemon.defaultIsolation)
437
+	return nil
438
+}
... ...
@@ -2,11 +2,13 @@ package daemon
2 2
 
3 3
 import (
4 4
 	"github.com/docker/docker/container"
5
-	"github.com/docker/docker/daemon/execdriver"
6
-	"github.com/docker/engine-api/types"
5
+	"github.com/docker/docker/daemon/exec"
6
+	"github.com/docker/docker/libcontainerd"
7 7
 )
8 8
 
9
-// setPlatformSpecificExecProcessConfig sets platform-specific fields in the
10
-// ProcessConfig structure. This is a no-op on Windows
11
-func setPlatformSpecificExecProcessConfig(config *types.ExecConfig, container *container.Container, pc *execdriver.ProcessConfig) {
9
+func execSetPlatformOpt(c *container.Container, ec *exec.Config, p *libcontainerd.Process) error {
10
+	// Process arguments need to be escaped before sending to OCI.
11
+	// TODO (jstarks): escape the entrypoint too once the tests are fixed to not rely on this behavior
12
+	p.Args = append([]string{p.Args[0]}, escapeArgs(p.Args[1:])...)
13
+	return nil
12 14
 }
... ...
@@ -33,8 +33,8 @@ func (daemon *Daemon) containerInspectPre120(name string) (*types.ContainerJSON,
33 33
 
34 34
 func inspectExecProcessConfig(e *exec.Config) *backend.ExecProcessConfig {
35 35
 	return &backend.ExecProcessConfig{
36
-		Tty:        e.ProcessConfig.Tty,
37
-		Entrypoint: e.ProcessConfig.Entrypoint,
38
-		Arguments:  e.ProcessConfig.Arguments,
36
+		Tty:        e.Tty,
37
+		Entrypoint: e.Entrypoint,
38
+		Arguments:  e.Args,
39 39
 	}
40 40
 }
41 41
new file mode 100644
... ...
@@ -0,0 +1,13 @@
0
+package daemon
1
+
2
+import (
3
+	"github.com/docker/docker/container"
4
+	"github.com/docker/docker/libcontainerd"
5
+)
6
+
7
+// platformConstructExitStatus returns a platform specific exit status structure
8
+func platformConstructExitStatus(e libcontainerd.StateInfo) *container.ExitStatus {
9
+	return &container.ExitStatus{
10
+		ExitCode: int(e.ExitCode),
11
+	}
12
+}
0 13
new file mode 100644
... ...
@@ -0,0 +1,204 @@
0
+package daemon
1
+
2
+import (
3
+	"fmt"
4
+	"strings"
5
+	"syscall"
6
+
7
+	"github.com/docker/docker/container"
8
+	"github.com/docker/docker/layer"
9
+	"github.com/docker/docker/libcontainerd"
10
+	"github.com/docker/docker/libcontainerd/windowsoci"
11
+	"github.com/docker/docker/oci"
12
+)
13
+
14
+func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, error) {
15
+	s := oci.DefaultSpec()
16
+
17
+	linkedEnv, err := daemon.setupLinkedContainers(c)
18
+	if err != nil {
19
+		return nil, err
20
+	}
21
+
22
+	// TODO Windows - this can be removed. Not used (UID/GID)
23
+	rootUID, rootGID := daemon.GetRemappedUIDGID()
24
+	if err := c.SetupWorkingDirectory(rootUID, rootGID); err != nil {
25
+		return nil, err
26
+	}
27
+
28
+	img, err := daemon.imageStore.Get(c.ImageID)
29
+	if err != nil {
30
+		return nil, fmt.Errorf("Failed to graph.Get on ImageID %s - %s", c.ImageID, err)
31
+	}
32
+
33
+	// In base spec
34
+	s.Hostname = c.FullHostname()
35
+
36
+	// In s.Mounts
37
+	mounts, err := daemon.setupMounts(c)
38
+	if err != nil {
39
+		return nil, err
40
+	}
41
+	for _, mount := range mounts {
42
+		s.Mounts = append(s.Mounts, windowsoci.Mount{
43
+			Source:      mount.Source,
44
+			Destination: mount.Destination,
45
+			Readonly:    !mount.Writable,
46
+		})
47
+	}
48
+
49
+	// Are we going to run as a Hyper-V container?
50
+	hv := false
51
+	if c.HostConfig.Isolation.IsDefault() {
52
+		// Container is set to use the default, so take the default from the daemon configuration
53
+		hv = daemon.defaultIsolation.IsHyperV()
54
+	} else {
55
+		// Container is requesting an isolation mode. Honour it.
56
+		hv = c.HostConfig.Isolation.IsHyperV()
57
+	}
58
+	if hv {
59
+		// TODO We don't yet have the ImagePath hooked up. But set to
60
+		// something non-nil to pickup in libcontainerd.
61
+		s.Windows.HvRuntime = &windowsoci.HvRuntime{}
62
+	}
63
+
64
+	// In s.Process
65
+	if c.Config.ArgsEscaped {
66
+		s.Process.Args = append([]string{c.Path}, c.Args...)
67
+	} else {
68
+		// TODO (jstarks): escape the entrypoint too once the tests are fixed to not rely on this behavior
69
+		s.Process.Args = append([]string{c.Path}, escapeArgs(c.Args)...)
70
+	}
71
+	s.Process.Cwd = c.Config.WorkingDir
72
+	s.Process.Env = c.CreateDaemonEnvironment(linkedEnv)
73
+	s.Process.InitialConsoleSize = c.HostConfig.ConsoleSize
74
+	s.Process.Terminal = c.Config.Tty
75
+	s.Process.User.User = c.Config.User
76
+
77
+	// In spec.Root
78
+	s.Root.Path = c.BaseFS
79
+	s.Root.Readonly = c.HostConfig.ReadonlyRootfs
80
+
81
+	// In s.Windows
82
+	s.Windows.FirstStart = !c.HasBeenStartedBefore
83
+
84
+	// s.Windows.LayerFolder.
85
+	m, err := c.RWLayer.Metadata()
86
+	if err != nil {
87
+		return nil, fmt.Errorf("Failed to get layer metadata - %s", err)
88
+	}
89
+	s.Windows.LayerFolder = m["dir"]
90
+
91
+	// s.Windows.LayerPaths
92
+	var layerPaths []string
93
+	if img.RootFS != nil && img.RootFS.Type == "layers+base" {
94
+		max := len(img.RootFS.DiffIDs)
95
+		for i := 0; i <= max; i++ {
96
+			img.RootFS.DiffIDs = img.RootFS.DiffIDs[:i]
97
+			path, err := layer.GetLayerPath(daemon.layerStore, img.RootFS.ChainID())
98
+			if err != nil {
99
+				return nil, fmt.Errorf("Failed to get layer path from graphdriver %s for ImageID %s - %s", daemon.layerStore, img.RootFS.ChainID(), err)
100
+			}
101
+			// Reverse order, expecting parent most first
102
+			layerPaths = append([]string{path}, layerPaths...)
103
+		}
104
+	}
105
+	s.Windows.LayerPaths = layerPaths
106
+
107
+	// In s.Windows.Networking (TP5+ libnetwork way of doing things)
108
+	// Connect all the libnetwork allocated networks to the container
109
+	var epList []string
110
+	if c.NetworkSettings != nil {
111
+		for n := range c.NetworkSettings.Networks {
112
+			sn, err := daemon.FindNetwork(n)
113
+			if err != nil {
114
+				continue
115
+			}
116
+
117
+			ep, err := c.GetEndpointInNetwork(sn)
118
+			if err != nil {
119
+				continue
120
+			}
121
+
122
+			data, err := ep.DriverInfo()
123
+			if err != nil {
124
+				continue
125
+			}
126
+			if data["hnsid"] != nil {
127
+				epList = append(epList, data["hnsid"].(string))
128
+			}
129
+		}
130
+	}
131
+	s.Windows.Networking = &windowsoci.Networking{
132
+		EndpointList: epList,
133
+	}
134
+
135
+	// In s.Windows.Networking (TP4 back compat)
136
+	// TODO Windows: Post TP4 - Remove this along with definitions from spec
137
+	// and changes to libcontainerd to not read these fields.
138
+	if daemon.netController == nil {
139
+		parts := strings.SplitN(string(c.HostConfig.NetworkMode), ":", 2)
140
+		switch parts[0] {
141
+		case "none":
142
+		case "default", "": // empty string to support existing containers
143
+			if !c.Config.NetworkDisabled {
144
+				s.Windows.Networking = &windowsoci.Networking{
145
+					MacAddress:   c.Config.MacAddress,
146
+					Bridge:       daemon.configStore.bridgeConfig.Iface,
147
+					PortBindings: c.HostConfig.PortBindings,
148
+				}
149
+			}
150
+		default:
151
+			return nil, fmt.Errorf("invalid network mode: %s", c.HostConfig.NetworkMode)
152
+		}
153
+	}
154
+
155
+	// In s.Windows.Resources
156
+	// @darrenstahlmsft implement these resources
157
+	cpuShares := uint64(c.HostConfig.CPUShares)
158
+	s.Windows.Resources = &windowsoci.Resources{
159
+		CPU: &windowsoci.CPU{
160
+			//TODO Count: ...,
161
+			//TODO Percent: ...,
162
+			Shares: &cpuShares,
163
+		},
164
+		Memory: &windowsoci.Memory{
165
+		//TODO Limit: ...,
166
+		//TODO Reservation: ...,
167
+		},
168
+		Network: &windowsoci.Network{
169
+		//TODO Bandwidth: ...,
170
+		},
171
+		Storage: &windowsoci.Storage{
172
+		//TODO Bps: ...,
173
+		//TODO Iops: ...,
174
+		//TODO SandboxSize: ...,
175
+		},
176
+	}
177
+
178
+	// BUGBUG - Next problem. This was an exec opt. Where do we now get these?
179
+	// Come back to this when add Xenon support.
180
+	//	var hvPartition bool
181
+	//	// Work out the isolation (whether it is a hypervisor partition)
182
+	//	if c.HostConfig.Isolation.IsDefault() {
183
+	//		// Not specified by caller. Take daemon default
184
+	//		hvPartition = windows.DefaultIsolation.IsHyperV()
185
+	//	} else {
186
+	//		// Take value specified by caller
187
+	//		hvPartition = c.HostConfig.Isolation.IsHyperV()
188
+	//	}
189
+
190
+	//		Isolation:   string(c.HostConfig.Isolation),
191
+	//		HvPartition: hvPartition,
192
+	//	}
193
+
194
+	return (*libcontainerd.Spec)(&s), nil
195
+}
196
+
197
+func escapeArgs(args []string) []string {
198
+	escapedArgs := make([]string, len(args))
199
+	for i, a := range args {
200
+		escapedArgs[i] = syscall.EscapeArg(a)
201
+	}
202
+	return escapedArgs
203
+}
0 204
new file mode 100644
... ...
@@ -0,0 +1,13 @@
0
+// +build windows
1
+
2
+package daemon
3
+
4
+import (
5
+	"github.com/docker/docker/libcontainerd"
6
+	"github.com/docker/engine-api/types/container"
7
+)
8
+
9
+func toContainerdResources(resources container.Resources) libcontainerd.Resources {
10
+	var r libcontainerd.Resources
11
+	return r
12
+}
... ...
@@ -7,18 +7,22 @@ import (
7 7
 	"sort"
8 8
 
9 9
 	"github.com/docker/docker/container"
10
-	"github.com/docker/docker/daemon/execdriver"
11 10
 	"github.com/docker/docker/volume"
12 11
 )
13 12
 
14 13
 // setupMounts configures the mount points for a container by appending each
15
-// of the configured mounts on the container to the execdriver mount structure
14
+// of the configured mounts on the container to the oci mount structure
16 15
 // which will ultimately be passed into the exec driver during container creation.
17 16
 // It also ensures each of the mounts are lexographically sorted.
18
-func (daemon *Daemon) setupMounts(container *container.Container) ([]execdriver.Mount, error) {
19
-	var mnts []execdriver.Mount
20
-	for _, mount := range container.MountPoints { // type is volume.MountPoint
21
-		if err := daemon.lazyInitializeVolume(container.ID, mount); err != nil {
17
+
18
+// BUGBUG TODO Windows containerd. This would be much better if it returned
19
+// an array of windowsoci mounts, not container mounts. Then no need to
20
+// do multiple transitions.
21
+
22
+func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, error) {
23
+	var mnts []container.Mount
24
+	for _, mount := range c.MountPoints { // type is volume.MountPoint
25
+		if err := daemon.lazyInitializeVolume(c.ID, mount); err != nil {
22 26
 			return nil, err
23 27
 		}
24 28
 		// If there is no source, take it from the volume path
... ...
@@ -29,7 +33,7 @@ func (daemon *Daemon) setupMounts(container *container.Container) ([]execdriver.
29 29
 		if s == "" {
30 30
 			return nil, fmt.Errorf("No source for mount name '%s' driver %q destination '%s'", mount.Name, mount.Driver, mount.Destination)
31 31
 		}
32
-		mnts = append(mnts, execdriver.Mount{
32
+		mnts = append(mnts, container.Mount{
33 33
 			Source:      s,
34 34
 			Destination: mount.Destination,
35 35
 			Writable:    mount.RW,
... ...
@@ -10,6 +10,7 @@ import (
10 10
 	"github.com/Sirupsen/logrus"
11 11
 	apiserver "github.com/docker/docker/api/server"
12 12
 	"github.com/docker/docker/daemon"
13
+	"github.com/docker/docker/libcontainerd"
13 14
 	"github.com/docker/docker/pkg/mflag"
14 15
 	"github.com/docker/docker/pkg/system"
15 16
 )
... ...
@@ -57,3 +58,7 @@ func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(
57 57
 		}
58 58
 	}()
59 59
 }
60
+
61
+func (cli *DaemonCli) getPlatformRemoteOptions() []libcontainerd.RemoteOption {
62
+	return nil
63
+}
60 64
new file mode 100644
... ...
@@ -0,0 +1,579 @@
0
+package libcontainerd
1
+
2
+import (
3
+	"encoding/json"
4
+	"errors"
5
+	"fmt"
6
+	"io"
7
+	"path/filepath"
8
+	"strconv"
9
+	"strings"
10
+
11
+	"syscall"
12
+	"time"
13
+
14
+	"github.com/Microsoft/hcsshim"
15
+	"github.com/Sirupsen/logrus"
16
+)
17
+
18
+type client struct {
19
+	clientCommon
20
+
21
+	// Platform specific properties below here (none presently on Windows)
22
+}
23
+
24
+// defaultContainerNAT is the default name of the container NAT device that is
25
+// preconfigured on the server. TODO Windows - Remove for TP5 support as not needed.
26
+const defaultContainerNAT = "ContainerNAT"
27
+
28
+// Win32 error codes that are used for various workarounds
29
+// These really should be ALL_CAPS to match golangs syscall library and standard
30
+// Win32 error conventions, but golint insists on CamelCase.
31
+const (
32
+	CoEClassstring     = syscall.Errno(0x800401F3) // Invalid class string
33
+	ErrorNoNetwork     = syscall.Errno(1222)       // The network is not present or not started
34
+	ErrorBadPathname   = syscall.Errno(161)        // The specified path is invalid
35
+	ErrorInvalidObject = syscall.Errno(0x800710D8) // The object identifier does not represent a valid object
36
+)
37
+
38
+type layer struct {
39
+	ID   string
40
+	Path string
41
+}
42
+
43
+type defConfig struct {
44
+	DefFile string
45
+}
46
+
47
+type portBinding struct {
48
+	Protocol     string
49
+	InternalPort int
50
+	ExternalPort int
51
+}
52
+
53
+type natSettings struct {
54
+	Name         string
55
+	PortBindings []portBinding
56
+}
57
+
58
+type networkConnection struct {
59
+	NetworkName string
60
+	//EnableNat bool
61
+	Nat natSettings
62
+}
63
+type networkSettings struct {
64
+	MacAddress string
65
+}
66
+
67
+type device struct {
68
+	DeviceType string
69
+	Connection interface{}
70
+	Settings   interface{}
71
+}
72
+
73
+type mappedDir struct {
74
+	HostPath      string
75
+	ContainerPath string
76
+	ReadOnly      bool
77
+}
78
+
79
+// TODO Windows RTM: @darrenstahlmsft Add ProcessorCount
80
+type containerInit struct {
81
+	SystemType              string      // HCS requires this to be hard-coded to "Container"
82
+	Name                    string      // Name of the container. We use the docker ID.
83
+	Owner                   string      // The management platform that created this container
84
+	IsDummy                 bool        // Used for development purposes.
85
+	VolumePath              string      // Windows volume path for scratch space
86
+	Devices                 []device    // Devices used by the container
87
+	IgnoreFlushesDuringBoot bool        // Optimization hint for container startup in Windows
88
+	LayerFolderPath         string      // Where the layer folders are located
89
+	Layers                  []layer     // List of storage layers
90
+	ProcessorWeight         uint64      `json:",omitempty"` // CPU Shares 0..10000 on Windows; where 0 will be omitted and HCS will default.
91
+	ProcessorMaximum        int64       `json:",omitempty"` // CPU maximum usage percent 1..100
92
+	StorageIOPSMaximum      uint64      `json:",omitempty"` // Maximum Storage IOPS
93
+	StorageBandwidthMaximum uint64      `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second
94
+	StorageSandboxSize      uint64      `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller
95
+	MemoryMaximumInMB       int64       `json:",omitempty"` // Maximum memory available to the container in Megabytes
96
+	HostName                string      // Hostname
97
+	MappedDirectories       []mappedDir // List of mapped directories (volumes/mounts)
98
+	SandboxPath             string      // Location of unmounted sandbox (used for Hyper-V containers)
99
+	HvPartition             bool        // True if it a Hyper-V Container
100
+	EndpointList            []string    // List of networking endpoints to be attached to container
101
+}
102
+
103
+// defaultOwner is a tag passed to HCS to allow it to differentiate between
104
+// container creator management stacks. We hard code "docker" in the case
105
+// of docker.
106
+const defaultOwner = "docker"
107
+
108
+// Create is the entrypoint to create a container from a spec, and if successfully
109
+// created, start it too.
110
+func (clnt *client) Create(containerID string, spec Spec, options ...CreateOption) error {
111
+	logrus.Debugln("LCD client.Create() with spec", spec)
112
+
113
+	cu := &containerInit{
114
+		SystemType: "Container",
115
+		Name:       containerID,
116
+		Owner:      defaultOwner,
117
+
118
+		VolumePath:              spec.Root.Path,
119
+		IgnoreFlushesDuringBoot: spec.Windows.FirstStart,
120
+		LayerFolderPath:         spec.Windows.LayerFolder,
121
+		HostName:                spec.Hostname,
122
+	}
123
+
124
+	if spec.Windows.Networking != nil {
125
+		cu.EndpointList = spec.Windows.Networking.EndpointList
126
+	}
127
+
128
+	if spec.Windows.Resources != nil {
129
+		if spec.Windows.Resources.CPU != nil {
130
+			if spec.Windows.Resources.CPU.Shares != nil {
131
+				cu.ProcessorWeight = *spec.Windows.Resources.CPU.Shares
132
+			}
133
+			if spec.Windows.Resources.CPU.Percent != nil {
134
+				cu.ProcessorMaximum = *spec.Windows.Resources.CPU.Percent * 100 // ProcessorMaximum is a value between 1 and 10000
135
+			}
136
+		}
137
+		if spec.Windows.Resources.Memory != nil {
138
+			if spec.Windows.Resources.Memory.Limit != nil {
139
+				cu.MemoryMaximumInMB = *spec.Windows.Resources.Memory.Limit / 1024 / 1024
140
+			}
141
+		}
142
+		if spec.Windows.Resources.Storage != nil {
143
+			if spec.Windows.Resources.Storage.Bps != nil {
144
+				cu.StorageBandwidthMaximum = *spec.Windows.Resources.Storage.Bps
145
+			}
146
+			if spec.Windows.Resources.Storage.Iops != nil {
147
+				cu.StorageIOPSMaximum = *spec.Windows.Resources.Storage.Iops
148
+			}
149
+			if spec.Windows.Resources.Storage.SandboxSize != nil {
150
+				cu.StorageSandboxSize = *spec.Windows.Resources.Storage.SandboxSize
151
+			}
152
+		}
153
+	}
154
+
155
+	// TODO Ultimately need to set the path from HvRuntime.ImagePath
156
+	cu.HvPartition = (spec.Windows.HvRuntime != nil)
157
+	//	if spec.Windows.HvRuntime != nil {
158
+	//		cu.HvPartition = len(spec.Windows.HvRuntime.ImagePath) > 0
159
+	//	}
160
+
161
+	if cu.HvPartition {
162
+		cu.SandboxPath = filepath.Dir(spec.Windows.LayerFolder)
163
+	} else {
164
+		cu.VolumePath = spec.Root.Path
165
+		cu.LayerFolderPath = spec.Windows.LayerFolder
166
+	}
167
+
168
+	for _, layerPath := range spec.Windows.LayerPaths {
169
+		_, filename := filepath.Split(layerPath)
170
+		g, err := hcsshim.NameToGuid(filename)
171
+		if err != nil {
172
+			return err
173
+		}
174
+		cu.Layers = append(cu.Layers, layer{
175
+			ID:   g.ToString(),
176
+			Path: layerPath,
177
+		})
178
+	}
179
+
180
+	// Add the mounts (volumes, bind mounts etc) to the structure
181
+	mds := make([]mappedDir, len(spec.Mounts))
182
+	for i, mount := range spec.Mounts {
183
+		mds[i] = mappedDir{
184
+			HostPath:      mount.Source,
185
+			ContainerPath: mount.Destination,
186
+			ReadOnly:      mount.Readonly}
187
+	}
188
+	cu.MappedDirectories = mds
189
+
190
+	// TODO Windows: vv START OF TP4 BLOCK OF CODE. REMOVE ONCE TP4 IS NO LONGER SUPPORTED
191
+	if hcsshim.IsTP4() &&
192
+		spec.Windows.Networking != nil &&
193
+		spec.Windows.Networking.Bridge != "" {
194
+		// Enumerate through the port bindings specified by the user and convert
195
+		// them into the internal structure matching the JSON blob that can be
196
+		// understood by the HCS.
197
+		var pbs []portBinding
198
+		for i, v := range spec.Windows.Networking.PortBindings {
199
+			proto := strings.ToUpper(i.Proto())
200
+			if proto != "TCP" && proto != "UDP" {
201
+				return fmt.Errorf("invalid protocol %s", i.Proto())
202
+			}
203
+
204
+			if len(v) > 1 {
205
+				return fmt.Errorf("Windows does not support more than one host port in NAT settings")
206
+			}
207
+
208
+			for _, v2 := range v {
209
+				var (
210
+					iPort, ePort int
211
+					err          error
212
+				)
213
+				if len(v2.HostIP) != 0 {
214
+					return fmt.Errorf("Windows does not support host IP addresses in NAT settings")
215
+				}
216
+				if ePort, err = strconv.Atoi(v2.HostPort); err != nil {
217
+					return fmt.Errorf("invalid container port %s: %s", v2.HostPort, err)
218
+				}
219
+				if iPort, err = strconv.Atoi(i.Port()); err != nil {
220
+					return fmt.Errorf("invalid internal port %s: %s", i.Port(), err)
221
+				}
222
+				if iPort < 0 || iPort > 65535 || ePort < 0 || ePort > 65535 {
223
+					return fmt.Errorf("specified NAT port is not in allowed range")
224
+				}
225
+				pbs = append(pbs,
226
+					portBinding{ExternalPort: ePort,
227
+						InternalPort: iPort,
228
+						Protocol:     proto})
229
+			}
230
+		}
231
+
232
+		dev := device{
233
+			DeviceType: "Network",
234
+			Connection: &networkConnection{
235
+				NetworkName: spec.Windows.Networking.Bridge,
236
+				Nat: natSettings{
237
+					Name:         defaultContainerNAT,
238
+					PortBindings: pbs,
239
+				},
240
+			},
241
+		}
242
+
243
+		if spec.Windows.Networking.MacAddress != "" {
244
+			windowsStyleMAC := strings.Replace(
245
+				spec.Windows.Networking.MacAddress, ":", "-", -1)
246
+			dev.Settings = networkSettings{
247
+				MacAddress: windowsStyleMAC,
248
+			}
249
+		}
250
+		cu.Devices = append(cu.Devices, dev)
251
+	} else {
252
+		logrus.Debugln("No network interface")
253
+	}
254
+	// TODO Windows: ^^ END OF TP4 BLOCK OF CODE. REMOVE ONCE TP4 IS NO LONGER SUPPORTED
255
+
256
+	configurationb, err := json.Marshal(cu)
257
+	if err != nil {
258
+		return err
259
+	}
260
+
261
+	configuration := string(configurationb)
262
+
263
+	// TODO Windows TP5 timeframe. Remove when TP4 is no longer supported.
264
+	// The following a workaround for Windows TP4 which has a networking
265
+	// bug which fairly frequently returns an error. Back off and retry.
266
+	if !hcsshim.IsTP4() {
267
+		if err := hcsshim.CreateComputeSystem(containerID, configuration); err != nil {
268
+			return err
269
+		}
270
+	} else {
271
+		maxAttempts := 5
272
+		for i := 1; i <= maxAttempts; i++ {
273
+			err = hcsshim.CreateComputeSystem(containerID, configuration)
274
+			if err == nil {
275
+				break
276
+			}
277
+
278
+			if herr, ok := err.(*hcsshim.HcsError); ok {
279
+				if herr.Err != syscall.ERROR_NOT_FOUND && // Element not found
280
+					herr.Err != syscall.ERROR_FILE_NOT_FOUND && // The system cannot find the file specified
281
+					herr.Err != ErrorNoNetwork && // The network is not present or not started
282
+					herr.Err != ErrorBadPathname && // The specified path is invalid
283
+					herr.Err != CoEClassstring && // Invalid class string
284
+					herr.Err != ErrorInvalidObject { // The object identifier does not represent a valid object
285
+					logrus.Debugln("Failed to create temporary container ", err)
286
+					return err
287
+				}
288
+				logrus.Warnf("Invoking Windows TP4 retry hack (%d of %d)", i, maxAttempts-1)
289
+				time.Sleep(50 * time.Millisecond)
290
+			}
291
+		}
292
+	}
293
+
294
+	// Construct a container object for calling start on it.
295
+	container := &container{
296
+		containerCommon: containerCommon{
297
+			process: process{
298
+				processCommon: processCommon{
299
+					containerID:  containerID,
300
+					client:       clnt,
301
+					friendlyName: InitFriendlyName,
302
+				},
303
+			},
304
+			processes: make(map[string]*process),
305
+		},
306
+		ociSpec: spec,
307
+	}
308
+
309
+	container.options = options
310
+	for _, option := range options {
311
+		if err := option.Apply(container); err != nil {
312
+			logrus.Error(err)
313
+		}
314
+	}
315
+
316
+	// Call start, and if it fails, delete the container from our
317
+	// internal structure, and also keep HCS in sync by deleting the
318
+	// container there.
319
+	logrus.Debugf("Create() id=%s, Calling start()", containerID)
320
+	if err := container.start(); err != nil {
321
+		clnt.deleteContainer(containerID)
322
+		return err
323
+	}
324
+
325
+	logrus.Debugf("Create() id=%s completed successfully", containerID)
326
+	return nil
327
+
328
+}
329
+
330
+// AddProcess is the handler for adding a process to an already running
331
+// container. It's called through docker exec.
332
+func (clnt *client) AddProcess(containerID, processFriendlyName string, procToAdd Process) error {
333
+
334
+	clnt.lock(containerID)
335
+	defer clnt.unlock(containerID)
336
+	container, err := clnt.getContainer(containerID)
337
+	if err != nil {
338
+		return err
339
+	}
340
+
341
+	createProcessParms := hcsshim.CreateProcessParams{
342
+		EmulateConsole: procToAdd.Terminal,
343
+		ConsoleSize:    procToAdd.InitialConsoleSize,
344
+	}
345
+
346
+	// Take working directory from the process to add if it is defined,
347
+	// otherwise take from the first process.
348
+	if procToAdd.Cwd != "" {
349
+		createProcessParms.WorkingDirectory = procToAdd.Cwd
350
+	} else {
351
+		createProcessParms.WorkingDirectory = container.ociSpec.Process.Cwd
352
+	}
353
+
354
+	// Configure the environment for the process
355
+	createProcessParms.Environment = setupEnvironmentVariables(procToAdd.Env)
356
+	createProcessParms.CommandLine = strings.Join(procToAdd.Args, " ")
357
+
358
+	logrus.Debugf("commandLine: %s", createProcessParms.CommandLine)
359
+
360
+	// Start the command running in the container. Note we always tell HCS to
361
+	// create stdout as it's required regardless of '-i' or '-t' options, so that
362
+	// docker can always grab the output through logs. We also tell HCS to always
363
+	// create stdin, even if it's not used - it will be closed shortly. Stderr
364
+	// is only created if it we're not -t.
365
+	var stdout, stderr io.ReadCloser
366
+	var pid uint32
367
+	iopipe := &IOPipe{Terminal: procToAdd.Terminal}
368
+	pid, iopipe.Stdin, stdout, stderr, err = hcsshim.CreateProcessInComputeSystem(
369
+		containerID,
370
+		true,
371
+		true,
372
+		!procToAdd.Terminal,
373
+		createProcessParms)
374
+	if err != nil {
375
+		logrus.Errorf("AddProcess %s CreateProcessInComputeSystem() failed %s", containerID, err)
376
+		return err
377
+	}
378
+
379
+	// Convert io.ReadClosers to io.Readers
380
+	if stdout != nil {
381
+		iopipe.Stdout = openReaderFromPipe(stdout)
382
+	}
383
+	if stderr != nil {
384
+		iopipe.Stderr = openReaderFromPipe(stderr)
385
+	}
386
+
387
+	// Add the process to the containers list of processes
388
+	container.processes[processFriendlyName] =
389
+		&process{
390
+			processCommon: processCommon{
391
+				containerID:  containerID,
392
+				friendlyName: processFriendlyName,
393
+				client:       clnt,
394
+				systemPid:    pid,
395
+			},
396
+		}
397
+
398
+	// Make sure the lock is not held while calling back into the daemon
399
+	clnt.unlock(containerID)
400
+
401
+	// Tell the engine to attach streams back to the client
402
+	if err := clnt.backend.AttachStreams(processFriendlyName, *iopipe); err != nil {
403
+		return err
404
+	}
405
+
406
+	// Lock again so that the defer unlock doesn't fail. (I really don't like this code)
407
+	clnt.lock(containerID)
408
+
409
+	// Spin up a go routine waiting for exit to handle cleanup
410
+	go container.waitExit(pid, processFriendlyName, false)
411
+
412
+	return nil
413
+}
414
+
415
+// Signal handles `docker stop` on Windows. While Linux has support for
416
+// the full range of signals, signals aren't really implemented on Windows.
417
+// We fake supporting regular stop and -9 to force kill.
418
+func (clnt *client) Signal(containerID string, sig int) error {
419
+	var (
420
+		cont *container
421
+		err  error
422
+	)
423
+
424
+	// Get the container as we need it to find the pid of the process.
425
+	clnt.lock(containerID)
426
+	defer clnt.unlock(containerID)
427
+	if cont, err = clnt.getContainer(containerID); err != nil {
428
+		return err
429
+	}
430
+
431
+	logrus.Debugf("lcd: Signal() containerID=%s sig=%d pid=%d", containerID, sig, cont.systemPid)
432
+	context := fmt.Sprintf("Signal: sig=%d pid=%d", sig, cont.systemPid)
433
+
434
+	if syscall.Signal(sig) == syscall.SIGKILL {
435
+		// Terminate the compute system
436
+		if err := hcsshim.TerminateComputeSystem(containerID, hcsshim.TimeoutInfinite, context); err != nil {
437
+			logrus.Errorf("Failed to terminate %s - %q", containerID, err)
438
+		}
439
+
440
+	} else {
441
+		// Terminate Process
442
+		if err = hcsshim.TerminateProcessInComputeSystem(containerID, cont.systemPid); err != nil {
443
+			logrus.Warnf("Failed to terminate pid %d in %s: %q", cont.systemPid, containerID, err)
444
+			// Ignore errors
445
+			err = nil
446
+		}
447
+
448
+		// Shutdown the compute system
449
+		if err := hcsshim.ShutdownComputeSystem(containerID, hcsshim.TimeoutInfinite, context); err != nil {
450
+			logrus.Errorf("Failed to shutdown %s - %q", containerID, err)
451
+		}
452
+	}
453
+	return nil
454
+}
455
+
456
+// Resize handles a CLI event to resize an interactive docker run or docker exec
457
+// window.
458
+func (clnt *client) Resize(containerID, processFriendlyName string, width, height int) error {
459
+	// Get the libcontainerd container object
460
+	clnt.lock(containerID)
461
+	defer clnt.unlock(containerID)
462
+	cont, err := clnt.getContainer(containerID)
463
+	if err != nil {
464
+		return err
465
+	}
466
+
467
+	if processFriendlyName == InitFriendlyName {
468
+		logrus.Debugln("Resizing systemPID in", containerID, cont.process.systemPid)
469
+		return hcsshim.ResizeConsoleInComputeSystem(containerID, cont.process.systemPid, height, width)
470
+	}
471
+
472
+	for _, p := range cont.processes {
473
+		if p.friendlyName == processFriendlyName {
474
+			logrus.Debugln("Resizing exec'd process", containerID, p.systemPid)
475
+			return hcsshim.ResizeConsoleInComputeSystem(containerID, p.systemPid, height, width)
476
+		}
477
+	}
478
+
479
+	return fmt.Errorf("Resize could not find containerID %s to resize", containerID)
480
+
481
+}
482
+
483
+// Pause handles pause requests for containers
484
+func (clnt *client) Pause(containerID string) error {
485
+	return errors.New("Windows: Containers cannot be paused")
486
+}
487
+
488
+// Resume handles resume requests for containers
489
+func (clnt *client) Resume(containerID string) error {
490
+	return errors.New("Windows: Containers cannot be paused")
491
+}
492
+
493
+// Stats handles stats requests for containers
494
+func (clnt *client) Stats(containerID string) (*Stats, error) {
495
+	return nil, errors.New("Windows: Stats not implemented")
496
+}
497
+
498
+// Restore is the handler for restoring a container
499
+func (clnt *client) Restore(containerID string, unusedOnWindows ...CreateOption) error {
500
+
501
+	logrus.Debugf("lcd Restore %s", containerID)
502
+	return clnt.backend.StateChanged(containerID, StateInfo{
503
+		State:    StateExit,
504
+		ExitCode: 1 << 31,
505
+	})
506
+
507
+	//	var err error
508
+	//	clnt.lock(containerID)
509
+	//	defer clnt.unlock(containerID)
510
+
511
+	//	logrus.Debugf("restore container %s state %s", containerID)
512
+
513
+	//	if _, err := clnt.getContainer(containerID); err == nil {
514
+	//		return fmt.Errorf("container %s is aleady active", containerID)
515
+	//	}
516
+
517
+	//	defer func() {
518
+	//		if err != nil {
519
+	//			clnt.deleteContainer(containerID)
520
+	//		}
521
+	//	}()
522
+
523
+	//	// ====> BUGBUG Where does linux get the pid from				systemPid:    pid,
524
+	//	container := &container{
525
+	//		containerCommon: containerCommon{
526
+	//			process: process{
527
+	//				processCommon: processCommon{
528
+	//					containerID:  containerID,
529
+	//					client:       clnt,
530
+	//					friendlyName: InitFriendlyName,
531
+	//				},
532
+	//			},
533
+	//			processes: make(map[string]*process),
534
+	//		},
535
+	//	}
536
+
537
+	//	container.systemPid = systemPid(cont)
538
+
539
+	//	var terminal bool
540
+	//	for _, p := range cont.Processes {
541
+	//		if p.Pid == InitFriendlyName {
542
+	//			terminal = p.Terminal
543
+	//		}
544
+	//	}
545
+
546
+	//	iopipe, err := container.openFifos(terminal)
547
+	//	if err != nil {
548
+	//		return err
549
+	//	}
550
+
551
+	//	if err := clnt.backend.AttachStreams(containerID, *iopipe); err != nil {
552
+	//		return err
553
+	//	}
554
+
555
+	//	clnt.appendContainer(container)
556
+
557
+	//	err = clnt.backend.StateChanged(containerID, StateInfo{
558
+	//		State: StateRestore,
559
+	//		Pid:   container.systemPid,
560
+	//	})
561
+
562
+	//	if err != nil {
563
+	//		return err
564
+	//	}
565
+
566
+	//	return nil
567
+}
568
+
569
+// GetPidsForContainers is not implemented on Windows.
570
+func (clnt *client) GetPidsForContainer(containerID string) ([]int, error) {
571
+	return nil, errors.New("GetPidsForContainer: GetPidsForContainer() not implemented")
572
+}
573
+
574
+func (clnt *client) UpdateResources(containerID string, resources Resources) error {
575
+	// Updating resource isn't supported on Windows
576
+	// but we should return nil for enabling updating container
577
+	return nil
578
+}
0 579
new file mode 100644
... ...
@@ -0,0 +1,204 @@
0
+package libcontainerd
1
+
2
+import (
3
+	"io"
4
+	"strings"
5
+	"syscall"
6
+
7
+	"github.com/Microsoft/hcsshim"
8
+	"github.com/Sirupsen/logrus"
9
+)
10
+
11
+type container struct {
12
+	containerCommon
13
+
14
+	// Platform specific fields are below here. There are none presently on Windows.
15
+	options []CreateOption
16
+
17
+	// The ociSpec is required, as client.Create() needs a spec,
18
+	// but can be called from the RestartManager context which does not
19
+	// otherwise have access to the Spec
20
+	ociSpec Spec
21
+}
22
+
23
+func (ctr *container) newProcess(friendlyName string) *process {
24
+	return &process{
25
+		processCommon: processCommon{
26
+			containerID:  ctr.containerID,
27
+			friendlyName: friendlyName,
28
+			client:       ctr.client,
29
+		},
30
+	}
31
+}
32
+
33
+func (ctr *container) start() error {
34
+	var err error
35
+
36
+	// Start the container
37
+	logrus.Debugln("Starting container ", ctr.containerID)
38
+	if err = hcsshim.StartComputeSystem(ctr.containerID); err != nil {
39
+		logrus.Errorf("Failed to start compute system: %s", err)
40
+		return err
41
+	}
42
+
43
+	createProcessParms := hcsshim.CreateProcessParams{
44
+		EmulateConsole:   ctr.ociSpec.Process.Terminal,
45
+		WorkingDirectory: ctr.ociSpec.Process.Cwd,
46
+		ConsoleSize:      ctr.ociSpec.Process.InitialConsoleSize,
47
+	}
48
+
49
+	// Configure the environment for the process
50
+	createProcessParms.Environment = setupEnvironmentVariables(ctr.ociSpec.Process.Env)
51
+	createProcessParms.CommandLine = strings.Join(ctr.ociSpec.Process.Args, " ")
52
+
53
+	iopipe := &IOPipe{Terminal: ctr.ociSpec.Process.Terminal}
54
+
55
+	// Start the command running in the container. Note we always tell HCS to
56
+	// create stdout as it's required regardless of '-i' or '-t' options, so that
57
+	// docker can always grab the output through logs. We also tell HCS to always
58
+	// create stdin, even if it's not used - it will be closed shortly. Stderr
59
+	// is only created if it we're not -t.
60
+	var pid uint32
61
+	var stdout, stderr io.ReadCloser
62
+	pid, iopipe.Stdin, stdout, stderr, err = hcsshim.CreateProcessInComputeSystem(
63
+		ctr.containerID,
64
+		true,
65
+		true,
66
+		!ctr.ociSpec.Process.Terminal,
67
+		createProcessParms)
68
+	if err != nil {
69
+		logrus.Errorf("CreateProcessInComputeSystem() failed %s", err)
70
+
71
+		// Explicitly terminate the compute system here.
72
+		if err2 := hcsshim.TerminateComputeSystem(ctr.containerID, hcsshim.TimeoutInfinite, "CreateProcessInComputeSystem failed"); err2 != nil {
73
+			// Ignore this error, there's not a lot we can do except log it
74
+			logrus.Warnf("Failed to TerminateComputeSystem after a failed CreateProcessInComputeSystem. Ignoring this.", err2)
75
+		} else {
76
+			logrus.Debugln("Cleaned up after failed CreateProcessInComputeSystem by calling TerminateComputeSystem")
77
+		}
78
+		return err
79
+	}
80
+
81
+	// Convert io.ReadClosers to io.Readers
82
+	if stdout != nil {
83
+		iopipe.Stdout = openReaderFromPipe(stdout)
84
+	}
85
+	if stderr != nil {
86
+		iopipe.Stderr = openReaderFromPipe(stderr)
87
+	}
88
+
89
+	// Save the PID
90
+	logrus.Debugf("Process started - PID %d", pid)
91
+	ctr.systemPid = uint32(pid)
92
+
93
+	// Spin up a go routine waiting for exit to handle cleanup
94
+	go ctr.waitExit(pid, InitFriendlyName, true)
95
+
96
+	ctr.client.appendContainer(ctr)
97
+
98
+	if err := ctr.client.backend.AttachStreams(ctr.containerID, *iopipe); err != nil {
99
+		// OK to return the error here, as waitExit will handle tear-down in HCS
100
+		return err
101
+	}
102
+
103
+	// Tell the docker engine that the container has started.
104
+	si := StateInfo{
105
+		State: StateStart,
106
+		Pid:   ctr.systemPid, // Not sure this is needed? Double-check monitor.go in daemon BUGBUG @jhowardmsft
107
+	}
108
+	return ctr.client.backend.StateChanged(ctr.containerID, si)
109
+
110
+}
111
+
112
+// waitExit runs as a goroutine waiting for the process to exit. It's
113
+// equivalent to (in the linux containerd world) where events come in for
114
+// state change notifications from containerd.
115
+func (ctr *container) waitExit(pid uint32, processFriendlyName string, isFirstProcessToStart bool) error {
116
+	logrus.Debugln("waitExit on pid", pid)
117
+
118
+	// Block indefinitely for the process to exit.
119
+	exitCode, err := hcsshim.WaitForProcessInComputeSystem(ctr.containerID, pid, hcsshim.TimeoutInfinite)
120
+	if err != nil {
121
+		if herr, ok := err.(*hcsshim.HcsError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {
122
+			logrus.Warnf("WaitForProcessInComputeSystem failed (container may have been killed): %s", err)
123
+		}
124
+		// Fall through here, do not return. This ensures we attempt to continue the
125
+		// shutdown in HCS nad tell the docker engine that the process/container
126
+		// has exited to avoid a container being dropped on the floor.
127
+	}
128
+
129
+	// Assume the container has exited
130
+	si := StateInfo{
131
+		State:     StateExit,
132
+		ExitCode:  uint32(exitCode),
133
+		Pid:       pid,
134
+		ProcessID: processFriendlyName,
135
+	}
136
+
137
+	// But it could have been an exec'd process which exited
138
+	if !isFirstProcessToStart {
139
+		si.State = StateExitProcess
140
+	}
141
+
142
+	// If this is the init process, always call into vmcompute.dll to
143
+	// shutdown the container after we have completed.
144
+	if isFirstProcessToStart {
145
+		logrus.Debugf("Shutting down container %s", ctr.containerID)
146
+		// Explicit timeout here rather than hcsshim.TimeoutInfinte to avoid a
147
+		// (remote) possibility that ShutdownComputeSystem hangs indefinitely.
148
+		const shutdownTimeout = 5 * 60 * 1000 // 5 minutes
149
+		if err := hcsshim.ShutdownComputeSystem(ctr.containerID, shutdownTimeout, "waitExit"); err != nil {
150
+			if herr, ok := err.(*hcsshim.HcsError); !ok ||
151
+				(herr.Err != hcsshim.ERROR_SHUTDOWN_IN_PROGRESS &&
152
+					herr.Err != ErrorBadPathname &&
153
+					herr.Err != syscall.ERROR_PATH_NOT_FOUND) {
154
+				logrus.Warnf("Ignoring error from ShutdownComputeSystem %s", err)
155
+			}
156
+		} else {
157
+			logrus.Debugf("Completed shutting down container %s", ctr.containerID)
158
+		}
159
+
160
+		// BUGBUG - Is taking the lock necessary here? Should it just be taken for
161
+		// the deleteContainer call, not for the restart logic? @jhowardmsft
162
+		ctr.client.lock(ctr.containerID)
163
+		defer ctr.client.unlock(ctr.containerID)
164
+
165
+		if si.State == StateExit && ctr.restartManager != nil {
166
+			restart, wait, err := ctr.restartManager.ShouldRestart(uint32(exitCode))
167
+			if err != nil {
168
+				logrus.Error(err)
169
+			} else if restart {
170
+				si.State = StateRestart
171
+				ctr.restarting = true
172
+				go func() {
173
+					err := <-wait
174
+					ctr.restarting = false
175
+					if err != nil {
176
+						si.State = StateExit
177
+						if err := ctr.client.backend.StateChanged(ctr.containerID, si); err != nil {
178
+							logrus.Error(err)
179
+						}
180
+						logrus.Error(err)
181
+					} else {
182
+						ctr.client.Create(ctr.containerID, ctr.ociSpec, ctr.options...)
183
+					}
184
+				}()
185
+			}
186
+		}
187
+
188
+		// Remove process from list if we have exited
189
+		// We need to do so here in case the Message Handler decides to restart it.
190
+		if si.State == StateExit {
191
+			ctr.client.deleteContainer(ctr.friendlyName)
192
+		}
193
+	}
194
+
195
+	// Call into the backend to notify it of the state change.
196
+	logrus.Debugf("waitExit() calling backend.StateChanged %v", si)
197
+	if err := ctr.client.backend.StateChanged(ctr.containerID, si); err != nil {
198
+		logrus.Error(err)
199
+	}
200
+
201
+	logrus.Debugln("waitExit() completed OK")
202
+	return nil
203
+}
0 204
new file mode 100644
... ...
@@ -0,0 +1,24 @@
0
+package libcontainerd
1
+
2
+import (
3
+	"io"
4
+)
5
+
6
+// process keeps the state for both main container process and exec process.
7
+
8
+// process keeps the state for both main container process and exec process.
9
+type process struct {
10
+	processCommon
11
+}
12
+
13
+func openReaderFromPipe(p io.ReadCloser) io.Reader {
14
+	r, w := io.Pipe()
15
+	go func() {
16
+		if _, err := io.Copy(w, p); err != nil {
17
+			r.CloseWithError(err)
18
+		}
19
+		w.Close()
20
+		p.Close()
21
+	}()
22
+	return r
23
+}
0 24
new file mode 100644
... ...
@@ -0,0 +1,28 @@
0
+package libcontainerd
1
+
2
+import "sync"
3
+
4
+type remote struct {
5
+}
6
+
7
+func (r *remote) Client(b Backend) (Client, error) {
8
+	c := &client{
9
+		clientCommon: clientCommon{
10
+			backend:          b,
11
+			containerMutexes: make(map[string]*sync.Mutex),
12
+			containers:       make(map[string]*container),
13
+		},
14
+	}
15
+	return c, nil
16
+}
17
+
18
+// Cleanup is a no-op on Windows. It is here to implement the same interface
19
+// to meet compilation requirements.
20
+func (r *remote) Cleanup() {
21
+}
22
+
23
+// New creates a fresh instance of libcontainerd remote. This is largely
24
+// a no-op on Windows.
25
+func New(_ string, _ ...RemoteOption) (Remote, error) {
26
+	return &remote{}, nil
27
+}
0 28
new file mode 100644
... ...
@@ -0,0 +1,18 @@
0
+package libcontainerd
1
+
2
+import "github.com/docker/docker/libcontainerd/windowsoci"
3
+
4
+// Spec is the base configuration for the container.
5
+type Spec windowsoci.WindowsSpec
6
+
7
+// Process contains information to start a specific application inside the container.
8
+type Process windowsoci.Process
9
+
10
+// User specifies user information for the containers main process.
11
+type User windowsoci.User
12
+
13
+// Stats contains a stats properties from containerd.
14
+type Stats struct{}
15
+
16
+// Resources defines updatable container resource values.
17
+type Resources struct{}
0 18
new file mode 100644
... ...
@@ -0,0 +1,16 @@
0
+package libcontainerd
1
+
2
+import "strings"
3
+
4
+// setupEnvironmentVariables convert a string array of environment variables
5
+// into a map as required by the HCS. Source array is in format [v1=k1] [v2=k2] etc.
6
+func setupEnvironmentVariables(a []string) map[string]string {
7
+	r := make(map[string]string)
8
+	for _, s := range a {
9
+		arr := strings.Split(s, "=")
10
+		if len(arr) == 2 {
11
+			r[arr[0]] = arr[1]
12
+		}
13
+	}
14
+	return r
15
+}
0 16
new file mode 100644
... ...
@@ -0,0 +1,188 @@
0
+package windowsoci
1
+
2
+// This file is a hack - essentially a mirror of OCI spec for Windows.
3
+
4
+import (
5
+	"fmt"
6
+
7
+	"github.com/docker/go-connections/nat"
8
+)
9
+
10
+// WindowsSpec is the full specification for Windows containers.
11
+type WindowsSpec struct {
12
+	Spec
13
+
14
+	// Windows is platform specific configuration for Windows based containers.
15
+	Windows Windows `json:"windows"`
16
+}
17
+
18
+// Spec is the base configuration for the container.  It specifies platform
19
+// independent configuration. This information must be included when the
20
+// bundle is packaged for distribution.
21
+type Spec struct {
22
+
23
+	// Version is the version of the specification that is supported.
24
+	Version string `json:"ociVersion"`
25
+	// Platform is the host information for OS and Arch.
26
+	Platform Platform `json:"platform"`
27
+	// Process is the container's main process.
28
+	Process Process `json:"process"`
29
+	// Root is the root information for the container's filesystem.
30
+	Root Root `json:"root"`
31
+	// Hostname is the container's host name.
32
+	Hostname string `json:"hostname,omitempty"`
33
+	// Mounts profile configuration for adding mounts to the container's filesystem.
34
+	Mounts []Mount `json:"mounts"`
35
+}
36
+
37
+// Windows contains platform specific configuration for Windows based containers.
38
+type Windows struct {
39
+	// Resources contain information for handling resource constraints for the container
40
+	Resources *Resources `json:"resources,omitempty"`
41
+	// Networking contains the platform specific network settings for the container.
42
+	Networking *Networking `json:"networking,omitempty"`
43
+	// FirstStart is used for an optimization on first boot of Windows
44
+	FirstStart bool `json:"first_start,omitempty"`
45
+	// LayerFolder is the path to the current layer folder
46
+	LayerFolder string `json:"layer_folder,omitempty"`
47
+	// Layer paths of the parent layers
48
+	LayerPaths []string `json:"layer_paths,omitempty"`
49
+	// HvRuntime contains settings specific to Hyper-V containers, omitted if not using Hyper-V isolation
50
+	HvRuntime *HvRuntime `json:"hv_runtime,omitempty"`
51
+}
52
+
53
+// Process contains information to start a specific application inside the container.
54
+type Process struct {
55
+	// Terminal indicates if stderr should NOT be attached for the container.
56
+	Terminal bool `json:"terminal"`
57
+	// ConsoleSize contains the initial h,w of the console size
58
+	InitialConsoleSize [2]int `json:"-"`
59
+	// User specifies user information for the process.
60
+	User User `json:"user"`
61
+	// Args specifies the binary and arguments for the application to execute.
62
+	Args []string `json:"args"`
63
+	// Env populates the process environment for the process.
64
+	Env []string `json:"env,omitempty"`
65
+	// Cwd is the current working directory for the process and must be
66
+	// relative to the container's root.
67
+	Cwd string `json:"cwd"`
68
+}
69
+
70
+// User contains the user information for Windows
71
+type User struct {
72
+	User string `json:"user,omitempty"`
73
+}
74
+
75
+// Root contains information about the container's root filesystem on the host.
76
+type Root struct {
77
+	// Path is the absolute path to the container's root filesystem.
78
+	Path string `json:"path"`
79
+	// Readonly makes the root filesystem for the container readonly before the process is executed.
80
+	Readonly bool `json:"readonly"`
81
+}
82
+
83
+// Platform specifies OS and arch information for the host system that the container
84
+// is created for.
85
+type Platform struct {
86
+	// OS is the operating system.
87
+	OS string `json:"os"`
88
+	// Arch is the architecture
89
+	Arch string `json:"arch"`
90
+}
91
+
92
+// Mount specifies a mount for a container.
93
+type Mount struct {
94
+	// Destination is the path where the mount will be placed relative to the container's root.  The path and child directories MUST exist, a runtime MUST NOT create directories automatically to a mount point.
95
+	Destination string `json:"destination"`
96
+	// Type specifies the mount kind.
97
+	Type string `json:"type"`
98
+	// Source specifies the source path of the mount.  In the case of bind mounts
99
+	// this would be the file on the host.
100
+	Source string `json:"source"`
101
+	// Readonly specifies if the mount should be read-only
102
+	Readonly bool `json:"readonly"`
103
+}
104
+
105
+// HvRuntime contains settings specific to Hyper-V containers
106
+type HvRuntime struct {
107
+	// ImagePath is the path to the Utility VM image for this container
108
+	ImagePath string `json:"image_path,omitempty"`
109
+}
110
+
111
+// Networking contains the platform specific network settings for the container
112
+type Networking struct {
113
+	// TODO Windows TP5. The following three fields are for 'legacy' non-
114
+	// libnetwork networking through HCS. They can be removed once TP4 is
115
+	// no longer supported. Also remove in libcontainerd\client_windows.go,
116
+	// function Create(), and in daemon\oci_windows.go, function CreateSpec()
117
+	MacAddress   string      `json:"mac,omitempty"`
118
+	Bridge       string      `json:"bridge,omitempty"`
119
+	PortBindings nat.PortMap `json:"port_bindings,omitempty"`
120
+	// End of TODO Windows TP5.
121
+
122
+	// List of endpoints to be attached to the container
123
+	EndpointList []string `json:"endpoints,omitempty"`
124
+}
125
+
126
+// Storage contains storage resource management settings
127
+type Storage struct {
128
+	// Specifies maximum Iops for the system drive
129
+	Iops *uint64 `json:"iops,omitempty"`
130
+	// Specifies maximum bytes per second for the system drive
131
+	Bps *uint64 `json:"bps,omitempty"`
132
+	// Sandbox size indicates the size to expand the system drive to if it is currently smaller
133
+	SandboxSize *uint64 `json:"sandbox_size,omitempty"`
134
+}
135
+
136
+// Memory contains memory settings for the container
137
+type Memory struct {
138
+	// Memory limit (in bytes).
139
+	Limit *int64 `json:"limit,omitempty"`
140
+	// Memory reservation (in bytes).
141
+	Reservation *uint64 `json:"reservation,omitempty"`
142
+}
143
+
144
+// CPU contains information for cpu resource management
145
+type CPU struct {
146
+	// Number of CPUs available to the container. This is an appoximation for Windows Server Containers.
147
+	Count *uint64 `json:"count,omitempty"`
148
+	// CPU shares (relative weight (ratio) vs. other containers with cpu shares). Range is from 1 to 10000.
149
+	Shares *uint64 `json:"shares,omitempty"`
150
+	// Percent of available CPUs usable by the container.
151
+	Percent *int64 `json:"percent,omitempty"`
152
+}
153
+
154
+// Network network resource management information
155
+type Network struct {
156
+	// Bandwidth is the maximum egress bandwidth in bytes per second
157
+	Bandwidth *uint64 `json:"bandwidth,omitempty"`
158
+}
159
+
160
+// Resources has container runtime resource constraints
161
+// TODO Windows containerd. This structure needs ratifying with the old resources
162
+// structure used on Windows and the latest OCI spec.
163
+type Resources struct {
164
+	// Memory restriction configuration
165
+	Memory *Memory `json:"memory,omitempty"`
166
+	// CPU resource restriction configuration
167
+	CPU *CPU `json:"cpu,omitempty"`
168
+	// Storage restriction configuration
169
+	Storage *Storage `json:"storage,omitempty"`
170
+	// Network restriction configuration
171
+	Network *Network `json:"network,omitempty"`
172
+}
173
+
174
+const (
175
+	// VersionMajor is for an API incompatible changes
176
+	VersionMajor = 0
177
+	// VersionMinor is for functionality in a backwards-compatible manner
178
+	VersionMinor = 3
179
+	// VersionPatch is for backwards-compatible bug fixes
180
+	VersionPatch = 0
181
+
182
+	// VersionDev indicates development branch. Releases will be empty string.
183
+	VersionDev = ""
184
+)
185
+
186
+// Version is the specification version that the package types support.
187
+var Version = fmt.Sprintf("%d.%d.%d%s (Windows)", VersionMajor, VersionMinor, VersionPatch, VersionDev)
0 188
new file mode 100644
... ...
@@ -0,0 +1,3 @@
0
+// +build !windows
1
+
2
+package windowsoci
0 3
new file mode 100644
... ...
@@ -0,0 +1,23 @@
0
+package oci
1
+
2
+import (
3
+	"runtime"
4
+
5
+	"github.com/docker/docker/libcontainerd/windowsoci"
6
+)
7
+
8
+// DefaultSpec returns default spec used by docker.
9
+func DefaultSpec() windowsoci.WindowsSpec {
10
+	s := windowsoci.Spec{
11
+		Version: windowsoci.Version,
12
+		Platform: windowsoci.Platform{
13
+			OS:   runtime.GOOS,
14
+			Arch: runtime.GOARCH,
15
+		},
16
+	}
17
+
18
+	return windowsoci.WindowsSpec{
19
+		Spec:    s,
20
+		Windows: windowsoci.Windows{},
21
+	}
22
+}