@@ -43,7 +43,7 @@ esac

 # the following lines are in sorted order, FYI

 clone git github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62

-clone git github.com/Microsoft/hcsshim v0.2.2

+clone git github.com/Microsoft/hcsshim v0.3.0

 clone git github.com/Microsoft/go-winio v0.3.4

 clone git github.com/Sirupsen/logrus v0.10.0 # logrus is a common dependency among multiple deps

 clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a

@@ -1,7 +1,6 @@

 package libcontainerd

 import (

-	"encoding/json"

 	"errors"

 	"fmt"

 	"io"

@@ -29,76 +28,6 @@ const (

 	ErrorInvalidObject = syscall.Errno(0x800710D8) // The object identifier does not represent a valid object

 )

-type layer struct {

-	ID   string

-	Path string

-}

-

-type defConfig struct {

-	DefFile string

-}

-

-type portBinding struct {

-	Protocol     string

-	InternalPort int

-	ExternalPort int

-}

-

-type natSettings struct {

-	Name         string

-	PortBindings []portBinding

-}

-

-type networkConnection struct {

-	NetworkName string

-	Nat         natSettings

-}

-type networkSettings struct {

-	MacAddress string

-}

-

-type device struct {

-	DeviceType string

-	Connection interface{}

-	Settings   interface{}

-}

-

-type mappedDir struct {

-	HostPath      string

-	ContainerPath string

-	ReadOnly      bool

-}

-

-type hvRuntime struct {

-	ImagePath string `json:",omitempty"`

-}

-

-// TODO Windows: @darrenstahlmsft Add ProcessorCount

-type containerInit struct {

-	SystemType              string      // HCS requires this to be hard-coded to "Container"

-	Name                    string      // Name of the container. We use the docker ID.

-	Owner                   string      // The management platform that created this container

-	IsDummy                 bool        // Used for development purposes.

-	VolumePath              string      // Windows volume path for scratch space

-	Devices                 []device    // Devices used by the container

-	IgnoreFlushesDuringBoot bool        // Optimization hint for container startup in Windows

-	LayerFolderPath         string      // Where the layer folders are located

-	Layers                  []layer     // List of storage layers

-	ProcessorWeight         uint64      `json:",omitempty"` // CPU Shares 0..10000 on Windows; where 0 will be omitted and HCS will default.

-	ProcessorMaximum        int64       `json:",omitempty"` // CPU maximum usage percent 1..100

-	StorageIOPSMaximum      uint64      `json:",omitempty"` // Maximum Storage IOPS

-	StorageBandwidthMaximum uint64      `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second

-	StorageSandboxSize      uint64      `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller

-	MemoryMaximumInMB       int64       `json:",omitempty"` // Maximum memory available to the container in Megabytes

-	HostName                string      // Hostname

-	MappedDirectories       []mappedDir // List of mapped directories (volumes/mounts)

-	SandboxPath             string      // Location of unmounted sandbox (used for Hyper-V containers)

-	HvPartition             bool        // True if it a Hyper-V Container

-	EndpointList            []string    // List of networking endpoints to be attached to container

-	HvRuntime               *hvRuntime  // Hyper-V container settings

-	Servicing               bool        // True if this container is for servicing

-}

-

 // defaultOwner is a tag passed to HCS to allow it to differentiate between

 // container creator management stacks. We hard code "docker" in the case

 // of docker.

@@ -109,7 +38,7 @@ const defaultOwner = "docker"

 func (clnt *client) Create(containerID string, spec Spec, options ...CreateOption) error {

 	logrus.Debugln("LCD client.Create() with spec", spec)

-	cu := &containerInit{

+	configuration := &hcsshim.ContainerConfig{

 		SystemType: "Container",

 		Name:       containerID,

 		Owner:      defaultOwner,

@@ -121,90 +50,84 @@ func (clnt *client) Create(containerID string, spec Spec, options ...CreateOptio

 	}

 	if spec.Windows.Networking != nil {

-		cu.EndpointList = spec.Windows.Networking.EndpointList

+		configuration.EndpointList = spec.Windows.Networking.EndpointList

 	}

 	if spec.Windows.Resources != nil {

 		if spec.Windows.Resources.CPU != nil {

 			if spec.Windows.Resources.CPU.Shares != nil {

-				cu.ProcessorWeight = *spec.Windows.Resources.CPU.Shares

+				configuration.ProcessorWeight = *spec.Windows.Resources.CPU.Shares

 			}

 			if spec.Windows.Resources.CPU.Percent != nil {

-				cu.ProcessorMaximum = *spec.Windows.Resources.CPU.Percent * 100 // ProcessorMaximum is a value between 1 and 10000

+				configuration.ProcessorMaximum = *spec.Windows.Resources.CPU.Percent * 100 // ProcessorMaximum is a value between 1 and 10000

 			}

 		}

 		if spec.Windows.Resources.Memory != nil {

 			if spec.Windows.Resources.Memory.Limit != nil {

-				cu.MemoryMaximumInMB = *spec.Windows.Resources.Memory.Limit / 1024 / 1024

+				configuration.MemoryMaximumInMB = *spec.Windows.Resources.Memory.Limit / 1024 / 1024

 			}

 		}

 		if spec.Windows.Resources.Storage != nil {

 			if spec.Windows.Resources.Storage.Bps != nil {

-				cu.StorageBandwidthMaximum = *spec.Windows.Resources.Storage.Bps

+				configuration.StorageBandwidthMaximum = *spec.Windows.Resources.Storage.Bps

 			}

 			if spec.Windows.Resources.Storage.Iops != nil {

-				cu.StorageIOPSMaximum = *spec.Windows.Resources.Storage.Iops

+				configuration.StorageIOPSMaximum = *spec.Windows.Resources.Storage.Iops

 			}

 			if spec.Windows.Resources.Storage.SandboxSize != nil {

-				cu.StorageSandboxSize = *spec.Windows.Resources.Storage.SandboxSize

+				configuration.StorageSandboxSize = *spec.Windows.Resources.Storage.SandboxSize

 			}

 		}

 	}

 	if spec.Windows.HvRuntime != nil {

-		cu.HvPartition = true

-		cu.HvRuntime = &hvRuntime{

+		configuration.HvPartition = true

+		configuration.HvRuntime = &hcsshim.HvRuntime{

 			ImagePath: spec.Windows.HvRuntime.ImagePath,

 		}

 	}

+	if configuration.HvPartition {

+		configuration.SandboxPath = filepath.Dir(spec.Windows.LayerFolder)

+	} else {

+		configuration.VolumePath = spec.Root.Path

+		configuration.LayerFolderPath = spec.Windows.LayerFolder

+	}

+

 	for _, option := range options {

 		if s, ok := option.(*ServicingOption); ok {

-			cu.Servicing = s.IsServicing

+			configuration.Servicing = s.IsServicing

 			break

 		}

 	}

-	if cu.HvPartition {

-		cu.SandboxPath = filepath.Dir(spec.Windows.LayerFolder)

-	} else {

-		cu.VolumePath = spec.Root.Path

-		cu.LayerFolderPath = spec.Windows.LayerFolder

-	}

-

 	for _, layerPath := range spec.Windows.LayerPaths {

 		_, filename := filepath.Split(layerPath)

 		g, err := hcsshim.NameToGuid(filename)

 		if err != nil {

 			return err

 		}

-		cu.Layers = append(cu.Layers, layer{

+		configuration.Layers = append(configuration.Layers, hcsshim.Layer{

 			ID:   g.ToString(),

 			Path: layerPath,

 		})

 	}

 	// Add the mounts (volumes, bind mounts etc) to the structure

-	mds := make([]mappedDir, len(spec.Mounts))

+	mds := make([]hcsshim.MappedDir, len(spec.Mounts))

 	for i, mount := range spec.Mounts {

-		mds[i] = mappedDir{

+		mds[i] = hcsshim.MappedDir{

 			HostPath:      mount.Source,

 			ContainerPath: mount.Destination,

 			ReadOnly:      mount.Readonly}

 	}

-	cu.MappedDirectories = mds

+	configuration.MappedDirectories = mds

-	configurationb, err := json.Marshal(cu)

+	hcsContainer, err := hcsshim.CreateContainer(containerID, configuration)

 	if err != nil {

 		return err

 	}

-	// Create the compute system

-	configuration := string(configurationb)

-	if err := hcsshim.CreateComputeSystem(containerID, configuration); err != nil {

-		return err

-	}

-

 	// Construct a container object for calling start on it.

 	container := &container{

 		containerCommon: containerCommon{

@@ -218,7 +141,8 @@ func (clnt *client) Create(containerID string, spec Spec, options ...CreateOptio

 			},

 			processes: make(map[string]*process),

 		},

-		ociSpec: spec,

+		ociSpec:      spec,

+		hcsContainer: hcsContainer,

 	}

 	container.options = options

@@ -252,10 +176,17 @@ func (clnt *client) AddProcess(containerID, processFriendlyName string, procToAd

 	if err != nil {

 		return err

 	}

-

-	createProcessParms := hcsshim.CreateProcessParams{

-		EmulateConsole: procToAdd.Terminal,

-		ConsoleSize:    procToAdd.InitialConsoleSize,

+	// Note we always tell HCS to

+	// create stdout as it's required regardless of '-i' or '-t' options, so that

+	// docker can always grab the output through logs. We also tell HCS to always

+	// create stdin, even if it's not used - it will be closed shortly. Stderr

+	// is only created if it we're not -t.

+	createProcessParms := hcsshim.ProcessConfig{

+		EmulateConsole:   procToAdd.Terminal,

+		ConsoleSize:      procToAdd.InitialConsoleSize,

+		CreateStdInPipe:  true,

+		CreateStdOutPipe: true,

+		CreateStdErrPipe: !procToAdd.Terminal,

 	}

 	// Take working directory from the process to add if it is defined,

@@ -272,25 +203,24 @@ func (clnt *client) AddProcess(containerID, processFriendlyName string, procToAd

 	logrus.Debugf("commandLine: %s", createProcessParms.CommandLine)

-	// Start the command running in the container. Note we always tell HCS to

-	// create stdout as it's required regardless of '-i' or '-t' options, so that

-	// docker can always grab the output through logs. We also tell HCS to always

-	// create stdin, even if it's not used - it will be closed shortly. Stderr

-	// is only created if it we're not -t.

+	// Start the command running in the container.

 	var stdout, stderr io.ReadCloser

-	var pid uint32

-	iopipe := &IOPipe{Terminal: procToAdd.Terminal}

-	pid, iopipe.Stdin, stdout, stderr, err = hcsshim.CreateProcessInComputeSystem(

-		containerID,

-		true,

-		true,

-		!procToAdd.Terminal,

-		createProcessParms)

+	var stdin io.WriteCloser

+	newProcess, err := container.hcsContainer.CreateProcess(&createProcessParms)

+	if err != nil {

+		logrus.Errorf("AddProcess %s CreateProcess() failed %s", containerID, err)

+		return err

+	}

+

+	stdin, stdout, stderr, err = newProcess.Stdio()

 	if err != nil {

-		logrus.Errorf("AddProcess %s CreateProcessInComputeSystem() failed %s", containerID, err)

+		logrus.Errorf("%s getting std pipes failed %s", containerID, err)

 		return err

 	}

+	iopipe := &IOPipe{Terminal: procToAdd.Terminal}

+	iopipe.Stdin = createStdInCloser(stdin, newProcess)

+

 	// TEMP: Work around Windows BS/DEL behavior.

 	iopipe.Stdin = fixStdinBackspaceBehavior(iopipe.Stdin, procToAdd.Terminal)

@@ -302,17 +232,21 @@ func (clnt *client) AddProcess(containerID, processFriendlyName string, procToAd

 		iopipe.Stderr = openReaderFromPipe(stderr)

 	}

-	// Add the process to the containers list of processes

-	container.processes[processFriendlyName] =

-		&process{

-			processCommon: processCommon{

-				containerID:  containerID,

-				friendlyName: processFriendlyName,

-				client:       clnt,

-				systemPid:    pid,

-			},

-			commandLine: createProcessParms.CommandLine,

-		}

+	pid := newProcess.Pid()

+

+	proc := &process{

+		processCommon: processCommon{

+			containerID:  containerID,

+			friendlyName: processFriendlyName,

+			client:       clnt,

+			systemPid:    uint32(pid),

+		},

+		commandLine: createProcessParms.CommandLine,

+		hcsProcess:  newProcess,

+	}

+

+	// Add the process to the container's list of processes

+	container.processes[processFriendlyName] = proc

 	// Make sure the lock is not held while calling back into the daemon

 	clnt.unlock(containerID)

@@ -326,7 +260,7 @@ func (clnt *client) AddProcess(containerID, processFriendlyName string, procToAd

 	clnt.lock(containerID)

 	// Spin up a go routine waiting for exit to handle cleanup

-	go container.waitExit(pid, processFriendlyName, false)

+	go container.waitExit(proc, false)

 	return nil

 }

@@ -350,16 +284,17 @@ func (clnt *client) Signal(containerID string, sig int) error {

 	cont.manualStopRequested = true

 	logrus.Debugf("lcd: Signal() containerID=%s sig=%d pid=%d", containerID, sig, cont.systemPid)

-	context := fmt.Sprintf("Signal: sig=%d pid=%d", sig, cont.systemPid)

 	if syscall.Signal(sig) == syscall.SIGKILL {

 		// Terminate the compute system

-		if err := hcsshim.TerminateComputeSystem(containerID, hcsshim.TimeoutInfinite, context); err != nil {

-			logrus.Errorf("Failed to terminate %s - %q", containerID, err)

+		if err := cont.hcsContainer.Terminate(); err != nil {

+			if err != hcsshim.ErrVmcomputeOperationPending {

+				logrus.Errorf("Failed to terminate %s - %q", containerID, err)

+			}

 		}

 	} else {

 		// Terminate Process

-		if err = hcsshim.TerminateProcessInComputeSystem(containerID, cont.systemPid); err != nil {

+		if err := cont.hcsProcess.Kill(); err != nil {

 			logrus.Warnf("Failed to terminate pid %d in %s: %q", cont.systemPid, containerID, err)

 			// Ignore errors

 			err = nil

@@ -380,15 +315,17 @@ func (clnt *client) Resize(containerID, processFriendlyName string, width, heigh

 		return err

 	}

+	h, w := uint16(height), uint16(width)

+

 	if processFriendlyName == InitFriendlyName {

 		logrus.Debugln("Resizing systemPID in", containerID, cont.process.systemPid)

-		return hcsshim.ResizeConsoleInComputeSystem(containerID, cont.process.systemPid, height, width)

+		return cont.process.hcsProcess.ResizeConsole(w, h)

 	}

 	for _, p := range cont.processes {

 		if p.friendlyName == processFriendlyName {

 			logrus.Debugln("Resizing exec'd process", containerID, p.systemPid)

-			return hcsshim.ResizeConsoleInComputeSystem(containerID, p.systemPid, height, width)

+			return p.hcsProcess.ResizeConsole(w, h)

 		}

 	}

@@ -22,6 +22,7 @@ type container struct {

 	ociSpec Spec

 	manualStopRequested bool

+	hcsContainer        hcsshim.Container

 }

 func (ctr *container) newProcess(friendlyName string) *process {

@@ -40,7 +41,7 @@ func (ctr *container) start() error {

 	// Start the container.  If this is a servicing container, this call will block

 	// until the container is done with the servicing execution.

 	logrus.Debugln("Starting container ", ctr.containerID)

-	if err = hcsshim.StartComputeSystem(ctr.containerID); err != nil {

+	if err = ctr.hcsContainer.Start(); err != nil {

 		logrus.Errorf("Failed to start compute system: %s", err)

 		return err

 	}

@@ -49,59 +50,61 @@ func (ctr *container) start() error {

 		if s, ok := option.(*ServicingOption); ok && s.IsServicing {

 			// Since the servicing operation is complete when StartCommputeSystem returns without error,

 			// we can shutdown (which triggers merge) and exit early.

-			const shutdownTimeout = 5 * 60 * 1000  // 4 minutes

-			const terminateTimeout = 1 * 60 * 1000 // 1 minute

-			if err := hcsshim.ShutdownComputeSystem(ctr.containerID, shutdownTimeout, ""); err != nil {

-				logrus.Errorf("Failed during cleanup of servicing container: %s", err)

-				// Terminate the container, ignoring errors.

-				if err2 := hcsshim.TerminateComputeSystem(ctr.containerID, terminateTimeout, ""); err2 != nil {

-					logrus.Errorf("Failed to terminate container %s after shutdown failure: %q", ctr.containerID, err2)

-				}

-				return err

-			}

-			return nil

+			return ctr.shutdown()

 		}

 	}

-	createProcessParms := hcsshim.CreateProcessParams{

+	// Note we always tell HCS to

+	// create stdout as it's required regardless of '-i' or '-t' options, so that

+	// docker can always grab the output through logs. We also tell HCS to always

+	// create stdin, even if it's not used - it will be closed shortly. Stderr

+	// is only created if it we're not -t.

+	createProcessParms := &hcsshim.ProcessConfig{

 		EmulateConsole:   ctr.ociSpec.Process.Terminal,

 		WorkingDirectory: ctr.ociSpec.Process.Cwd,

 		ConsoleSize:      ctr.ociSpec.Process.InitialConsoleSize,

+		CreateStdInPipe:  true,

+		CreateStdOutPipe: true,

+		CreateStdErrPipe: !ctr.ociSpec.Process.Terminal,

 	}

 	// Configure the environment for the process

 	createProcessParms.Environment = setupEnvironmentVariables(ctr.ociSpec.Process.Env)

 	createProcessParms.CommandLine = strings.Join(ctr.ociSpec.Process.Args, " ")

-	iopipe := &IOPipe{Terminal: ctr.ociSpec.Process.Terminal}

-

-	// Start the command running in the container. Note we always tell HCS to

-	// create stdout as it's required regardless of '-i' or '-t' options, so that

-	// docker can always grab the output through logs. We also tell HCS to always

-	// create stdin, even if it's not used - it will be closed shortly. Stderr

-	// is only created if it we're not -t.

-	var pid uint32

-	var stdout, stderr io.ReadCloser

-	pid, iopipe.Stdin, stdout, stderr, err = hcsshim.CreateProcessInComputeSystem(

-		ctr.containerID,

-		true,

-		true,

-		!ctr.ociSpec.Process.Terminal,

-		createProcessParms)

+	// Start the command running in the container.

+	hcsProcess, err := ctr.hcsContainer.CreateProcess(createProcessParms)

 	if err != nil {

-		logrus.Errorf("CreateProcessInComputeSystem() failed %s", err)

-

-		// Explicitly terminate the compute system here.

-		if err2 := hcsshim.TerminateComputeSystem(ctr.containerID, hcsshim.TimeoutInfinite, "CreateProcessInComputeSystem failed"); err2 != nil {

-			// Ignore this error, there's not a lot we can do except log it

-			logrus.Warnf("Failed to TerminateComputeSystem after a failed CreateProcessInComputeSystem. Ignoring this.", err2)

+		logrus.Errorf("CreateProcess() failed %s", err)

+		if err2 := ctr.terminate(); err2 != nil {

+			logrus.Debugf("Failed to cleanup after a failed CreateProcess. Ignoring this. %s", err2)

 		} else {

-			logrus.Debugln("Cleaned up after failed CreateProcessInComputeSystem by calling TerminateComputeSystem")

+			logrus.Debugln("Cleaned up after failed CreateProcess by calling Terminate")

 		}

 		return err

 	}

 	ctr.startedAt = time.Now()

+	// Save the hcs Process and PID

+	ctr.process.friendlyName = InitFriendlyName

+	pid := hcsProcess.Pid()

+	ctr.process.hcsProcess = hcsProcess

+

+	var stdout, stderr io.ReadCloser

+	var stdin io.WriteCloser

+	stdin, stdout, stderr, err = hcsProcess.Stdio()

+	if err != nil {

+		logrus.Errorf("failed to get stdio pipes: %s", err)

+		if err := ctr.terminate(); err != nil {

+			logrus.Debugf("Failed to cleanup after a failed CreateProcess. Ignoring this. %s", err)

+		}

+		return err

+	}

+

+	iopipe := &IOPipe{Terminal: ctr.ociSpec.Process.Terminal}

+

+	iopipe.Stdin = createStdInCloser(stdin, hcsProcess)

+

 	// TEMP: Work around Windows BS/DEL behavior.

 	iopipe.Stdin = fixStdinBackspaceBehavior(iopipe.Stdin, ctr.ociSpec.Process.Terminal)

@@ -118,7 +121,7 @@ func (ctr *container) start() error {

 	ctr.systemPid = uint32(pid)

 	// Spin up a go routine waiting for exit to handle cleanup

-	go ctr.waitExit(pid, InitFriendlyName, true)

+	go ctr.waitExit(&ctr.process, true)

 	ctr.client.appendContainer(ctr)

@@ -140,17 +143,27 @@ func (ctr *container) start() error {

 // waitExit runs as a goroutine waiting for the process to exit. It's

 // equivalent to (in the linux containerd world) where events come in for

 // state change notifications from containerd.

-func (ctr *container) waitExit(pid uint32, processFriendlyName string, isFirstProcessToStart bool) error {

-	logrus.Debugln("waitExit on pid", pid)

+func (ctr *container) waitExit(process *process, isFirstProcessToStart bool) error {

+	logrus.Debugln("waitExit on pid", process.systemPid)

 	// Block indefinitely for the process to exit.

-	exitCode, err := hcsshim.WaitForProcessInComputeSystem(ctr.containerID, pid, hcsshim.TimeoutInfinite)

+	err := process.hcsProcess.Wait()

 	if err != nil {

-		if herr, ok := err.(*hcsshim.HcsError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {

+		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {

 			logrus.Warnf("WaitForProcessInComputeSystem failed (container may have been killed): %s", err)

 		}

 		// Fall through here, do not return. This ensures we attempt to continue the

-		// shutdown in HCS nad tell the docker engine that the process/container

+		// shutdown in HCS and tell the docker engine that the process/container

+		// has exited to avoid a container being dropped on the floor.

+	}

+

+	exitCode, err := process.hcsProcess.ExitCode()

+	if err != nil {

+		if herr, ok := err.(*hcsshim.ProcessError); ok && herr.Err != syscall.ERROR_BROKEN_PIPE {

+			logrus.Warnf("Unable to get exit code from container %s", ctr.containerID)

+		}

+		// Fall through here, do not return. This ensures we attempt to continue the

+		// shutdown in HCS and tell the docker engine that the process/container

 		// has exited to avoid a container being dropped on the floor.

 	}

@@ -159,46 +172,35 @@ func (ctr *container) waitExit(pid uint32, processFriendlyName string, isFirstPr

 		CommonStateInfo: CommonStateInfo{

 			State:     StateExit,

 			ExitCode:  uint32(exitCode),

-			Pid:       pid,

-			ProcessID: processFriendlyName,

+			Pid:       process.systemPid,

+			ProcessID: process.friendlyName,

 		},

 		UpdatePending: false,

 	}

 	// But it could have been an exec'd process which exited

 	if !isFirstProcessToStart {

+		if err := process.hcsProcess.Close(); err != nil {

+			logrus.Error(err)

+		}

 		si.State = StateExitProcess

 	} else {

-		// Since this is the init process, always call into vmcompute.dll to

-		// shutdown the container after we have completed.

-

-		propertyCheckFlag := 1 // Include update pending check.

-		csProperties, err := hcsshim.GetComputeSystemProperties(ctr.containerID, uint32(propertyCheckFlag))

+		updatePending, err := ctr.hcsContainer.HasPendingUpdates()

 		if err != nil {

-			logrus.Warnf("GetComputeSystemProperties failed (container may have been killed): %s", err)

+			logrus.Warnf("HasPendingUpdates failed (container may have been killed): %s", err)

 		} else {

-			si.UpdatePending = csProperties.AreUpdatesPending

+			si.UpdatePending = updatePending

 		}

 		logrus.Debugf("Shutting down container %s", ctr.containerID)

-		// Explicit timeout here rather than hcsshim.TimeoutInfinte to avoid a

-		// (remote) possibility that ShutdownComputeSystem hangs indefinitely.

-		const shutdownTimeout = 5 * 60 * 1000 // 5 minutes

-		if err := hcsshim.ShutdownComputeSystem(ctr.containerID, shutdownTimeout, "waitExit"); err != nil {

-			if herr, ok := err.(*hcsshim.HcsError); !ok ||

-				(herr.Err != hcsshim.ERROR_SHUTDOWN_IN_PROGRESS &&

-					herr.Err != ErrorBadPathname &&

-					herr.Err != syscall.ERROR_PATH_NOT_FOUND) {

-				logrus.Debugf("waitExit - error from ShutdownComputeSystem on %s %v. Calling TerminateComputeSystem", ctr.containerCommon, err)

-				if err := hcsshim.TerminateComputeSystem(ctr.containerID, shutdownTimeout, "waitExit"); err != nil {

-					logrus.Debugf("waitExit - ignoring error from TerminateComputeSystem %s %v", ctr.containerID, err)

-				} else {

-					logrus.Debugf("Successful TerminateComputeSystem after failed ShutdownComputeSystem on %s in waitExit", ctr.containerID)

-				}

-			}

+		if err := ctr.shutdown(); err != nil {

+			logrus.Debugf("Failed to shutdown container %s", ctr.containerID)

 		} else {

 			logrus.Debugf("Completed shutting down container %s", ctr.containerID)

 		}

+		if err := ctr.hcsContainer.Close(); err != nil {

+			logrus.Error(err)

+		}

 		if !ctr.manualStopRequested && ctr.restartManager != nil {

 			restart, wait, err := ctr.restartManager.ShouldRestart(uint32(exitCode), false, time.Since(ctr.startedAt))

@@ -227,6 +229,9 @@ func (ctr *container) waitExit(pid uint32, processFriendlyName string, isFirstPr

 		// Remove process from list if we have exited

 		// We need to do so here in case the Message Handler decides to restart it.

 		if si.State == StateExit {

+			if err := ctr.hcsContainer.Close(); err != nil {

+				logrus.Error(err)

+			}

 			ctr.client.deleteContainer(ctr.friendlyName)

 		}

 	}

@@ -240,3 +245,37 @@ func (ctr *container) waitExit(pid uint32, processFriendlyName string, isFirstPr

 	logrus.Debugln("waitExit() completed OK")

 	return nil

 }

+

+func (ctr *container) shutdown() error {

+	const shutdownTimeout = time.Minute * 5

+	err := ctr.hcsContainer.Shutdown()

+	if err == hcsshim.ErrVmcomputeOperationPending {

+		// Explicit timeout to avoid a (remote) possibility that shutdown hangs indefinitely.

+		err = ctr.hcsContainer.WaitTimeout(shutdownTimeout)

+	}

+

+	if err != nil {

+		logrus.Debugf("error shutting down container %s %v calling terminate", ctr.containerID, err)

+		if err := ctr.terminate(); err != nil {

+			return err

+		}

+		return err

+	}

+

+	return nil

+}

+

+func (ctr *container) terminate() error {

+	const terminateTimeout = time.Minute * 5

+	err := ctr.hcsContainer.Terminate()

+

+	if err == hcsshim.ErrVmcomputeOperationPending {

+		err = ctr.hcsContainer.WaitTimeout(terminateTimeout)

+	}

+

+	if err != nil {

+		return err

+	}

+

+	return nil

+}

@@ -3,6 +3,7 @@ package libcontainerd

 import (

 	"io"

+	"github.com/Microsoft/hcsshim"

 	"github.com/docker/docker/pkg/system"

 )

@@ -14,6 +15,7 @@ type process struct {

 	// commandLine is to support returning summary information for docker top

 	commandLine string

+	hcsProcess  hcsshim.Process

 }

 func openReaderFromPipe(p io.ReadCloser) io.Reader {

@@ -57,3 +59,27 @@ func (w *delToBsWriter) Write(b []byte) (int, error) {

 	}

 	return w.WriteCloser.Write(bc)

 }

+

+type stdInCloser struct {

+	io.WriteCloser

+	hcsshim.Process

+}

+

+func createStdInCloser(pipe io.WriteCloser, process hcsshim.Process) *stdInCloser {

+	return &stdInCloser{

+		WriteCloser: pipe,

+		Process:     process,

+	}

+}

+

+func (stdin *stdInCloser) Close() error {

+	if err := stdin.WriteCloser.Close(); err != nil {

+		return err

+	}

+

+	return stdin.Process.CloseStdin()

+}

+

+func (stdin *stdInCloser) Write(p []byte) (n int, err error) {

+	return stdin.WriteCloser.Write(p)

+}

@@ -62,20 +62,17 @@ func (w *baseLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) (err e

 		}

 	}()

-	err = winio.RunWithPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}, func() (err error) {

-		createmode := uint32(syscall.CREATE_NEW)

-		if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {

-			err := os.Mkdir(path, 0)

-			if err != nil && !os.IsExist(err) {

-				return err

-			}

-			createmode = syscall.OPEN_EXISTING

+	createmode := uint32(syscall.CREATE_NEW)

+	if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {

+		err := os.Mkdir(path, 0)

+		if err != nil && !os.IsExist(err) {

+			return err

 		}

+		createmode = syscall.OPEN_EXISTING

+	}

-		mode := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | winio.WRITE_DAC | winio.WRITE_OWNER | winio.ACCESS_SYSTEM_SECURITY)

-		f, err = winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createmode)

-		return

-	})

+	mode := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | winio.WRITE_DAC | winio.WRITE_OWNER | winio.ACCESS_SYSTEM_SECURITY)

+	f, err = winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createmode)

 	if err != nil {

 		return err

 	}

@@ -113,9 +110,7 @@ func (w *baseLayerWriter) AddLink(name string, target string) (err error) {

 		return err

 	}

-	return winio.RunWithPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}, func() (err error) {

-		return os.Link(linktarget, linkpath)

-	})

+	return os.Link(linktarget, linkpath)

 }

 func (w *baseLayerWriter) Remove(name string) error {

@@ -123,11 +118,7 @@ func (w *baseLayerWriter) Remove(name string) error {

 }

 func (w *baseLayerWriter) Write(b []byte) (int, error) {

-	var n int

-	err := winio.RunWithPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}, func() (err error) {

-		n, err = w.bw.Write(b)

-		return

-	})

+	n, err := w.bw.Write(b)

 	if err != nil {

 		w.err = err

 	}

new file mode 100644

@@ -0,0 +1,78 @@

+package hcsshim

+

+import (

+	"errors"

+	"sync"

+	"syscall"

+)

+

+var (

+	nextCallback    uintptr

+	callbackMap     = map[uintptr]*notifcationWatcherContext{}

+	callbackMapLock = sync.RWMutex{}

+

+	notificationWatcherCallback = syscall.NewCallback(notificationWatcher)

+

+	// Notifications for HCS_SYSTEM handles

+	hcsNotificationSystemExited          hcsNotification = 0x00000001

+	hcsNotificationSystemCreateCompleted hcsNotification = 0x00000002

+	hcsNotificationSystemStartCompleted  hcsNotification = 0x00000003

+	hcsNotificationSystemPauseCompleted  hcsNotification = 0x00000004

+	hcsNotificationSystemResumeCompleted hcsNotification = 0x00000005

+

+	// Notifications for HCS_PROCESS handles

+	hcsNotificationProcessExited hcsNotification = 0x00010000

+

+	// Common notifications

+	hcsNotificationInvalid           hcsNotification = 0x00000000

+	hcsNotificationServiceDisconnect hcsNotification = 0x01000000

+

+	// ErrUnexpectedContainerExit is the error returned when a container exits while waiting for

+	// a different expected notification

+	ErrUnexpectedContainerExit = errors.New("unexpected container exit")

+

+	// ErrUnexpectedProcessAbort is the error returned when communication with the compute service

+	// is lost while waiting for a notification

+	ErrUnexpectedProcessAbort = errors.New("lost communication with compute service")

+)

+

+type hcsNotification uint32

+type notificationChannel chan error

+

+type notifcationWatcherContext struct {

+	channel              notificationChannel

+	expectedNotification hcsNotification

+	handle               hcsCallback

+}

+

+func notificationWatcher(notificationType hcsNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr {

+	var (

+		result       error

+		completeWait = false

+	)

+

+	callbackMapLock.RLock()

+	context := callbackMap[callbackNumber]

+	callbackMapLock.RUnlock()

+

+	if notificationType == context.expectedNotification {

+		if int32(notificationStatus) < 0 {

+			result = syscall.Errno(win32FromHresult(notificationStatus))

+		} else {

+			result = nil

+		}

+		completeWait = true

+	} else if notificationType == hcsNotificationSystemExited {

+		result = ErrUnexpectedContainerExit

+		completeWait = true

+	} else if notificationType == hcsNotificationServiceDisconnect {

+		result = ErrUnexpectedProcessAbort

+		completeWait = true

+	}

+

+	if completeWait {

+		context.channel <- result

+	}

+

+	return 0

+}

new file mode 100644

@@ -0,0 +1,513 @@

+package hcsshim

+

+import (

+	"encoding/json"

+	"errors"

+	"fmt"

+	"runtime"

+	"syscall"

+	"time"

+

+	"github.com/Sirupsen/logrus"

+)

+

+var (

+	defaultTimeout = time.Minute * 4

+

+	// ErrTimeout is an error encountered when waiting on a notification times out

+	ErrTimeout = errors.New("hcsshim: timeout waiting for notification")

+)

+

+type ContainerError struct {

+	Container *container

+	Operation string

+	ExtraInfo string

+	Err       error

+}

+

+type container struct {

+	handle hcsSystem

+	id     string

+}

+

+type containerProperties struct {

+	ID                string `json:"Id"`

+	Name              string

+	SystemType        string

+	Owner             string

+	SiloGUID          string `json:"SiloGuid,omitempty"`

+	IsDummy           bool   `json:",omitempty"`

+	RuntimeID         string `json:"RuntimeId,omitempty"`

+	Stopped           bool   `json:",omitempty"`

+	ExitType          string `json:",omitempty"`

+	AreUpdatesPending bool   `json:",omitempty"`

+}

+

+// CreateContainer creates a new container with the given configuration but does not start it.

+func CreateContainer(id string, c *ContainerConfig) (Container, error) {

+	operation := "CreateContainer"

+	title := "HCSShim::" + operation

+	logrus.Debugf(title+" id=%s", id)

+

+	container := &container{

+		id: id,

+	}

+

+	configurationb, err := json.Marshal(c)

+	if err != nil {

+		return nil, err

+	}

+

+	configuration := string(configurationb)

+

+	var (

+		handle      hcsSystem

+		resultp     *uint16