@@ -43,7 +43,7 @@ esac

 # the following lines are in sorted order, FYI

 clone git github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62

-clone git github.com/Microsoft/hcsshim v0.3.0

+clone git github.com/Microsoft/hcsshim v0.3.1

 clone git github.com/Microsoft/go-winio v0.3.4

 clone git github.com/Sirupsen/logrus v0.10.0 # logrus is a common dependency among multiple deps

 clone git github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a

@@ -6740,3 +6740,18 @@ func (s *DockerSuite) TestBuildLabelsOverride(c *check.C) {

 	}

 }

+

+// Test case for #22855

+func (s *DockerSuite) TestBuildDeleteCommittedFile(c *check.C) {

+	name := "test-delete-committed-file"

+

+	_, err := buildImage(name,

+		`FROM busybox

+		RUN echo test > file

+		RUN test -e file

+		RUN rm file

+		RUN sh -c "! test -e file"`, false)

+	if err != nil {

+		c.Fatal(err)

+	}

+}

@@ -40,39 +40,36 @@ type hcsNotification uint32

 type notificationChannel chan error

 type notifcationWatcherContext struct {

-	channel              notificationChannel

-	expectedNotification hcsNotification

-	handle               hcsCallback

+	channels notificationChannels

+	handle   hcsCallback

+}

+

+type notificationChannels map[hcsNotification]notificationChannel

+

+func newChannels() notificationChannels {

+	channels := make(notificationChannels)

+

+	channels[hcsNotificationSystemExited] = make(notificationChannel, 1)

+	channels[hcsNotificationSystemCreateCompleted] = make(notificationChannel, 1)

+	channels[hcsNotificationSystemStartCompleted] = make(notificationChannel, 1)

+	channels[hcsNotificationSystemPauseCompleted] = make(notificationChannel, 1)

+	channels[hcsNotificationSystemResumeCompleted] = make(notificationChannel, 1)

+	channels[hcsNotificationProcessExited] = make(notificationChannel, 1)

+	channels[hcsNotificationServiceDisconnect] = make(notificationChannel, 1)

+	return channels

 }

 func notificationWatcher(notificationType hcsNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr {

-	var (

-		result       error

-		completeWait = false

-	)

+	var result error

+	if int32(notificationStatus) < 0 {

+		result = syscall.Errno(win32FromHresult(notificationStatus))

+	}

 	callbackMapLock.RLock()

-	context := callbackMap[callbackNumber]

+	channels := callbackMap[callbackNumber].channels

 	callbackMapLock.RUnlock()

-	if notificationType == context.expectedNotification {

-		if int32(notificationStatus) < 0 {

-			result = syscall.Errno(win32FromHresult(notificationStatus))

-		} else {

-			result = nil

-		}

-		completeWait = true

-	} else if notificationType == hcsNotificationSystemExited {

-		result = ErrUnexpectedContainerExit

-		completeWait = true

-	} else if notificationType == hcsNotificationServiceDisconnect {

-		result = ErrUnexpectedProcessAbort

-		completeWait = true

-	}

-

-	if completeWait {

-		context.channel <- result

-	}

+	channels[notificationType] <- result

 	return 0

 }

@@ -26,8 +26,9 @@ type ContainerError struct {

 }

 type container struct {

-	handle hcsSystem

-	id     string

+	handle         hcsSystem

+	id             string

+	callbackNumber uintptr

 }

 type containerProperties struct {

@@ -47,7 +48,6 @@ type containerProperties struct {

 func CreateContainer(id string, c *ContainerConfig) (Container, error) {

 	operation := "CreateContainer"

 	title := "HCSShim::" + operation

-	logrus.Debugf(title+" id=%s", id)

 	container := &container{

 		id: id,

@@ -59,21 +59,28 @@ func CreateContainer(id string, c *ContainerConfig) (Container, error) {

 	}

 	configuration := string(configurationb)

+	logrus.Debugf(title+" id=%s config=%s", id, configuration)

 	var (

-		handle      hcsSystem

 		resultp     *uint16

 		createError error

 	)

 	if hcsCallbacksSupported {

 		var identity syscall.Handle

-		createError = hcsCreateComputeSystem(id, configuration, identity, &handle, &resultp)

+		createError = hcsCreateComputeSystem(id, configuration, identity, &container.handle, &resultp)

+

+		if createError == nil || createError == ErrVmcomputeOperationPending {

+			if err := container.registerCallback(); err != nil {

+				err := &ContainerError{Container: container, Operation: operation, Err: err}

+				logrus.Error(err)

+				return nil, err

+			}

+		}

 	} else {

-		createError = hcsCreateComputeSystemTP5(id, configuration, &handle, &resultp)

+		createError = hcsCreateComputeSystemTP5(id, configuration, &container.handle, &resultp)

 	}

-	container.handle = handle

-	err = processAsyncHcsResult(container, createError, resultp, hcsNotificationSystemCreateCompleted, &defaultTimeout)

+	err = processAsyncHcsResult(createError, resultp, container.callbackNumber, hcsNotificationSystemCreateCompleted, &defaultTimeout)

 	if err != nil {

 		err := &ContainerError{Container: container, Operation: operation, ExtraInfo: configuration, Err: err}

 		logrus.Error(err)

@@ -122,7 +129,7 @@ func (container *container) Start() error {

 	var resultp *uint16

 	err := hcsStartComputeSystemTP5(container.handle, nil, &resultp)

-	err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemStartCompleted, &defaultTimeout)

+	err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemStartCompleted, &defaultTimeout)

 	if err != nil {

 		err := &ContainerError{Container: container, Operation: operation, Err: err}

 		logrus.Error(err)

@@ -186,7 +193,7 @@ func (container *container) Wait() error {

 	logrus.Debugf(title+" id=%s", container.id)

 	if hcsCallbacksSupported {

-		err := registerAndWaitForCallback(container, hcsNotificationSystemExited)

+		err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, nil)

 		if err != nil {

 			err := &ContainerError{Container: container, Operation: operation, Err: err}

 			logrus.Error(err)

@@ -217,7 +224,7 @@ func (container *container) WaitTimeout(timeout time.Duration) error {

 	logrus.Debugf(title+" id=%s", container.id)

 	if hcsCallbacksSupported {

-		err := registerAndWaitForCallbackTimeout(container, hcsNotificationSystemExited, timeout)

+		err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, &timeout)

 		if err == ErrTimeout {

 			return ErrTimeout

 		} else if err != nil {

@@ -304,7 +311,7 @@ func (container *container) Pause() error {

 	var resultp *uint16

 	err := hcsPauseComputeSystemTP5(container.handle, nil, &resultp)

-	err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemPauseCompleted, &defaultTimeout)

+	err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemPauseCompleted, &defaultTimeout)

 	if err != nil {

 		err := &ContainerError{Container: container, Operation: operation, Err: err}

 		logrus.Error(err)

@@ -325,7 +332,7 @@ func (container *container) Resume() error {

 	)

 	err := hcsResumeComputeSystemTP5(container.handle, nil, &resultp)

-	err = processAsyncHcsResult(container, err, resultp, hcsNotificationSystemResumeCompleted, &defaultTimeout)

+	err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemResumeCompleted, &defaultTimeout)

 	if err != nil {

 		err := &ContainerError{Container: container, Operation: operation, Err: err}

 		logrus.Error(err)

@@ -340,7 +347,6 @@ func (container *container) Resume() error {

 func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {

 	operation := "CreateProcess"

 	title := "HCSShim::Container::" + operation

-	logrus.Debugf(title+" id=%s", container.id)

 	var (

 		processInfo   hcsProcessInformation

 		processHandle hcsProcess

@@ -359,6 +365,7 @@ func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {

 	}

 	configuration := string(configurationb)

+	logrus.Debugf(title+" id=%s config=%s", container.id, configuration)

 	err = hcsCreateProcess(container.handle, configuration, &processInfo, &processHandle, &resultp)

 	err = processHcsResult(err, resultp)

@@ -379,6 +386,14 @@ func (container *container) CreateProcess(c *ProcessConfig) (Process, error) {

 		},

 	}

+	if hcsCallbacksSupported {

+		if err := process.registerCallback(); err != nil {

+			err = &ContainerError{Container: container, Operation: operation, Err: err}

+			logrus.Error(err)

+			return nil, err

+		}

+	}

+

 	logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID)

 	runtime.SetFinalizer(process, closeProcess)

 	return process, nil

@@ -408,6 +423,12 @@ func (container *container) OpenProcess(pid int) (Process, error) {

 		container: container,

 	}

+	if err := process.registerCallback(); err != nil {

+		err = &ContainerError{Container: container, Operation: operation, Err: err}

+		logrus.Error(err)

+		return nil, err

+	}

+

 	logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID)

 	runtime.SetFinalizer(process, closeProcess)

 	return process, nil

@@ -424,6 +445,14 @@ func (container *container) Close() error {

 		return nil

 	}

+	if hcsCallbacksSupported {

+		if err := container.unregisterCallback(); err != nil {

+			err = &ContainerError{Container: container, Operation: operation, Err: err}

+			logrus.Error(err)

+			return err

+		}

+	}

+

 	if err := hcsCloseComputeSystem(container.handle); err != nil {

 		err = &ContainerError{Container: container, Operation: operation, Err: err}

 		logrus.Error(err)

@@ -441,7 +470,7 @@ func closeContainer(container *container) {

 	container.Close()

 }

-func (container *container) registerCallback(expectedNotification hcsNotification) (uintptr, error) {

+func (container *container) registerCallback() error {

 	callbackMapLock.Lock()

 	defer callbackMapLock.Unlock()

@@ -449,22 +478,24 @@ func (container *container) registerCallback(expectedNotification hcsNotificatio

 	nextCallback++

 	context := &notifcationWatcherContext{

-		expectedNotification: expectedNotification,

-		channel:              make(chan error, 1),

+		channels: newChannels(),

 	}

 	callbackMap[callbackNumber] = context

 	var callbackHandle hcsCallback

 	err := hcsRegisterComputeSystemCallback(container.handle, notificationWatcherCallback, callbackNumber, &callbackHandle)

 	if err != nil {

-		return 0, err

+		return err

 	}

 	context.handle = callbackHandle

+	container.callbackNumber = callbackNumber

-	return callbackNumber, nil

+	return nil

 }

-func (container *container) unregisterCallback(callbackNumber uintptr) error {

+func (container *container) unregisterCallback() error {

+	callbackNumber := container.callbackNumber

+

 	callbackMapLock.Lock()

 	defer callbackMapLock.Unlock()

@@ -4,6 +4,7 @@ import (

 	"bufio"

 	"encoding/binary"

 	"errors"

+	"fmt"

 	"io"

 	"os"

 	"path/filepath"

@@ -77,7 +78,7 @@ func readTombstones(path string) (map[string]([]string), error) {

 	ts := make(map[string]([]string))

 	for s.Scan() {

-		t := s.Text()[1:] // skip leading `\`

+		t := filepath.Join("Files", s.Text()[1:]) // skip leading `\`

 		dir := filepath.Dir(t)

 		ts[dir] = append(ts[dir], t)

 	}

@@ -107,6 +108,7 @@ func (r *legacyLayerReader) walkUntilCancelled() error {

 		if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") {

 			return nil

 		}

+

 		r.result <- &fileEntry{path, info, nil}

 		if !<-r.proceed {

 			return errorIterationCanceled

@@ -120,7 +122,7 @@ func (r *legacyLayerReader) walkUntilCancelled() error {

 			}

 			if dts, ok := ts[relPath]; ok {

 				for _, t := range dts {

-					r.result <- &fileEntry{t, nil, nil}

+					r.result <- &fileEntry{filepath.Join(r.root, t), nil, nil}

 					if !<-r.proceed {

 						return errorIterationCanceled

 					}

@@ -397,7 +399,10 @@ func (w *legacyLayerWriter) AddLink(name string, target string) error {

 }

 func (w *legacyLayerWriter) Remove(name string) error {

-	w.tombstones = append(w.tombstones, name)

+	if !strings.HasPrefix(name, `Files\`) {

+		return fmt.Errorf("invalid tombstone %s", name)

+	}

+	w.tombstones = append(w.tombstones, name[len(`Files\`):])

 	return nil

 }

@@ -22,11 +22,11 @@ type ProcessError struct {

 }

 type process struct {

-	handle             hcsProcess

-	processID          int

-	container          *container

-	cachedPipes        *cachedPipes

-	killCallbackNumber uintptr

+	handle         hcsProcess

+	processID      int

+	container      *container

+	cachedPipes    *cachedPipes

+	callbackNumber uintptr

 }

 type cachedPipes struct {

@@ -101,7 +101,7 @@ func (process *process) Wait() error {

 	logrus.Debugf(title+" processid=%d", process.processID)

 	if hcsCallbacksSupported {

-		err := registerAndWaitForCallback(process, hcsNotificationProcessExited)

+		err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, nil)

 		if err != nil {

 			err := &ProcessError{Operation: operation, Process: process, Err: err}

 			logrus.Error(err)

@@ -128,7 +128,7 @@ func (process *process) WaitTimeout(timeout time.Duration) error {

 	logrus.Debugf(title+" processid=%d", process.processID)

 	if hcsCallbacksSupported {

-		err := registerAndWaitForCallbackTimeout(process, hcsNotificationProcessExited, timeout)

+		err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, &timeout)

 		if err == ErrTimeout {

 			return ErrTimeout

 		} else if err != nil {

@@ -344,6 +344,14 @@ func (process *process) Close() error {

 		return nil

 	}

+	if hcsCallbacksSupported {

+		if err := process.unregisterCallback(); err != nil {

+			err = &ProcessError{Operation: operation, Process: process, Err: err}

+			logrus.Error(err)

+			return err

+		}

+	}

+

 	if err := hcsCloseProcess(process.handle); err != nil {

 		err = &ProcessError{Operation: operation, Process: process, Err: err}

 		logrus.Error(err)

@@ -361,7 +369,7 @@ func closeProcess(process *process) {

 	process.Close()

 }

-func (process *process) registerCallback(expectedNotification hcsNotification) (uintptr, error) {

+func (process *process) registerCallback() error {

 	callbackMapLock.Lock()

 	defer callbackMapLock.Unlock()

@@ -369,22 +377,24 @@ func (process *process) registerCallback(expectedNotification hcsNotification) (

 	nextCallback++

 	context := &notifcationWatcherContext{

-		expectedNotification: expectedNotification,

-		channel:              make(chan error, 1),

+		channels: newChannels(),

 	}

 	callbackMap[callbackNumber] = context

 	var callbackHandle hcsCallback

 	err := hcsRegisterProcessCallback(process.handle, notificationWatcherCallback, callbackNumber, &callbackHandle)

 	if err != nil {

-		return 0, err

+		return err

 	}

 	context.handle = callbackHandle

+	process.callbackNumber = callbackNumber

-	return callbackNumber, nil

+	return nil

 }

-func (process *process) unregisterCallback(callbackNumber uintptr) error {

+func (process *process) unregisterCallback() error {

+	callbackNumber := process.callbackNumber

+

 	callbackMapLock.Lock()

 	defer callbackMapLock.Unlock()

 	handle := callbackMap[callbackNumber].handle

@@ -1,6 +1,7 @@

 package hcsshim

 import (

+	"github.com/Sirupsen/logrus"

 	"syscall"

 	"time"

 )

@@ -10,11 +11,6 @@ type waitable interface {

 	hcsWait(timeout uint32) (bool, error)

 }

-type callbackable interface {

-	registerCallback(expectedNotification hcsNotification) (uintptr, error)

-	unregisterCallback(callbackNumber uintptr) error

-}

-

 func waitTimeoutHelper(object waitable, timeout time.Duration) (bool, error) {

 	var (

 		millis uint32

@@ -52,62 +48,62 @@ func waitForSingleObject(handle syscall.Handle, timeout uint32) (bool, error) {

 	}

 }

-func processAsyncHcsResult(object callbackable, err error, resultp *uint16, expectedNotification hcsNotification, timeout *time.Duration) error {

+func processAsyncHcsResult(err error, resultp *uint16, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error {

 	err = processHcsResult(err, resultp)

 	if err == ErrVmcomputeOperationPending {

-		if timeout != nil {

-			err = registerAndWaitForCallbackTimeout(object, expectedNotification, *timeout)

-		} else {

-			err = registerAndWaitForCallback(object, expectedNotification)

-		}

+		return waitForNotification(callbackNumber, expectedNotification, timeout)

 	}

 	return err

 }

-func registerAndWaitForCallbackTimeout(object callbackable, expectedNotification hcsNotification, timeout time.Duration) error {

-	callbackNumber, err := object.registerCallback(expectedNotification)

-	if err != nil {

-		return err

-	}

-	defer object.unregisterCallback(callbackNumber)

-

-	return waitForNotificationTimeout(callbackNumber, timeout)

-}

-

-func registerAndWaitForCallback(object callbackable, expectedNotification hcsNotification) error {

-	callbackNumber, err := object.registerCallback(expectedNotification)

-	if err != nil {

-		return err

-	}

-	defer object.unregisterCallback(callbackNumber)

-

-	return waitForNotification(callbackNumber)

-}

-

-func waitForNotificationTimeout(callbackNumber uintptr, timeout time.Duration) error {

+func waitForNotification(callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error {

 	callbackMapLock.RLock()

-	channel := callbackMap[callbackNumber].channel

+	channels := callbackMap[callbackNumber].channels

 	callbackMapLock.RUnlock()

-	timer := time.NewTimer(timeout)

-	defer timer.Stop()

-

-	select {

-	case err := <-channel:

-		return err

-	case <-timer.C:

-		return ErrTimeout

+	expectedChannel := channels[expectedNotification]

+	if expectedChannel == nil {

+		logrus.Errorf("unknown notification type in waitForNotification %x", expectedNotification)

 	}

-}

-

-func waitForNotification(callbackNumber uintptr) error {

-	callbackMapLock.RLock()

-	channel := callbackMap[callbackNumber].channel

-	callbackMapLock.RUnlock()

+	if timeout != nil {

+		timer := time.NewTimer(*timeout)

+		defer timer.Stop()

+

+		select {

+		case err := <-expectedChannel:

+			return err

+		case err := <-channels[hcsNotificationSystemExited]:

+			// If the expected notification is hcsNotificationSystemExited which of the two selects

+			// chosen is random. Return the raw error if hcsNotificationSystemExited is expected

+			if channels[hcsNotificationSystemExited] == expectedChannel {

+				return err

+			}

+			return ErrUnexpectedContainerExit

+		case err := <-channels[hcsNotificationServiceDisconnect]:

+			// hcsNotificationServiceDisconnect should never be an expected notification

+			// it does not need the same handling as hcsNotificationSystemExited

+			logrus.Error(err)

+			return ErrUnexpectedProcessAbort

+		case <-timer.C:

+			return ErrTimeout

+		}

+	}

 	select {

-	case err := <-channel:

+	case err := <-expectedChannel:

 		return err

+	case err := <-channels[hcsNotificationSystemExited]:

+		// If the expected notification is hcsNotificationSystemExited which of the two selects

+		// chosen is random. Return the raw error if hcsNotificationSystemExited is expected

+		if channels[hcsNotificationSystemExited] == expectedChannel {

+			return err

+		}

+		return ErrUnexpectedContainerExit

+	case err := <-channels[hcsNotificationServiceDisconnect]:

+		// hcsNotificationServiceDisconnect should never be an expected notification

+		// it does not need the same handling as hcsNotificationSystemExited

+		logrus.Error(err)

+		return ErrUnexpectedProcessAbort

 	}

 }