Browse code

Merge pull request #15898 from Microsoft/15775-buildcontextfix

Windows: Fix long path handling for docker build

David Calavera authored on 2015/09/18 01:19:23
Showing 15 changed files
... ...
@@ -34,6 +34,7 @@ import (
34 34
 	"github.com/docker/docker/pkg/progressreader"
35 35
 	"github.com/docker/docker/pkg/stringid"
36 36
 	"github.com/docker/docker/pkg/stringutils"
37
+	"github.com/docker/docker/pkg/symlink"
37 38
 	"github.com/docker/docker/pkg/system"
38 39
 	"github.com/docker/docker/pkg/tarsum"
39 40
 	"github.com/docker/docker/pkg/urlutil"
... ...
@@ -42,7 +43,7 @@ import (
42 42
 )
43 43
 
44 44
 func (b *builder) readContext(context io.Reader) (err error) {
45
-	tmpdirPath, err := ioutil.TempDir("", "docker-build")
45
+	tmpdirPath, err := getTempDir("", "docker-build")
46 46
 	if err != nil {
47 47
 		return
48 48
 	}
... ...
@@ -305,7 +306,7 @@ func calcCopyInfo(b *builder, cmdName string, cInfos *[]*copyInfo, origPath stri
305 305
 		}
306 306
 
307 307
 		// Create a tmp dir
308
-		tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-remote")
308
+		tmpDirName, err := getTempDir(b.contextPath, "docker-remote")
309 309
 		if err != nil {
310 310
 			return err
311 311
 		}
... ...
@@ -684,14 +685,14 @@ func (b *builder) run(c *daemon.Container) error {
684 684
 
685 685
 func (b *builder) checkPathForAddition(orig string) error {
686 686
 	origPath := filepath.Join(b.contextPath, orig)
687
-	origPath, err := filepath.EvalSymlinks(origPath)
687
+	origPath, err := symlink.EvalSymlinks(origPath)
688 688
 	if err != nil {
689 689
 		if os.IsNotExist(err) {
690 690
 			return fmt.Errorf("%s: no such file or directory", orig)
691 691
 		}
692 692
 		return err
693 693
 	}
694
-	contextPath, err := filepath.EvalSymlinks(b.contextPath)
694
+	contextPath, err := symlink.EvalSymlinks(b.contextPath)
695 695
 	if err != nil {
696 696
 		return err
697 697
 	}
... ...
@@ -3,10 +3,15 @@
3 3
 package builder
4 4
 
5 5
 import (
6
+	"io/ioutil"
6 7
 	"os"
7 8
 	"path/filepath"
8 9
 )
9 10
 
11
+func getTempDir(dir, prefix string) (string, error) {
12
+	return ioutil.TempDir(dir, prefix)
13
+}
14
+
10 15
 func fixPermissions(source, destination string, uid, gid int, destExisted bool) error {
11 16
 	// If the destination didn't already exist, or the destination isn't a
12 17
 	// directory, then we should Lchown the destination. Otherwise, we shouldn't
... ...
@@ -2,6 +2,20 @@
2 2
 
3 3
 package builder
4 4
 
5
+import (
6
+	"io/ioutil"
7
+
8
+	"github.com/docker/docker/pkg/longpath"
9
+)
10
+
11
+func getTempDir(dir, prefix string) (string, error) {
12
+	tempDir, err := ioutil.TempDir(dir, prefix)
13
+	if err != nil {
14
+		return "", err
15
+	}
16
+	return longpath.AddPrefix(tempDir), nil
17
+}
18
+
5 19
 func fixPermissions(source, destination string, uid, gid int, destExisted bool) error {
6 20
 	// chown is not supported on Windows
7 21
 	return nil
... ...
@@ -8,15 +8,14 @@ import (
8 8
 	"os"
9 9
 	"path/filepath"
10 10
 	"strings"
11
+
12
+	"github.com/docker/docker/pkg/longpath"
11 13
 )
12 14
 
13 15
 // fixVolumePathPrefix does platform specific processing to ensure that if
14 16
 // the path being passed in is not in a volume path format, convert it to one.
15 17
 func fixVolumePathPrefix(srcPath string) string {
16
-	if !strings.HasPrefix(srcPath, `\\?\`) {
17
-		srcPath = `\\?\` + srcPath
18
-	}
19
-	return srcPath
18
+	return longpath.AddPrefix(srcPath)
20 19
 }
21 20
 
22 21
 // getWalkRoot calculates the root path when performing a TarWithOptions.
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"io"
5 5
 
6 6
 	"github.com/docker/docker/pkg/archive"
7
+	"github.com/docker/docker/pkg/longpath"
7 8
 )
8 9
 
9 10
 // chroot is not supported by Windows
... ...
@@ -17,5 +18,5 @@ func invokeUnpack(decompressedArchive io.ReadCloser,
17 17
 	// Windows is different to Linux here because Windows does not support
18 18
 	// chroot. Hence there is no point sandboxing a chrooted process to
19 19
 	// do the unpack. We call inline instead within the daemon process.
20
-	return archive.Unpack(decompressedArchive, dest, options)
20
+	return archive.Unpack(decompressedArchive, longpath.AddPrefix(dest), options)
21 21
 }
... ...
@@ -5,9 +5,9 @@ import (
5 5
 	"io/ioutil"
6 6
 	"os"
7 7
 	"path/filepath"
8
-	"strings"
9 8
 
10 9
 	"github.com/docker/docker/pkg/archive"
10
+	"github.com/docker/docker/pkg/longpath"
11 11
 )
12 12
 
13 13
 // applyLayerHandler parses a diff in the standard layer format from `layer`, and
... ...
@@ -17,9 +17,7 @@ func applyLayerHandler(dest string, layer archive.Reader, decompress bool) (size
17 17
 	dest = filepath.Clean(dest)
18 18
 
19 19
 	// Ensure it is a Windows-style volume path
20
-	if !strings.HasPrefix(dest, `\\?\`) {
21
-		dest = `\\?\` + dest
22
-	}
20
+	dest = longpath.AddPrefix(dest)
23 21
 
24 22
 	if decompress {
25 23
 		decompressed, err := archive.DecompressStream(layer)
... ...
@@ -5,10 +5,18 @@ package directory
5 5
 import (
6 6
 	"os"
7 7
 	"path/filepath"
8
+	"strings"
9
+
10
+	"github.com/docker/docker/pkg/longpath"
8 11
 )
9 12
 
10 13
 // Size walks a directory tree and returns its total size in bytes.
11 14
 func Size(dir string) (size int64, err error) {
15
+	fixedPath, err := filepath.Abs(dir)
16
+	if err != nil {
17
+		return
18
+	}
19
+	fixedPath = longpath.AddPrefix(fixedPath)
12 20
 	err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, e error) error {
13 21
 		// Ignore directory sizes
14 22
 		if fileInfo == nil {
15 23
new file mode 100644
... ...
@@ -0,0 +1,21 @@
0
+// longpath introduces some constants and helper functions for handling long paths
1
+// in Windows, which are expected to be prepended with `\\?\` and followed by either
2
+// a drive letter, a UNC server\share, or a volume identifier.
3
+
4
+package longpath
5
+
6
+import (
7
+	"strings"
8
+)
9
+
10
+// Prefix is the longpath prefix for Windows file paths.
11
+const Prefix = `\\?\`
12
+
13
+// AddPrefix will add the Windows long path prefix to the path provided if
14
+// it does not already have it.
15
+func AddPrefix(path string) string {
16
+	if !strings.HasPrefix(path, Prefix) {
17
+		path = Prefix + path
18
+	}
19
+	return path
20
+}
... ...
@@ -1,4 +1,5 @@
1
-Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks
1
+Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks,
2
+as well as a Windows long-path aware version of filepath.EvalSymlinks
2 3
 from the [Go standard library](https://golang.org/pkg/path/filepath).
3 4
 
4 5
 The code from filepath.EvalSymlinks has been adapted in fs.go.
... ...
@@ -132,3 +132,12 @@ func evalSymlinksInScope(path, root string) (string, error) {
132 132
 	// what's happening here
133 133
 	return filepath.Clean(root + filepath.Clean(string(filepath.Separator)+b.String())), nil
134 134
 }
135
+
136
+// EvalSymlinks returns the path name after the evaluation of any symbolic
137
+// links.
138
+// If path is relative the result will be relative to the current directory,
139
+// unless one of the components is an absolute symbolic link.
140
+// This version has been updated to support long paths prepended with `\\?\`.
141
+func EvalSymlinks(path string) (string, error) {
142
+	return evalSymlinks(path)
143
+}
135 144
new file mode 100644
... ...
@@ -0,0 +1,11 @@
0
+// +build !windows
1
+
2
+package symlink
3
+
4
+import (
5
+	"path/filepath"
6
+)
7
+
8
+func evalSymlinks(path string) (string, error) {
9
+	return filepath.EvalSymlinks(path)
10
+}
0 11
new file mode 100644
... ...
@@ -0,0 +1,156 @@
0
+package symlink
1
+
2
+import (
3
+	"bytes"
4
+	"errors"
5
+	"os"
6
+	"path/filepath"
7
+	"strings"
8
+	"syscall"
9
+
10
+	"github.com/docker/docker/pkg/longpath"
11
+)
12
+
13
+func toShort(path string) (string, error) {
14
+	p, err := syscall.UTF16FromString(path)
15
+	if err != nil {
16
+		return "", err
17
+	}
18
+	b := p // GetShortPathName says we can reuse buffer
19
+	n, err := syscall.GetShortPathName(&p[0], &b[0], uint32(len(b)))
20
+	if err != nil {
21
+		return "", err
22
+	}
23
+	if n > uint32(len(b)) {
24
+		b = make([]uint16, n)
25
+		n, err = syscall.GetShortPathName(&p[0], &b[0], uint32(len(b)))
26
+		if err != nil {
27
+			return "", err
28
+		}
29
+	}
30
+	return syscall.UTF16ToString(b), nil
31
+}
32
+
33
+func toLong(path string) (string, error) {
34
+	p, err := syscall.UTF16FromString(path)
35
+	if err != nil {
36
+		return "", err
37
+	}
38
+	b := p // GetLongPathName says we can reuse buffer
39
+	n, err := syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
40
+	if err != nil {
41
+		return "", err
42
+	}
43
+	if n > uint32(len(b)) {
44
+		b = make([]uint16, n)
45
+		n, err = syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
46
+		if err != nil {
47
+			return "", err
48
+		}
49
+	}
50
+	b = b[:n]
51
+	return syscall.UTF16ToString(b), nil
52
+}
53
+
54
+func evalSymlinks(path string) (string, error) {
55
+	path, err := walkSymlinks(path)
56
+	if err != nil {
57
+		return "", err
58
+	}
59
+
60
+	p, err := toShort(path)
61
+	if err != nil {
62
+		return "", err
63
+	}
64
+	p, err = toLong(p)
65
+	if err != nil {
66
+		return "", err
67
+	}
68
+	// syscall.GetLongPathName does not change the case of the drive letter,
69
+	// but the result of EvalSymlinks must be unique, so we have
70
+	// EvalSymlinks(`c:\a`) == EvalSymlinks(`C:\a`).
71
+	// Make drive letter upper case.
72
+	if len(p) >= 2 && p[1] == ':' && 'a' <= p[0] && p[0] <= 'z' {
73
+		p = string(p[0]+'A'-'a') + p[1:]
74
+	} else if len(p) >= 6 && p[5] == ':' && 'a' <= p[4] && p[4] <= 'z' {
75
+		p = p[:3] + string(p[4]+'A'-'a') + p[5:]
76
+	}
77
+	return filepath.Clean(p), nil
78
+}
79
+
80
+const utf8RuneSelf = 0x80
81
+
82
+func walkSymlinks(path string) (string, error) {
83
+	const maxIter = 255
84
+	originalPath := path
85
+	// consume path by taking each frontmost path element,
86
+	// expanding it if it's a symlink, and appending it to b
87
+	var b bytes.Buffer
88
+	for n := 0; path != ""; n++ {
89
+		if n > maxIter {
90
+			return "", errors.New("EvalSymlinks: too many links in " + originalPath)
91
+		}
92
+
93
+		// A path beginnging with `\\?\` represents the root, so automatically
94
+		// skip that part and begin processing the next segment.
95
+		if strings.HasPrefix(path, longpath.Prefix) {
96
+			b.WriteString(longpath.Prefix)
97
+			path = path[4:]
98
+			continue
99
+		}
100
+
101
+		// find next path component, p
102
+		var i = -1
103
+		for j, c := range path {
104
+			if c < utf8RuneSelf && os.IsPathSeparator(uint8(c)) {
105
+				i = j
106
+				break
107
+			}
108
+		}
109
+		var p string
110
+		if i == -1 {
111
+			p, path = path, ""
112
+		} else {
113
+			p, path = path[:i], path[i+1:]
114
+		}
115
+
116
+		if p == "" {
117
+			if b.Len() == 0 {
118
+				// must be absolute path
119
+				b.WriteRune(filepath.Separator)
120
+			}
121
+			continue
122
+		}
123
+
124
+		// If this is the first segment after the long path prefix, accept the
125
+		// current segment as a volume root or UNC share and move on to the next.
126
+		if b.String() == longpath.Prefix {
127
+			b.WriteString(p)
128
+			b.WriteRune(filepath.Separator)
129
+			continue
130
+		}
131
+
132
+		fi, err := os.Lstat(b.String() + p)
133
+		if err != nil {
134
+			return "", err
135
+		}
136
+		if fi.Mode()&os.ModeSymlink == 0 {
137
+			b.WriteString(p)
138
+			if path != "" || (b.Len() == 2 && len(p) == 2 && p[1] == ':') {
139
+				b.WriteRune(filepath.Separator)
140
+			}
141
+			continue
142
+		}
143
+
144
+		// it's a symlink, put it at the front of path
145
+		dest, err := os.Readlink(b.String() + p)
146
+		if err != nil {
147
+			return "", err
148
+		}
149
+		if filepath.IsAbs(dest) || os.IsPathSeparator(dest[0]) {
150
+			b.Reset()
151
+		}
152
+		path = dest + string(filepath.Separator) + path
153
+	}
154
+	return filepath.Clean(b.String()), nil
155
+}
... ...
@@ -200,9 +200,13 @@ func ReplaceOrAppendEnvValues(defaults, overrides []string) []string {
200 200
 // can be read and returns an error if some files can't be read
201 201
 // symlinks which point to non-existing files don't trigger an error
202 202
 func ValidateContextDirectory(srcPath string, excludes []string) error {
203
-	return filepath.Walk(filepath.Join(srcPath, "."), func(filePath string, f os.FileInfo, err error) error {
203
+	contextRoot, err := getContextRoot(srcPath)
204
+	if err != nil {
205
+		return err
206
+	}
207
+	return filepath.Walk(contextRoot, func(filePath string, f os.FileInfo, err error) error {
204 208
 		// skip this directory/file if it's not in the path, it won't get added to the context
205
-		if relFilePath, err := filepath.Rel(srcPath, filePath); err != nil {
209
+		if relFilePath, err := filepath.Rel(contextRoot, filePath); err != nil {
206 210
 			return err
207 211
 		} else if skip, err := fileutils.Matches(relFilePath, excludes); err != nil {
208 212
 			return err
209 213
new file mode 100644
... ...
@@ -0,0 +1,11 @@
0
+// +build !windows
1
+
2
+package utils
3
+
4
+import (
5
+	"path/filepath"
6
+)
7
+
8
+func getContextRoot(srcPath string) (string, error) {
9
+	return filepath.Join(srcPath, "."), nil
10
+}
0 11
new file mode 100644
... ...
@@ -0,0 +1,17 @@
0
+// +build windows
1
+
2
+package utils
3
+
4
+import (
5
+	"path/filepath"
6
+
7
+	"github.com/docker/docker/pkg/longpath"
8
+)
9
+
10
+func getContextRoot(srcPath string) (string, error) {
11
+	cr, err := filepath.Abs(srcPath)
12
+	if err != nil {
13
+		return "", err
14
+	}
15
+	return longpath.AddPrefix(cr), nil
16
+}