Browse code

Merge pull request #9122 from dqminh/debug-huge-expose

Expose a large number of ports should not slow down builder

Arnaud Porterie authored on 2014/12/17 03:03:31
Showing 4 changed files
... ...
@@ -12,6 +12,7 @@ import (
12 12
 	"io/ioutil"
13 13
 	"path/filepath"
14 14
 	"regexp"
15
+	"sort"
15 16
 	"strings"
16 17
 
17 18
 	log "github.com/Sirupsen/logrus"
... ...
@@ -326,14 +327,21 @@ func expose(b *Builder, args []string, attributes map[string]bool, original stri
326 326
 		return err
327 327
 	}
328 328
 
329
+	// instead of using ports directly, we build a list of ports and sort it so
330
+	// the order is consistent. This prevents cache burst where map ordering
331
+	// changes between builds
332
+	portList := make([]string, len(ports))
333
+	var i int
329 334
 	for port := range ports {
330 335
 		if _, exists := b.Config.ExposedPorts[port]; !exists {
331 336
 			b.Config.ExposedPorts[port] = struct{}{}
332 337
 		}
338
+		portList[i] = string(port)
339
+		i++
333 340
 	}
341
+	sort.Strings(portList)
334 342
 	b.Config.PortSpecs = nil
335
-
336
-	return b.commit("", b.Config.Cmd, fmt.Sprintf("EXPOSE %v", ports))
343
+	return b.commit("", b.Config.Cmd, fmt.Sprintf("EXPOSE %s", strings.Join(portList, " ")))
337 344
 }
338 345
 
339 346
 // USER foo
... ...
@@ -212,6 +212,21 @@ func (b *Builder) dispatch(stepN int, ast *parser.Node) error {
212 212
 		msg += " " + ast.Value
213 213
 	}
214 214
 
215
+	// count the number of nodes that we are going to traverse first
216
+	// so we can pre-create the argument and message array. This speeds up the
217
+	// allocation of those list a lot when they have a lot of arguments
218
+	cursor := ast
219
+	var n int
220
+	for cursor.Next != nil {
221
+		cursor = cursor.Next
222
+		n++
223
+	}
224
+	l := len(strs)
225
+	strList := make([]string, n+l)
226
+	copy(strList, strs)
227
+	msgList := make([]string, n)
228
+
229
+	var i int
215 230
 	for ast.Next != nil {
216 231
 		ast = ast.Next
217 232
 		var str string
... ...
@@ -219,16 +234,18 @@ func (b *Builder) dispatch(stepN int, ast *parser.Node) error {
219 219
 		if _, ok := replaceEnvAllowed[cmd]; ok {
220 220
 			str = b.replaceEnv(ast.Value)
221 221
 		}
222
-		strs = append(strs, str)
223
-		msg += " " + ast.Value
222
+		strList[i+l] = str
223
+		msgList[i] = ast.Value
224
+		i++
224 225
 	}
225 226
 
227
+	msg += " " + strings.Join(msgList, " ")
226 228
 	fmt.Fprintln(b.OutStream, msg)
227 229
 
228 230
 	// XXX yes, we skip any cmds that are not valid; the parser should have
229 231
 	// picked these out already.
230 232
 	if f, ok := evaluateTable[cmd]; ok {
231
-		return f(b, strs, attrs, original)
233
+		return f(b, strList, attrs, original)
232 234
 	}
233 235
 
234 236
 	fmt.Fprintf(b.ErrStream, "# Skipping unknown instruction %s\n", strings.ToUpper(cmd))
... ...
@@ -1099,9 +1099,9 @@ func (daemon *Daemon) ImageGetCached(imgID string, config *runconfig.Config) (*i
1099 1099
 	// Loop on the children of the given image and check the config
1100 1100
 	var match *image.Image
1101 1101
 	for elem := range imageMap[imgID] {
1102
-		img, err := daemon.Graph().Get(elem)
1103
-		if err != nil {
1104
-			return nil, err
1102
+		img, ok := images[elem]
1103
+		if !ok {
1104
+			return nil, fmt.Errorf("unable to find image %q", elem)
1105 1105
 		}
1106 1106
 		if runconfig.Compare(&img.ContainerConfig, config) {
1107 1107
 			if match == nil || match.Created.Before(img.Created) {
... ...
@@ -2,6 +2,7 @@ package main
2 2
 
3 3
 import (
4 4
 	"archive/tar"
5
+	"bytes"
5 6
 	"encoding/json"
6 7
 	"fmt"
7 8
 	"io/ioutil"
... ...
@@ -11,9 +12,11 @@ import (
11 11
 	"path/filepath"
12 12
 	"reflect"
13 13
 	"regexp"
14
+	"strconv"
14 15
 	"strings"
15 16
 	"syscall"
16 17
 	"testing"
18
+	"text/template"
17 19
 	"time"
18 20
 
19 21
 	"github.com/docker/docker/pkg/archive"
... ...
@@ -1942,6 +1945,85 @@ func TestBuildExpose(t *testing.T) {
1942 1942
 	logDone("build - expose")
1943 1943
 }
1944 1944
 
1945
+func TestBuildExposeMorePorts(t *testing.T) {
1946
+	// start building docker file with a large number of ports
1947
+	portList := make([]string, 50)
1948
+	line := make([]string, 100)
1949
+	expectedPorts := make([]int, len(portList)*len(line))
1950
+	for i := 0; i < len(portList); i++ {
1951
+		for j := 0; j < len(line); j++ {
1952
+			p := i*len(line) + j + 1
1953
+			line[j] = strconv.Itoa(p)
1954
+			expectedPorts[p-1] = p
1955
+		}
1956
+		if i == len(portList)-1 {
1957
+			portList[i] = strings.Join(line, " ")
1958
+		} else {
1959
+			portList[i] = strings.Join(line, " ") + ` \`
1960
+		}
1961
+	}
1962
+
1963
+	dockerfile := `FROM scratch
1964
+	EXPOSE {{range .}} {{.}}
1965
+	{{end}}`
1966
+	tmpl := template.Must(template.New("dockerfile").Parse(dockerfile))
1967
+	buf := bytes.NewBuffer(nil)
1968
+	tmpl.Execute(buf, portList)
1969
+
1970
+	name := "testbuildexpose"
1971
+	defer deleteImages(name)
1972
+	_, err := buildImage(name, buf.String(), true)
1973
+	if err != nil {
1974
+		t.Fatal(err)
1975
+	}
1976
+
1977
+	// check if all the ports are saved inside Config.ExposedPorts
1978
+	res, err := inspectFieldJSON(name, "Config.ExposedPorts")
1979
+	if err != nil {
1980
+		t.Fatal(err)
1981
+	}
1982
+	var exposedPorts map[string]interface{}
1983
+	if err := json.Unmarshal([]byte(res), &exposedPorts); err != nil {
1984
+		t.Fatal(err)
1985
+	}
1986
+
1987
+	for _, p := range expectedPorts {
1988
+		ep := fmt.Sprintf("%d/tcp", p)
1989
+		if _, ok := exposedPorts[ep]; !ok {
1990
+			t.Errorf("Port(%s) is not exposed", ep)
1991
+		} else {
1992
+			delete(exposedPorts, ep)
1993
+		}
1994
+	}
1995
+	if len(exposedPorts) != 0 {
1996
+		t.Errorf("Unexpected extra exposed ports %v", exposedPorts)
1997
+	}
1998
+	logDone("build - expose large number of ports")
1999
+}
2000
+
2001
+func TestBuildExposeOrder(t *testing.T) {
2002
+	buildID := func(name, exposed string) string {
2003
+		_, err := buildImage(name, fmt.Sprintf(`FROM scratch
2004
+		EXPOSE %s`, exposed), true)
2005
+		if err != nil {
2006
+			t.Fatal(err)
2007
+		}
2008
+		id, err := inspectField(name, "Id")
2009
+		if err != nil {
2010
+			t.Fatal(err)
2011
+		}
2012
+		return id
2013
+	}
2014
+
2015
+	id1 := buildID("testbuildexpose1", "80 2375")
2016
+	id2 := buildID("testbuildexpose2", "2375 80")
2017
+	defer deleteImages("testbuildexpose1", "testbuildexpose2")
2018
+	if id1 != id2 {
2019
+		t.Errorf("EXPOSE should invalidate the cache only when ports actually changed")
2020
+	}
2021
+	logDone("build - expose order")
2022
+}
2023
+
1945 2024
 func TestBuildEmptyEntrypointInheritance(t *testing.T) {
1946 2025
 	name := "testbuildentrypointinheritance"
1947 2026
 	name2 := "testbuildentrypointinheritance2"