Browse code

Add a bunch more tweaks to the parallel test compilation

- put all the precompiled test binaries in $DEST so they show up in bundles and can be re-run individually afterwards
- support cases where parallel is not installed (when using dyntest-unit, for example, this is much more common, since it's designed to be run outside the Dockerfile)
- use "mktemp -d" instead of "/tmp" directly for our temporary parallel HOME
- update the default PARALLEL_JOBS to be the value of "nproc" instead of 0, since "0 means as many as possible" (see https://www.gnu.org/software/parallel/man.html#jobs_n)

Docker-DCO-1.1-Signed-off-by: Andrew Page <admwiggin@gmail.com> (github: tianon)

Tianon Gravi authored on 2014/07/04 06:11:32
Showing 2 changed files
... ...
@@ -159,6 +159,7 @@ go_test_dir() {
159 159
 # Compile phase run by parallel in test-unit. No support for coverpkg
160 160
 go_compile_test_dir() {
161 161
 	dir=$1
162
+	out_file="$DEST/precompiled/$dir.test"
162 163
 	testcover=()
163 164
 	if [ "$HAVE_GO_TEST_COVER" ]; then
164 165
 		# if our current go install has -cover, we want to use it :)
... ...
@@ -167,12 +168,16 @@ go_compile_test_dir() {
167 167
 		coverprofile="$DEST/coverprofiles/${coverprofile//\//-}"
168 168
 		testcover=( -cover -coverprofile "$coverprofile" ) # missing $coverpkg
169 169
 	fi
170
-	(
170
+	if [ "$BUILDFLAGS_FILE" ]; then
171 171
 		readarray -t BUILDFLAGS < "$BUILDFLAGS_FILE"
172
+	fi
173
+	(
172 174
 		cd "$dir"
173 175
 		go test "${testcover[@]}" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS -c
174
-		echo "$dir"
175 176
 	)
177
+	mkdir -p "$(dirname "$out_file")"
178
+	mv "$dir/$(basename "$dir").test" "$out_file"
179
+	echo "Precompiled: github.com/dotcloud/docker${dir#.}"
176 180
 }
177 181
 
178 182
 # This helper function walks the current directory looking for directories
... ...
@@ -2,11 +2,11 @@
2 2
 set -e
3 3
 
4 4
 DEST=$1
5
+: ${PARALLEL_JOBS:=$(nproc)}
5 6
 
6 7
 RED=$'\033[31m'
7 8
 GREEN=$'\033[32m'
8 9
 TEXTRESET=$'\033[0m' # reset the foreground colour
9
-: ${PARALLEL_JOBS:=0}
10 10
 
11 11
 # Run Docker's test suite, including sub-packages, and store their output as a bundle
12 12
 # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
... ...
@@ -23,10 +23,10 @@ bundle_test_unit() {
23 23
 			TESTDIRS=$(find_dirs '*_test.go')
24 24
 		fi
25 25
 
26
-		(
26
+		if command -v parallel &> /dev/null; then (
27 27
 			# accomodate parallel to be able to access variables
28
-			export SHELL=/bin/bash 
29
-			export HOME=/tmp
28
+			export SHELL="$BASH"
29
+			export HOME="$(mktemp -d)"
30 30
 			mkdir -p "$HOME/.parallel"
31 31
 			touch "$HOME/.parallel/ignored_vars"
32 32
 			export -f go_compile_test_dir
... ...
@@ -38,8 +38,15 @@ bundle_test_unit() {
38 38
 			export BUILDFLAGS_FILE="$HOME/buildflags_file"
39 39
 			( IFS=$'\n'; echo "${BUILDFLAGS[*]}" ) > "$BUILDFLAGS_FILE"
40 40
 
41
-			echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --env _ go_compile_test_dir | go_run_test_dir
42
-		)
41
+			echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --env _ go_compile_test_dir
42
+			rm -rf "$HOME"
43
+		) else
44
+			# aww, no "parallel" available - fall back to boring
45
+			for test_dir in $TESTDIRS; do
46
+				go_compile_test_dir "$test_dir"
47
+			done
48
+		fi
49
+		echo "$TESTDIRS" | go_run_test_dir
43 50
 	}
44 51
 }
45 52
 
... ...
@@ -48,16 +55,13 @@ go_run_test_dir() {
48 48
 	while read dir; do
49 49
 		echo
50 50
 		echo '+ go test' $TESTFLAGS "github.com/dotcloud/docker${dir#.}"
51
-		pushd "$dir" > /dev/null
52
-		t=$(basename "$dir").test
53
-		if ! ./$t; then
51
+		precompiled="$DEST/precompiled/$dir.test"
52
+		if ! "$precompiled"; then
54 53
 			TESTS_FAILED+=("$dir")
55 54
 			echo
56 55
 			echo "${RED}Tests failed: $dir${TEXTRESET}"
57 56
 			sleep 1 # give it a second, so observers watching can take note
58 57
 		fi
59
-		rm ./$t || true
60
-		popd > /dev/null
61 58
 	done
62 59
 
63 60
 	echo