Browse code

Merge pull request #6943 from tiborvass/parallel_compile_test

Parallel compile test binaries

Tianon Gravi authored on 2014/07/29 03:32:56
Showing 3 changed files
... ...
@@ -44,6 +44,7 @@ RUN	apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \
44 44
 	lxc=1.0* \
45 45
 	mercurial \
46 46
 	pandoc \
47
+	parallel \
47 48
 	reprepro \
48 49
 	ruby1.9.1 \
49 50
 	ruby1.9.1-dev \
... ...
@@ -42,17 +42,17 @@ echo
42 42
 DEFAULT_BUNDLES=(
43 43
 	validate-dco
44 44
 	validate-gofmt
45
-	
45
+
46 46
 	binary
47
-	
47
+
48 48
 	test-unit
49 49
 	test-integration
50 50
 	test-integration-cli
51
-	
51
+
52 52
 	dynbinary
53 53
 	dyntest-unit
54 54
 	dyntest-integration
55
-	
55
+
56 56
 	cover
57 57
 	cross
58 58
 	tgz
... ...
@@ -156,6 +156,31 @@ go_test_dir() {
156 156
 	)
157 157
 }
158 158
 
159
+# Compile phase run by parallel in test-unit. No support for coverpkg
160
+go_compile_test_dir() {
161
+	dir=$1
162
+	out_file="$DEST/precompiled/$dir.test"
163
+	testcover=()
164
+	if [ "$HAVE_GO_TEST_COVER" ]; then
165
+		# if our current go install has -cover, we want to use it :)
166
+		mkdir -p "$DEST/coverprofiles"
167
+		coverprofile="docker${dir#.}"
168
+		coverprofile="$DEST/coverprofiles/${coverprofile//\//-}"
169
+		testcover=( -cover -coverprofile "$coverprofile" ) # missing $coverpkg
170
+	fi
171
+	if [ "$BUILDFLAGS_FILE" ]; then
172
+		readarray -t BUILDFLAGS < "$BUILDFLAGS_FILE"
173
+	fi
174
+	(
175
+		cd "$dir"
176
+		go test "${testcover[@]}" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS -c
177
+	)
178
+	[ $? -ne 0 ] && return 1
179
+	mkdir -p "$(dirname "$out_file")"
180
+	mv "$dir/$(basename "$dir").test" "$out_file"
181
+	echo "Precompiled: github.com/dotcloud/docker${dir#.}"
182
+}
183
+
159 184
 # This helper function walks the current directory looking for directories
160 185
 # holding certain files ($1 parameter), and prints their paths on standard
161 186
 # output, one per line.
... ...
@@ -2,6 +2,7 @@
2 2
 set -e
3 3
 
4 4
 DEST=$1
5
+: ${PARALLEL_JOBS:=$(nproc)}
5 6
 
6 7
 RED=$'\033[31m'
7 8
 GREEN=$'\033[32m'
... ...
@@ -22,34 +23,62 @@ bundle_test_unit() {
22 22
 			TESTDIRS=$(find_dirs '*_test.go')
23 23
 		fi
24 24
 
25
-		TESTS_FAILED=()
26
-		for test_dir in $TESTDIRS; do
27
-			echo
25
+		if command -v parallel &> /dev/null; then (
26
+			# accomodate parallel to be able to access variables
27
+			export SHELL="$BASH"
28
+			export HOME="$(mktemp -d)"
29
+			mkdir -p "$HOME/.parallel"
30
+			touch "$HOME/.parallel/ignored_vars"
31
+			export -f go_compile_test_dir
32
+			export LDFLAGS="$LDFLAGS $LDFLAGS_STATIC_DOCKER"
33
+			export TESTFLAGS
34
+			export HAVE_GO_TEST_COVER
35
+			export DEST
36
+			# some hack to export array variables
37
+			export BUILDFLAGS_FILE="$HOME/buildflags_file"
38
+			( IFS=$'\n'; echo "${BUILDFLAGS[*]}" ) > "$BUILDFLAGS_FILE"
28 39
 
29
-			if ! LDFLAGS="$LDFLAGS $LDFLAGS_STATIC_DOCKER" go_test_dir "$test_dir"; then
30
-				TESTS_FAILED+=("$test_dir")
31
-				echo
32
-				echo "${RED}Tests failed: $test_dir${TEXTRESET}"
33
-				sleep 1 # give it a second, so observers watching can take note
34
-			fi
35
-		done
40
+			echo "$TESTDIRS" | parallel --jobs "$PARALLEL_JOBS" --halt 2 --env _ go_compile_test_dir
41
+			rm -rf "$HOME"
42
+		) else
43
+			# aww, no "parallel" available - fall back to boring
44
+			for test_dir in $TESTDIRS; do
45
+				go_compile_test_dir "$test_dir"
46
+			done
47
+		fi
48
+		echo "$TESTDIRS" | go_run_test_dir
49
+	}
50
+}
36 51
 
52
+go_run_test_dir() {
53
+	TESTS_FAILED=()
54
+	while read dir; do
37 55
 		echo
38
-		echo
39
-		echo
40
-
41
-		# if some tests fail, we want the bundlescript to fail, but we want to
42
-		# try running ALL the tests first, hence TESTS_FAILED
43
-		if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then
44
-			echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}"
56
+		echo '+ go test' $TESTFLAGS "github.com/dotcloud/docker${dir#.}"
57
+		precompiled="$DEST/precompiled/$dir.test"
58
+		if ! ( cd "$dir" && "$precompiled" ); then
59
+			TESTS_FAILED+=("$dir")
45 60
 			echo
46
-			false
47
-		else
48
-			echo "${GREEN}Test success${TEXTRESET}"
49
-			echo
50
-			true
61
+			echo "${RED}Tests failed: $dir${TEXTRESET}"
62
+			sleep 1 # give it a second, so observers watching can take note
51 63
 		fi
52
-	}
64
+	done
65
+
66
+	echo
67
+	echo
68
+	echo
69
+
70
+	# if some tests fail, we want the bundlescript to fail, but we want to
71
+	# try running ALL the tests first, hence TESTS_FAILED
72
+	if [ "${#TESTS_FAILED[@]}" -gt 0 ]; then
73
+		echo "${RED}Test failures in: ${TESTS_FAILED[@]}${TEXTRESET}"
74
+		echo
75
+		false
76
+	else
77
+		echo "${GREEN}Test success${TEXTRESET}"
78
+		echo
79
+		true
80
+	fi
53 81
 }
54 82
 
55 83
 exec > >(tee -a $DEST/test.log) 2>&1