Browse code

Improvements to the test runners

1. Use `go list` to get list of integration dirs to build. This means we
do not need to have a valid `.go` in every subdirectory and also
filters out other dirs like "bundles" which may have been created.
2. Add option to specify custom flags for integration and
integration-cli. This is needed so both suites can be run AND set
custom flags... since the cli suite does not support standard go
flags.
3. Add options to skip an entire integration suite.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>
(cherry picked from commit abece9b56221bd682f723d5049f7cc047cd522ba)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Brian Goff authored on 2019/07/30 07:00:55
Showing 9 changed files
... ...
@@ -54,9 +54,13 @@ DOCKER_ENVS := \
54 54
 	-e DOCKER_USERLANDPROXY \
55 55
 	-e DOCKERD_ARGS \
56 56
 	-e TEST_INTEGRATION_DIR \
57
+	-e TEST_SKIP_INTEGRATION \
58
+	-e TEST_SKIP_INTEGRATION_CLI \
57 59
 	-e TESTDEBUG \
58 60
 	-e TESTDIRS \
59 61
 	-e TESTFLAGS \
62
+	-e TESTFLAGS_INTEGRATION \
63
+	-e TESTFLAGS_INTEGRATION_CLI \
60 64
 	-e TIMEOUT \
61 65
 	-e VALIDATE_REPO \
62 66
 	-e VALIDATE_BRANCH \
... ...
@@ -181,8 +185,13 @@ test-docker-py: build ## run the docker-py tests
181 181
 
182 182
 test-integration-cli: test-integration ## (DEPRECATED) use test-integration
183 183
 
184
+ifneq ($(and $(TEST_SKIP_INTEGRATION),$(TEST_SKIP_INTEGRATION_CLI)),)
185
+test-integration:
186
+	@echo Both integrations suites skipped per environment variables
187
+else
184 188
 test-integration: build ## run the integration tests
185 189
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
190
+endif
186 191
 
187 192
 test-integration-flaky: build ## run the stress test for all new integration tests
188 193
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky
... ...
@@ -327,19 +327,16 @@ Function Run-UnitTests() {
327 327
 # Run the integration tests
328 328
 Function Run-IntegrationTests() {
329 329
     $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
330
-    $dirs =  Get-ChildItem -Path integration -Directory -Recurse
330
+    $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
331 331
     $integration_api_dirs = @()
332 332
     ForEach($dir in $dirs) {
333
-        $RelativePath = "." + $dir.FullName -replace "$($PWD.Path -replace "\\","\\")",""
334
-        If ($RelativePath -notmatch '(^.\\integration($|\\internal)|\\testdata)') {
335
-            $integration_api_dirs += $dir
336
-            Write-Host "Building test suite binary $RelativePath"
337
-            go test -c -o "$RelativePath\test.exe" $RelativePath
338
-        }
333
+        $integration_api_dirs += $dir
334
+        Write-Host "Building test suite binary $dir"
335
+        go test -c -o "$dir\test.exe" $dir
339 336
     }
340 337
 
341 338
     ForEach($dir in $integration_api_dirs) {
342
-        Set-Location $dir.FullName
339
+        Set-Location $dir
343 340
         Write-Host "Running $($PWD.Path)"
344 341
         $pinfo = New-Object System.Diagnostics.ProcessStartInfo
345 342
         $pinfo.FileName = "$($PWD.Path)\test.exe"
... ...
@@ -5,6 +5,18 @@
5 5
 #
6 6
 #     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
7 7
 #
8
+
9
+if [[ "${TESTFLAGS}" = *-check.f* ]]; then
10
+	echo Skipping integration tests since TESTFLAGS includes integration-cli only flags
11
+	TEST_SKIP_INTEGRATION=1
12
+fi
13
+
14
+if [[ "${TESTFLAGS}" = *-test.run* ]]; then
15
+	echo Skipping integration-cli tests since TESTFLAGS includes integration only flags
16
+	TEST_SKIP_INTEGRATION_CLI=1
17
+fi
18
+
19
+
8 20
 if [ -z ${MAKEDIR} ]; then
9 21
 	export MAKEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
10 22
 fi
... ...
@@ -15,26 +27,24 @@ source "$MAKEDIR/.go-autogen"
15 15
 : ${TESTFLAGS:=}
16 16
 : ${TESTDEBUG:=}
17 17
 
18
-integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(
19
-	find ./integration -type d |
20
-	grep -vE '(^./integration($|/internal)|/testdata)')"}
18
+integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(go list  -test -f '{{- if ne .ForTest "" -}}{{- .Dir -}}{{- end -}}'  ./integration/...)"}
21 19
 
22 20
 run_test_integration() {
23 21
 	set_platform_timeout
24
-	if [[ "$TESTFLAGS" != *-check.f* ]]; then
22
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
25 23
 		run_test_integration_suites
26 24
 	fi
27
-	if [[ "$TESTFLAGS" != *-test.run* ]]; then
25
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
28 26
 		run_test_integration_legacy_suites
29 27
 	fi
30 28
 }
31 29
 
32 30
 run_test_integration_suites() {
33
-	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
31
+	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS ${TESTFLAGS_INTEGRATION}"
34 32
 	for dir in ${integration_api_dirs}; do
35 33
 		if ! (
36 34
 			cd "$dir"
37
-			echo "Running $PWD"
35
+			echo "Running $PWD flags=${flags}"
38 36
 			test_env ./test.main ${flags}
39 37
 		); then exit 1; fi
40 38
 	done
... ...
@@ -42,9 +52,9 @@ run_test_integration_suites() {
42 42
 
43 43
 run_test_integration_legacy_suites() {
44 44
 	(
45
-		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
45
+		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS ${TESTFLAGS_INTEGRATION_CLI}"
46 46
 		cd integration-cli
47
-		echo "Running $PWD"
47
+		echo "Running $PWD flags=${flags}"
48 48
 		test_env ./test.main $flags
49 49
 	)
50 50
 }
... ...
@@ -54,10 +64,14 @@ build_test_suite_binaries() {
54 54
 		echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set"
55 55
 		return
56 56
 	fi
57
-	build_test_suite_binary ./integration-cli "test.main"
58
-	for dir in ${integration_api_dirs}; do
59
-		build_test_suite_binary "$dir" "test.main"
60
-	done
57
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
58
+		build_test_suite_binary ./integration-cli "test.main"
59
+	fi
60
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
61
+		for dir in ${integration_api_dirs}; do
62
+			build_test_suite_binary "$dir" "test.main"
63
+		done
64
+	fi
61 65
 }
62 66
 
63 67
 # Build a binary for a test suite package
... ...
@@ -3,6 +3,12 @@ set -e -o pipefail
3 3
 
4 4
 source hack/make/.integration-test-helpers
5 5
 
6
+if [ ! -z "${TEST_SKIP_INTEGRATION}" ] && [ ! -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
7
+	echo integration and integraiton-cli skipped according to env vars
8
+	exit 0
9
+fi
10
+
11
+
6 12
 (
7 13
 	build_test_suite_binaries
8 14
 	bundle .integration-daemon-start
9 15
deleted file mode 100644
... ...
@@ -1 +0,0 @@
1
-package main
2 1
deleted file mode 100644
... ...
@@ -1 +0,0 @@
1
-package cmd
2 1
deleted file mode 100644
... ...
@@ -1 +0,0 @@
1
-package main
2 1
deleted file mode 100644
... ...
@@ -1 +0,0 @@
1
-package cmd
2 1
deleted file mode 100644
... ...
@@ -1 +0,0 @@
1
-package main