Browse code

hack/unit: Rerun failed flaky libnetwork tests

libnetwork tests tend to be flaky (namely `TestNetworkDBIslands` and
`TestNetworkDBCRUDTableEntries`).

Move execution of tests which name has `TestFlaky` prefix to a separate
gotestsum pass which allows them to be reran 4 times.

On Windows, the libnetwork test execution is not split into a separate
pass.

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

Paweł Gronowski authored on 2024/03/12 20:08:20
Showing 2 changed files
... ...
@@ -327,10 +327,26 @@ Function Run-UnitTests() {
327 327
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
328 328
     $pkgList = $pkgList -replace "`r`n", " "
329 329
 
330
+    $jsonFilePath = $bundlesDir + "\go-test-report-unit-flaky-tests.json"
331
+    $xmlFilePath = $bundlesDir + "\junit-report-unit-flaky-tests.xml"
332
+    $coverageFilePath = $bundlesDir + "\coverage-report-unit-flaky-tests.txt"
333
+    $goTestArg = "--rerun-fails=4  --format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath """ + "--packages=$pkgList" + """ -- " + $raceParm + " -coverprofile=$coverageFilePath -covermode=atomic -ldflags -w -a -test.timeout=10m -test.run=TestFlaky.*"
334
+    Write-Host "INFO: Invoking unit tests run with $GOTESTSUM_LOCATION\gotestsum.exe $goTestArg"
335
+    $pinfo = New-Object System.Diagnostics.ProcessStartInfo
336
+    $pinfo.FileName = "$GOTESTSUM_LOCATION\gotestsum.exe"
337
+    $pinfo.WorkingDirectory = "$($PWD.Path)"
338
+    $pinfo.UseShellExecute = $false
339
+    $pinfo.Arguments = $goTestArg
340
+    $p = New-Object System.Diagnostics.Process
341
+    $p.StartInfo = $pinfo
342
+    $p.Start() | Out-Null
343
+    $p.WaitForExit()
344
+    if ($p.ExitCode -ne 0) { Throw "Unit tests (flaky) failed" }
345
+
330 346
     $jsonFilePath = $bundlesDir + "\go-test-report-unit-tests.json"
331 347
     $xmlFilePath = $bundlesDir + "\junit-report-unit-tests.xml"
332 348
     $coverageFilePath = $bundlesDir + "\coverage-report-unit-tests.txt"
333
-    $goTestArg = "--format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- " + $raceParm + " -coverprofile=$coverageFilePath -covermode=atomic -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList"
349
+    $goTestArg = "--format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- " + $raceParm + " -coverprofile=$coverageFilePath -covermode=atomic -ldflags -w -a -test.timeout=10m -test.skip=TestFlaky.*" + " $pkgList"
334 350
     Write-Host "INFO: Invoking unit tests run with $GOTESTSUM_LOCATION\gotestsum.exe $goTestArg"
335 351
     $pinfo = New-Object System.Diagnostics.ProcessStartInfo
336 352
     $pinfo.FileName = "$GOTESTSUM_LOCATION\gotestsum.exe"
... ...
@@ -38,15 +38,38 @@ if [ -n "${base_pkg_list}" ]; then
38 38
 		${base_pkg_list}
39 39
 fi
40 40
 if [ -n "${libnetwork_pkg_list}" ]; then
41
+	rerun_flaky=1
42
+
43
+	gotest_extra_flags="-skip=TestFlaky.*"
44
+	# Custom -run passed, don't run flaky tests separately.
45
+	if echo "$TESTFLAGS" | grep -Eq '(-run|-test.run)[= ]'; then
46
+		rerun_flaky=0
47
+		gotest_extra_flags=""
48
+	fi
49
+
41 50
 	# libnetwork tests invoke iptables, and cannot be run in parallel. Execute
42 51
 	# tests within /libnetwork with '-p=1' to run them sequentially. See
43 52
 	# https://github.com/moby/moby/issues/42458#issuecomment-873216754 for details.
44
-	gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report-libnetwork.json --junitfile=bundles/junit-report-libnetwork.xml -- \
45
-		"${BUILDFLAGS[@]}" \
53
+	gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report-libnetwork.json --junitfile=bundles/junit-report-libnetwork.xml \
54
+		-- "${BUILDFLAGS[@]}" \
46 55
 		-cover \
47 56
 		-coverprofile=bundles/coverage-libnetwork.out \
48 57
 		-covermode=atomic \
49 58
 		-p=1 \
59
+		${gotest_extra_flags} \
50 60
 		${TESTFLAGS} \
51 61
 		${libnetwork_pkg_list}
62
+
63
+	if [ $rerun_flaky -eq 1 ]; then
64
+		gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report-libnetwork-flaky.json --junitfile=bundles/junit-report-libnetwork-flaky.xml \
65
+			--packages "${libnetwork_pkg_list}" \
66
+			--rerun-fails=4 \
67
+			-- "${BUILDFLAGS[@]}" \
68
+			-cover \
69
+			-coverprofile=bundles/coverage-libnetwork-flaky.out \
70
+			-covermode=atomic \
71
+			-p=1 \
72
+			-test.run 'TestFlaky.*' \
73
+			${TESTFLAGS}
74
+	fi
52 75
 fi