libnetwork tests tend to be flaky (namely `TestNetworkDBIslands` and
`TestNetworkDBCRUDTableEntries`).
Move execution of tests which name has `TestFlaky` prefix to a separate
gotestsum pass which allows them to be reran 4 times.
On Windows, the libnetwork test execution is not split into a separate
pass.
Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
| ... | ... |
@@ -327,10 +327,26 @@ Function Run-UnitTests() {
|
| 327 | 327 |
$pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration" |
| 328 | 328 |
$pkgList = $pkgList -replace "`r`n", " " |
| 329 | 329 |
|
| 330 |
+ $jsonFilePath = $bundlesDir + "\go-test-report-unit-flaky-tests.json" |
|
| 331 |
+ $xmlFilePath = $bundlesDir + "\junit-report-unit-flaky-tests.xml" |
|
| 332 |
+ $coverageFilePath = $bundlesDir + "\coverage-report-unit-flaky-tests.txt" |
|
| 333 |
+ $goTestArg = "--rerun-fails=4 --format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath """ + "--packages=$pkgList" + """ -- " + $raceParm + " -coverprofile=$coverageFilePath -covermode=atomic -ldflags -w -a -test.timeout=10m -test.run=TestFlaky.*" |
|
| 334 |
+ Write-Host "INFO: Invoking unit tests run with $GOTESTSUM_LOCATION\gotestsum.exe $goTestArg" |
|
| 335 |
+ $pinfo = New-Object System.Diagnostics.ProcessStartInfo |
|
| 336 |
+ $pinfo.FileName = "$GOTESTSUM_LOCATION\gotestsum.exe" |
|
| 337 |
+ $pinfo.WorkingDirectory = "$($PWD.Path)" |
|
| 338 |
+ $pinfo.UseShellExecute = $false |
|
| 339 |
+ $pinfo.Arguments = $goTestArg |
|
| 340 |
+ $p = New-Object System.Diagnostics.Process |
|
| 341 |
+ $p.StartInfo = $pinfo |
|
| 342 |
+ $p.Start() | Out-Null |
|
| 343 |
+ $p.WaitForExit() |
|
| 344 |
+ if ($p.ExitCode -ne 0) { Throw "Unit tests (flaky) failed" }
|
|
| 345 |
+ |
|
| 330 | 346 |
$jsonFilePath = $bundlesDir + "\go-test-report-unit-tests.json" |
| 331 | 347 |
$xmlFilePath = $bundlesDir + "\junit-report-unit-tests.xml" |
| 332 | 348 |
$coverageFilePath = $bundlesDir + "\coverage-report-unit-tests.txt" |
| 333 |
- $goTestArg = "--format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- " + $raceParm + " -coverprofile=$coverageFilePath -covermode=atomic -ldflags -w -a """ + "-test.timeout=10m" + """ $pkgList" |
|
| 349 |
+ $goTestArg = "--format=standard-verbose --jsonfile=$jsonFilePath --junitfile=$xmlFilePath -- " + $raceParm + " -coverprofile=$coverageFilePath -covermode=atomic -ldflags -w -a -test.timeout=10m -test.skip=TestFlaky.*" + " $pkgList" |
|
| 334 | 350 |
Write-Host "INFO: Invoking unit tests run with $GOTESTSUM_LOCATION\gotestsum.exe $goTestArg" |
| 335 | 351 |
$pinfo = New-Object System.Diagnostics.ProcessStartInfo |
| 336 | 352 |
$pinfo.FileName = "$GOTESTSUM_LOCATION\gotestsum.exe" |
| ... | ... |
@@ -38,15 +38,38 @@ if [ -n "${base_pkg_list}" ]; then
|
| 38 | 38 |
${base_pkg_list}
|
| 39 | 39 |
fi |
| 40 | 40 |
if [ -n "${libnetwork_pkg_list}" ]; then
|
| 41 |
+ rerun_flaky=1 |
|
| 42 |
+ |
|
| 43 |
+ gotest_extra_flags="-skip=TestFlaky.*" |
|
| 44 |
+ # Custom -run passed, don't run flaky tests separately. |
|
| 45 |
+ if echo "$TESTFLAGS" | grep -Eq '(-run|-test.run)[= ]'; then |
|
| 46 |
+ rerun_flaky=0 |
|
| 47 |
+ gotest_extra_flags="" |
|
| 48 |
+ fi |
|
| 49 |
+ |
|
| 41 | 50 |
# libnetwork tests invoke iptables, and cannot be run in parallel. Execute |
| 42 | 51 |
# tests within /libnetwork with '-p=1' to run them sequentially. See |
| 43 | 52 |
# https://github.com/moby/moby/issues/42458#issuecomment-873216754 for details. |
| 44 |
- gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report-libnetwork.json --junitfile=bundles/junit-report-libnetwork.xml -- \ |
|
| 45 |
- "${BUILDFLAGS[@]}" \
|
|
| 53 |
+ gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report-libnetwork.json --junitfile=bundles/junit-report-libnetwork.xml \ |
|
| 54 |
+ -- "${BUILDFLAGS[@]}" \
|
|
| 46 | 55 |
-cover \ |
| 47 | 56 |
-coverprofile=bundles/coverage-libnetwork.out \ |
| 48 | 57 |
-covermode=atomic \ |
| 49 | 58 |
-p=1 \ |
| 59 |
+ ${gotest_extra_flags} \
|
|
| 50 | 60 |
${TESTFLAGS} \
|
| 51 | 61 |
${libnetwork_pkg_list}
|
| 62 |
+ |
|
| 63 |
+ if [ $rerun_flaky -eq 1 ]; then |
|
| 64 |
+ gotestsum --format=standard-quiet --jsonfile=bundles/go-test-report-libnetwork-flaky.json --junitfile=bundles/junit-report-libnetwork-flaky.xml \ |
|
| 65 |
+ --packages "${libnetwork_pkg_list}" \
|
|
| 66 |
+ --rerun-fails=4 \ |
|
| 67 |
+ -- "${BUILDFLAGS[@]}" \
|
|
| 68 |
+ -cover \ |
|
| 69 |
+ -coverprofile=bundles/coverage-libnetwork-flaky.out \ |
|
| 70 |
+ -covermode=atomic \ |
|
| 71 |
+ -p=1 \ |
|
| 72 |
+ -test.run 'TestFlaky.*' \ |
|
| 73 |
+ ${TESTFLAGS}
|
|
| 74 |
+ fi |
|
| 52 | 75 |
fi |