Jenkins JUnit does not handle skips in different files that have the
same name, leading to invalid results. Merge all the JUnit output files
into one by suite and test name. Failures override Successes, both
override Skips.
... | ... |
@@ -29,4 +29,6 @@ TEST_PARALLEL="${PARALLEL_NODES:-5}" FOCUS="${pf}" SKIP="${ps}" TEST_REPORT_FILE |
29 | 29 |
os::log::info "Running serial tests" |
30 | 30 |
FOCUS="${sf}" SKIP="${ss}" TEST_REPORT_FILE_NAME=conformance_serial os::test::extended::run -- -ginkgo.noColor -ginkgo.v -test.timeout 2h ${TEST_EXTENDED_ARGS-} || exitstatus=$? |
31 | 31 |
|
32 |
+os::test::extended::merge_junit |
|
33 |
+ |
|
32 | 34 |
exit $exitstatus |
... | ... |
@@ -34,4 +34,6 @@ os::log::info "" |
34 | 34 |
os::log::info "Running serial tests" |
35 | 35 |
FOCUS="${sf}" SKIP="${ss}" TEST_REPORT_FILE_NAME=core_serial os::test::extended::run -- -ginkgo.noColor -ginkgo.v -test.timeout 2h ${TEST_EXTENDED_ARGS-} || exitstatus=$? |
36 | 36 |
|
37 |
+os::test::extended::merge_junit |
|
38 |
+ |
|
37 | 39 |
exit $exitstatus |
... | ... |
@@ -3,7 +3,7 @@ |
3 | 3 |
# This abstracts starting up an extended server. |
4 | 4 |
|
5 | 5 |
# If invoked with arguments, executes the test directly. |
6 |
-function os::test::extended::focus { |
|
6 |
+function os::test::extended::focus () { |
|
7 | 7 |
if [[ $# -ne 0 ]]; then |
8 | 8 |
os::log::info "Running custom: $*" |
9 | 9 |
os::test::extended::test_list "$@" |
... | ... |
@@ -27,6 +27,7 @@ function os::test::extended::setup () { |
27 | 27 |
os::util::ensure::built_binary_exists 'openshift' |
28 | 28 |
os::util::ensure::built_binary_exists 'oadm' |
29 | 29 |
os::util::ensure::built_binary_exists 'oc' |
30 |
+ os::util::ensure::built_binary_exists 'junitmerge' 'tools/junitmerge' |
|
30 | 31 |
|
31 | 32 |
# ensure proper relative directories are set |
32 | 33 |
export EXTENDED_TEST_PATH="${OS_ROOT}/test/extended" |
... | ... |
@@ -244,6 +245,18 @@ function os::test::extended::test_list () { |
244 | 244 |
} |
245 | 245 |
readonly -f os::test::extended::test_list |
246 | 246 |
|
247 |
+# Merge all of the JUnit output files in the TEST_REPORT_DIR into a single file. |
|
248 |
+# This works around a gap in Jenkins JUnit reporter output that double counts skipped |
|
249 |
+# files until https://github.com/jenkinsci/junit-plugin/pull/54 is merged. |
|
250 |
+function os::test::extended::merge_junit () { |
|
251 |
+ local output |
|
252 |
+ output="$( mktemp )" |
|
253 |
+ "$( os::util::find::built_binary junitmerge )" "${TEST_REPORT_DIR}"/*.xml > "${output}" |
|
254 |
+ rm "${TEST_REPORT_DIR}"/*.xml |
|
255 |
+ mv "${output}" "${TEST_REPORT_DIR}/junit.xml" |
|
256 |
+} |
|
257 |
+readonly -f os::test::extended::merge_junit |
|
258 |
+ |
|
247 | 259 |
# Not run by any suite |
248 | 260 |
readonly EXCLUDED_TESTS=( |
249 | 261 |
"\[Skipped\]" |
250 | 262 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,155 @@ |
0 |
+package main |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "encoding/xml" |
|
4 |
+ "log" |
|
5 |
+ "os" |
|
6 |
+ |
|
7 |
+ "fmt" |
|
8 |
+ "github.com/openshift/origin/tools/junitreport/pkg/api" |
|
9 |
+ "sort" |
|
10 |
+) |
|
11 |
+ |
|
12 |
+type uniqueSuites map[string]*suiteRuns |
|
13 |
+ |
|
14 |
+func (s uniqueSuites) Merge(namePrefix string, suite *api.TestSuite) { |
|
15 |
+ name := suite.Name |
|
16 |
+ if len(namePrefix) > 0 { |
|
17 |
+ name = namePrefix + "/" |
|
18 |
+ } |
|
19 |
+ existing, ok := s[name] |
|
20 |
+ if !ok { |
|
21 |
+ existing = newSuiteRuns(suite) |
|
22 |
+ s[name] = existing |
|
23 |
+ } |
|
24 |
+ |
|
25 |
+ existing.Merge(suite.TestCases) |
|
26 |
+ |
|
27 |
+ for _, suite := range suite.Children { |
|
28 |
+ s.Merge(name, suite) |
|
29 |
+ } |
|
30 |
+} |
|
31 |
+ |
|
32 |
+type suiteRuns struct { |
|
33 |
+ suite *api.TestSuite |
|
34 |
+ runs map[string]*api.TestCase |
|
35 |
+} |
|
36 |
+ |
|
37 |
+func newSuiteRuns(suite *api.TestSuite) *suiteRuns { |
|
38 |
+ return &suiteRuns{ |
|
39 |
+ suite: suite, |
|
40 |
+ runs: make(map[string]*api.TestCase), |
|
41 |
+ } |
|
42 |
+} |
|
43 |
+ |
|
44 |
+func (r *suiteRuns) Merge(testCases []*api.TestCase) { |
|
45 |
+ for _, testCase := range testCases { |
|
46 |
+ existing, ok := r.runs[testCase.Name] |
|
47 |
+ if !ok { |
|
48 |
+ r.runs[testCase.Name] = testCase |
|
49 |
+ continue |
|
50 |
+ } |
|
51 |
+ switch { |
|
52 |
+ case testCase.SkipMessage != nil: |
|
53 |
+ // if the new test is a skip, ignore it |
|
54 |
+ case existing.SkipMessage != nil && testCase.SkipMessage == nil: |
|
55 |
+ // always replace a skip with a non-skip |
|
56 |
+ r.runs[testCase.Name] = testCase |
|
57 |
+ case existing.FailureOutput == nil && testCase.FailureOutput != nil: |
|
58 |
+ // replace a passing test with a failing test |
|
59 |
+ r.runs[testCase.Name] = testCase |
|
60 |
+ } |
|
61 |
+ } |
|
62 |
+} |
|
63 |
+ |
|
64 |
+func main() { |
|
65 |
+ log.SetFlags(0) |
|
66 |
+ suites := make(uniqueSuites) |
|
67 |
+ |
|
68 |
+ for _, arg := range os.Args[1:] { |
|
69 |
+ f, err := os.Open(arg) |
|
70 |
+ if err != nil { |
|
71 |
+ log.Fatal(err) |
|
72 |
+ } |
|
73 |
+ defer f.Close() |
|
74 |
+ d := xml.NewDecoder(f) |
|
75 |
+ |
|
76 |
+ for { |
|
77 |
+ t, err := d.Token() |
|
78 |
+ if err != nil { |
|
79 |
+ log.Fatal(err) |
|
80 |
+ } |
|
81 |
+ if t == nil { |
|
82 |
+ log.Fatalf("input file %s does not appear to be a JUnit XML file", arg) |
|
83 |
+ } |
|
84 |
+ // Inspect the top level DOM element and perform the appropriate action |
|
85 |
+ switch se := t.(type) { |
|
86 |
+ case xml.StartElement: |
|
87 |
+ switch se.Name.Local { |
|
88 |
+ case "testsuites": |
|
89 |
+ input := &api.TestSuites{} |
|
90 |
+ if err := d.DecodeElement(input, &se); err != nil { |
|
91 |
+ log.Fatal(err) |
|
92 |
+ } |
|
93 |
+ for _, suite := range input.Suites { |
|
94 |
+ suites.Merge("", suite) |
|
95 |
+ } |
|
96 |
+ case "testsuite": |
|
97 |
+ input := &api.TestSuite{} |
|
98 |
+ if err := d.DecodeElement(input, &se); err != nil { |
|
99 |
+ log.Fatal(err) |
|
100 |
+ } |
|
101 |
+ suites.Merge("", input) |
|
102 |
+ default: |
|
103 |
+ log.Fatal(fmt.Errorf("unexpected top level element in %s: %s", arg, se.Name.Local)) |
|
104 |
+ } |
|
105 |
+ default: |
|
106 |
+ continue |
|
107 |
+ } |
|
108 |
+ break |
|
109 |
+ } |
|
110 |
+ } |
|
111 |
+ |
|
112 |
+ var suiteNames []string |
|
113 |
+ for k := range suites { |
|
114 |
+ suiteNames = append(suiteNames, k) |
|
115 |
+ } |
|
116 |
+ sort.Sort(sort.StringSlice(suiteNames)) |
|
117 |
+ output := &api.TestSuites{} |
|
118 |
+ |
|
119 |
+ for _, name := range suiteNames { |
|
120 |
+ suite := suites[name] |
|
121 |
+ |
|
122 |
+ out := &api.TestSuite{ |
|
123 |
+ Name: name, |
|
124 |
+ NumTests: uint(len(suite.runs)), |
|
125 |
+ } |
|
126 |
+ |
|
127 |
+ var keys []string |
|
128 |
+ for k := range suite.runs { |
|
129 |
+ keys = append(keys, k) |
|
130 |
+ } |
|
131 |
+ sort.Sort(sort.StringSlice(keys)) |
|
132 |
+ |
|
133 |
+ for _, k := range keys { |
|
134 |
+ testCase := suite.runs[k] |
|
135 |
+ out.TestCases = append(out.TestCases, testCase) |
|
136 |
+ switch { |
|
137 |
+ case testCase.SkipMessage != nil: |
|
138 |
+ out.NumSkipped++ |
|
139 |
+ case testCase.FailureOutput != nil: |
|
140 |
+ out.NumFailed++ |
|
141 |
+ } |
|
142 |
+ out.Duration += testCase.Duration |
|
143 |
+ } |
|
144 |
+ output.Suites = append(output.Suites, out) |
|
145 |
+ } |
|
146 |
+ |
|
147 |
+ e := xml.NewEncoder(os.Stdout) |
|
148 |
+ e.Indent("", "\t") |
|
149 |
+ if err := e.Encode(output); err != nil { |
|
150 |
+ log.Fatal(err) |
|
151 |
+ } |
|
152 |
+ e.Flush() |
|
153 |
+ fmt.Fprintln(os.Stdout) |
|
154 |
+} |
... | ... |
@@ -60,6 +60,9 @@ type TestCase struct { |
60 | 60 |
// Name is the name of the test case |
61 | 61 |
Name string `xml:"name,attr"` |
62 | 62 |
|
63 |
+ // Classname is an attribute set by the package type and is required |
|
64 |
+ Classname string `xml:"classname,attr,omitempty"` |
|
65 |
+ |
|
63 | 66 |
// Duration is the time taken in seconds to run the test |
64 | 67 |
Duration float64 `xml:"time,attr"` |
65 | 68 |
|
... | ... |
@@ -68,6 +71,12 @@ type TestCase struct { |
68 | 68 |
|
69 | 69 |
// FailureOutput holds the output from a failing test |
70 | 70 |
FailureOutput *FailureOutput `xml:"failure"` |
71 |
+ |
|
72 |
+ // SystemOut is output written to stdout during the execution of this test case |
|
73 |
+ SystemOut string `xml:"system-out,omitempty"` |
|
74 |
+ |
|
75 |
+ // SystemErr is output written to stderr during the execution of this test case |
|
76 |
+ SystemErr string `xml:"system-err,omitempty"` |
|
71 | 77 |
} |
72 | 78 |
|
73 | 79 |
// SkipMessage holds a message explaining why a test was skipped |
... | ... |
@@ -75,7 +84,7 @@ type SkipMessage struct { |
75 | 75 |
XMLName xml.Name `xml:"skipped"` |
76 | 76 |
|
77 | 77 |
// Message explains why the test was skipped |
78 |
- Message string `xml:"message,attr"` |
|
78 |
+ Message string `xml:"message,attr,omitempty"` |
|
79 | 79 |
} |
80 | 80 |
|
81 | 81 |
// FailureOutput holds the output from a failing test |