diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ab1493b6279f..95363bd97a5f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -266,6 +266,7 @@ jobs: hadoop-ozone/dev-support/checks/_summary.sh target/$SCRIPT/summary.txt - name: Archive build results + id: archive if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -274,6 +275,19 @@ jobs: path: target/${{ inputs.script }} continue-on-error: true + - name: Test summary + if: ${{ !cancelled() }} + continue-on-error: true + run: | + if [[ "${QUARANTINE}" == "true" ]]; then + python3 dev-support/ci/junit_summary.py --quarantine >> "$GITHUB_STEP_SUMMARY" + else + python3 dev-support/ci/junit_summary.py >> "$GITHUB_STEP_SUMMARY" + fi + env: + JUNIT_REPORT_URL: ${{ steps.archive.outputs.artifact-url }} + QUARANTINE: ${{ contains(inputs.script-args, 'test-flaky') }} + # The following steps are hard-coded to be run only for 'build' check, # to avoid the need for 3 more inputs. - name: Store binaries for tests diff --git a/dev-support/ci/junit_summary.bats b/dev-support/ci/junit_summary.bats new file mode 100644 index 000000000000..4f6003c6013f --- /dev/null +++ b/dev-support/ci/junit_summary.bats @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +@test "junit_summary.py unit tests" { + cd "${BATS_TEST_DIRNAME}" + run python3 -m unittest test_junit_summary + [ "$status" -eq 0 ] +} diff --git a/dev-support/ci/junit_summary.py b/dev-support/ci/junit_summary.py new file mode 100755 index 000000000000..f8de6fc8cd29 --- /dev/null +++ b/dev-support/ci/junit_summary.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Parse Maven Surefire/Failsafe JUnit XML reports and print a Markdown test summary. + +Adapted from Apache Kafka's .github/scripts/junit.py (summary format), reworked for Maven Surefire XML. + +Intended for GitHub Actions: junit_summary.py [--path DIR] [--quarantine] >> "$GITHUB_STEP_SUMMARY" +Optional env: JUNIT_REPORT_URL (link to the archived test report artifact). +Prints nothing and exits 0 when no reports are found. Always exits 0. +""" + +import argparse +import dataclasses +import html +import os +import sys +import xml.etree.ElementTree as ET +from glob import glob + +PASSED = "PASSED ✅" +FAILED = "FAILED ❌" +FLAKY = "FLAKY ⚠️" +SKIPPED = "SKIPPED 🙈" +QUARANTINED = "QUARANTINED 😷" + +FAIL_TAGS = frozenset(("failure", "error")) +FLAKY_TAGS = frozenset(("flakyFailure", "flakyError")) +MESSAGE_LIMIT = 300 + + +@dataclasses.dataclass +class TestCase: + module: str + class_name: str + test_name: str + time: float + status: str # passed | failed | flaky | skipped + message: str = "" + + +def module_name(xml_path): + # Works for both layouts: /target/surefire-reports/TEST-*.xml (normal run) + # and target///TEST-*.xml (failed tests moved by _mvn_unit_report.sh). + parts = os.path.normpath(xml_path).split(os.sep)[:-1] + while parts and parts[-1] in ("surefire-reports", "failsafe-reports", "target"): + parts.pop() + return parts[-1] if parts else "-" + + +def clean_message(elem): + message = elem.get("message") or (elem.text or "") + message = " ".join(message.split()) + if len(message) > MESSAGE_LIMIT: + message = message[:MESSAGE_LIMIT] + "..." + return message + + +def parse_report(xml_path): + module = module_name(xml_path) + cases = [] + for testcase in ET.parse(xml_path).getroot().iter("testcase"): + failure = next((c for c in testcase if c.tag in FAIL_TAGS), None) + flaky = next((c for c in testcase if c.tag in FLAKY_TAGS), None) + skipped = next((c for c in testcase if c.tag == "skipped"), None) + status, message = "passed", "" + if failure is not None: + status, message = "failed", clean_message(failure) + elif flaky is not None: + status, message = "flaky", clean_message(flaky) + elif skipped is not None: + status = "skipped" + cases.append(TestCase(module, testcase.get("classname") or "", testcase.get("name") or "", + float(testcase.get("time") or 0), status, message)) + return cases + + +def format_time(seconds): + minutes, secs = divmod(int(seconds), 60) + hours, minutes = divmod(minutes, 60) + if hours: + return "%dh%dm%ds" % (hours, minutes, secs) + if minutes: + return "%dm%ds" % (minutes, secs) + return "%ds" % secs + + +def cell(text): + return html.escape(" ".join(str(text).split())).replace("|", "\\|") + + +def render_table(title, header, rows): + lines = ["
%s (%d)" % (title, len(rows)), ""] + lines.append("|" + "|".join(header) + "|") + lines.append("|" + "|".join("---" for _ in header) + "|") + lines.extend("|" + "|".join(cell(value) for value in row) + "|" for row in rows) + lines.extend(["", "
", ""]) + return lines + + +def render_summary(cases, quarantine=False): + def select(status): + return [c for c in cases if c.status == status] + + def full_name(case): + return "%s.%s" % (case.class_name, case.test_name) + + passed, failed, flaky, skipped = select("passed"), select("failed"), select("flaky"), select("skipped") + lines = ["## Test Summary", ""] + # sum of per-test times, not wall clock (parallel forks make wall clock much shorter) + lines.append("%d tests run in %s (total test time): %d %s, %d %s, %d %s, %d %s." % ( + len(cases), format_time(sum(c.time for c in cases)), + len(passed), PASSED, len(failed), FAILED, len(flaky), FLAKY, len(skipped), SKIPPED)) + lines.append("") + report_url = os.environ.get("JUNIT_REPORT_URL") + if report_url: + lines.extend(["[Download test artifacts](%s)" % report_url, ""]) + if failed: + lines.extend(render_table(FAILED, ["Module", "Test", "Message", "Time"], + [[c.module, full_name(c), c.message, format_time(c.time)] for c in failed])) + if flaky: + lines.extend(render_table(FLAKY, ["Module", "Test", "Message", "Time"], + [[c.module, full_name(c), c.message, format_time(c.time)] for c in flaky])) + if skipped: + lines.extend(render_table(SKIPPED, ["Module", "Test"], + [[c.module, full_name(c)] for c in skipped])) + if quarantine: + ran = [c for c in cases if c.status != "skipped"] + lines.extend(render_table(QUARANTINED, ["Module", "Test"], + [[c.module, full_name(c)] for c in ran])) + return "\n".join(lines) + "\n" + + +REPORT_PATTERNS = ( + os.path.join("**", "surefire-reports", "TEST-*.xml"), + os.path.join("**", "failsafe-reports", "TEST-*.xml"), + os.path.join("target", "*", "**", "TEST-*.xml"), +) + + +def find_reports(base): + found = set() + for pattern in REPORT_PATTERNS: + found.update(glob(os.path.join(base, pattern), recursive=True)) + # junit.sh keeps per-iteration copies under target//iterationN/ when ITERATIONS>1; + # exclude them like _mvn_unit_report.sh does, so reruns are not counted multiple times. + return sorted(path for path in found if "/iteration" not in path.replace(os.sep, "/")) + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Print a Markdown summary of JUnit XML test reports.") + parser.add_argument("--path", default=".", help="directory to scan for TEST-*.xml reports") + parser.add_argument("--quarantine", action="store_true", + help="label all tests in this run as quarantined (flaky split)") + try: + args = parser.parse_args(argv) + except SystemExit: + return 0 + # This script only decorates the step summary; it must NEVER fail the check, so exit 0 no matter what. + try: + cases = [] + for report in find_reports(args.path): + try: + cases.extend(parse_report(report)) + except Exception as e: + print("Skipping unreadable report %s: %s" % (report, e), file=sys.stderr) + if cases: + print(render_summary(cases, args.quarantine), end="") + except Exception as e: + print("junit_summary failed: %s" % e, file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev-support/ci/test_junit_summary.py b/dev-support/ci/test_junit_summary.py new file mode 100644 index 000000000000..eeb74be2dc51 --- /dev/null +++ b/dev-support/ci/test_junit_summary.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for junit_summary.py. Run: cd dev-support/ci && python3 -m unittest test_junit_summary -v""" + +import contextlib +import io +import os +import tempfile +import unittest + +import junit_summary + +SAMPLE_XML = """ + + + + stack trace here + + + stack + + + + + +""" + + +class TestParseReport(unittest.TestCase): + + def write_report(self, tmp, relpath, content=SAMPLE_XML): + path = os.path.join(tmp, relpath) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(content) + return path + + def test_classification(self): + with tempfile.TemporaryDirectory() as tmp: + path = self.write_report(tmp, "hadoop-hdds/common/target/surefire-reports/TEST-x.xml") + cases = junit_summary.parse_report(path) + by_name = {c.test_name: c for c in cases} + self.assertEqual(len(cases), 4) + self.assertEqual(by_name["testPasses"].status, "passed") + self.assertEqual(by_name["testFails"].status, "failed") + self.assertEqual(by_name["testFlaky"].status, "flaky") + self.assertEqual(by_name["testSkipped"].status, "skipped") + self.assertEqual(by_name["testFails"].class_name, "org.apache.hadoop.ozone.TestExample") + self.assertAlmostEqual(by_name["testFlaky"].time, 3.0) + + def test_message_is_collapsed_raw_text(self): + with tempfile.TemporaryDirectory() as tmp: + path = self.write_report(tmp, "m/target/surefire-reports/TEST-x.xml") + cases = junit_summary.parse_report(path) + failed = next(c for c in cases if c.status == "failed") + self.assertEqual(failed.message, "expected: <1> but was: |2|") + flaky = next(c for c in cases if c.status == "flaky") + self.assertEqual(flaky.message, "Connection refused") + + def test_module_name_from_path(self): + # normal surefire layout: module is the dir above target/ + self.assertEqual(junit_summary.module_name("hadoop-hdds/common/target/surefire-reports/TEST-a.xml"), "common") + # layout after _mvn_unit_report.sh moves a failed test's XML under target/// + self.assertEqual(junit_summary.module_name("target/unit/hadoop-hdds/common/TEST-a.xml"), "common") + self.assertEqual(junit_summary.module_name("TEST-a.xml"), "-") + + +class TestRender(unittest.TestCase): + + def make_cases(self): + tc = junit_summary.TestCase + return [ + tc("common", "org.X", "ok", 1.0, "passed"), + tc("common", "org.X", "bad", 2.0, "failed", "boom"), + tc("common", "org.X", "shaky", 3.0, "flaky", "flap"), + tc("common", "org.X", "skip", 0.0, "skipped"), + ] + + def test_format_time(self): + self.assertEqual(junit_summary.format_time(59), "59s") + self.assertEqual(junit_summary.format_time(779), "12m59s") + self.assertEqual(junit_summary.format_time(3725), "1h2m5s") + + def test_render_counts_line(self): + md = junit_summary.render_summary(self.make_cases()) + self.assertIn("## Test Summary", md) + self.assertIn("4 tests run in 6s (total test time): 1 PASSED ✅, 1 FAILED ❌, 1 FLAKY ⚠️, 1 SKIPPED 🙈.", md) + + def test_render_tables(self): + md = junit_summary.render_summary(self.make_cases()) + self.assertIn("FAILED ❌ (1)", md) + self.assertIn("|common|org.X.bad|boom|2s|", md) + self.assertIn("FLAKY ⚠️ (1)", md) + self.assertIn("|common|org.X.shaky|flap|3s|", md) + self.assertIn("SKIPPED 🙈 (1)", md) + + def test_render_escapes_every_cell(self): + # parameterized test names can contain pipes and HTML-significant chars + cases = [junit_summary.TestCase("common", "org.X", "test[a|b]", 1.0, "failed", "got |x| ")] + md = junit_summary.render_summary(cases) + self.assertIn("|common|org.X.test[a\\|b]<c>|got \\|x\\| <y>|1s|", md) + + def test_render_report_url(self): + os.environ["JUNIT_REPORT_URL"] = "https://example.com/artifact" + try: + md = junit_summary.render_summary(self.make_cases()) + finally: + del os.environ["JUNIT_REPORT_URL"] + self.assertIn("[Download test artifacts](https://example.com/artifact)", md) + + def test_render_quarantine(self): + md = junit_summary.render_summary(self.make_cases(), quarantine=True) + self.assertIn("QUARANTINED 😷 (3)", md) + + +class TestMain(unittest.TestCase): + + write_report = TestParseReport.write_report + + def run_main(self, argv): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = junit_summary.main(argv) + return code, out.getvalue() + + def test_find_reports_dedupes_both_trees(self): + with tempfile.TemporaryDirectory() as tmp: + self.write_report(tmp, "hadoop-hdds/common/target/surefire-reports/TEST-a.xml") + self.write_report(tmp, "target/unit/surefire-reports/TEST-b.xml") + reports = junit_summary.find_reports(tmp) + self.assertEqual(len(reports), 2) + + def test_find_reports_skips_iteration_dirs(self): + with tempfile.TemporaryDirectory() as tmp: + self.write_report(tmp, "target/unit/TEST-a.xml") + self.write_report(tmp, "target/unit/iteration2/TEST-a.xml") + reports = junit_summary.find_reports(tmp) + self.assertEqual(len(reports), 1) + + def test_main_no_reports_is_silent_noop(self): + with tempfile.TemporaryDirectory() as tmp: + code, out = self.run_main(["--path", tmp]) + self.assertEqual(code, 0) + self.assertEqual(out, "") + + def test_main_prints_summary(self): + with tempfile.TemporaryDirectory() as tmp: + self.write_report(tmp, "m/target/surefire-reports/TEST-a.xml") + code, out = self.run_main(["--path", tmp]) + self.assertEqual(code, 0) + self.assertIn("## Test Summary", out) + self.assertIn("4 tests run", out) + + def test_main_survives_malformed_xml(self): + with tempfile.TemporaryDirectory() as tmp: + self.write_report(tmp, "m/target/surefire-reports/TEST-a.xml") + self.write_report(tmp, "m/target/surefire-reports/TEST-bad.xml", content="