From f60311a4501a570cb3c61811d0cf5e0db0efe1d0 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Wed, 20 May 2026 11:01:16 +0200 Subject: [PATCH 1/2] fix: stop tagging "test exception" testcases as synthetic skip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "test exception" is not a synthetic placeholder emitted by any test framework — JUnit 4, JUnit Platform, Gradle's adapters, Spock and Kotest all reserve "initializationError"/"executionError" for that purpose, not generic English names. In dd-trace-java the entries observed in CI came from a real Spock feature method (`def "test exception"()` in HttpServerTest.groovy), not a framework synthetic. Tagging that name unconditionally silently masks real pass/fail outcomes for any Spock/JUnit 5 @DisplayName/Kotest test that happens to use it. Drop "test exception" from JUnitReport.tagSyntheticFailures and document the criteria for future synthetic entries (must be framework-emitted, must link to the source pinned to a release tag). Annotate the two remaining entries — initializationError (JUnit 4 + Gradle) and executionError (Gradle) — with the canonical source locations. --- .gitlab/collect-result/JUnitReport.java | 41 ++++++++++++++++++++----- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/.gitlab/collect-result/JUnitReport.java b/.gitlab/collect-result/JUnitReport.java index 0c74f4c934b..adbfcba25fb 100644 --- a/.gitlab/collect-result/JUnitReport.java +++ b/.gitlab/collect-result/JUnitReport.java @@ -66,16 +66,43 @@ boolean normalizeStableTestNames() { return changed; } + /// Tags framework-emitted synthetic testcases so Test Optimization does not treat them as + /// real failures. + /// + /// **Criteria for new entries:** + /// - Must be a name the framework/runner emits itself — never a name a user-authored test + /// could legitimately have. + /// - Must link to the source that emits the literal, pinned to a release tag (not a commit + /// hash). + /// + /// **Frameworks/libraries to audit before adding a name:** JUnit 4, JUnit Platform engines + /// (Vintage / Jupiter), Gradle's JUnit and JUnit Platform adapters, TestNG, Spock, Kotest, + /// Maven Surefire/Failsafe. JUnit 5's main sources do not define these literals — Vintage + /// forwards `"initializationError"` from JUnit 4 as-is, and `"executionError"` is + /// Gradle-specific. + /// + /// **Do not add** generic English names such as `"test exception"`. Spock feature methods + /// (`def "..."()`), JUnit 5 `@DisplayName`, and Kotest specs can all produce them, so the + /// tagger would silently mask real pass/fail outcomes. void tagSyntheticFailures() { Map> initializationErrorsByClassname = new LinkedHashMap<>(); for (var testcase : testcases()) { - var name = testcase.getAttribute("name"); - if ("initializationError".equals(name)) { - initializationErrorsByClassname - .computeIfAbsent(testcase.getAttribute("classname"), ignored -> new ArrayList<>()) - .add(testcase); - } else if ("executionError".equals(name) || "test exception".equals(name)) { - addFinalStatusProperty(testcase, "skip", MissingPropertiesPlacement.APPEND_TO_TESTCASE); + switch (testcase.getAttribute("name")) { + // JUnit 4 ErrorReportingRunner — initializationError + // https://github.com/junit-team/junit4/blob/r4.13.2/src/main/java/org/junit/internal/runners/ErrorReportingRunner.java#L83 + // Gradle JUnit Platform listener — reuses the same name for synthetic container failures + // https://github.com/gradle/gradle/blob/v8.14.5/platforms/jvm/testing-junit-platform/src/main/java/org/gradle/api/internal/tasks/testing/junitplatform/JUnitPlatformTestExecutionListener.java#L248 + // https://github.com/gradle/gradle/blob/v9.5.0/platforms/jvm/testing-jvm-infrastructure/src/main/java/org/gradle/api/internal/tasks/testing/junitplatform/JUnitPlatformTestExecutionListener.java#L426 + case "initializationError" -> + initializationErrorsByClassname + .computeIfAbsent(testcase.getAttribute("classname"), ignored -> new ArrayList<>()) + .add(testcase); + // Gradle JUnit Platform listener — executionError, used when descendants already started + // https://github.com/gradle/gradle/blob/v8.14.5/platforms/jvm/testing-junit-platform/src/main/java/org/gradle/api/internal/tasks/testing/junitplatform/JUnitPlatformTestExecutionListener.java#L248 + // https://github.com/gradle/gradle/blob/v9.5.0/platforms/jvm/testing-jvm-infrastructure/src/main/java/org/gradle/api/internal/tasks/testing/junitplatform/JUnitPlatformTestExecutionListener.java#L426 + case "executionError" -> + addFinalStatusProperty(testcase, "skip", MissingPropertiesPlacement.APPEND_TO_TESTCASE); + default -> {} } } From bc8f0c92c40cb96a61ca3b06dae301f447a7e2cd Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 21 May 2026 09:17:38 +0200 Subject: [PATCH 2/2] Update JUnitReport.java Co-authored-by: Sarah Chen --- .gitlab/collect-result/JUnitReport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/collect-result/JUnitReport.java b/.gitlab/collect-result/JUnitReport.java index adbfcba25fb..b9f8dad568b 100644 --- a/.gitlab/collect-result/JUnitReport.java +++ b/.gitlab/collect-result/JUnitReport.java @@ -70,7 +70,7 @@ boolean normalizeStableTestNames() { /// real failures. /// /// **Criteria for new entries:** - /// - Must be a name the framework/runner emits itself — never a name a user-authored test + /// - Must be a name the framework/runner emits itself — never use a name that a user-authored test /// could legitimately have. /// - Must link to the source that emits the literal, pinned to a release tag (not a commit /// hash).