diff --git a/.gitlab/TagInitializationErrors.java b/.gitlab/TagInitializationErrors.java index 3fdf8357a74..ad6edf267f6 100644 --- a/.gitlab/TagInitializationErrors.java +++ b/.gitlab/TagInitializationErrors.java @@ -1,44 +1,51 @@ -import org.w3c.dom.Element; -import javax.xml.XMLConstants; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; import java.io.File; import java.nio.file.Files; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.w3c.dom.Document; +import org.w3c.dom.Element; -/// Tags intermediate `initializationError` retries with `dd_tags[test.final_status]=skip`. +/// Tags synthetic testcases (`initializationError`, `executionError`, `test exception`) with +/// `dd_tags[test.final_status]=skip` so Test Optimization does not treat them as real failures. +/// The script is idempotent — testcases that already carry a `dd_tags[test.final_status]` property +/// are left unchanged. +/// +/// **`initializationError`** — Gradle generates these for setup methods. When retried and +// eventually +/// successful, multiple testcases appear; only the last one passes. All intermediate attempts are +/// tagged skip. Groups with only one (or zero) `initializationError` entries per classname are left +// unmodified. /// -/// Gradle generates synthetic "initializationError" testcases in JUnit reports for setup methods. -/// When a setup is retried and eventually succeeds, multiple testcases are created, with only the -/// last one passing. All intermediate attempts are marked skip so Test Optimization is not misled. +/// **`executionError`** and **`test exception`** — Framework-level synthetic failures that do not +/// represent real test results. Tagged skip unconditionally so Test Optimization treats them as +// non-failures. /// -/// For any suite with multiple `initializationError` test cases (when retries occurred), all entries -/// but the last one are tagged by this script with `dd_tags[test.final_status]=skip`. The last -/// entry is left unmodified, allowing **Test Optimization** to apply its default status inference based -/// on the actual outcome. Files with only one (or zero) `initializationError` test cases are left unmodified. +/// Before (two retries of the same class — first is intermediate, second is the final outcome): /// -/// Before: -/// /// ``` -/// +/// +/// /// ``` -/// -/// After: -/// +/// +/// After (only the intermediate attempt is tagged; the last entry is left untouched): +/// /// ``` -/// +/// /// -/// +/// /// /// +/// /// ``` -/// +/// /// Usage (Java 25): `java TagInitializationErrors.java junit-report.xml` class TagInitializationErrors { @@ -61,54 +68,73 @@ public static void main(String[] args) throws Exception { var doc = dbf.newDocumentBuilder().parse(xmlFile); var testcases = doc.getElementsByTagName("testcase"); Map> byClassname = new LinkedHashMap<>(); + boolean modified = false; for (int i = 0; i < testcases.getLength(); i++) { var e = (Element) testcases.item(i); - if ("initializationError".equals(e.getAttribute("name"))) { + var name = e.getAttribute("name"); + if ("initializationError".equals(name)) { byClassname.computeIfAbsent(e.getAttribute("classname"), k -> new ArrayList<>()).add(e); + } else if ("executionError".equals(name) || "test exception".equals(name)) { + if (tagSkip(doc, e)) modified = true; } } - boolean modified = false; for (var group : byClassname.values()) { - if (group.size() <= 1) continue; for (int i = 0; i < group.size() - 1; i++) { - var testcase = group.get(i); - var existingProperties = testcase.getElementsByTagName("properties"); - if (existingProperties.getLength() > 0) { - var props = (Element) existingProperties.item(0); - var existingProps = props.getElementsByTagName("property"); - boolean alreadyTagged = false; - for (int j = 0; j < existingProps.getLength(); j++) { - if ("dd_tags[test.final_status]".equals(((Element) existingProps.item(j)).getAttribute("name"))) { - alreadyTagged = true; - break; - } - } - if (alreadyTagged) continue; - var property = doc.createElement("property"); - property.setAttribute("name", "dd_tags[test.final_status]"); - property.setAttribute("value", "skip"); - props.appendChild(property); - } else { - var properties = doc.createElement("properties"); - var property = doc.createElement("property"); - property.setAttribute("name", "dd_tags[test.final_status]"); - property.setAttribute("value", "skip"); - properties.appendChild(property); - testcase.appendChild(properties); + if (tagSkip(doc, group.get(i))) { + modified = true; } - modified = true; } } - if (!modified) return; + if (!modified) { + return; + } var tmpFile = File.createTempFile("TagInitializationErrors", ".xml", xmlFile.getParentFile()); try { var transformer = TransformerFactory.newInstance().newTransformer(); transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); transformer.transform(new DOMSource(doc), new StreamResult(tmpFile)); - Files.move(tmpFile.toPath(), xmlFile.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); + Files.move( + tmpFile.toPath(), xmlFile.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); } catch (Exception e) { tmpFile.delete(); throw e; } } + + static Element firstChildElement(Element parent, String tagName) { + var children = parent.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + var child = children.item(i); + if (child instanceof Element e && tagName.equals(e.getTagName())) { + return e; + } + } + return null; + } + + static boolean tagSkip(Document doc, Element testcase) { + var props = firstChildElement(testcase, "properties"); + if (props != null) { + var children = props.getChildNodes(); + for (int j = 0; j < children.getLength(); j++) { + if (children.item(j) instanceof Element e + && "property".equals(e.getTagName()) + && "dd_tags[test.final_status]".equals(e.getAttribute("name"))) { + return false; + } + } + var property = doc.createElement("property"); + property.setAttribute("name", "dd_tags[test.final_status]"); + property.setAttribute("value", "skip"); + props.appendChild(property); + } else { + var properties = doc.createElement("properties"); + var property = doc.createElement("property"); + property.setAttribute("name", "dd_tags[test.final_status]"); + property.setAttribute("value", "skip"); + properties.appendChild(property); + testcase.appendChild(properties); + } + return true; + } } diff --git a/.gitlab/collect_results.sh b/.gitlab/collect_results.sh index dec34ed5d59..0df4a213cfa 100755 --- a/.gitlab/collect_results.sh +++ b/.gitlab/collect_results.sh @@ -90,7 +90,7 @@ do echo " (non-stable test names detected)" fi - echo "Add dd_tags[test.final_status] property on retried synthetics testcase initializationErrors" + echo "Add dd_tags[test.final_status] property on retried synthetics testcase initializationErrors, and all executionError and test exception synthetic testcases" $JAVA_25_HOME/bin/java "$(dirname "$0")/TagInitializationErrors.java" "$TARGET_DIR/$AGGREGATED_FILE_NAME" echo "Add dd_tags[test.final_status] property to each testcase on $TARGET_DIR/$AGGREGATED_FILE_NAME" diff --git a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy index c30c8c394b8..03fced9538e 100644 --- a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy +++ b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy @@ -1248,7 +1248,7 @@ abstract class HttpServerTest extends WithHttpServer { } @Flaky(value = "https://github.com/DataDog/dd-trace-java/issues/9396", suites = ["PekkoHttpServerInstrumentationAsyncHttp2Test"]) - def "test exception"() { + def "Instrumentation test exception"() { setup: def method = "GET" def body = null