Add error-comparison engine to the comparator (PR 1/7, default off)

src/test/java/com/databricks/jdbc/comparator/ComparisonResult.java

@@ -28,9 +28,11 @@ public boolean hasDifferences() {
   public String csvSummary() {
     List<String> parts = new ArrayList<>();
 
-    // Exception vs ResultSet — already concise, keep as is
+    // Exception vs ResultSet — already concise, keep as is. Legacy diffs use the " vs class "
+    // marker; deep error comparison uses the stable "Error one-sided: " prefix (which survives
+    // regardless of whether the non-throwing side renders to a class, a scalar, or null).
     for (String d : metadataDifferences) {
-      if (d.contains(" vs class ")) {
+      if (d.contains(" vs class ") || d.startsWith("Error one-sided: ")) {
         parts.add(d);
       }
     }
@@ -52,6 +54,14 @@ public String csvSummary() {
             .count();
     if (rowMismatches > 0) parts.add(rowMismatches + " data mismatches");
 
+    // Count error-field mismatches (e.g., "Error SQLState mismatch: ...") emitted by
+    // ErrorComparator for both-threw cases.
+    long errorMismatches =
+        dataDifferences.stream()
+            .filter(d -> d.startsWith("Error ") && d.contains("mismatch"))
+            .count();
+    if (errorMismatches > 0) parts.add(errorMismatches + " error mismatches");
+
     return parts.isEmpty() ? "" : String.join(", ", parts);
   }
 

src/test/java/com/databricks/jdbc/comparator/ResultSetComparator.java

@@ -1,5 +1,9 @@
 package com.databricks.jdbc.comparator;
 
+import com.databricks.jdbc.comparator.error.CapturedOutcome;
+import com.databricks.jdbc.comparator.error.ErrorComparator;
+import com.databricks.jdbc.comparator.error.ErrorComparison;
+import com.databricks.jdbc.comparator.error.ErrorPolicy;
 import java.io.IOException;
 import java.io.Reader;
 import java.sql.ResultSet;
@@ -121,6 +125,10 @@ public static ComparisonResult compare(
       } else {
         result.dataDifferences = compareData(rs1, rs2);
       }
+    } else if (compareErrorsDeeply(
+        queryType, queryOrMethod, methodArgs, result1, result2, result)) {
+      // Deep error comparison handled it (at least one side is a Throwable and the
+      // ERROR_COMPARISON_MODE gate is on). Diffs, if any, were folded into `result`.
     } else if (!(result1 instanceof ResultSet) && !(result2 instanceof ResultSet)) {
       // Both are not of type ResultSet
       if (result1 == null || !resultIsSame(result1, result2)) {
@@ -146,7 +154,46 @@ public static ComparisonResult compare(
     return result;
   }
 
+  /**
+   * When {@code ERROR_COMPARISON_MODE} is on and at least one side is a {@link Throwable}, compares
+   * the errors deeply (class + SQLState + code + serverCode + message) and folds any diffs into
+   * {@code result}. Returns true when it handled the comparison; false (a no-op) when the gate is
+   * off or neither side threw, so the legacy branches below take over unchanged.
+   */
+  private static boolean compareErrorsDeeply(
+      String queryType,
+      String queryOrMethod,
+      Object[] methodArgs,
+      Object result1,
+      Object result2,
+      ComparisonResult result) {
+    ErrorPolicy policy = ErrorPolicy.active();
+    if (!policy.isDeepComparisonEnabled()) {
+      return false;
+    }
+    if (!(result1 instanceof Throwable) && !(result2 instanceof Throwable)) {
+      return false;
+    }
+    CapturedOutcome left = toOutcome(result1);
+    CapturedOutcome right = toOutcome(result2);
+    ErrorComparison comparison =
+        ErrorComparator.compare(left, right, ResultSetComparator::describeResult);
+    result.metadataDifferences.addAll(comparison.metadataDiffs);
+    result.dataDifferences.addAll(comparison.dataDiffs);
+    return true;
+  }
+
+  private static CapturedOutcome toOutcome(Object result) {
+    if (result instanceof Throwable) {
+      return CapturedOutcome.threw((Throwable) result);
+    }
+    return CapturedOutcome.returned(result);
+  }
+
   private static String describeResult(Object result) {
+    if (result == null) {
+      return "null";
+    }
     if (result instanceof ResultSet) {
       try {
         ResultSet rs = (ResultSet) result;

src/test/java/com/databricks/jdbc/comparator/error/CapturedOutcome.java

@@ -0,0 +1,64 @@
+package com.databricks.jdbc.comparator.error;
+
+/**
+ * What one side of a comparison did for a single JDBC call: it {@link Kind#THREW threw}, {@link
+ * Kind#RETURNED returned} a value, or returned {@link Kind#NULL null}.
+ *
+ * <p>This lets a thrown error be treated as a first-class, comparable result rather than
+ * propagating and aborting the case. Capture a call with {@link Captures#capture}.
+ */
+public final class CapturedOutcome {
+
+  public enum Kind {
+    THREW,
+    RETURNED,
+    NULL
+  }
+
+  private final Kind kind;
+  private final Object value; // non-null only when RETURNED (ResultSet, update count, Boolean, ...)
+  private final Throwable throwable; // non-null only when THREW
+  private final ErrorFacts error; // non-null only when THREW
+
+  private CapturedOutcome(Kind kind, Object value, Throwable throwable, ErrorFacts error) {
+    this.kind = kind;
+    this.value = value;
+    this.throwable = throwable;
+    this.error = error;
+  }
+
+  public static CapturedOutcome threw(Throwable t) {
+    return new CapturedOutcome(Kind.THREW, null, t, ErrorFacts.from(t));
+  }
+
+  public static CapturedOutcome returned(Object value) {
+    return value == null ? nul() : new CapturedOutcome(Kind.RETURNED, value, null, null);
+  }
+
+  public static CapturedOutcome nul() {
+    return new CapturedOutcome(Kind.NULL, null, null, null);
+  }
+
+  public Kind kind() {
+    return kind;
+  }
+
+  public boolean threw() {
+    return kind == Kind.THREW;
+  }
+
+  /** The returned value (may be null); meaningful only when {@link #kind()} is RETURNED. */
+  public Object value() {
+    return value;
+  }
+
+  /** The raw Throwable; non-null only when {@link #threw()} is true. */
+  public Throwable throwable() {
+    return throwable;
+  }
+
+  /** The extracted error fields; non-null only when {@link #threw()} is true. */
+  public ErrorFacts error() {
+    return error;
+  }
+}

src/test/java/com/databricks/jdbc/comparator/error/Captures.java

@@ -0,0 +1,53 @@
+package com.databricks.jdbc.comparator.error;
+
+import com.databricks.jdbc.comparator.ComparisonResult;
+import java.util.function.Function;
+
+/**
+ * Helper that lets a suite provider capture a JDBC call's outcome (value or thrown error) instead
+ * of letting the error propagate and abort the case.
+ *
+ * <p>Wrap ONLY the single driver call in {@link #capture} — provider bookkeeping (building SQL,
+ * opening connections, temp files) must stay outside, so a harness bug still propagates and fails
+ * the test loudly rather than being silently captured.
+ */
+public final class Captures {
+
+  private Captures() {}
+
+  @FunctionalInterface
+  public interface JdbcCall {
+    Object call() throws Exception;
+  }
+
+  /** Runs a driver call, capturing any Throwable as a {@link CapturedOutcome}. */
+  public static CapturedOutcome capture(JdbcCall call) {
+    try {
+      return CapturedOutcome.returned(call.call());
+    } catch (Throwable t) {
+      return CapturedOutcome.threw(t);
+    }
+  }
+
+  /**
+   * Compares two captured outcomes and folds the result into a {@link ComparisonResult}. Error
+   * diffs land in the metadata/data difference lists in the exact formats {@code csvSummary()}
+   * understands.
+   *
+   * @param describeReturned renders the non-throwing side of a one-sided diff (see {@link
+   *     ErrorComparator#compare})
+   */
+  public static ComparisonResult compareCall(
+      String queryType,
+      String queryOrMethod,
+      Object[] args,
+      CapturedOutcome left,
+      CapturedOutcome right,
+      Function<Object, String> describeReturned) {
+    ComparisonResult result = new ComparisonResult(queryType, queryOrMethod, args);
+    ErrorComparison comparison = ErrorComparator.compare(left, right, describeReturned);
+    result.metadataDifferences.addAll(comparison.metadataDiffs);
+    result.dataDifferences.addAll(comparison.dataDiffs);
+    return result;
+  }
+}

src/test/java/com/databricks/jdbc/comparator/error/ErrorComparator.java

@@ -0,0 +1,95 @@
+package com.databricks.jdbc.comparator.error;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * Compares two {@link CapturedOutcome}s field-by-field and produces an {@link ErrorComparison}
+ * whose diff strings are kept by {@code ComparisonResult.csvSummary()}:
+ *
+ * <ul>
+ *   <li>One-sided (throw vs result) → a {@code metadataDifferences} entry prefixed with {@link
+ *       #ONE_SIDED_PREFIX}, so it survives to the CSV regardless of what the non-throwing side
+ *       renders to (a class, a scalar, or {@code null}).
+ *   <li>Both threw, field mismatch → {@code dataDifferences} entries of the form {@code "Error
+ *       <field> mismatch: <left> vs <right>"} (each contains {@code mismatch}).
+ *   <li>Both threw, all fields match → no diffs (MATCH).
+ *   <li>Neither threw → NOT_APPLICABLE, no diffs (caller does normal data/metadata comparison).
+ * </ul>
+ */
+public final class ErrorComparator {
+
+  private ErrorComparator() {}
+
+  /**
+   * @param describeReturned renders the non-throwing side for one-sided diffs. Receives the
+   *     returned value, which may be {@code null} for the NULL outcome, so it must be null-safe.
+   */
+  public static ErrorComparison compare(
+      CapturedOutcome left, CapturedOutcome right, Function<Object, String> describeReturned) {
+    List<String> metadataDiffs = new ArrayList<>();
+    List<String> dataDiffs = new ArrayList<>();
+
+    boolean leftThrew = left.threw();
+    boolean rightThrew = right.threw();
+
+    if (!leftThrew && !rightThrew) {
+      return new ErrorComparison(ErrorComparison.Verdict.NOT_APPLICABLE, metadataDiffs, dataDiffs);
+    }
+
+    if (leftThrew != rightThrew) {
+      metadataDiffs.add(oneSidedDiff(left, right, describeReturned));
+      return new ErrorComparison(ErrorComparison.Verdict.ONE_SIDED, metadataDiffs, dataDiffs);
+    }
+
+    // Both threw — compare the raw fields. No normalization or format assumptions.
+    ErrorFacts l = left.error();
+    ErrorFacts r = right.error();
+
+    if (!l.exceptionClass.equals(r.exceptionClass)) {
+      dataDiffs.add("Error class mismatch: " + l.simpleClassName() + " vs " + r.simpleClassName());
+    }
+    if (!equalsNullSafe(l.sqlState, r.sqlState)) {
+      dataDiffs.add("Error SQLState mismatch: " + l.sqlState + " vs " + r.sqlState);
+    }
+    if (l.vendorCode != r.vendorCode) {
+      dataDiffs.add("Error code mismatch: " + l.vendorCode + " vs " + r.vendorCode);
+    }
+    if (!equalsNullSafe(l.message, r.message)) {
+      dataDiffs.add("Error message mismatch: '" + l.message + "' vs '" + r.message + "'");
+    }
+
+    ErrorComparison.Verdict verdict =
+        dataDiffs.isEmpty() ? ErrorComparison.Verdict.MATCH : ErrorComparison.Verdict.MISMATCH;
+    return new ErrorComparison(verdict, metadataDiffs, dataDiffs);
+  }
+
+  private static boolean equalsNullSafe(Object a, Object b) {
+    return a == null ? b == null : a.equals(b);
+  }
+
+  /**
+   * Stable prefix on one-sided diffs so {@code csvSummary()} always keeps them (regardless of
+   * whether the non-throwing side renders to a ResultSet, a scalar, or {@code null}).
+   */
+  public static final String ONE_SIDED_PREFIX = "Error one-sided: ";
+
+  /**
+   * Builds a one-sided diff string carrying {@link #ONE_SIDED_PREFIX}. The thrower is rendered with
+   * its extracted facts; the non-throwing side via {@code describeReturned} (which may render to
+   * {@code "null"}). Left/right order is preserved so the report shows which side threw.
+   */
+  private static String oneSidedDiff(
+      CapturedOutcome left, CapturedOutcome right, Function<Object, String> describeReturned) {
+    String body =
+        left.threw()
+            ? renderThrower(left.error()) + " vs " + describeReturned.apply(right.value())
+            : describeReturned.apply(left.value()) + " vs " + renderThrower(right.error());
+    return ONE_SIDED_PREFIX + body;
+  }
+
+  private static String renderThrower(ErrorFacts f) {
+    return f.exceptionClass + " (SQLSTATE=" + f.sqlState + ", code=" + f.vendorCode + ")";
+  }
+}