diff --git a/src/test/java/com/databricks/jdbc/comparator/COMPARATOR_README.md b/src/test/java/com/databricks/jdbc/comparator/COMPARATOR_README.md index 0060a0780..4a172f690 100644 --- a/src/test/java/com/databricks/jdbc/comparator/COMPARATOR_README.md +++ b/src/test/java/com/databricks/jdbc/comparator/COMPARATOR_README.md @@ -260,6 +260,9 @@ If both are present for a method, **runOnly takes precedence**: argument combina | `NEGATIVE_STATEMENT_BATCH` | executeBatch partial/full failure + per-element BatchUpdateException counts | | `NEGATIVE_CONNECTION_STATE` | setCatalog/setSchema/setClientInfo/USE to nonexistent targets (own fresh connections) | | `NEGATIVE_TRANSACTION` | commit/rollback with autocommit on; DDL inside a manual transaction (own fresh connections) | +| `NEGATIVE_CONNECTION` | connect failures — bad token, unknown host, bad warehouse, bad ConnCatalog/ConnSchema (own broken connections) | +| `NEGATIVE_CANCEL_TIMEOUT` | cancel() mid/after/double + setQueryTimeout(1) on a slow query (own fresh connections) | +| `NEGATIVE_VOLUME` | GET/PUT/REMOVE failures + op-after-close (runs under the `VOLUME_OPERATIONS` config) | Negative suites compare each endpoint's **error behavior** (exception class, SQLState, vendor code, message) via the `ERROR_COMPARISON_MODE` gate (default `shadow`). See diff --git a/src/test/java/com/databricks/jdbc/comparator/JDBCDriverComparisonTest.java b/src/test/java/com/databricks/jdbc/comparator/JDBCDriverComparisonTest.java index a25ad4860..afe89e36d 100644 --- a/src/test/java/com/databricks/jdbc/comparator/JDBCDriverComparisonTest.java +++ b/src/test/java/com/databricks/jdbc/comparator/JDBCDriverComparisonTest.java @@ -153,10 +153,28 @@ static Stream provideAllTests() { // config's resolved LEFT/RIGHT URLs; the returned connections are NOT cached, so the suite // owns closing them and cannot poison the shared connections above. ConnectionFactory factory = - side -> { - if ("LEFT".equalsIgnoreCase(side)) return connectionManager.openUncached(leftUrl); - if ("RIGHT".equalsIgnoreCase(side)) return connectionManager.openUncached(rightUrl); - throw new IllegalArgumentException("side must be LEFT or RIGHT, got: " + side); + new ConnectionFactory() { + @Override + public Connection openFresh(String side) throws SQLException { + return connectionManager.openUncached(urlFor(side)); + } + + @Override + public String urlFor(String side) { + if ("LEFT".equalsIgnoreCase(side)) return leftUrl; + if ("RIGHT".equalsIgnoreCase(side)) return rightUrl; + throw new IllegalArgumentException("side must be LEFT or RIGHT, got: " + side); + } + + @Override + public String token() { + return connectionManager.getToken(); + } + + @Override + public Connection open(String url, String token) throws SQLException { + return connectionManager.openUncached(url, token); + } }; for (TestSuite suite : config.getApplicableSuites()) { diff --git a/src/test/java/com/databricks/jdbc/comparator/config/ConnectionConfig.java b/src/test/java/com/databricks/jdbc/comparator/config/ConnectionConfig.java index 0677be74d..1f04f418d 100644 --- a/src/test/java/com/databricks/jdbc/comparator/config/ConnectionConfig.java +++ b/src/test/java/com/databricks/jdbc/comparator/config/ConnectionConfig.java @@ -21,7 +21,11 @@ public enum ConnectionConfig { "Default params", Map.of(), null, - allExcept(TestSuite.COMPLEX_TYPES, TestSuite.GEOSPATIAL, TestSuite.VOLUME_OPERATIONS)), + allExcept( + TestSuite.COMPLEX_TYPES, + TestSuite.GEOSPATIAL, + TestSuite.VOLUME_OPERATIONS, + TestSuite.NEGATIVE_VOLUME)), COMPRESSION_DISABLED( "Compression disabled", @@ -69,7 +73,7 @@ public enum ConnectionConfig { "Volume operations", Map.of("VolumeOperationAllowedLocalPaths", "/tmp"), null, - EnumSet.of(TestSuite.VOLUME_OPERATIONS)), + EnumSet.of(TestSuite.VOLUME_OPERATIONS, TestSuite.NEGATIVE_VOLUME)), PRO_WAREHOUSE( "Pro warehouse", diff --git a/src/test/java/com/databricks/jdbc/comparator/config/ConnectionFactory.java b/src/test/java/com/databricks/jdbc/comparator/config/ConnectionFactory.java index 4a44d6b32..ce5f94745 100644 --- a/src/test/java/com/databricks/jdbc/comparator/config/ConnectionFactory.java +++ b/src/test/java/com/databricks/jdbc/comparator/config/ConnectionFactory.java @@ -23,4 +23,23 @@ public interface ConnectionFactory { * @param side "LEFT" or "RIGHT" (case-insensitive) */ Connection openFresh(String side) throws SQLException; + + /** + * The resolved (healthy) JDBC URL for the named side. Exposed so suites that deliberately open + * broken connections (e.g. NEGATIVE_CONNECTION) can start from a good URL and corrupt one piece + * of it (host, warehouse, a connection param) while leaving the rest valid. + * + * @param side "LEFT" or "RIGHT" (case-insensitive) + */ + String urlFor(String side); + + /** The shared PAT used for all connections. Exposed for suites that test bad-token auth. */ + String token(); + + /** + * Opens a new, uncached connection for an arbitrary URL + token — the escape hatch for suites + * that build a deliberately-broken URL/token and need to capture the resulting failure. The + * caller owns closing whatever is returned. + */ + Connection open(String url, String token) throws SQLException; } diff --git a/src/test/java/com/databricks/jdbc/comparator/config/ConnectionManager.java b/src/test/java/com/databricks/jdbc/comparator/config/ConnectionManager.java index 6b7f17612..2d6038a3c 100644 --- a/src/test/java/com/databricks/jdbc/comparator/config/ConnectionManager.java +++ b/src/test/java/com/databricks/jdbc/comparator/config/ConnectionManager.java @@ -42,6 +42,19 @@ public Connection openUncached(String url) throws SQLException { return DriverManager.getConnection(url, "token", token); } + /** + * Opens a NEW, uncached connection for an arbitrary URL and token — used by suites that build a + * deliberately-broken URL/token to capture the resulting failure. The caller owns closing it. + */ + public Connection openUncached(String url, String tokenOverride) throws SQLException { + return DriverManager.getConnection(url, "token", tokenOverride); + } + + /** The shared PAT. */ + public String getToken() { + return token; + } + /** Returns all active connection URLs, useful for report headers. */ public List getActiveUrls() { return new ArrayList<>(cache.keySet()); diff --git a/src/test/java/com/databricks/jdbc/comparator/config/TestSuite.java b/src/test/java/com/databricks/jdbc/comparator/config/TestSuite.java index b7ce3f429..2dad8bd46 100644 --- a/src/test/java/com/databricks/jdbc/comparator/config/TestSuite.java +++ b/src/test/java/com/databricks/jdbc/comparator/config/TestSuite.java @@ -38,7 +38,15 @@ public enum TestSuite { // Negative suites that mutate connection/session state — each opens its OWN fresh connections // via ConnectionFactory (closed in a finally), so they never poison the shared connections. NEGATIVE_CONNECTION_STATE(new NegativeConnectionStateProvider()), - NEGATIVE_TRANSACTION(new NegativeTransactionProvider()); + NEGATIVE_TRANSACTION(new NegativeTransactionProvider()), + + // Negative suites that open their own deliberately-broken or slow connections via + // ConnectionFactory. NEGATIVE_VOLUME runs under the VOLUME_OPERATIONS config (needs + // VolumeOperationAllowedLocalPaths), so it is excluded from DEFAULT_PARAMS like the positive + // VOLUME_OPERATIONS suite. + NEGATIVE_CONNECTION(new NegativeConnectionProvider()), + NEGATIVE_CANCEL_TIMEOUT(new NegativeCancelTimeoutProvider()), + NEGATIVE_VOLUME(new NegativeVolumeProvider()); private final SuiteProvider provider; diff --git a/src/test/java/com/databricks/jdbc/comparator/suite/NegativeCancelTimeoutProvider.java b/src/test/java/com/databricks/jdbc/comparator/suite/NegativeCancelTimeoutProvider.java new file mode 100644 index 000000000..104dc7a6d --- /dev/null +++ b/src/test/java/com/databricks/jdbc/comparator/suite/NegativeCancelTimeoutProvider.java @@ -0,0 +1,154 @@ +package com.databricks.jdbc.comparator.suite; + +import com.databricks.jdbc.comparator.ComparisonResult; +import com.databricks.jdbc.comparator.config.ConnectionFactory; +import com.databricks.jdbc.comparator.error.CapturedOutcome; +import com.databricks.jdbc.comparator.error.Captures; +import com.databricks.jdbc.comparator.error.ErrorDiffs; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +/** + * Negative cancel/timeout cases — {@code cancel()} mid-query, {@code cancel()} after completion, + * double {@code cancel()}, and {@code setQueryTimeout(1)} on a slow query. Compares how the two + * endpoints surface each outcome via {@link ErrorDiffs}. + * + *

Isolation: cancel/timeout affect statement/connection state, so the suite opens its OWN fresh + * connections per side (closed in a {@code finally}). The "slow query" is a bounded range + * aggregation (~a few seconds) — no server-side sleep is available, so it is intentionally modest + * to keep load and flakiness low; {@code setQueryTimeout(1)} and mid-flight {@code cancel()} still + * act well within it. + */ +public class NegativeCancelTimeoutProvider implements SuiteProvider { + + // Bounded slow query: a cross-join aggregation that runs a few seconds on a small warehouse + // without any explicit sleep. Kept modest to limit shared-warehouse load. + private static final String SLOW_QUERY = + "SELECT COUNT(*) FROM range(0, 100000000) a CROSS JOIN range(0, 20) b"; + + private static final String CANCEL_MID = "CANCEL_MID_QUERY"; + private static final String CANCEL_AFTER = "CANCEL_AFTER_COMPLETE"; + private static final String CANCEL_TWICE = "CANCEL_ALREADY_CANCELLED"; + private static final String QUERY_TIMEOUT = "SET_QUERY_TIMEOUT_1"; + + @Override + public List getTestCases() { + return Arrays.asList( + new TestCase(CANCEL_MID, "cancel() mid slow query"), + new TestCase(CANCEL_AFTER, "cancel() after the query completed"), + new TestCase(CANCEL_TWICE, "cancel() on an already-cancelled statement"), + new TestCase(QUERY_TIMEOUT, "setQueryTimeout(1) on a slow query")); + } + + /** Runs one case's action against a single connection; returns a value or throws. */ + private Object runCase(String id, Connection conn) throws Exception { + switch (id) { + case CANCEL_MID: + return cancelMid(conn); + case CANCEL_AFTER: + return cancelAfterComplete(conn); + case CANCEL_TWICE: + return cancelTwice(conn); + case QUERY_TIMEOUT: + return queryTimeout(conn); + default: + throw new IllegalArgumentException("Unknown case: " + id); + } + } + + /** Starts the slow query on a background thread, cancels it after a short delay, joins. */ + private Object cancelMid(Connection conn) throws Exception { + try (Statement stmt = conn.createStatement()) { + final Object[] box = new Object[1]; + Thread runner = + new Thread( + () -> { + try { + stmt.executeQuery(SLOW_QUERY); + box[0] = "completed"; + } catch (Throwable t) { + box[0] = t; + } + }); + runner.start(); + Thread.sleep(1500); // let the query start, then cancel it mid-flight + stmt.cancel(); + runner.join(60000); + if (box[0] instanceof Throwable) { + throw (Exception) box[0]; // surface the cancellation error for comparison + } + return box[0]; + } + } + + /** Runs the query to completion, then cancels the (finished) statement. */ + private Object cancelAfterComplete(Connection conn) throws Exception { + try (Statement stmt = conn.createStatement()) { + stmt.executeQuery("SELECT 1").close(); + stmt.cancel(); + return "cancelled after complete"; + } + } + + /** Cancels a statement twice in a row. */ + private Object cancelTwice(Connection conn) throws Exception { + try (Statement stmt = conn.createStatement()) { + stmt.executeQuery("SELECT 1").close(); + stmt.cancel(); + stmt.cancel(); + return "double cancel ok"; + } + } + + /** setQueryTimeout(1) on the slow query — expect a timeout error. */ + private Object queryTimeout(Connection conn) throws Exception { + try (Statement stmt = conn.createStatement()) { + stmt.setQueryTimeout(1); + stmt.executeQuery(SLOW_QUERY).close(); + return "completed within timeout"; + } + } + + /** Not used — this suite requires the ConnectionFactory overload below. */ + @Override + public ComparisonResult execute( + Connection conn1, Connection conn2, TestCase testCase, String label) throws Exception { + throw new UnsupportedOperationException( + "NEGATIVE_CANCEL_TIMEOUT requires the ConnectionFactory overload"); + } + + @Override + public ComparisonResult execute( + Connection conn1, + Connection conn2, + ConnectionFactory factory, + TestCase testCase, + String label) + throws Exception { + String id = testCase.getIdentifier(); + Connection left = factory.openFresh("LEFT"); + Connection right = factory.openFresh("RIGHT"); + try { + CapturedOutcome lo = Captures.capture(() -> runCase(id, left)); + CapturedOutcome ro = Captures.capture(() -> runCase(id, right)); + ComparisonResult result = + new ComparisonResult(label, testCase.getDescription(), testCase.getArgs()); + ErrorDiffs.foldInto(result, lo, ro, "result ", ""); + return result; + } finally { + closeQuietly(left); + closeQuietly(right); + } + } + + private static void closeQuietly(Connection conn) { + try { + if (conn != null) conn.close(); + } catch (SQLException ignored) { + // best-effort + } + } +} diff --git a/src/test/java/com/databricks/jdbc/comparator/suite/NegativeConnectionProvider.java b/src/test/java/com/databricks/jdbc/comparator/suite/NegativeConnectionProvider.java new file mode 100644 index 000000000..801909b67 --- /dev/null +++ b/src/test/java/com/databricks/jdbc/comparator/suite/NegativeConnectionProvider.java @@ -0,0 +1,134 @@ +package com.databricks.jdbc.comparator.suite; + +import com.databricks.jdbc.comparator.ComparisonResult; +import com.databricks.jdbc.comparator.config.ConnectionFactory; +import com.databricks.jdbc.comparator.error.CapturedOutcome; +import com.databricks.jdbc.comparator.error.Captures; +import com.databricks.jdbc.comparator.error.ErrorDiffs; +import java.sql.Connection; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Negative connection cases — opening a connection is itself the test. Each case starts from a + * side's healthy resolved URL/token and corrupts exactly one piece (token, host, warehouse id, or a + * connection param), then compares how the two endpoints surface the connect failure. + * + *

These do not use {@code openFresh} (that opens a HEALTHY connection); they build the broken + * URL/token from {@link ConnectionFactory#urlFor}/{@link ConnectionFactory#token} and open via + * {@link ConnectionFactory#open}. Any connection that does unexpectedly open is closed in a {@code + * finally}. Nothing touches the shared connections. + */ +public class NegativeConnectionProvider implements SuiteProvider { + + /** Corrupts a (url, token) pair for one side into a deliberately-broken one. */ + @FunctionalInterface + private interface Corruptor { + String[] brokenUrlAndToken(String goodUrl, String goodToken); + } + + private static final class Case { + final String description; + final Corruptor corruptor; + + Case(String description, Corruptor corruptor) { + this.description = description; + this.corruptor = corruptor; + } + } + + private static final List CASES = + Arrays.asList( + new Case( + "Bad token (auth failure)", + (url, token) -> new String[] {url, "dapi0000000000000000000000000000badx"}), + new Case( + "Unknown host", + (url, token) -> + new String[] { + url.replaceFirst( + "jdbc:databricks://[^:/;]+", "jdbc:databricks://no-such-host.invalid"), + token + }), + new Case( + "Malformed URL (bad httpPath)", + (url, token) -> + new String[] { + url.replaceFirst("httpPath=[^;]*", "httpPath=/not/a/valid/path"), token + }), + new Case( + "Non-existent warehouse id", + (url, token) -> + new String[] { + url.replaceFirst( + "/sql/1.0/warehouses/[^;]*", "/sql/1.0/warehouses/0000000000000000"), + token + }), + new Case( + "Non-existent ConnCatalog", + (url, token) -> new String[] {url + ";ConnCatalog=__no_such_catalog__", token}), + new Case( + "Non-existent ConnSchema", + (url, token) -> + new String[] { + url + ";ConnCatalog=comparator_tests;ConnSchema=__no_such_schema__", token + })); + + @Override + public List getTestCases() { + return CASES.stream() + .map(c -> new TestCase(c.description, c.description)) + .collect(Collectors.toList()); + } + + /** Not used — this suite requires the ConnectionFactory overload below. */ + @Override + public ComparisonResult execute( + Connection conn1, Connection conn2, TestCase testCase, String label) throws Exception { + throw new UnsupportedOperationException( + "NEGATIVE_CONNECTION requires the ConnectionFactory overload"); + } + + @Override + public ComparisonResult execute( + Connection conn1, + Connection conn2, + ConnectionFactory factory, + TestCase testCase, + String label) + throws Exception { + Case c = + CASES.stream() + .filter(x -> x.description.equals(testCase.getIdentifier())) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException("Unknown case: " + testCase.getIdentifier())); + + String[] left = c.corruptor.brokenUrlAndToken(factory.urlFor("LEFT"), factory.token()); + String[] right = c.corruptor.brokenUrlAndToken(factory.urlFor("RIGHT"), factory.token()); + + // The captured driver call is the connect itself. A connection that unexpectedly opens is a + // real (one-sided) divergence; close whatever opened afterward. + CapturedOutcome lo = Captures.capture(() -> factory.open(left[0], left[1])); + CapturedOutcome ro = Captures.capture(() -> factory.open(right[0], right[1])); + try { + ComparisonResult result = new ComparisonResult(label, c.description, testCase.getArgs()); + ErrorDiffs.foldInto(result, lo, ro, "connection ", ""); + return result; + } finally { + closeIfConnection(lo); + closeIfConnection(ro); + } + } + + private static void closeIfConnection(CapturedOutcome outcome) { + if (!outcome.threw() && outcome.value() instanceof Connection) { + try { + ((Connection) outcome.value()).close(); + } catch (Exception ignored) { + // best-effort + } + } + } +} diff --git a/src/test/java/com/databricks/jdbc/comparator/suite/NegativeVolumeProvider.java b/src/test/java/com/databricks/jdbc/comparator/suite/NegativeVolumeProvider.java new file mode 100644 index 000000000..99072bc77 --- /dev/null +++ b/src/test/java/com/databricks/jdbc/comparator/suite/NegativeVolumeProvider.java @@ -0,0 +1,85 @@ +package com.databricks.jdbc.comparator.suite; + +import com.databricks.jdbc.comparator.ComparisonResult; +import com.databricks.jdbc.comparator.error.CapturedOutcome; +import com.databricks.jdbc.comparator.error.Captures; +import com.databricks.jdbc.comparator.error.ErrorDiffs; +import java.sql.Connection; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; + +/** + * Negative UC Volume cases — GET a missing file, PUT outside the allowed local paths, PUT a missing + * local file, REMOVE a missing file, and a volume op after the statement is closed. Compares how + * the two endpoints surface each failure via {@link ErrorDiffs}. + * + *

Runs under the {@code VOLUME_OPERATIONS} connection config (which sets {@code + * VolumeOperationAllowedLocalPaths=/tmp}), matching the positive VOLUME_OPERATIONS suite. Uses the + * shared connections — these ops fail without mutating connection/session state. + */ +public class NegativeVolumeProvider implements SuiteProvider { + + private static final String VOLUME_PATH = + "/Volumes/comparator_tests/oss_jdbc_tests/comparator_volume"; + + private static final String GET_MISSING = "GET missing volume file"; + private static final String PUT_DISALLOWED = "PUT from a disallowed local path"; + private static final String PUT_MISSING_LOCAL = "PUT a missing local file"; + private static final String REMOVE_MISSING = "REMOVE a missing file"; + private static final String OP_AFTER_CLOSE = "Volume op after statement close"; + + @Override + public List getTestCases() { + return Arrays.asList( + new TestCase(GET_MISSING, GET_MISSING), + new TestCase(PUT_DISALLOWED, PUT_DISALLOWED), + new TestCase(PUT_MISSING_LOCAL, PUT_MISSING_LOCAL), + new TestCase(REMOVE_MISSING, REMOVE_MISSING), + new TestCase(OP_AFTER_CLOSE, OP_AFTER_CLOSE)); + } + + @Override + public ComparisonResult execute( + Connection conn1, Connection conn2, TestCase testCase, String label) throws Exception { + String id = testCase.getIdentifier(); + CapturedOutcome left = Captures.capture(() -> runOp(conn1, id)); + CapturedOutcome right = Captures.capture(() -> runOp(conn2, id)); + ComparisonResult result = new ComparisonResult(label, id, testCase.getArgs()); + ErrorDiffs.foldInto(result, left, right, "result ", ""); + return result; + } + + private Object runOp(Connection conn, String id) throws Exception { + switch (id) { + case GET_MISSING: + return exec(conn, "GET '" + VOLUME_PATH + "/__no_such_file__.txt' TO '/tmp/neg_get.txt'"); + case PUT_DISALLOWED: + // /etc is outside VolumeOperationAllowedLocalPaths=/tmp -> driver-side rejection. + return exec(conn, "PUT '/etc/hostname' INTO '" + VOLUME_PATH + "/neg_put.txt' OVERWRITE"); + case PUT_MISSING_LOCAL: + return exec( + conn, + "PUT '/tmp/__no_such_local_file__.txt' INTO '" + + VOLUME_PATH + + "/neg_put2.txt' OVERWRITE"); + case REMOVE_MISSING: + return exec(conn, "REMOVE '" + VOLUME_PATH + "/__no_such_file__.txt'"); + case OP_AFTER_CLOSE: + { + Statement stmt = conn.createStatement(); + stmt.close(); + // Using a closed statement — expect a "statement closed" style error. + return stmt.execute("REMOVE '" + VOLUME_PATH + "/whatever.txt'"); + } + default: + throw new IllegalArgumentException("Unknown case: " + id); + } + } + + private Object exec(Connection conn, String sql) throws Exception { + try (Statement stmt = conn.createStatement()) { + return stmt.execute(sql); + } + } +}