From b2cdc3d60460426622fae960a4ce0fa3d1d2c15f Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Mon, 6 Jul 2026 20:32:16 +0000 Subject: [PATCH 1/6] perf(bigquery-jdbc): optimize metadata introspection performance and parallel project scanning --- .../jdbc/BigQueryDatabaseMetaData.java | 165 ++++++++++-------- 1 file changed, 90 insertions(+), 75 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 273d07b75602..8f0969671889 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -20,8 +20,12 @@ import com.google.cloud.Tuple; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetListOption; +import com.google.cloud.bigquery.BigQuery.RoutineField; import com.google.cloud.bigquery.BigQuery.RoutineListOption; +import com.google.cloud.bigquery.BigQuery.RoutineOption; +import com.google.cloud.bigquery.BigQuery.TableField; import com.google.cloud.bigquery.BigQuery.TableListOption; +import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.DatasetId; @@ -1056,7 +1060,14 @@ public ResultSet getProcedureColumns( getRoutineDetailsExecutor = connection.getMetadataExecutor(); List fullRoutines = - fetchFullRoutineDetailsForIds(procedureIdsToGet, getRoutineDetailsExecutor, LOG); + fetchFullRoutineDetailsForIds( + procedureIdsToGet, + getRoutineDetailsExecutor, + LOG, + RoutineOption.fields( + RoutineField.ROUTINE_REFERENCE, + RoutineField.ARGUMENTS, + RoutineField.ROUTINE_TYPE)); getRoutineDetailsExecutor = null; if (fullRoutines.isEmpty() || Thread.currentThread().isInterrupted()) { @@ -1174,7 +1185,8 @@ List listMatchingProcedureIdsFromDatasets( List fetchFullRoutineDetailsForIds( List procedureIdsToGet, ExecutorService getRoutineDetailsExecutor, - BigQueryJdbcCustomLogger logger) + BigQueryJdbcCustomLogger logger, + RoutineOption... options) throws InterruptedException, ExecutionException { logger.fine("Fetching full details for %d procedure IDs.", procedureIdsToGet.size()); final List> getRoutineFutures = new ArrayList<>(); @@ -1191,7 +1203,7 @@ List fetchFullRoutineDetailsForIds( Callable getCallable = () -> { try { - return bigquery.getRoutine(currentProcId); + return bigquery.getRoutine(currentProcId, options); } catch (Exception e) { logger.warning( "Failed to get full details for routine " @@ -1616,7 +1628,11 @@ public ResultSet getTables( TableId.of( currentDatasetId.getProject(), currentDatasetId.getDataset(), - name)), + name), + TableOption.fields( + TableField.TABLE_REFERENCE, + TableField.TYPE, + TableField.DESCRIPTION)), (tbl) -> tbl.getTableId().getTable(), tableNamePattern, tableNameRegex, @@ -1930,7 +1946,9 @@ public ResultSet getColumns( datasetId, TableListOption.pageSize(DEFAULT_PAGE_SIZE)), (name) -> bigquery.getTable( - TableId.of(datasetId.getProject(), datasetId.getDataset(), name)), + TableId.of(datasetId.getProject(), datasetId.getDataset(), name), + TableOption.fields( + TableField.TABLE_REFERENCE, TableField.TYPE, TableField.SCHEMA)), (tbl) -> tbl.getTableId().getTable(), tableNamePattern, tableNameRegex, @@ -1999,7 +2017,10 @@ private void processTableColumns( "Schema not included in table object for " + tableId + ", fetching full table details..."); - Table fullTable = bigquery.getTable(tableId); + Table fullTable = + bigquery.getTable( + tableId, + TableOption.fields(TableField.TABLE_REFERENCE, TableField.TYPE, TableField.SCHEMA)); if (fullTable != null) { definition = fullTable.getDefinition(); tableSchema = (definition != null) ? definition.getSchema() : null; @@ -3402,70 +3423,16 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce final Schema resultSchema = defineGetSchemasSchema(); final FieldList resultSchemaFields = resultSchema.getFields(); - if (catalog != null) { - // Single-Catalog Path: completely synchronous on caller thread - final BlockingQueue queue = new LinkedBlockingQueue<>(); - List datasets = fetchMatchingDatasets(catalog, schemaPattern, schemaRegex); - List collectedResults = new ArrayList<>(); - for (Dataset dataset : datasets) { - processSchemaInfo(dataset, collectedResults, resultSchemaFields); - } - Comparator comparator = defineGetSchemasComparator(resultSchemaFields); - sortResults(collectedResults, comparator, "getSchemas", LOG); - populateQueue(collectedResults, queue, resultSchemaFields); - signalEndOfData(queue, resultSchemaFields); - return BigQueryJsonResultSet.of(resultSchema, -1, queue, null); - } - - // Multi-Catalog Path: fan out using connection-scoped metadataExecutor final BlockingQueue queue = new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); - Runnable multiSchemaFetcher = + Runnable schemaFetcher = () -> { final FieldList localResultSchemaFields = resultSchemaFields; - final List>> apiFutures = new ArrayList<>(); - final List collectedDatasets = new ArrayList<>(); final List collectedResults = new ArrayList<>(); try { - List projectsToScanList = getAccessibleCatalogNames(); - - if (projectsToScanList.isEmpty()) { - LOG.info( - "No valid projects to scan (primary, specified, or additional). Returning empty" - + " resultset."); - return; - } - - ExecutorService apiExecutor = connection.getMetadataExecutor(); - - LOG.fine("Submitting parallel fetchMatchingDatasets tasks..."); - for (String currentProjectToScan : projectsToScanList) { - if (Thread.currentThread().isInterrupted()) { - LOG.warning("Fetcher interrupted during project iteration submission."); - break; - } - - Callable> apiCallable = - () -> fetchMatchingDatasets(currentProjectToScan, schemaPattern, schemaRegex); - Future> apiFuture = apiExecutor.submit(apiCallable); - apiFutures.add(apiFuture); - } - LOG.fine("Finished submitting " + apiFutures.size() + " fetchMatchingDatasets tasks."); - - LOG.fine("Processing results from fetchMatchingDatasets tasks..."); - for (Future> apiFuture : apiFutures) { - if (Thread.currentThread().isInterrupted()) { - LOG.warning("Fetcher interrupted while processing API futures."); - break; - } - List datasetsResult = apiFuture.get(); - if (datasetsResult != null) { - collectedDatasets.addAll(datasetsResult); - } - } - - for (Dataset dataset : collectedDatasets) { + List datasets = fetchMatchingDatasets(catalog, schemaPattern, schemaRegex); + for (Dataset dataset : datasets) { if (Thread.currentThread().isInterrupted()) { break; } @@ -3480,16 +3447,15 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce } catch (Throwable t) { handleFetcherException(t, queue, "getSchemas"); } finally { - apiFutures.forEach(f -> f.cancel(true)); - finalizeFetcher(queue, localResultSchemaFields, "Multi-schema fetcher"); + finalizeFetcher(queue, localResultSchemaFields, "Schema fetcher"); } }; - Future fetcherFuture = connection.getExecutorService().submit(multiSchemaFetcher); + Future fetcherFuture = connection.getExecutorService().submit(schemaFetcher); BigQueryJsonResultSet resultSet = BigQueryJsonResultSet.of(resultSchema, -1, queue, null, fetcherFuture); - LOG.info("Submitted background task for multi-catalog getSchemas to query executor"); + LOG.info("Submitted background task for getSchemas to query executor"); return resultSet; } @@ -4838,11 +4804,13 @@ private List fetchMatchingDatasets( String catalog, String schemaPattern, Pattern schemaRegex) throws SQLException { List projects = (catalog != null) ? Collections.singletonList(catalog) : getAccessibleCatalogNames(); - List allDatasets = new ArrayList<>(); - for (String project : projects) { - if (Thread.currentThread().isInterrupted()) { - throw new SQLException("Interrupted while fetching matching datasets"); - } + + if (projects.isEmpty()) { + return Collections.emptyList(); + } + + if (projects.size() == 1) { + String project = projects.get(0); try { List datasets = findMatchingBigQueryObjects( @@ -4853,9 +4821,7 @@ private List fetchMatchingDatasets( schemaPattern, schemaRegex, LOG); - if (datasets != null) { - allDatasets.addAll(datasets); - } + return datasets != null ? datasets : Collections.emptyList(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SQLException("Interrupted while fetching matching datasets", e); @@ -4863,6 +4829,55 @@ private List fetchMatchingDatasets( throw new SQLException("Failed to fetch matching datasets for project " + project, e); } } + + final List allDatasets = Collections.synchronizedList(new ArrayList<>()); + final List> taskFutures = new ArrayList<>(); + ExecutorService executor = connection.getMetadataExecutor(); + + for (String project : projects) { + Callable task = + () -> { + try { + List datasets = + findMatchingBigQueryObjects( + "Dataset", + () -> + bigquery.listDatasets( + project, DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)), + (name) -> bigquery.getDataset(DatasetId.of(project, name)), + (ds) -> ds.getDatasetId().getDataset(), + schemaPattern, + schemaRegex, + LOG); + if (datasets != null) { + allDatasets.addAll(datasets); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SQLException( + "Interrupted while fetching datasets for project " + project, e); + } catch (Exception e) { + throw new SQLException("Failed to fetch datasets for project " + project, e); + } + return null; + }; + taskFutures.add(executor.submit(task)); + } + + try { + waitForTasksCompletion(taskFutures); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof SQLException) { + throw (SQLException) cause; + } + throw new SQLException("Error parallel-fetching matching datasets", e); + } + + if (Thread.currentThread().isInterrupted()) { + throw new SQLException("Interrupted while parallel-fetching matching datasets"); + } + return allDatasets; } From 7139d253fcbaa9c4a92ae292177320fba135939e Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Mon, 6 Jul 2026 20:44:07 +0000 Subject: [PATCH 2/6] fix: resource leak --- .../cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 8f0969671889..9dc19bf16563 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -4866,16 +4866,17 @@ private List fetchMatchingDatasets( try { waitForTasksCompletion(taskFutures); + if (Thread.currentThread().isInterrupted()) { + throw new SQLException("Interrupted while parallel-fetching matching datasets"); + } } catch (ExecutionException e) { Throwable cause = e.getCause(); if (cause instanceof SQLException) { throw (SQLException) cause; } throw new SQLException("Error parallel-fetching matching datasets", e); - } - - if (Thread.currentThread().isInterrupted()) { - throw new SQLException("Interrupted while parallel-fetching matching datasets"); + } finally { + taskFutures.forEach(f -> f.cancel(true)); } return allDatasets; From 60af5b126fc2518541668fad063f661210014a8d Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Mon, 6 Jul 2026 21:11:43 +0000 Subject: [PATCH 3/6] chore: address pr feedback --- .../jdbc/BigQueryDatabaseMetaData.java | 83 ++++++++----------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 9dc19bf16563..c1db14b56d49 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -4800,6 +4800,28 @@ private void signalEndOfData( } } + private List fetchDatasetsForProject( + String project, String schemaPattern, Pattern schemaRegex) throws SQLException { + try { + List datasets = + findMatchingBigQueryObjects( + "Dataset", + () -> bigquery.listDatasets(project, DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)), + (name) -> bigquery.getDataset(DatasetId.of(project, name)), + (ds) -> ds.getDatasetId().getDataset(), + schemaPattern, + schemaRegex, + LOG); + return datasets != null ? datasets : Collections.emptyList(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SQLException( + "Interrupted while fetching matching datasets for project " + project, e); + } catch (Exception e) { + throw new SQLException("Failed to fetch matching datasets for project " + project, e); + } + } + private List fetchMatchingDatasets( String catalog, String schemaPattern, Pattern schemaRegex) throws SQLException { List projects = @@ -4809,62 +4831,27 @@ private List fetchMatchingDatasets( return Collections.emptyList(); } + // Single project path if (projects.size() == 1) { - String project = projects.get(0); - try { - List datasets = - findMatchingBigQueryObjects( - "Dataset", - () -> bigquery.listDatasets(project, DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)), - (name) -> bigquery.getDataset(DatasetId.of(project, name)), - (ds) -> ds.getDatasetId().getDataset(), - schemaPattern, - schemaRegex, - LOG); - return datasets != null ? datasets : Collections.emptyList(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SQLException("Interrupted while fetching matching datasets", e); - } catch (Exception e) { - throw new SQLException("Failed to fetch matching datasets for project " + project, e); - } + return fetchDatasetsForProject(projects.get(0), schemaPattern, schemaRegex); } + // Multi-project path final List allDatasets = Collections.synchronizedList(new ArrayList<>()); final List> taskFutures = new ArrayList<>(); ExecutorService executor = connection.getMetadataExecutor(); - for (String project : projects) { - Callable task = - () -> { - try { - List datasets = - findMatchingBigQueryObjects( - "Dataset", - () -> - bigquery.listDatasets( - project, DatasetListOption.pageSize(DEFAULT_PAGE_SIZE)), - (name) -> bigquery.getDataset(DatasetId.of(project, name)), - (ds) -> ds.getDatasetId().getDataset(), - schemaPattern, - schemaRegex, - LOG); - if (datasets != null) { - allDatasets.addAll(datasets); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SQLException( - "Interrupted while fetching datasets for project " + project, e); - } catch (Exception e) { - throw new SQLException("Failed to fetch datasets for project " + project, e); - } - return null; - }; - taskFutures.add(executor.submit(task)); - } - try { + for (String project : projects) { + Callable task = + () -> { + List datasets = fetchDatasetsForProject(project, schemaPattern, schemaRegex); + allDatasets.addAll(datasets); + return null; + }; + taskFutures.add(executor.submit(task)); + } + waitForTasksCompletion(taskFutures); if (Thread.currentThread().isInterrupted()) { throw new SQLException("Interrupted while parallel-fetching matching datasets"); From fd49c6274a7f2b04ecc581ae76dfafe56ffaa595 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Mon, 6 Jul 2026 23:08:10 +0000 Subject: [PATCH 4/6] revert missing single catalog path in getSchemas --- .../bigquery/jdbc/BigQueryDatabaseMetaData.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index c1db14b56d49..e72e052ed82e 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -3423,6 +3423,22 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce final Schema resultSchema = defineGetSchemasSchema(); final FieldList resultSchemaFields = resultSchema.getFields(); + if (catalog != null) { + // Single-Catalog Path: completely synchronous on caller thread + final BlockingQueue queue = + new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); + List datasets = fetchMatchingDatasets(catalog, schemaPattern, schemaRegex); + List collectedResults = new ArrayList<>(); + for (Dataset dataset : datasets) { + processSchemaInfo(dataset, collectedResults, resultSchemaFields); + } + Comparator comparator = defineGetSchemasComparator(resultSchemaFields); + sortResults(collectedResults, comparator, "getSchemas", LOG); + populateQueue(collectedResults, queue, resultSchemaFields); + signalEndOfData(queue, resultSchemaFields); + return BigQueryJsonResultSet.of(resultSchema, -1, queue, null); + } + final BlockingQueue queue = new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); Runnable schemaFetcher = From d1f486352ecdf8ed89f10fd56b959cb8570aa23e Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Mon, 6 Jul 2026 23:37:43 +0000 Subject: [PATCH 5/6] revert naming to avoid confusion --- .../cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index e72e052ed82e..a0677caafe89 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -3425,8 +3425,7 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce if (catalog != null) { // Single-Catalog Path: completely synchronous on caller thread - final BlockingQueue queue = - new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); + final BlockingQueue queue = new LinkedBlockingQueue<>(); List datasets = fetchMatchingDatasets(catalog, schemaPattern, schemaRegex); List collectedResults = new ArrayList<>(); for (Dataset dataset : datasets) { @@ -3441,7 +3440,7 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce final BlockingQueue queue = new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); - Runnable schemaFetcher = + Runnable multiSchemaFetcher = () -> { final FieldList localResultSchemaFields = resultSchemaFields; final List collectedResults = new ArrayList<>(); @@ -3463,15 +3462,15 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce } catch (Throwable t) { handleFetcherException(t, queue, "getSchemas"); } finally { - finalizeFetcher(queue, localResultSchemaFields, "Schema fetcher"); + finalizeFetcher(queue, localResultSchemaFields, "Multi-schema fetcher"); } }; - Future fetcherFuture = connection.getExecutorService().submit(schemaFetcher); + Future fetcherFuture = connection.getExecutorService().submit(multiSchemaFetcher); BigQueryJsonResultSet resultSet = BigQueryJsonResultSet.of(resultSchema, -1, queue, null, fetcherFuture); - LOG.info("Submitted background task for getSchemas to query executor"); + LOG.info("Submitted background task for multi-catalog getSchemas to query executor"); return resultSet; } From b64c66bcc10bc1c7ac44e18ddf4d7066a93b7244 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Mon, 6 Jul 2026 23:39:59 +0000 Subject: [PATCH 6/6] revert comment --- .../com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index a0677caafe89..752100f4888c 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -3438,6 +3438,7 @@ public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLExce return BigQueryJsonResultSet.of(resultSchema, -1, queue, null); } + // Multi-Catalog Path: fan out using connection-scoped metadataExecutor final BlockingQueue queue = new LinkedBlockingQueue<>(DEFAULT_QUEUE_CAPACITY); Runnable multiSchemaFetcher =