diff --git a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java index 0233951a044d..79f6bdb2a2f6 100644 --- a/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java +++ b/java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaData.java @@ -836,7 +836,7 @@ public ResultSet getProcedures( (rt) -> rt.getRoutineId().getRoutine(), procedureNamePattern, procedureNameRegex, - LOG); + false); Future> apiFuture = apiExecutor.submit(apiCallable); apiFutures.add(apiFuture); } @@ -1138,7 +1138,7 @@ List listMatchingProcedureIdsFromDatasets( (rt) -> rt.getRoutineId().getRoutine(), procedureNamePattern, procedureNameRegex, - logger); + false); listRoutineFutures.add(listRoutinesExecutor.submit(listCallable)); } logger.fine( @@ -1637,7 +1637,7 @@ public ResultSet getTables( (tbl) -> tbl.getTableId().getTable(), tableNamePattern, tableNameRegex, - LOG); + false); Future> apiFuture = apiExecutor.submit(apiCallable); apiFutures.add(apiFuture); } @@ -1951,7 +1951,7 @@ public ResultSet getColumns( (tbl) -> tbl.getTableId().getTable(), tableNamePattern, tableNameRegex, - LOG); + false); for (Table table : tablesToScan) { if (Thread.currentThread().isInterrupted()) { @@ -2428,13 +2428,11 @@ public ResultSet getPrimaryKeys(String catalog, String schema, String table) thr final List collectedResults = Collections.synchronizedList(new ArrayList<>()); List targetDatasets = getTargetDatasets(catalog, schema); - boolean ignoreAccessErrors = (catalog == null); processTargetTablesConcurrently( targetDatasets, table, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); processPrimaryKey(constraints, bqTable.getTableId(), results, fields); @@ -2525,13 +2523,11 @@ public ResultSet getImportedKeys(String catalog, String schema, String table) final List collectedResults = Collections.synchronizedList(new ArrayList<>()); List targetDatasets = getTargetDatasets(catalog, schema); - boolean ignoreAccessErrors = (catalog == null); processTargetTablesConcurrently( targetDatasets, table, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); if (constraints == null || constraints.getForeignKeys() == null) { @@ -2578,15 +2574,13 @@ public ResultSet getExportedKeys(String catalog, String schema, String table) // Fallback Path: If catalog or schema is null, fall back to REST API metadata scan. if (catalog == null || schema == null) { final List collectedResults = Collections.synchronizedList(new ArrayList<>()); - List targetDatasets = getTargetDatasets(catalog, schema); + List targetDatasets = getTargetDatasets(catalog, null); - boolean ignoreAccessErrors = (catalog == null); processTargetTablesConcurrently( targetDatasets, null, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); if (constraints == null || constraints.getForeignKeys() == null) { @@ -2656,13 +2650,11 @@ public ResultSet getCrossReference( final List collectedResults = Collections.synchronizedList(new ArrayList<>()); List targetDatasets = getTargetDatasets(foreignCatalog, foreignSchema); - boolean ignoreAccessErrors = (foreignCatalog == null); processTargetTablesConcurrently( targetDatasets, foreignTable, collectedResults, resultSchemaFields, - ignoreAccessErrors, (bqTable, results, fields) -> { TableConstraints constraints = bqTable.getTableConstraints(); if (constraints == null || constraints.getForeignKeys() == null) { @@ -3869,7 +3861,7 @@ public ResultSet getFunctions(String catalog, String schemaPattern, String funct (rt) -> rt.getRoutineId().getRoutine(), functionNamePattern, functionNameRegex, - LOG); + false); }; Future> apiFuture = apiExecutor.submit(apiCallable); apiFutures.add(apiFuture); @@ -4220,7 +4212,7 @@ List listMatchingFunctionIdsFromDatasets( (rt) -> rt.getRoutineId().getRoutine(), functionNamePattern, functionNameRegex, - logger); + false); listRoutineFutures.add(listRoutinesExecutor.submit(listCallable)); } logger.fine( @@ -4656,7 +4648,7 @@ List findMatchingBigQueryObjects( Function nameExtractor, String pattern, Pattern regex, - BigQueryJdbcCustomLogger logger) + boolean throwOn404) throws InterruptedException { boolean needsList = needsListing(pattern); @@ -4665,30 +4657,29 @@ List findMatchingBigQueryObjects( try { Iterable objects; if (needsList) { - logger.info( + LOG.info( "Listing all %ss (pattern: %s)...", objectTypeName, pattern == null ? "" : pattern); Page firstPage = listAllOperation.get(); objects = firstPage.iterateAll(); - logger.fine( - "Retrieved initial %s list, iterating & filtering if needed...", objectTypeName); + LOG.fine("Retrieved initial %s list, iterating & filtering if needed...", objectTypeName); } else { - logger.info("Getting specific %s: '%s'", objectTypeName, pattern); + LOG.info("Getting specific %s: '%s'", objectTypeName, pattern); T specificObject = getSpecificOperation.apply(pattern); objects = (specificObject == null) ? Collections.emptyList() : Collections.singletonList(specificObject); if (specificObject == null) { - logger.info("Specific %s not found: '%s'", objectTypeName, pattern); + LOG.info("Specific %s not found: '%s'", objectTypeName, pattern); } } boolean wasListing = needsList; for (T obj : objects) { if (Thread.currentThread().isInterrupted()) { - logger.warning("Thread interrupted during " + objectTypeName + " processing loop."); + LOG.warning("Thread interrupted during " + objectTypeName + " processing loop."); throw new InterruptedException( "Interrupted during " + objectTypeName + " processing loop"); } @@ -4705,20 +4696,21 @@ List findMatchingBigQueryObjects( } } catch (BigQueryException e) { - if (!needsList && e.getCode() == 404) { - logger.info("%s '%s' not found (API error 404).", objectTypeName, pattern); + if (e.getCode() == 404 && !throwOn404) { + LOG.info("%s '%s' not found (API error 404).", objectTypeName, pattern); + return Collections.emptyList(); } else { - logger.warning( + LOG.warning( "BigQueryException finding %ss for pattern '%s': %s (Code: %d)", objectTypeName, pattern, e.getMessage(), e.getCode()); throw e; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - logger.warning("Interrupted while finding " + objectTypeName + "s."); + LOG.warning("Interrupted while finding " + objectTypeName + "s."); throw e; } catch (Exception e) { - logger.severe( + LOG.severe( "Unexpected exception finding %ss for pattern '%s': %s", objectTypeName, pattern, e.getMessage()); throw new RuntimeException(e); @@ -5022,7 +5014,8 @@ private void signalEndOfData( } private List fetchDatasetsForProject( - String project, String schemaPattern, Pattern schemaRegex) throws SQLException { + String project, String schemaPattern, Pattern schemaRegex, boolean throwOn404) + throws SQLException { try { List datasets = findMatchingBigQueryObjects( @@ -5032,7 +5025,7 @@ private List fetchDatasetsForProject( (ds) -> ds.getDatasetId().getDataset(), schemaPattern, schemaRegex, - LOG); + throwOn404); return datasets != null ? datasets : Collections.emptyList(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -5052,9 +5045,11 @@ private List fetchMatchingDatasets( return Collections.emptyList(); } + boolean isBroadScan = (catalog == null); + // Single project path if (projects.size() == 1) { - return fetchDatasetsForProject(projects.get(0), schemaPattern, schemaRegex); + return fetchDatasetsForProject(projects.get(0), schemaPattern, schemaRegex, isBroadScan); } // Multi-project path @@ -5066,7 +5061,8 @@ private List fetchMatchingDatasets( for (String project : projects) { Callable task = () -> { - List datasets = fetchDatasetsForProject(project, schemaPattern, schemaRegex); + List datasets = + fetchDatasetsForProject(project, schemaPattern, schemaRegex, isBroadScan); allDatasets.addAll(datasets); return null; }; @@ -5179,7 +5175,6 @@ private void processSingleTable( String tableName, List collectedResults, FieldList resultSchemaFields, - boolean ignoreAccessErrors, TableProcessor processor) throws SQLException { Table bqTable; @@ -5187,10 +5182,10 @@ private void processSingleTable( bqTable = bigquery.getTable(TableId.of(datasetId.getProject(), datasetId.getDataset(), tableName)); } catch (BigQueryException e) { - if (ignoreAccessErrors && (e.getCode() == 404 || e.getCode() == 403)) { + if (e.getCode() == 404) { LOG.info( - "Table '%s' or dataset '%s' not found/accessible in project '%s' (API error %d). Skipping.", - tableName, datasetId.getDataset(), datasetId.getProject(), e.getCode()); + "Table '%s' or dataset '%s' not found in project '%s' (API error 404). Skipping.", + tableName, datasetId.getDataset(), datasetId.getProject()); bqTable = null; } else { throw new SQLException("Error while fetching table metadata: " + e.getMessage(), e); @@ -5208,17 +5203,11 @@ private void processTargetTablesConcurrently( String tableName, List collectedResults, FieldList resultSchemaFields, - boolean ignoreAccessErrors, TableProcessor processor) throws SQLException { if (targetDatasets.size() == 1 && tableName != null) { processSingleTable( - targetDatasets.get(0), - tableName, - collectedResults, - resultSchemaFields, - ignoreAccessErrors, - processor); + targetDatasets.get(0), tableName, collectedResults, resultSchemaFields, processor); return; } @@ -5232,12 +5221,7 @@ private void processTargetTablesConcurrently( tasks.add( () -> { processSingleTable( - datasetId, - tableName, - collectedResults, - resultSchemaFields, - ignoreAccessErrors, - processor); + datasetId, tableName, collectedResults, resultSchemaFields, processor); return null; }); continue; @@ -5261,16 +5245,15 @@ private void processTargetTablesConcurrently( table.getTableId().getTable(), collectedResults, resultSchemaFields, - ignoreAccessErrors, processor); return null; }); } } catch (BigQueryException e) { - if (ignoreAccessErrors && (e.getCode() == 404 || e.getCode() == 403)) { + if (e.getCode() == 404) { LOG.info( - "Dataset '%s' not found/accessible in project '%s' (API error %d). Skipping.", - datasetId.getDataset(), datasetId.getProject(), e.getCode()); + "Dataset '%s' not found in project '%s' (API error 404). Skipping.", + datasetId.getDataset(), datasetId.getProject()); continue; } throw new SQLException("Error while listing tables: " + e.getMessage(), e); diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java index b8e207e0db44..3ef43bf648fb 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryDatabaseMetaDataTest.java @@ -1028,7 +1028,7 @@ public void testFindMatchingBigQueryObjects_Routines_ListWithPattern() throws Ex (rt) -> rt.getRoutineId().getRoutine(), pattern, regex, - dbMetadata.LOG); + false); verify(bigqueryClient, times(1)) .listRoutines(eq(datasetId), any(BigQuery.RoutineListOption[].class)); @@ -1070,7 +1070,7 @@ public void testFindMatchingBigQueryObjects_Routines_ListNoPattern() throws Exce (rt) -> rt.getRoutineId().getRoutine(), pattern, regex, - dbMetadata.LOG); + false); verify(bigqueryClient, times(1)) .listRoutines(eq(datasetId), any(BigQuery.RoutineListOption[].class)); @@ -1105,7 +1105,7 @@ public void testFindMatchingBigQueryObjects_Routines_GetSpecific() throws Except (rt) -> rt.getRoutineId().getRoutine(), procNameExact, regex, - dbMetadata.LOG); + false); verify(bigqueryClient, times(1)).getRoutine(eq(routineId)); verify(bigqueryClient, never()) @@ -1117,6 +1117,57 @@ public void testFindMatchingBigQueryObjects_Routines_GetSpecific() throws Except assertSame(mockRoutine, resultList.get(0)); } + private List invokeFindMatchingObjectsWithException( + BigQueryException bqe, String pattern, boolean throwOn404) throws Exception { + return dbMetadata.findMatchingBigQueryObjects( + "Table", + () -> { + throw bqe; + }, + (name) -> { + throw bqe; + }, + (table) -> "name", + pattern, + dbMetadata.compileSqlLikePattern(pattern), + throwOn404); + } + + @Test + public void testFindMatchingBigQueryObjects_Swallows404_TargetedScan() throws Exception { + List
results = + invokeFindMatchingObjectsWithException( + new BigQueryException(404, "Not Found"), "exact_match", false); + assertTrue(results.isEmpty()); + } + + @Test + public void testFindMatchingBigQueryObjects_Throws404_BroadScan() { + assertThrows( + BigQueryException.class, + () -> + invokeFindMatchingObjectsWithException( + new BigQueryException(404, "Not Found"), "%", true)); + } + + @Test + public void testFindMatchingBigQueryObjects_Throws403_BroadScan() { + assertThrows( + BigQueryException.class, + () -> + invokeFindMatchingObjectsWithException( + new BigQueryException(403, "Access Denied"), "%", true)); + } + + @Test + public void testFindMatchingBigQueryObjects_Throws403_TargetedScan() { + assertThrows( + BigQueryException.class, + () -> + invokeFindMatchingObjectsWithException( + new BigQueryException(403, "Access Denied"), "exact_match", false)); + } + @Test public void testDefineGetProcedureColumnsSchema() { Schema schema = dbMetadata.defineGetProcedureColumnsSchema(); diff --git a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java index d83b5e440326..036fb086074d 100644 --- a/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java +++ b/java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITDatabaseMetadataTest.java @@ -852,10 +852,9 @@ public void testDatabaseMetadataGetSchemas() throws SQLException { Assertions.assertFalse(rsNoMatch.next()); // Test case 4: Get schemas with non-existent catalog - Assertions.assertThrows( - SQLException.class, - () -> databaseMetaData.getSchemas("invalid-catalog", null), - "Should throw SQLException for non-existent catalog"); + ResultSet rsInvalid = databaseMetaData.getSchemas("invalid-catalog", null); + Assertions.assertFalse( + rsInvalid.next(), "Should return empty ResultSet for non-existent catalog"); connection.close(); }