Skip to content

Commit b062acb

Browse files
michaelpri10michaelpri10
authored
Merge branch 'main' into pub-impl-test
2 parents fa99806 + 15467c8 commit b062acb

22 files changed

Lines changed: 797 additions & 329 deletions

File tree

‎gapic-libraries-bom/pom.xml‎

Lines changed: 220 additions & 220 deletions
Large diffs are not rendered by default.

‎java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ protected boolean removeEldestEntry(Map.Entry<String, Map<String, String>> eldes
7272
static final String QUERY_PROPERTIES_NAME = "QueryProperties";
7373
static final int DEFAULT_HTAPI_ACTIVATION_RATIO_VALUE = 2;
7474
static final String HTAPI_MIN_TABLE_SIZE_PROPERTY_NAME = "HighThroughputMinTableSize";
75-
static final int DEFAULT_HTAPI_MIN_TABLE_SIZE_VALUE = 100;
75+
static final int DEFAULT_HTAPI_MIN_TABLE_SIZE_VALUE = 10000;
7676
static final int DEFAULT_OAUTH_TYPE_VALUE = -1;
7777
static final String LOCATION_PROPERTY_NAME = "Location";
7878
static final String ENDPOINT_OVERRIDES_PROPERTY_NAME = "EndpointOverrides";

‎java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryStatement.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@
5555
import com.google.cloud.bigquery.storage.v1.ReadSession;
5656
import com.google.common.annotations.VisibleForTesting;
5757
import com.google.common.collect.ImmutableList;
58-
import com.google.common.collect.Iterators;
5958
import com.google.common.util.concurrent.Uninterruptibles;
6059
import java.lang.ref.ReferenceQueue;
6160
import java.sql.Connection;
@@ -944,15 +943,21 @@ private boolean meetsReadRatio(TableResult results) {
944943
LOG.finest("++enter++");
945944
long totalRows = results.getTotalRows();
946945

947-
if (totalRows == 0 || totalRows < querySettings.getHighThroughputMinTableSize()) {
946+
// SAFEGUARD: If all data has already been retrieved in the first page,
947+
// NEVER switch to the Read API as it would discard in-memory data and cause a double-fetch.
948+
if (totalRows == 0
949+
|| totalRows < querySettings.getHighThroughputMinTableSize()
950+
|| !results.hasNextPage()) {
948951
return false;
949952
}
950953

951-
// TODO(BQ Team): TableResult doesnt expose the number of records in the current page, hence the
952-
// below log iterates and counts. This is inefficient and we may eventually want to expose
953-
// PageSize with TableResults
954-
// TODO(Obada): Scope for performance optimization.
955-
int pageSize = Iterators.size(results.getValues().iterator());
954+
long pageSize = querySettings.getMaxResultPerPage();
955+
956+
// Prevent division by zero due to potential overflows/empty sets:
957+
if (pageSize <= 0) {
958+
pageSize = 1;
959+
}
960+
956961
return totalRows / pageSize > querySettings.getHighThroughputActivationRatio();
957962
}
958963

‎java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryStatementTest.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.google.cloud.bigquery.BigQueryOptions;
3333
import com.google.cloud.bigquery.Field;
3434
import com.google.cloud.bigquery.FieldList;
35+
import com.google.cloud.bigquery.FieldValueList;
3536
import com.google.cloud.bigquery.Job;
3637
import com.google.cloud.bigquery.JobId;
3738
import com.google.cloud.bigquery.JobInfo;
@@ -55,6 +56,7 @@
5556
import java.io.IOException;
5657
import java.sql.ResultSet;
5758
import java.sql.SQLException;
59+
import java.util.ArrayList;
5860
import java.util.HashMap;
5961
import java.util.List;
6062
import java.util.Map;
@@ -494,4 +496,98 @@ public void testGetStatementType(boolean isReadOnlyTokenUsed) throws Exception {
494496
verify(bigquery, isReadOnlyTokenUsed ? Mockito.never() : Mockito.times(1))
495497
.create(any(JobInfo.class));
496498
}
499+
500+
@Test
501+
public void testUseReadAPI_SafeguardSmallDataset() throws SQLException {
502+
// Setup: totalRows < MinTableSize, so it should not activate the Read API
503+
doReturn(true).when(bigQueryConnection).isEnableHighThroughputAPI();
504+
doReturn(100).when(bigQueryConnection).getHighThroughputMinTableSize();
505+
doReturn(2).when(bigQueryConnection).getHighThroughputActivationRatio();
506+
doReturn(1000L).when(bigQueryConnection).getMaxResults();
507+
508+
BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
509+
TableResult tableResult = mock(TableResult.class);
510+
doReturn(50L).when(tableResult).getTotalRows();
511+
512+
// Standard java collection in values
513+
List<FieldValueList> valuesList = new ArrayList<>();
514+
for (int i = 0; i < 50; i++) {
515+
valuesList.add(mock(FieldValueList.class));
516+
}
517+
doReturn(valuesList).when(tableResult).getValues();
518+
519+
boolean useReadApi = statement.useReadAPI(tableResult);
520+
assertThat(useReadApi).isFalse();
521+
}
522+
523+
@Test
524+
public void testUseReadAPI_SafeguardNoNextPage() throws SQLException {
525+
// Setup: totalRows = 500 > MinTableSize (100), but hasNextPage() is false.
526+
// Safeguard should prevent double-fetching and not activate the Read API.
527+
doReturn(true).when(bigQueryConnection).isEnableHighThroughputAPI();
528+
doReturn(100).when(bigQueryConnection).getHighThroughputMinTableSize();
529+
doReturn(2).when(bigQueryConnection).getHighThroughputActivationRatio();
530+
doReturn(1000L).when(bigQueryConnection).getMaxResults();
531+
532+
BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
533+
TableResult tableResult = mock(TableResult.class);
534+
doReturn(500L).when(tableResult).getTotalRows();
535+
doReturn(false).when(tableResult).hasNextPage();
536+
537+
boolean useReadApi = statement.useReadAPI(tableResult);
538+
assertThat(useReadApi).isFalse();
539+
}
540+
541+
@Test
542+
public void testUseReadAPI_MeetsRatio() throws SQLException {
543+
// Setup: totalRows = 500, maxResultPerPage = 100, MinTableSize = 100, ActivationRatio = 2
544+
// ratio = 5 > 2, should activate Read API
545+
doReturn(true).when(bigQueryConnection).isEnableHighThroughputAPI();
546+
doReturn(100).when(bigQueryConnection).getHighThroughputMinTableSize();
547+
doReturn(2).when(bigQueryConnection).getHighThroughputActivationRatio();
548+
doReturn(100L).when(bigQueryConnection).getMaxResults();
549+
550+
BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
551+
TableResult tableResult = mock(TableResult.class);
552+
doReturn(500L).when(tableResult).getTotalRows();
553+
doReturn(true).when(tableResult).hasNextPage();
554+
555+
boolean useReadApi = statement.useReadAPI(tableResult);
556+
assertThat(useReadApi).isTrue();
557+
}
558+
559+
@Test
560+
public void testUseReadAPI_FailsMinTableSize() throws SQLException {
561+
// Setup: totalRows = 80 < MinTableSize (100)
562+
doReturn(true).when(bigQueryConnection).isEnableHighThroughputAPI();
563+
doReturn(100).when(bigQueryConnection).getHighThroughputMinTableSize();
564+
doReturn(2).when(bigQueryConnection).getHighThroughputActivationRatio();
565+
doReturn(1000L).when(bigQueryConnection).getMaxResults();
566+
567+
BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
568+
TableResult tableResult = mock(TableResult.class);
569+
doReturn(80L).when(tableResult).getTotalRows();
570+
571+
boolean useReadApi = statement.useReadAPI(tableResult);
572+
assertThat(useReadApi).isFalse();
573+
}
574+
575+
@Test
576+
public void testUseReadAPI_ZeroPageSizeDivisionByZeroSafeguard() throws SQLException {
577+
// Setup: totalRows = 500, MinTableSize = 100, ActivationRatio = 2, maxResultPerPage = 0
578+
// Verify that the division by zero check safely guards and falls back to pageSize = 1
579+
doReturn(true).when(bigQueryConnection).isEnableHighThroughputAPI();
580+
doReturn(100).when(bigQueryConnection).getHighThroughputMinTableSize();
581+
doReturn(2).when(bigQueryConnection).getHighThroughputActivationRatio();
582+
doReturn(0L).when(bigQueryConnection).getMaxResults(); // maxResultPerPage = 0
583+
584+
BigQueryStatement statement = new BigQueryStatement(bigQueryConnection);
585+
TableResult tableResult = mock(TableResult.class);
586+
doReturn(500L).when(tableResult).getTotalRows();
587+
doReturn(true).when(tableResult).hasNextPage();
588+
589+
// This should not throw ArithmeticException (/ by zero) and should evaluate safely
590+
boolean useReadApi = statement.useReadAPI(tableResult);
591+
assertThat(useReadApi).isTrue(); // ratio = 500 / 1 = 500 > 2 -> true
592+
}
497593
}

‎java-datastore/datastore-v1-proto-client/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@
1919
<modelVersion>4.0.0</modelVersion>
2020
<groupId>com.google.cloud.datastore</groupId>
2121
<artifactId>datastore-v1-proto-client</artifactId>
22-
<version>3.0.0</version><!-- {x-version-update:datastore-v1-proto-client:current} -->
22+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:datastore-v1-proto-client:current} -->
2323

2424
<parent>
2525
<groupId>com.google.cloud</groupId>
2626
<artifactId>google-cloud-datastore-parent</artifactId>
27-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
27+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
2828
</parent>
2929

3030
<packaging>jar</packaging>

‎java-datastore/google-cloud-datastore-bom/pom.xml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>com.google.cloud</groupId>
55
<artifactId>google-cloud-datastore-bom</artifactId>
6-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore-bom:current} -->
6+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore-bom:current} -->
77
<packaging>pom</packaging>
88
<parent>
99
<groupId>com.google.cloud</groupId>
1010
<artifactId>google-cloud-pom-parent</artifactId>
11-
<version>1.86.0</version><!-- {x-version-update:google-cloud-java:current} -->
11+
<version>1.87.0-SNAPSHOT</version><!-- {x-version-update:google-cloud-java:current} -->
1212
<relativePath>../../google-cloud-pom-parent/pom.xml</relativePath>
1313
</parent>
1414

@@ -53,27 +53,27 @@
5353
<dependency>
5454
<groupId>com.google.cloud</groupId>
5555
<artifactId>google-cloud-datastore</artifactId>
56-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
56+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
5757
</dependency>
5858
<dependency>
5959
<groupId>com.google.api.grpc</groupId>
6060
<artifactId>grpc-google-cloud-datastore-admin-v1</artifactId>
61-
<version>3.0.0</version><!-- {x-version-update:grpc-google-cloud-datastore-admin-v1:current} -->
61+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:grpc-google-cloud-datastore-admin-v1:current} -->
6262
</dependency>
6363
<dependency>
6464
<groupId>com.google.api.grpc</groupId>
6565
<artifactId>grpc-google-cloud-datastore-v1</artifactId>
66-
<version>3.0.0</version><!-- {x-version-update:grpc-google-cloud-datastore-v1:current} -->
66+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:grpc-google-cloud-datastore-v1:current} -->
6767
</dependency>
6868
<dependency>
6969
<groupId>com.google.api.grpc</groupId>
7070
<artifactId>proto-google-cloud-datastore-v1</artifactId>
71-
<version>0.133.0</version><!-- {x-version-update:proto-google-cloud-datastore-v1:current} -->
71+
<version>0.134.0-SNAPSHOT</version><!-- {x-version-update:proto-google-cloud-datastore-v1:current} -->
7272
</dependency>
7373
<dependency>
7474
<groupId>com.google.api.grpc</groupId>
7575
<artifactId>proto-google-cloud-datastore-admin-v1</artifactId>
76-
<version>3.0.0</version><!-- {x-version-update:proto-google-cloud-datastore-admin-v1:current} -->
76+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:proto-google-cloud-datastore-admin-v1:current} -->
7777
</dependency>
7878
</dependencies>
7979
</dependencyManagement>

‎java-datastore/google-cloud-datastore-utils/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>com.google.cloud</groupId>
55
<artifactId>google-cloud-datastore-utils</artifactId>
6-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
6+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
77
<packaging>jar</packaging>
88
<name>Google Cloud Datastore Utilities</name>
99
<url>https://github.com/googleapis/google-cloud-java</url>
@@ -13,7 +13,7 @@
1313
<parent>
1414
<groupId>com.google.cloud</groupId>
1515
<artifactId>google-cloud-datastore-parent</artifactId>
16-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
16+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
1717
</parent>
1818
<properties>
1919
<site.installationModule>google-cloud-datastore-utils</site.installationModule>

‎java-datastore/google-cloud-datastore/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>com.google.cloud</groupId>
55
<artifactId>google-cloud-datastore</artifactId>
6-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
6+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
77
<packaging>jar</packaging>
88
<name>Google Cloud Datastore</name>
99
<url>https://github.com/googleapis/google-cloud-java</url>
@@ -13,7 +13,7 @@
1313
<parent>
1414
<groupId>com.google.cloud</groupId>
1515
<artifactId>google-cloud-datastore-parent</artifactId>
16-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
16+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
1717
</parent>
1818
<properties>
1919
<site.installationModule>google-cloud-datastore</site.installationModule>

‎java-datastore/grpc-google-cloud-datastore-admin-v1/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
<modelVersion>4.0.0</modelVersion>
55
<groupId>com.google.api.grpc</groupId>
66
<artifactId>grpc-google-cloud-datastore-admin-v1</artifactId>
7-
<version>3.0.0</version><!-- {x-version-update:grpc-google-cloud-datastore-admin-v1:current} -->
7+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:grpc-google-cloud-datastore-admin-v1:current} -->
88
<name>grpc-google-cloud-datastore-admin-v1</name>
99
<description>GRPC library for google-cloud-datastore</description>
1010
<parent>
1111
<groupId>com.google.cloud</groupId>
1212
<artifactId>google-cloud-datastore-parent</artifactId>
13-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
13+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
1414
</parent>
1515
<dependencies>
1616
<dependency>

‎java-datastore/grpc-google-cloud-datastore-v1/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
<modelVersion>4.0.0</modelVersion>
55
<groupId>com.google.api.grpc</groupId>
66
<artifactId>grpc-google-cloud-datastore-v1</artifactId>
7-
<version>3.0.0</version><!-- {x-version-update:grpc-google-cloud-datastore-v1:current} -->
7+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:grpc-google-cloud-datastore-v1:current} -->
88
<name>grpc-google-cloud-datastore-v1</name>
99
<description>GRPC library for google-cloud-datastore</description>
1010
<parent>
1111
<groupId>com.google.cloud</groupId>
1212
<artifactId>google-cloud-datastore-parent</artifactId>
13-
<version>3.0.0</version><!-- {x-version-update:google-cloud-datastore:current} -->
13+
<version>3.0.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-datastore:current} -->
1414
</parent>
1515
<dependencies>
1616
<dependency>

0 commit comments

Comments
 (0)