From ef20f5ef51daae8002530a905faae7d1dab26b12 Mon Sep 17 00:00:00 2001 From: dlarocque Date: Mon, 15 Jun 2026 11:33:34 -0500 Subject: [PATCH 1/5] feat(firestore): Add support for 16MB documents --- .../firestore/it/ITLargeDocumentTest.java | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java diff --git a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java new file mode 100644 index 000000000000..441a443f088f --- /dev/null +++ b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java @@ -0,0 +1,230 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.it; + +import static com.google.cloud.firestore.LocalFirestoreHelper.autoId; +import static com.google.cloud.firestore.it.ITQueryTest.map; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.cloud.firestore.CollectionReference; +import com.google.cloud.firestore.DocumentReference; +import com.google.cloud.firestore.DocumentSnapshot; +import com.google.cloud.firestore.FieldPath; +import com.google.cloud.firestore.FieldValue; +import com.google.cloud.firestore.FirestoreException; +import com.google.cloud.firestore.ListenerRegistration; +import com.google.cloud.firestore.QuerySnapshot; +import com.google.cloud.firestore.WriteResult; +import io.grpc.Status; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ITLargeDocumentTest extends ITBaseTest { + + private String collectionName; + private String unicodePayload; + private String asciiPayload; + + private DocumentReference docRef; + private DocumentReference docA; + private DocumentReference docB; + + static boolean runLargeDocTests() { + String propertyName = "FIRESTORE_RUN_LARGE_DOC_TESTS"; + String runLargeTests = System.getProperty(propertyName); + if (runLargeTests == null) { + runLargeTests = System.getenv(propertyName); + } + return "YES".equalsIgnoreCase(runLargeTests) || "true".equalsIgnoreCase(runLargeTests); + } + + private static String generateUnicodeString(int targetUtf8Bytes) { + StringBuilder sb = new StringBuilder(); + String emoji = "🚀"; // 4 bytes in UTF-8 + int bytes = 0; + while (bytes < targetUtf8Bytes) { + if (bytes % 2 == 0 && bytes + 4 <= targetUtf8Bytes) { + sb.append(emoji); + bytes += 4; + } else { + sb.append('a'); + bytes += 1; + } + } + return sb.toString(); + } + + private static String generateAsciiString(int sizeInBytes) { + char[] chars = new char[sizeInBytes]; + Arrays.fill(chars, 'a'); + return new String(chars); + } + + @Before + @Override + public void before() throws Exception { + // Check preconditions before setting up + assumeTrue(runLargeDocTests()); + assumeTrue("NIGHTLY".equalsIgnoreCase(getTargetBackend())); + assumeTrue(getFirestoreEdition() == FirestoreEdition.ENTERPRISE); + + // Call base class before() to initialize firestore + super.before(); + + collectionName = "large_doc_tests_" + autoId(); + CollectionReference colRef = firestore.collection(collectionName); + docRef = colRef.document("doc_15_9MB_unicode"); + docA = colRef.document("doc_a"); + docB = colRef.document("doc_b"); + + int targetBytes = (int) Math.floor(15.9 * 1024 * 1024); + unicodePayload = generateUnicodeString(targetBytes); + asciiPayload = generateAsciiString(targetBytes); + + // Write documents in parallel + ApiFuture f1 = docRef.set(map("chunk", unicodePayload)); + ApiFuture f2 = docA.set(map("chunk", asciiPayload)); + ApiFuture f3 = docB.set(map("chunk", asciiPayload)); + + ApiFutures.allAsList(Arrays.asList(f1, f2, f3)).get(90, TimeUnit.SECONDS); + } + + @After + @Override + public void after() throws Exception { + if (firestore != null && collectionName != null) { + try { + // Delete documents in parallel + ApiFuture d1 = docRef.delete(); + ApiFuture d2 = docA.delete(); + ApiFuture d3 = docB.delete(); + ApiFutures.allAsList(Arrays.asList(d1, d2, d3)).get(30, TimeUnit.SECONDS); + } catch (Exception e) { + // Suppress errors during cleanup to not mask test failures + } + } + super.after(); + } + + @Test + public void testReadLargeUnicodeDocument() throws Exception { + DocumentSnapshot snapshot = docRef.get().get(); + assertTrue(snapshot.exists()); + String chunk = snapshot.getString("chunk"); + assertNotNull(chunk); + assertEquals(unicodePayload.length(), chunk.length()); + assertEquals(unicodePayload, chunk); + } + + @Test + public void testQueryMultipleLargeDocuments() throws Exception { + CollectionReference colRef = firestore.collection(collectionName); + QuerySnapshot querySnapshot = + colRef.whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b")).get().get(); + assertEquals(2, querySnapshot.size()); + + DocumentSnapshot snapshotA = querySnapshot.getDocuments().get(0); + DocumentSnapshot snapshotB = querySnapshot.getDocuments().get(1); + assertEquals(asciiPayload, snapshotA.getString("chunk")); + assertEquals(asciiPayload, snapshotB.getString("chunk")); + } + + @Test + public void testWatchStreamInitialization() throws Exception { + CompletableFuture snapshotFuture = new CompletableFuture<>(); + ListenerRegistration registration = + docRef.addSnapshotListener( + (snapshot, error) -> { + if (error != null) { + snapshotFuture.completeExceptionally(error); + } else if (snapshot != null && snapshot.exists()) { + snapshotFuture.complete(snapshot); + } + }); + + try { + DocumentSnapshot snapshot = snapshotFuture.get(60, TimeUnit.SECONDS); + assertTrue(snapshot.exists()); + assertEquals(unicodePayload, snapshot.getString("chunk")); + } finally { + registration.remove(); + } + } + + @Test + public void testTransactionReadModifyWrite() throws Exception { + firestore + .runTransaction( + transaction -> { + DocumentSnapshot snapshot = transaction.get(docRef).get(); + assertTrue(snapshot.exists()); + transaction.update(docRef, map("transaction_timestamp", FieldValue.serverTimestamp())); + return null; + }) + .get(60, TimeUnit.SECONDS); + } + + @Test + public void testPaginateLargeDocuments() throws Exception { + CollectionReference colRef = firestore.collection(collectionName); + com.google.cloud.firestore.Query q = + colRef.whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b")).orderBy(FieldPath.documentId()); + + QuerySnapshot firstPage = q.limit(1).get().get(); + assertEquals(1, firstPage.size()); + DocumentSnapshot doc1 = firstPage.getDocuments().get(0); + assertEquals(asciiPayload, doc1.getString("chunk")); + + QuerySnapshot secondPage = q.startAfter(doc1).limit(1).get().get(); + assertEquals(1, secondPage.size()); + DocumentSnapshot doc2 = secondPage.getDocuments().get(0); + assertEquals(asciiPayload, doc2.getString("chunk")); + } + + @Test + public void testOversizedPayloadRejection() { + DocumentReference oversizedDoc = firestore.collection(collectionName).document("temp_oversized_doc"); + int targetBytes = 16 * 1024 * 1024 + 102400; + String largePayload = generateAsciiString(targetBytes); + Map data = new HashMap<>(); + data.put("chunk", largePayload); + + try { + oversizedDoc.set(data).get(60, TimeUnit.SECONDS); + fail("Setting a document exceeding the 16MB limit should fail."); + } catch (Exception e) { + Throwable cause = e.getCause(); + assertTrue(cause instanceof com.google.api.gax.rpc.InvalidArgumentException); + } + } +} From 753c46c78c446127ead209395209fa0b7bae2cab Mon Sep 17 00:00:00 2001 From: dlarocque Date: Mon, 29 Jun 2026 16:45:13 -0400 Subject: [PATCH 2/5] optimize 16mb string allocation --- .../com/google/cloud/firestore/it/ITLargeDocumentTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java index 441a443f088f..30f73a2e84bc 100644 --- a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java +++ b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java @@ -215,9 +215,9 @@ public void testPaginateLargeDocuments() throws Exception { public void testOversizedPayloadRejection() { DocumentReference oversizedDoc = firestore.collection(collectionName).document("temp_oversized_doc"); int targetBytes = 16 * 1024 * 1024 + 102400; - String largePayload = generateAsciiString(targetBytes); - Map data = new HashMap<>(); - data.put("chunk", largePayload); + char[] chars = new char[targetBytes]; + Arrays.fill(chars, 'a'); + String largePayload = new String(chars); try { oversizedDoc.set(data).get(60, TimeUnit.SECONDS); From 8034945ad2f846528d59bbf0f666ff7ef9a4b4f0 Mon Sep 17 00:00:00 2001 From: dlarocque Date: Mon, 13 Jul 2026 14:39:17 -0400 Subject: [PATCH 3/5] format and review comment --- .../cloud/firestore/it/ITLargeDocumentTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java index 30f73a2e84bc..28f66220d457 100644 --- a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java +++ b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java @@ -18,7 +18,6 @@ import static com.google.cloud.firestore.LocalFirestoreHelper.autoId; import static com.google.cloud.firestore.it.ITQueryTest.map; -import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -32,14 +31,10 @@ import com.google.cloud.firestore.DocumentSnapshot; import com.google.cloud.firestore.FieldPath; import com.google.cloud.firestore.FieldValue; -import com.google.cloud.firestore.FirestoreException; import com.google.cloud.firestore.ListenerRegistration; import com.google.cloud.firestore.QuerySnapshot; import com.google.cloud.firestore.WriteResult; -import io.grpc.Status; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.junit.After; @@ -188,7 +183,8 @@ public void testTransactionReadModifyWrite() throws Exception { transaction -> { DocumentSnapshot snapshot = transaction.get(docRef).get(); assertTrue(snapshot.exists()); - transaction.update(docRef, map("transaction_timestamp", FieldValue.serverTimestamp())); + transaction.update( + docRef, map("transaction_timestamp", FieldValue.serverTimestamp())); return null; }) .get(60, TimeUnit.SECONDS); @@ -198,22 +194,27 @@ public void testTransactionReadModifyWrite() throws Exception { public void testPaginateLargeDocuments() throws Exception { CollectionReference colRef = firestore.collection(collectionName); com.google.cloud.firestore.Query q = - colRef.whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b")).orderBy(FieldPath.documentId()); + colRef + .whereIn(FieldPath.documentId(), Arrays.asList("doc_a", "doc_b")) + .orderBy(FieldPath.documentId()); QuerySnapshot firstPage = q.limit(1).get().get(); assertEquals(1, firstPage.size()); DocumentSnapshot doc1 = firstPage.getDocuments().get(0); + assertEquals("doc_a", doc1.getId()); assertEquals(asciiPayload, doc1.getString("chunk")); QuerySnapshot secondPage = q.startAfter(doc1).limit(1).get().get(); assertEquals(1, secondPage.size()); DocumentSnapshot doc2 = secondPage.getDocuments().get(0); + assertEquals("doc_b", doc2.getId()); assertEquals(asciiPayload, doc2.getString("chunk")); } @Test public void testOversizedPayloadRejection() { - DocumentReference oversizedDoc = firestore.collection(collectionName).document("temp_oversized_doc"); + DocumentReference oversizedDoc = + firestore.collection(collectionName).document("temp_oversized_doc"); int targetBytes = 16 * 1024 * 1024 + 102400; char[] chars = new char[targetBytes]; Arrays.fill(chars, 'a'); From 3b3efe102a0b086aa102088c5fec96855a46bb1f Mon Sep 17 00:00:00 2001 From: dlarocque Date: Mon, 13 Jul 2026 14:52:19 -0400 Subject: [PATCH 4/5] fix --- .../java/com/google/cloud/firestore/it/ITLargeDocumentTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java index 28f66220d457..46876096ea82 100644 --- a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java +++ b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java @@ -221,7 +221,7 @@ public void testOversizedPayloadRejection() { String largePayload = new String(chars); try { - oversizedDoc.set(data).get(60, TimeUnit.SECONDS); + oversizedDoc.set(map("chunk", largePayload)).get(60, TimeUnit.SECONDS); fail("Setting a document exceeding the 16MB limit should fail."); } catch (Exception e) { Throwable cause = e.getCause(); From fe5837692fcb3eaeee9467b2e69a3d4698153756 Mon Sep 17 00:00:00 2001 From: dlarocque Date: Wed, 22 Jul 2026 16:10:32 -0400 Subject: [PATCH 5/5] test(firestore): guard super.after() in ITLargeDocumentTest when firestore is null --- .../firestore/it/ITLargeDocumentTest.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java index 46876096ea82..5391d016b850 100644 --- a/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java +++ b/java-firestore/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITLargeDocumentTest.java @@ -117,18 +117,20 @@ public void before() throws Exception { @After @Override public void after() throws Exception { - if (firestore != null && collectionName != null) { - try { - // Delete documents in parallel - ApiFuture d1 = docRef.delete(); - ApiFuture d2 = docA.delete(); - ApiFuture d3 = docB.delete(); - ApiFutures.allAsList(Arrays.asList(d1, d2, d3)).get(30, TimeUnit.SECONDS); - } catch (Exception e) { - // Suppress errors during cleanup to not mask test failures + if (firestore != null) { + if (collectionName != null) { + try { + // Delete documents in parallel + ApiFuture d1 = docRef.delete(); + ApiFuture d2 = docA.delete(); + ApiFuture d3 = docB.delete(); + ApiFutures.allAsList(Arrays.asList(d1, d2, d3)).get(30, TimeUnit.SECONDS); + } catch (Exception e) { + // Suppress errors during cleanup to not mask test failures + } } + super.after(); } - super.after(); } @Test