From 7d7d6a82f9e4fc0bfa48ef553b185766e8f67bbc Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:48:58 +0200 Subject: [PATCH 1/5] Document server-side batching ingest and refactor the batch import page - Add the one-shot SSB entry points to the batch import docs: data.ingest() (Python sync+async, TypeScript) and Batch.InsertMany (C#), alongside the streaming examples - Add an FAQ entry for the insert_many 'message larger than max' (RESOURCE_EXHAUSTED) failure with the verified client error and the ingest/SSB fix - State the GRPC_MAX_MESSAGE_SIZE default only in the env-vars table (row anchor added); other pages link to it; correct the stale 10MB value - Refactor manage-objects/import.mdx: remove the gRPC API and Python-specific sections, promote Error handling to its own section, rewrite Stream data from large files as tips, trim Asynchronous imports, and merge the four Specify sections under Customize imported objects with snippets moved to ingest/SSB (Go remains manual batching) - Rank SSB first on the Python client pages and best practices --- .../code/csharp/ManageObjectsImportTest.cs | 80 +++- _includes/code/howto/manage-data.import.py | 186 ++++++---- _includes/code/howto/manage-data.import.ts | 15 +- .../test/java/ManageObjectsImportTest.java | 97 +++-- docs/deploy/configuration/env-vars/index.md | 2 +- docs/weaviate/best-practices/index.md | 14 +- .../weaviate/client-libraries/python/async.md | 9 +- .../client-libraries/python/index.mdx | 14 + .../python/notes-best-practices.mdx | 2 + docs/weaviate/concepts/data-import.mdx | 2 +- docs/weaviate/manage-objects/import.mdx | 342 ++++-------------- docs/weaviate/more-resources/faq.md | 25 ++ 12 files changed, 386 insertions(+), 402 deletions(-) diff --git a/_includes/code/csharp/ManageObjectsImportTest.cs b/_includes/code/csharp/ManageObjectsImportTest.cs index c556d141e..822c8e5dd 100644 --- a/_includes/code/csharp/ManageObjectsImportTest.cs +++ b/_includes/code/csharp/ManageObjectsImportTest.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using CsvHelper; using Weaviate.Client; +using Weaviate.Client.Batch; using Weaviate.Client.Models; using Xunit; @@ -146,8 +147,55 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); - // Use `Batch.InsertMany` for server-side batching. The client sends - // data in batches at a rate controlled by the server. + // Use `Batch.StartBatch` for server-side batching. The client streams + // objects to the server, which paces the import based on its own load. + // highlight-start + await using var batch = await collection.Batch.StartBatch(); + + var handles = new List(); + foreach (var dataRow in dataRows) + { + handles.Add(await batch.Add(dataRow)); + } + + await batch.Close(); + // highlight-end + + var results = await Task.WhenAll(handles.Select(h => h.Result)); + var failedObjects = results.Where(r => !r.Success).ToList(); + if (failedObjects.Any()) + { + Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); + } + // END ServerSideBatchImportExample + + var result = await collection.Aggregate.OverAll(totalCount: true); + Assert.Equal(5, result.TotalCount); + } + + [Fact] + public async Task TestServerSideIngest() + { + await BeforeEach(); + await client.Collections.Create( + new CollectionCreateParams + { + Name = "MyCollection", + VectorConfig = Configure.Vector("default", v => v.SelfProvided()), + } + ); + + // START ServerSideIngestExample + var dataRows = Enumerable + .Range(0, 5) + .Select(i => new { title = $"Object {i + 1}" }) + .ToList(); + + var collection = client.Collections.Use("MyCollection"); + + // `Batch.InsertMany` is the one-shot server-side ingest of an + // in-memory list. The client streams the list to the server + // using server-side batching under the hood. // highlight-start var response = await collection.Batch.InsertMany(dataRows); // highlight-end @@ -158,7 +206,7 @@ await client.Collections.Create( Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); Console.WriteLine($"First failed object: {failedObjects.First().Error}"); } - // END ServerSideBatchImportExample + // END ServerSideIngestExample var result = await collection.Aggregate.OverAll(totalCount: true); Assert.Equal(5, result.TotalCount); @@ -194,8 +242,9 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); + // `Batch.InsertMany` imports the list using server-side batching. // highlight-start - var response = await collection.Data.InsertMany(dataToInsert); + var response = await collection.Batch.InsertMany(dataToInsert); // highlight-end var failedObjects = response.Where(r => r.Error != null).ToList(); @@ -245,13 +294,17 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); - var response = await collection.Data.InsertMany(dataToInsert); + // `Batch.InsertMany` imports the list using server-side batching. + // highlight-start + var response = await collection.Batch.InsertMany(dataToInsert); + // highlight-end // Handle errors - if (response.HasErrors) + var failedObjects = response.Where(r => r.Error != null).ToList(); + if (failedObjects.Any()) { - Console.WriteLine($"Number of failed imports: {response.Errors.Count()}"); - Console.WriteLine($"First failed object: {response.Errors.First().Message}"); + Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); + Console.WriteLine($"First failed object: {failedObjects.First().Error}"); } // END BatchImportWithVectorExample @@ -342,16 +395,17 @@ await client.Collections.Create( var collection = client.Collections.Use("MyCollection"); - // Insert the data using InsertMany + // `Batch.InsertMany` imports the list using server-side batching. // highlight-start - var response = await collection.Data.InsertMany(dataToInsert); + var response = await collection.Batch.InsertMany(dataToInsert); // highlight-end // Handle errors - if (response.HasErrors) + var failedObjects = response.Where(r => r.Error != null).ToList(); + if (failedObjects.Any()) { - Console.WriteLine($"Number of failed imports: {response.Errors.Count()}"); - Console.WriteLine($"First failed object error: {response.Errors.First().Message}"); + Console.WriteLine($"Number of failed imports: {failedObjects.Count}"); + Console.WriteLine($"First failed object error: {failedObjects.First().Error}"); } // END BatchImportWithNamedVectors } diff --git a/_includes/code/howto/manage-data.import.py b/_includes/code/howto/manage-data.import.py index fa191b5b7..4a7bf34f0 100644 --- a/_includes/code/howto/manage-data.import.py +++ b/_includes/code/howto/manage-data.import.py @@ -157,38 +157,43 @@ # ===== Insert many with custom ID ===== # ======================================= +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + # START BatchImportWithIDExample # highlight-start from weaviate.util import generate_uuid5 # Generate a deterministic ID +from weaviate.classes.data import DataObject # highlight-end -# START BatchImportWithIDExample data_rows = [{"title": f"Object {i+1}"} for i in range(5)] collection = client.collections.use("MyCollection") # highlight-start -with collection.batch.fixed_size(batch_size=200) as batch: - for data_row in data_rows: - obj_uuid = generate_uuid5(data_row) - batch.add_object( - properties=data_row, - uuid=obj_uuid - ) +data_objects = [ + DataObject( + properties=data_row, + uuid=generate_uuid5(data_row) + ) + for data_row in data_rows +] + +result = collection.data.ingest(data_objects) # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break -failed_objects = collection.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithIDExample -result = collection.aggregate.over_all(total_count=True) -assert result.total_count == 5 -resp_obj = collection.query.fetch_object_by_id(obj_uuid) +# Tests +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 +resp_obj = collection.query.fetch_object_by_id(generate_uuid5(data_rows[-1])) assert resp_obj != None # Clean up client.collections.delete(collection.name) @@ -197,32 +202,40 @@ # ===== Batch import with custom vector ===== # =========================================== +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + # START BatchImportWithVectorExample +from weaviate.classes.data import DataObject + data_rows = [{"title": f"Object {i+1}"} for i in range(5)] vectors = [[0.1] * 1536 for i in range(5)] collection = client.collections.use("MyCollection") # highlight-start -with collection.batch.fixed_size(batch_size=200) as batch: - for i, data_row in enumerate(data_rows): - batch.add_object( - properties=data_row, - vector=vectors[i] - ) +data_objects = [ + DataObject( + properties=data_row, + vector=vectors[i] + ) + for i, data_row in enumerate(data_rows) +] + +result = collection.data.ingest(data_objects) # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break -failed_objects = collection.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithVectorExample -result = collection.aggregate.over_all(total_count=True) -assert result.total_count == 5 +# Tests +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 # Clean up client.collections.delete(collection.name) @@ -255,6 +268,8 @@ ) # START BatchImportWithNamedVectors +from weaviate.classes.data import DataObject + data_rows = [{ "title": f"Object {i+1}", "body": f"Body {i+1}" @@ -266,24 +281,22 @@ collection = client.collections.use("MyCollection") # highlight-start -with collection.batch.fixed_size(batch_size=200) as batch: - for i, data_row in enumerate(data_rows): - batch.add_object( - properties=data_row, - vector={ - "title": title_vectors[i], - "body": body_vectors[i], - } - ) +data_objects = [ + DataObject( + properties=data_row, + vector={ + "title": title_vectors[i], + "body": body_vectors[i], + } + ) + for i, data_row in enumerate(data_rows) +] + +result = collection.data.ingest(data_objects) # highlight-end - if batch.number_errors > 10: - print("Batch import stopped due to excessive errors.") - break -failed_objects = collection.batch.failed_objects -if failed_objects: - print(f"Number of failed imports: {len(failed_objects)}") - print(f"First failed object: {failed_objects[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithNamedVectors response = collection.query.fetch_objects(include_vector=True) @@ -318,36 +331,37 @@ ] ) -from_uuid = authors.data.insert( - properties={"name": "Jane Austen"} -) - publications.data.insert( {"title": "Ye Olde Times"} ) target_uuid = publications.query.fetch_objects(limit=1).objects[0].uuid -# BatchImportWithRefExample +# START BatchImportWithRefExample +from weaviate.classes.data import DataObject + collection = client.collections.use("Author") -with collection.batch.fixed_size(batch_size=100) as batch: - batch.add_reference( - from_property="writesFor", - from_uuid=from_uuid, - to=target_uuid, - ) +# highlight-start +data_objects = [ + DataObject( + properties={"name": "Jane Austen"}, + references={"writesFor": target_uuid}, + ), +] + +result = collection.data.ingest(data_objects) +# highlight-end -failed_references = collection.batch.failed_references -if failed_references: - print(f"Number of failed imports: {len(failed_references)}") - print(f"First failed reference: {failed_references[0]}") +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") # END BatchImportWithRefExample # Tests from weaviate.classes.query import QueryReference +new_uuid = result.uuids[0] response = collection.query.fetch_object_by_id( - from_uuid, + new_uuid, return_references=QueryReference(link_on="writesFor", return_properties=["title"]) ) @@ -602,4 +616,46 @@ def add_object(obj) -> None: client.collections.delete(collection.name) +# ================================================== +# ===== Server-side one-shot ingest ===== +# ================================================== + +# Re-create the collection +client.collections.delete("MyCollection") +client.collections.create( + "MyCollection", + vector_config=Configure.Vectors.self_provided() +) + +# START ServerSideIngestExample +data_rows = [ + {"title": f"Object {i+1}"} for i in range(5) +] + +collection = client.collections.use("MyCollection") + +# highlight-start +# `ingest` imports the whole list with server-side batching in a single call +result = collection.data.ingest(data_rows) +# highlight-end + +# The return object is the same as for `insert_many` +if result.errors: + print(f"Number of failed imports: {len(result.errors)}") + # `errors` is a dict keyed by the index of the failed object + for index, error in result.errors.items(): + print(f"Failed object at index {index}: {error.message}") +# END ServerSideIngestExample + +# Tests +assert len(result.errors) == 0 +assert len(result.uuids) == 5 + +agg_result = collection.aggregate.over_all(total_count=True) +assert agg_result.total_count == 5 + +# Clean up +client.collections.delete(collection.name) + + client.close() diff --git a/_includes/code/howto/manage-data.import.ts b/_includes/code/howto/manage-data.import.ts index d5b438d2d..a0fdc8788 100644 --- a/_includes/code/howto/manage-data.import.ts +++ b/_includes/code/howto/manage-data.import.ts @@ -111,7 +111,10 @@ let dataObjects = [ ] const myCollection = client.collections.use('MyCollection') -await myCollection.data.insertMany(dataObject) +// highlight-start +// `ingest` imports the list using server-side batching +await myCollection.data.ingest(dataObjects) +// highlight-end // END BatchImportWithIDExample // result = await client.graphql.aggregate().withClassName(className).withFields('meta { count }').do(); @@ -146,7 +149,10 @@ let dataObjects = [ // ... ] -await jeopardy.data.insertMany(dataObjects) +// highlight-start +// `ingest` imports the list using server-side batching +await myCollection.data.ingest(dataObjects) +// highlight-end // END BatchImportWithVectorExample // result = await client.graphql.aggregate().withClassName(className).withFields('meta { count }').do(); @@ -204,7 +210,10 @@ let dataObjects = [ // ... ] -await myCollection.data.insertMany(dataObjects) +// highlight-start +// `ingest` imports the list using server-side batching +await myCollection.data.ingest(dataObjects) +// highlight-end } // END BatchImportWithNamedVectors diff --git a/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java b/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java index 4e0c5140f..0211cb63a 100644 --- a/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java +++ b/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java @@ -7,7 +7,6 @@ import io.weaviate.client6.v1.api.collections.data.BatchReference; import io.weaviate.client6.v1.api.collections.Vectors; import io.weaviate.client6.v1.api.collections.WeaviateObject; -import io.weaviate.client6.v1.api.collections.data.InsertManyResponse; import io.weaviate.client6.v1.api.collections.query.QueryReference; import org.junit.jupiter.api.AfterAll; @@ -158,32 +157,35 @@ void testBatchImportWithID() throws IOException { col -> col.vectorConfig(VectorConfig.selfProvided())); // START BatchImportWithIDExample - List>> dataObjects = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - Map dataRow = Map.of("title", "Object " + (i + 1)); - UUID objUuid = generateUuid5(dataRow.toString()); - - dataObjects.add(WeaviateObject.>of( - obj -> obj.properties(dataRow).uuid(objUuid.toString()))); - } - var collection = client.collections.use("MyCollection"); + // Add objects with custom IDs to a server-side batch import. // highlight-start - var response = collection.data.insertMany(dataObjects); + BatchContext> batch = collection.batch.start(); + try (batch) { + for (int i = 0; i < 5; i++) { + Map dataRow = Map.of("title", "Object " + (i + 1)); + UUID objUuid = generateUuid5(dataRow.toString()); + + batch.add(WeaviateObject.>of( + obj -> obj.properties(dataRow).uuid(objUuid.toString()))); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } // highlight-end - if (!response.errors().isEmpty()) { + if (batch.numberOfErrors() > 0) { System.err - .println("Number of failed imports: " + response.errors().size()); - System.err.println("First failed object: " + response.errors().get(0)); + .println("Number of failed imports: " + batch.numberOfErrors()); } // END BatchImportWithIDExample var result = collection.aggregate.overAll(agg -> agg.includeTotalCount(true)); assertThat(result.totalCount()).isEqualTo(5); - String lastUuid = dataObjects.get(4).uuid(); + String lastUuid = + generateUuid5(Map.of("title", "Object 5").toString()).toString(); assertThat(collection.data.exists(lastUuid)).isTrue(); client.collections.delete("MyCollection"); @@ -195,30 +197,29 @@ void testBatchImportWithVector() throws IOException { col -> col.vectorConfig(VectorConfig.selfProvided())); // START BatchImportWithVectorExample - List>> dataObjects = new ArrayList<>(); float[] vector = new float[10]; // Using a small vector for demonstration Arrays.fill(vector, 0.1f); - for (int i = 0; i < 5; i++) { - Map dataRow = Map.of("title", "Object " + (i + 1)); - UUID objUuid = generateUuid5(dataRow.toString()); - - dataObjects.add( - WeaviateObject.>of(obj -> obj.properties(dataRow) - .uuid(objUuid.toString()) - .vectors(Vectors.of(vector)))); - } - var collection = client.collections.use("MyCollection"); + // Add objects with custom vectors to a server-side batch import. // highlight-start - var response = collection.data.insertMany(dataObjects); + BatchContext> batch = collection.batch.start(); + try (batch) { + for (int i = 0; i < 5; i++) { + Map dataRow = Map.of("title", "Object " + (i + 1)); + + batch.add(WeaviateObject.>of( + obj -> obj.properties(dataRow).vectors(Vectors.of(vector)))); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } // highlight-end - if (!response.errors().isEmpty()) { + if (batch.numberOfErrors() > 0) { System.err - .println("Number of failed imports: " + response.errors().size()); - System.err.println("First failed object: " + response.errors().get(0)); + .println("Number of failed imports: " + batch.numberOfErrors()); } // END BatchImportWithVectorExample @@ -297,32 +298,26 @@ void testImportWithNamedVectors() throws IOException { CollectionHandle> collection = client.collections.use("MyCollection"); - List>> objectsToInsert = - new ArrayList<>(); - for (int i = 0; i < dataRows.size(); i++) { - int index = i; - objectsToInsert.add( - // highlight-start - // Use the Builder with the EXACT matching generic types - WeaviateObject - .>of(v -> v.properties(dataRows.get(index)) - .vectors(Vectors.of("title", titleVectors.get(index))) - .vectors(Vectors.of("body", bodyVectors.get(index))))); - // highlight-end - - } - - // Insert the data using insertMany with the List + // Add objects with named vectors to a server-side batch import. // highlight-start - InsertManyResponse response = collection.data.insertMany(objectsToInsert); + BatchContext> batch = collection.batch.start(); + try (batch) { + for (int i = 0; i < dataRows.size(); i++) { + int index = i; + batch.add(WeaviateObject + .>of(v -> v.properties(dataRows.get(index)) + .vectors(Vectors.of("title", titleVectors.get(index))) + .vectors(Vectors.of("body", bodyVectors.get(index))))); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } // highlight-end // Check for errors - if (!response.errors().isEmpty()) { + if (batch.numberOfErrors() > 0) { System.err.printf("Number of failed imports: %d\n", - response.errors().size()); - System.err.printf("First failed object error: %s\n", - response.errors().get(0)); + batch.numberOfErrors()); } // END BatchImportWithNamedVectors } diff --git a/docs/deploy/configuration/env-vars/index.md b/docs/deploy/configuration/env-vars/index.md index e5cca50b4..a0c6d1028 100644 --- a/docs/deploy/configuration/env-vars/index.md +++ b/docs/deploy/configuration/env-vars/index.md @@ -58,7 +58,7 @@ import APITable from '@site/src/components/APITable'; | `GO_PROFILING_DISABLE` | If `true`, disables Go profiling. Default: `false`. | `boolean` | `false` | | `GO_PROFILING_PORT` | Sets the port for the Go profiler. Default: `6060` | `integer` | `6060` | | `DEBUG_ENDPOINTS_ENABLED` | Gate for the debug HTTP listener (the profiling port set by `GO_PROFILING_PORT`, default `6060`), which serves Weaviate's **unauthenticated** internal debug and profiling endpoints — `/debug/config`, Go profiling (`/debug/pprof/*`, `/debug/fgprof`), and various maintenance and diagnostic routes. [Runtime-configurable](/deploy/configuration/env-vars/runtime-config.md) via the `debug_endpoints_enabled` override. Default: `false`. `GO_PROFILING_DISABLE` still controls whether the listener binds at all.
Added in `v1.37.9` | `boolean` | `true` | -| `GRPC_MAX_MESSAGE_SIZE` | Maximum gRPC message size in bytes. Default: 10MB | `string - number` | `2000000000` | +| `GRPC_MAX_MESSAGE_SIZE` | Maximum gRPC message size in bytes. Requests larger than this limit (e.g. a large `insert_many` call) are rejected. Default: `104858000` (approximately 100 MB) | `string - number` | `2000000000` | | `GRPC_PORT` | The port on which Weaviate's gRPC server listens for incoming requests. Default: `50051` | `string - number` | `50052` | | `HNSW_GEO_INDEX_EF` | Balance geo index search speed and recall. This value controls the search depth for geo-based queries. Default: `800`
Added in `v1.31.22` | `string - number` | `1000` | | `LAZY_LOAD_SHARD_COUNT_THRESHOLD` | Number of shards (tenants) in a collection before lazy shard loading activates. Set to `0` to force lazy loading for all collections. Default: `1000`. See [dynamic lazy shard loading](/weaviate/concepts/storage#dynamic-lazy-shard-loading).
Added in `v1.36.6` | `string - number` | `1000` | diff --git a/docs/weaviate/best-practices/index.md b/docs/weaviate/best-practices/index.md index 8403e0162..b218148ff 100644 --- a/docs/weaviate/best-practices/index.md +++ b/docs/weaviate/best-practices/index.md @@ -295,14 +295,24 @@ When importing any significant amount of data (i.e. more than 10 objects), use b for obj in objects: collection.data.insert(properties=obj) -# ✅ Do this -with collection.batch.fixed_size(batch_size=200) as batch: +# ✅ Do this: server-side batching (recommended) - the server +# tells the client how much data to send next +with collection.batch.stream() as batch: for obj in objects: batch.add_object(properties=obj) + +# ✅ Or, if your objects are already in an in-memory list, +# ingest the whole list with a single call +result = collection.data.ingest(objects) ``` +[Server-side batching](../concepts/data-import.mdx#server-side-batching) requires Weaviate `v1.36` or later and a client that supports it. If it is not available, use a client-side batching method such as `collection.batch.fixed_size(batch_size=200)` or `collection.batch.dynamic()` instead. + +Avoid passing large lists to `collection.data.insert_many()`. It sends all objects in a single request, which fails if the request exceeds the server's [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit. `collection.data.ingest()` is a drop-in replacement that does not have this limitation. + :::tip Further resources - [How-to: Batch import data](../manage-objects/import.mdx) +- [Concepts: Data import](../concepts/data-import.mdx) ::: ### Minimize costs by offloading inactive tenants diff --git a/docs/weaviate/client-libraries/python/async.md b/docs/weaviate/client-libraries/python/async.md index 2af0897e3..fdb65dc9b 100644 --- a/docs/weaviate/client-libraries/python/async.md +++ b/docs/weaviate/client-libraries/python/async.md @@ -130,7 +130,8 @@ Methods that involve sending requests to Weaviate will be async functions. For e - `async_client.connect()`: Connect to a Weaviate server - `async_client.collections.create()`: Create a new collection -- `.data.insert_many()`: Insert a list of objects into a collection +- `.data.insert_many()`: Insert a list of objects into a collection in a single request +- `.data.ingest()`: Insert a list of objects into a collection using [server-side batching](../../manage-objects/import.mdx#server-side-batching) ### Example sync methods @@ -193,7 +194,11 @@ Note the use of a context manager in the async function. The context manager is The async client supports server-side batching through the `stream()` method, which uses the same feedback-based flow as the synchronous client. For client-side batching methods (`dynamic`, `fixed_size`, `rate_limit`), use the synchronous client. -The async client also offers `insert` and `insert_many` methods for data insertion, which can be used in an async context. +The async client also offers `insert` and `insert_many` methods for data insertion, which can be used in an async context. The one-shot `data.ingest()` method is also available on the async client and is preferred over `insert_many` for large lists. + +:::caution `insert_many` and large lists +`insert_many` sends all objects in a **single request**. The server rejects requests larger than its [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit, so the whole call fails for large lists. Use `data.ingest()` instead: it is a drop-in replacement that splits the list into server-paced batches. +::: ### Application-level example diff --git a/docs/weaviate/client-libraries/python/index.mdx b/docs/weaviate/client-libraries/python/index.mdx index b4b4cb635..7a4b47a5c 100644 --- a/docs/weaviate/client-libraries/python/index.mdx +++ b/docs/weaviate/client-libraries/python/index.mdx @@ -116,6 +116,20 @@ For more code examples, check out the [How-to manuals & Guides](../../guides.mdx The Python client library provides a synchronous API by default through the `WeaviateClient` class, which is covered on this page. An asynchronous API is also available through the `WeaviateAsyncClient` class (from `weaviate-client` `v4.7.0` and up). See the [async client API page](./async.md) for further details. +## Batch imports + +The client provides several ways to [import data in bulk](../../manage-objects/import.mdx): + +- `collection.batch.stream()`: [Server-side batching](../../concepts/data-import.mdx#server-side-batching), the recommended starting point. The server tells the client how much data to send next, so you don't have to tune batch parameters yourself. +- `collection.data.ingest()`: A one-shot convenience for importing a list of objects that you already hold in memory. It uses server-side batching under the hood and returns the same return object as `insert_many`. +- `collection.batch.dynamic()`, `fixed_size()`, and `rate_limit()`: Client-side batching methods for when you want manual control over batch size and concurrency, or when server-side batching is not available. + +:::caution `insert_many` and large lists +`collection.data.insert_many()` sends all objects in a **single request**. The server rejects requests larger than its [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit, so the whole call fails for large lists. Use `collection.data.ingest()` instead: it is a drop-in replacement that splits the list into server-paced batches. +::: + +For guidance on choosing a batching method, see the [best practices page](./notes-best-practices.mdx#batch-imports). + ## Releases Go to the [GitHub releases page](https://github.com/weaviate/weaviate-python-client/releases) to see the history of the Python client library releases and change logs. diff --git a/docs/weaviate/client-libraries/python/notes-best-practices.mdx b/docs/weaviate/client-libraries/python/notes-best-practices.mdx index e2ec89f33..55d611e18 100644 --- a/docs/weaviate/client-libraries/python/notes-best-practices.mdx +++ b/docs/weaviate/client-libraries/python/notes-best-practices.mdx @@ -202,6 +202,8 @@ There are four methods to configure the batching behavior. They are `stream`, `d | `fixed_size` | The batch size and number of concurrent requests are fixed to sizes specified by the user. | When you want to specify fixed parameters. | | `rate_limit` | The number of objects sent to Weaviate is rate limited (specified as n_objects per minute). | When you want to avoid hitting third-party vectorization API rate limits. | +If your objects are already in an in-memory list, `collection.data.ingest(objs)` is a one-shot convenience that uses server-side batching under the hood (no batching context required). It accepts plain property dicts or `DataObject` instances and returns the same `BatchObjectReturn` object as `insert_many`, making it a drop-in replacement for `insert_many` for large lists. + #### Usage We recommend using a context manager as shown below. diff --git a/docs/weaviate/concepts/data-import.mdx b/docs/weaviate/concepts/data-import.mdx index 92713352f..b6858e502 100644 --- a/docs/weaviate/concepts/data-import.mdx +++ b/docs/weaviate/concepts/data-import.mdx @@ -50,7 +50,7 @@ This architecture centralizes the complex batching logic on the server, resultin - **Simplified client code**: No need to tweak the batch size and the number of concurrent requests manually. The server determines the optimal batch size based on its current workload. - **Improved stability**: The system automatically applies **backpressure**. If the server is busy, it will instruct the client to send less data, preventing overloads and request timeouts, which is especially useful during long-running vectorization tasks. -- **Enhanced resilience**: It's designed to handle cluster events like node scaling more gracefully, reducing the risk of interrupted batches. +- **Enhanced resilience**: It's designed to handle cluster events like node scaling more gracefully, reducing the risk of interrupted batches. Because the server paces the client, the import load tracks the actual server capacity, which behaves well on autoscaling clusters and under memory pressure. ::: diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index 0fd7b1823..992ac3cbf 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -9,9 +9,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; import PyCode from '!!raw-loader!/_includes/code/howto/manage-data.import.py'; -import PySuppCode from '!!raw-loader!/_includes/code/howto/sample-data.py'; import TSCode from '!!raw-loader!/_includes/code/howto/manage-data.import.ts'; -import TsSuppCode from '!!raw-loader!/_includes/code/howto/sample-data.ts'; import JavaV6Code from '!!raw-loader!/_includes/code/java-v6/src/test/java/ManageObjectsImportTest.java'; import CSharpCode from '!!raw-loader!/_includes/code/csharp/ManageObjectsImportTest.cs'; import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.import_test.go'; @@ -25,9 +23,16 @@ import SsbStatus from '/_includes/feature-notes/ssb-status.mdx'; -With [server-side batch imports](../concepts/data-import.mdx#server-side-batching) (also called "automatic" batching), the client sends data in batch sizes determined by feedback from the server. This simplifies your code and helps the server manage its own load. The following example adds objects to a collection named `MyCollection`. +With [server-side batch imports](../concepts/data-import.mdx#server-side-batching) (also called "automatic" batching), the client sends data in batch sizes determined by feedback from the server. This simplifies your code and helps the server manage its own load. Server-side batching offers two entry points: -Server-side batching uses the [gRPC API](#use-the-grpc-api), which current client versions enable by default. +- **Stream from a data source** (recommended for large datasets): Open a batching context (`batch.stream()` in Python) and add objects one at a time as you read them, so the full dataset never has to fit in memory. +- **[Ingest an in-memory list](#ingest-an-in-memory-list)**: If your objects are already in a list, call `data.ingest()` to import the whole list in a single call. The client streams the list to the server using server-side batching under the hood. + +Server-side batching uses the [gRPC API](../api/index.mdx), which current client versions enable by default. If you use the Go client, configure its gRPC connection parameters as described on the [connection configuration page](../connections/connect-custom.mdx). + +The [async Python client](../client-libraries/python/async.md#bulk-data-insertion) also supports server-side batching through the `stream()` method and the one-shot `ingest()` method. + +The following example adds objects to a collection named `MyCollection`. @@ -69,20 +74,59 @@ The Go client does not support server-side batching; use [manual batching](#manu -## Manual batching +### Ingest an in-memory list -Use manual (client-side) batching when you want to control the batch size and concurrency yourself, or when using a client that does not yet support server-side batching (such as the Go client). The following example adds objects to the `MyCollection` collection. +In the Python and TypeScript clients, `ingest` is the safe replacement for passing a large list to `insert_many`. `insert_many` sends all objects in a single request, which fails if the request exceeds the server's [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit, while `ingest` splits the list into server-paced batches. -
- Additional information + + + +`ingest` accepts plain property dicts or `DataObject` instances (to set object IDs, vectors, or references) and returns the same return object as `insert_many`. -To create a bulk import job manually, follow these steps: + + + + +In TypeScript, `data.ingest()` is the server-side batching API, with no separate streaming context. It accepts any `Iterable`, so you can also pass a lazy source instead of a list. + + + + + +The Go client does not support server-side batching; use [manual batching](#manual-batching) instead. + + + + +The Java client does not provide a one-shot ingest method. Use the streaming context `collection.batch.start()` shown in the [server-side batching example](#server-side-batching) above. Note that `collection.data.insertMany(...)` sends all objects in a single request and does not use server-side batching. + + + + +In the C# client, `Batch.InsertMany` uses server-side batching under the hood. + + + + -1. Initialize a batch object. -1. Add items to the batch object. -1. Ensure that the last batch is sent (flushed). +## Manual batching -
+Use manual (client-side) batching when you want to control the batch size and concurrency yourself, or when using a client that does not yet support server-side batching (such as the Go client). The following example adds objects to the `MyCollection` collection. @@ -92,17 +136,6 @@ To create a bulk import job manually, follow these steps: endMarker="# END BasicBatchImportExample" language="py" /> - -### Error handling - - - -During a batch import, any failed objects or references will be stored and can be obtained through `batch.failed_objects` and `batch.failed_references`. -Additionally, a running count of failed objects and references is maintained and can be accessed through `batch.number_errors` within the context manager. -This counter can be used to stop the import process in order to investigate the failed objects or references. - -Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python). - -## Use the gRPC API +## Error handling -The [gRPC API](../api/index.mdx) is faster than the REST API. Use the gRPC API to improve import speeds. - - - - - - - -The Python client uses gRPC by default. - -
-The legacy Python client does not support gRPC. - - - - -The TypeScript client v3 uses gRPC by default. - -
-The legacy TypeScript client does not support gRPC. - - - - -The Java client v6 uses gRPC by default. - - - - -To use the gRPC API with the Go client, add the `GrpcConfig` field to your client connection code. Update `Secured` if you use an encrypted connection.

- -```go -cfg := weaviate.Config{ - Host: fmt.Sprintf("localhost:%v", "8080"), - Scheme: "http", - // highlight-start - GrpcConfig: &grpc.Config{ - Host: "localhost:50051", - Secured: false, - }, - // highlight-end -} + -client, err := weaviate.NewClient(cfg) -if err != nil { - require.Nil(t, err) - } -``` +Errors are reported the same way in server-side and manual batching. During a batch import, any failed objects or references are stored and can be inspected after the batching context closes. In the Python client: - - +- Within the context manager, `batch.number_errors` holds a running count of failed objects and references. You can use this counter to stop the import process and investigate the failures. +- After the context closes, `collection.batch.failed_objects` and `collection.batch.failed_references` contain the failed items. +- The one-shot `data.ingest()` method returns the same result object as `insert_many`: its `errors` dict maps the original index of each failed object to its error. -The C# uses gRPC by default. - - - - -To use the gRPC API with the [Spark connector](https://github.com/weaviate/spark-connector), add the `grpc:host` field to your client connection code. Update `grpc:secured` if you use an encrypted connection.

+Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python). -```java - df.write - .format("io.weaviate.spark.Weaviate") - .option("scheme", "http") - .option("host", "localhost:8080") - // highlight-start - .option("grpc:host", "localhost:50051") - .option("grpc:secured", "false") - // highlight-start - .option("className", className) - .mode("append") - .save() -``` +## Customize imported objects - - +Batch-imported objects support the same parameters as individually created objects, such as custom IDs, vectors, and cross-references. These parameters work the same way in server-side and manual batching. -## Specify an ID value +### Specify an ID value Weaviate generates an UUID for each object. Object IDs must be unique. If you set object IDs, use one of these deterministic UUID methods to prevent duplicate IDs: @@ -266,7 +237,7 @@ Weaviate generates an UUID for each object. Object IDs must be unique. If you se -## Specify a vector +### Specify a vector Use the `vector` property to specify a vector for each object. @@ -313,7 +284,7 @@ Use the `vector` property to specify a vector for each object. -## Specify named vectors +### Specify named vectors When you create an object, you can specify named vectors (if [configured in your collection](../manage-collections/vector-config.mdx#define-named-vectors)). @@ -352,7 +323,7 @@ When you create an object, you can specify named vectors (if [configured in your -## Import with references +### Import with references You can batch create links from an object to another other object through cross-references. @@ -360,7 +331,7 @@ You can batch create links from an object to another other object through cross- @@ -383,142 +354,15 @@ You can batch create links from an object to another other object through cross- -## Python-specific considerations - -The Python clients have built-in batching methods to help you optimize import speed. For details, see the client documentation: - - -- [Python client](../client-libraries/python/notes-best-practices.mdx) - -### Async Python client and batching - -The [async Python client](../client-libraries/python/async.md#bulk-data-insertion) supports server-side batching through the `stream()` method. For client-side batching, use the sync Python client. - ## Stream data from large files -If your dataset is large, consider streaming the import to avoid out-of-memory issues. +If your dataset does not fit in memory, do not load it all at once. Instead, read the source file lazily and add objects to the import as you go: -To try the example code, download the sample data and create the sample input files. +- With the server-side streaming context (`batch.stream()` in Python), add each object as you read it from the file. The client sends data at the pace the server requests, so memory usage stays flat. +- In TypeScript, `data.ingest()` accepts any `Iterable`, so you can pass a lazy source (such as a generator that reads the file record by record) instead of a fully loaded array. +- With manual batching, apply the same pattern: add objects to the batch as you read them. -
- Get the sample data - - - - - - - - - - - - - - - - -
- -
- Stream JSON files example code - - - - - - - - - - - - - - - - -
- -
- Stream CSV files example code - - - - - - - - - - - - - - - - -
+For JSON files, use a streaming parser that yields one object at a time (such as `ijson` in Python). For CSV files, read the file in chunks (such as `pandas` with the `chunksize` parameter) rather than loading it whole. ## Batch vectorization @@ -541,22 +385,6 @@ Note that each provider exposes different configuration options. language="py" /> - - - - - - ## Additional considerations @@ -565,22 +393,7 @@ Data imports can be resource intensive. Consider the following when you import l ### Asynchronous imports -:::caution Experimental -Available starting in `v1.22`. This is an experimental feature. Use with caution. -::: - -To maximize import speed, enable [asynchronous indexing](/weaviate/config-refs/indexing/vector-index.mdx#asynchronous-indexing). - -To enable asynchronous indexing, set the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration file. - -```yaml -weaviate: - image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| - ... - environment: - ASYNC_INDEXING: 'true' - ... -``` +To maximize import speed, enable [asynchronous indexing](/weaviate/config-refs/indexing/vector-index.mdx#asynchronous-indexing) by setting the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration. This decouples vector index construction from object creation, so imports are not slowed down by index building. ### Automatically add new tenants @@ -594,6 +407,7 @@ For details, see [auto-tenant](/weaviate/manage-collections/multi-tenancy#automa - [Connect to Weaviate](/weaviate/connections/index.mdx) - [How-to: Create objects](./create.mdx) +- [Python client: batch import notes and best practices](../client-libraries/python/notes-best-practices.mdx) - References: REST - /v1/batch diff --git a/docs/weaviate/more-resources/faq.md b/docs/weaviate/more-resources/faq.md index 01fd0cb98..032f58deb 100644 --- a/docs/weaviate/more-resources/faq.md +++ b/docs/weaviate/more-resources/faq.md @@ -541,6 +541,31 @@ client.collections.create( +#### Q: Why does `insert_many` fail with a "message larger than max" (`RESOURCE_EXHAUSTED`) error? + +
+ Answer + +`insert_many` (Python) and `insertMany` (TypeScript, Java) send all objects in a **single gRPC request**. Requests larger than the server's [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit are rejected with the gRPC status `RESOURCE_EXHAUSTED`, so a sufficiently large list fails as a whole with an error similar to: + +```text +WeaviateBatchError: Query call with protocol GRPC batch failed with message +CLIENT: Sent message larger than max (3002340 vs. 1000000). +``` + +The two numbers are the size of your request and the server's limit (the values shown here are from a test with a 1 MB limit). Recent Python clients read the server's limit at connect time and reject oversized requests before sending them. With older clients, the server-side variant `grpc: received message larger than max` may appear instead. + +For large lists, use [server-side batching](../manage-objects/import.mdx#server-side-batching) instead, which splits the data into server-paced batches: + +- **Python**: Use `collection.data.ingest(objects)`, a drop-in replacement for `insert_many` that returns the same return object. Alternatively, use the `collection.batch.stream()` context manager. +- **TypeScript**: `collection.data.ingest(objects)`. +- **Java** (`v6`): the `collection.batch.start()` streaming context. +- **C#**: `collection.Batch.InsertMany(items)` (already uses server-side batching). + +Alternatively, you can raise the `GRPC_MAX_MESSAGE_SIZE` [environment variable](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) on the server, but batching is the recommended solution. + +
+ ## Miscellaneous #### Q: Can I request a feature in Weaviate? From a9253880460fca2bfb7273bea8ca56b8c085eafc Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:39:25 +0200 Subject: [PATCH 2/5] Address review: differentiate TypeScript SSB examples, drop extras - TypeScript: the server-side batching example now streams from a sync generator passed to data.ingest(), and a separate ServerSideIngestExample shows the in-memory array flavor (both executed against a live 1.38.0 with persistence checks) - Remove the Batch imports section from the Python client page - Remove the redundant lowercase anchor from the GRPC_MAX_MESSAGE_SIZE row (links use the APITable-generated anchor) - Add the data import best practices blog post to Further resources --- _includes/code/howto/manage-data.import.ts | 56 +++++++++++++++++-- docs/deploy/configuration/env-vars/index.md | 2 +- .../client-libraries/python/index.mdx | 14 ----- docs/weaviate/manage-objects/import.mdx | 10 +++- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/_includes/code/howto/manage-data.import.ts b/_includes/code/howto/manage-data.import.ts index a0fdc8788..04c56fc55 100644 --- a/_includes/code/howto/manage-data.import.ts +++ b/_includes/code/howto/manage-data.import.ts @@ -343,6 +343,53 @@ try { { // START ServerSideBatchImportExample +const myCollection = client.collections.use('MyCollection') + +// highlight-start +// `ingest` is the TypeScript server-side batching API. It accepts any +// Iterable, so passing a generator streams objects to the server +// without building the full list in memory. +function* generateData(): Generator<{ properties: { title: string } }> { + for (let i = 1; i <= 5; i++) { + yield { properties: { title: `Object ${i}` } } + } +} + +const result = await myCollection.data.ingest(generateData()) +// highlight-end + +console.log(result) +// END ServerSideBatchImportExample + +// Verify the import (not shown in the docs snippet): all 5 objects and +// their `title` property must have persisted. +const check = await myCollection.query.fetchObjects({ limit: 5 }) +if (check.objects.length !== 5) + throw new Error(`SSB import: expected 5 objects, got ${check.objects.length}`) +if (!check.objects.every((o) => typeof o.properties.title === 'string' && o.properties.title.length > 0)) + throw new Error('SSB import did not persist the title property') +} + +await client.collections.delete('MyCollection'); + +// ================================================== +// ===== Server-side one-shot ingest ===== +// ================================================== + +// Clean slate +try { + await client.collections.delete('MyCollection'); +} catch (e) { + // ignore error if class doesn't exist +} finally { + await client.collections.create({ + name: 'MyCollection', + vectorizers: weaviate.configure.vectors.selfProvided(), + }) +} + +{ +// START ServerSideIngestExample const dataObjects = [ { properties: { title: 'Object 1' } }, { properties: { title: 'Object 2' } }, @@ -354,21 +401,20 @@ const dataObjects = [ const myCollection = client.collections.use('MyCollection') // highlight-start -// Use `ingest` for server-side batching. The client sends data -// in batches at a rate controlled by the server. +// `ingest` imports the whole list with server-side batching in a single call const result = await myCollection.data.ingest(dataObjects) // highlight-end console.log(result) -// END ServerSideBatchImportExample +// END ServerSideIngestExample // Verify the import (not shown in the docs snippet): all 5 objects and // their `title` property must have persisted. const check = await myCollection.query.fetchObjects({ limit: 5 }) if (check.objects.length !== 5) - throw new Error(`SSB import: expected 5 objects, got ${check.objects.length}`) + throw new Error(`Ingest import: expected 5 objects, got ${check.objects.length}`) if (!check.objects.every((o) => typeof o.properties.title === 'string' && o.properties.title.length > 0)) - throw new Error('SSB import did not persist the title property') + throw new Error('Ingest import did not persist the title property') } await client.collections.delete('MyCollection'); diff --git a/docs/deploy/configuration/env-vars/index.md b/docs/deploy/configuration/env-vars/index.md index a0c6d1028..51defbdd0 100644 --- a/docs/deploy/configuration/env-vars/index.md +++ b/docs/deploy/configuration/env-vars/index.md @@ -58,7 +58,7 @@ import APITable from '@site/src/components/APITable'; | `GO_PROFILING_DISABLE` | If `true`, disables Go profiling. Default: `false`. | `boolean` | `false` | | `GO_PROFILING_PORT` | Sets the port for the Go profiler. Default: `6060` | `integer` | `6060` | | `DEBUG_ENDPOINTS_ENABLED` | Gate for the debug HTTP listener (the profiling port set by `GO_PROFILING_PORT`, default `6060`), which serves Weaviate's **unauthenticated** internal debug and profiling endpoints — `/debug/config`, Go profiling (`/debug/pprof/*`, `/debug/fgprof`), and various maintenance and diagnostic routes. [Runtime-configurable](/deploy/configuration/env-vars/runtime-config.md) via the `debug_endpoints_enabled` override. Default: `false`. `GO_PROFILING_DISABLE` still controls whether the listener binds at all.
Added in `v1.37.9` | `boolean` | `true` | -| `GRPC_MAX_MESSAGE_SIZE` | Maximum gRPC message size in bytes. Requests larger than this limit (e.g. a large `insert_many` call) are rejected. Default: `104858000` (approximately 100 MB) | `string - number` | `2000000000` | +| `GRPC_MAX_MESSAGE_SIZE` | Maximum gRPC message size in bytes. Requests larger than this limit (e.g. a large `insert_many` call) are rejected. Default: `104858000` (approximately 100 MB) | `string - number` | `2000000000` | | `GRPC_PORT` | The port on which Weaviate's gRPC server listens for incoming requests. Default: `50051` | `string - number` | `50052` | | `HNSW_GEO_INDEX_EF` | Balance geo index search speed and recall. This value controls the search depth for geo-based queries. Default: `800`
Added in `v1.31.22` | `string - number` | `1000` | | `LAZY_LOAD_SHARD_COUNT_THRESHOLD` | Number of shards (tenants) in a collection before lazy shard loading activates. Set to `0` to force lazy loading for all collections. Default: `1000`. See [dynamic lazy shard loading](/weaviate/concepts/storage#dynamic-lazy-shard-loading).
Added in `v1.36.6` | `string - number` | `1000` | diff --git a/docs/weaviate/client-libraries/python/index.mdx b/docs/weaviate/client-libraries/python/index.mdx index 7a4b47a5c..b4b4cb635 100644 --- a/docs/weaviate/client-libraries/python/index.mdx +++ b/docs/weaviate/client-libraries/python/index.mdx @@ -116,20 +116,6 @@ For more code examples, check out the [How-to manuals & Guides](../../guides.mdx The Python client library provides a synchronous API by default through the `WeaviateClient` class, which is covered on this page. An asynchronous API is also available through the `WeaviateAsyncClient` class (from `weaviate-client` `v4.7.0` and up). See the [async client API page](./async.md) for further details. -## Batch imports - -The client provides several ways to [import data in bulk](../../manage-objects/import.mdx): - -- `collection.batch.stream()`: [Server-side batching](../../concepts/data-import.mdx#server-side-batching), the recommended starting point. The server tells the client how much data to send next, so you don't have to tune batch parameters yourself. -- `collection.data.ingest()`: A one-shot convenience for importing a list of objects that you already hold in memory. It uses server-side batching under the hood and returns the same return object as `insert_many`. -- `collection.batch.dynamic()`, `fixed_size()`, and `rate_limit()`: Client-side batching methods for when you want manual control over batch size and concurrency, or when server-side batching is not available. - -:::caution `insert_many` and large lists -`collection.data.insert_many()` sends all objects in a **single request**. The server rejects requests larger than its [`GRPC_MAX_MESSAGE_SIZE`](/deploy/configuration/env-vars/index.md#GRPC_MAX_MESSAGE_SIZE) limit, so the whole call fails for large lists. Use `collection.data.ingest()` instead: it is a drop-in replacement that splits the list into server-paced batches. -::: - -For guidance on choosing a batching method, see the [best practices page](./notes-best-practices.mdx#batch-imports). - ## Releases Go to the [GitHub releases page](https://github.com/weaviate/weaviate-python-client/releases) to see the history of the Python client library releases and change logs. diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index 992ac3cbf..5b1beff62 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -44,6 +44,9 @@ The following example adds objects to a collection named `MyCollection`. /> + +In TypeScript, `data.ingest()` is the server-side batching API, with no separate streaming context. It accepts any `Iterable`, so passing a generator streams objects to the server without building the full list in memory. + -In TypeScript, `data.ingest()` is the server-side batching API, with no separate streaming context. It accepts any `Iterable`, so you can also pass a lazy source instead of a list. +When your objects are already in an array, pass the array directly to `data.ingest()` to import the whole list in a single call. @@ -408,6 +411,7 @@ For details, see [auto-tenant](/weaviate/manage-collections/multi-tenancy#automa - [Connect to Weaviate](/weaviate/connections/index.mdx) - [How-to: Create objects](./create.mdx) - [Python client: batch import notes and best practices](../client-libraries/python/notes-best-practices.mdx) +- [Blog: Data import best practices](https://weaviate.io/blog/data-import-best-practices) - References: REST - /v1/batch From 96e58dfa0fccd122a57d5484c1a3c6359fd612fb Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:59:47 +0200 Subject: [PATCH 3/5] Scope language-specific prose to the language tabs The server-side batching, ingest, and error-handling sections mixed Python-specific method names into shared paragraphs. The shared prose is now language-neutral and each language tab carries its own wording, so a selected language reads as one coherent story. Error handling gains a per-language tab block; the Go tab stays generic because the Go example does not consume the batch response. --- docs/weaviate/manage-objects/import.mdx | 49 +++++++++++++++++++++---- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index 5b1beff62..ab08e63cb 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -25,17 +25,18 @@ import SsbStatus from '/_includes/feature-notes/ssb-status.mdx'; With [server-side batch imports](../concepts/data-import.mdx#server-side-batching) (also called "automatic" batching), the client sends data in batch sizes determined by feedback from the server. This simplifies your code and helps the server manage its own load. Server-side batching offers two entry points: -- **Stream from a data source** (recommended for large datasets): Open a batching context (`batch.stream()` in Python) and add objects one at a time as you read them, so the full dataset never has to fit in memory. -- **[Ingest an in-memory list](#ingest-an-in-memory-list)**: If your objects are already in a list, call `data.ingest()` to import the whole list in a single call. The client streams the list to the server using server-side batching under the hood. +- **Stream from a data source** (recommended for large datasets): Add objects to the import one at a time as you read them from the source, so the full dataset never has to fit in memory. +- **[Ingest an in-memory list](#ingest-an-in-memory-list)**: Import a list of objects that you already hold in memory with a single call. Server-side batching uses the [gRPC API](../api/index.mdx), which current client versions enable by default. If you use the Go client, configure its gRPC connection parameters as described on the [connection configuration page](../connections/connect-custom.mdx). -The [async Python client](../client-libraries/python/async.md#bulk-data-insertion) also supports server-side batching through the `stream()` method and the one-shot `ingest()` method. - The following example adds objects to a collection named `MyCollection`. + +Open the `batch.stream()` context manager and add objects one at a time; the client sends them at the pace the server requests. The [async Python client](../client-libraries/python/async.md#bulk-data-insertion) also supports server-side batching through the `stream()` method and the one-shot `ingest()` method. + + +Open a streaming context with `collection.batch.start()` and add objects one at a time. The batch is flushed and closed automatically when the try-with-resources block exits. + + +Open a streaming batch with `collection.Batch.StartBatch()` and add objects one at a time with `Add`. Call `Close` to flush the batch. + -`ingest` accepts plain property dicts or `DataObject` instances (to set object IDs, vectors, or references) and returns the same return object as `insert_many`. +`data.ingest()` is the safe replacement for passing a large list to `insert_many`, which sends all objects in a single request. `ingest` accepts plain property dicts or `DataObject` instances (to set object IDs, vectors, or references) and returns the same return object as `insert_many`. -Errors are reported the same way in server-side and manual batching. During a batch import, any failed objects or references are stored and can be inspected after the batching context closes. In the Python client: +Batch imports report failures per object: a problem with one object does not abort the rest of the import. Errors are reported the same way in server-side and manual batching. Inspect the failed items during and after the import to catch data issues early. -- Within the context manager, `batch.number_errors` holds a running count of failed objects and references. You can use this counter to stop the import process and investigate the failures. + + + +- Within a batching context manager, `batch.number_errors` holds a running count of failed objects and references. You can use this counter to stop the import process and investigate the failures. - After the context closes, `collection.batch.failed_objects` and `collection.batch.failed_references` contain the failed items. - The one-shot `data.ingest()` method returns the same result object as `insert_many`: its `errors` dict maps the original index of each failed object to its error. Find out more about error handling on the Python client [reference page](/weaviate/client-libraries/python). + + + +`data.ingest()` returns a result object. Inspect its `errors` field for the objects that failed to import. + + + + +Inspect the per-object errors on the result returned by the batcher. + + + + +Within a `batch.start()` streaming context, `batch.numberOfErrors()` reports the number of objects that could not be imported. The response returned by `insertMany` exposes the failed objects through its `errors()` method. + + + + +The response returned by `Batch.InsertMany` is a collection of per-object entries. Filter for entries where `Error` is not null to find the failed objects. With `Batch.StartBatch()`, each `Add` returns a handle whose result reports whether the object succeeded. + + + + ## Customize imported objects Batch-imported objects support the same parameters as individually created objects, such as custom IDs, vectors, and cross-references. These parameters work the same way in server-side and manual batching. From 2ce190ff4e14f383ee4f3f92fea7c29fa0124138 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:18:50 +0200 Subject: [PATCH 4/5] Move the Go gRPC connection note into the manual batching Go tab --- docs/weaviate/manage-objects/import.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index ab08e63cb..f993f1712 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -28,7 +28,7 @@ With [server-side batch imports](../concepts/data-import.mdx#server-side-batchin - **Stream from a data source** (recommended for large datasets): Add objects to the import one at a time as you read them from the source, so the full dataset never has to fit in memory. - **[Ingest an in-memory list](#ingest-an-in-memory-list)**: Import a list of objects that you already hold in memory with a single call. -Server-side batching uses the [gRPC API](../api/index.mdx), which current client versions enable by default. If you use the Go client, configure its gRPC connection parameters as described on the [connection configuration page](../connections/connect-custom.mdx). +Server-side batching uses the [gRPC API](../api/index.mdx), which current client versions enable by default. The following example adds objects to a collection named `MyCollection`. @@ -156,6 +156,9 @@ Use manual (client-side) batching when you want to control the batch size and co /> + +Configure the Go client's gRPC connection parameters as described on the [connection configuration](../connections/connect-custom.mdx) page. + Date: Thu, 23 Jul 2026 15:51:44 +0200 Subject: [PATCH 5/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docs/weaviate/manage-objects/import.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/weaviate/manage-objects/import.mdx b/docs/weaviate/manage-objects/import.mdx index f993f1712..b2642662b 100644 --- a/docs/weaviate/manage-objects/import.mdx +++ b/docs/weaviate/manage-objects/import.mdx @@ -364,7 +364,7 @@ When you create an object, you can specify named vectors (if [configured in your ### Import with references -You can batch create links from an object to another other object through cross-references. +You can batch create links from an object to another object through cross-references.