diff --git a/_includes/code/howto/manage-data.collections.py b/_includes/code/howto/manage-data.collections.py index 994eac79..356c891a 100644 --- a/_includes/code/howto/manage-data.collections.py +++ b/_includes/code/howto/manage-data.collections.py @@ -308,6 +308,166 @@ client.collections.delete("Article") +# ========================================================== +# ===== UPDATE PROPERTY INDEXES ON A LIVE COLLECTION ===== +# ========================================================== + +from weaviate.classes.config import Property, DataType + +client.collections.create( + "Article", + properties=[ + Property( + name="title", + data_type=DataType.TEXT, + index_filterable=True, + index_searchable=False, + ), + Property( + name="chunk_number", + data_type=DataType.INT, + index_filterable=True, + index_range_filters=False, + ), + ], +) + +articles = client.collections.use("Article") +with articles.batch.fixed_size(batch_size=50) as batch: + for i in range(50): + batch.add_object({"title": f"Title {i}", "chunk_number": i}) + +# Runtime property index updates require Weaviate `1.39` or higher. The docs test +# suite pins an older server in `tests/docker-compose-anon.yml`, so the blocks below +# are skipped there. Bump that image to a released `1.39` to execute them in CI. +server_version = tuple( + int(part) for part in client.get_meta()["version"].split("-")[0].split(".")[:2] +) + +if server_version >= (1, 39): + # START AddPropertyIndex + from weaviate.classes.config import Tokenization, InvertedIndexType + + collection = client.collections.use("Article") + + # highlight-start + # Add a searchable index to the existing "title" property. + # Enabling a searchable index always requires a tokenization. + task = collection.config.update_property_index( + "title", + InvertedIndexType.SEARCHABLE, + tokenization=Tokenization.WORD, + ) + # highlight-end + + print(task.status) # InvertedIndexTaskStatus.STARTED + print(task.task_id) # "Article:enable-searchable:title:a1ec" + + # Or block until the index is ready and return its final status + # highlight-start + status = collection.config.update_property_index( + "chunk_number", + InvertedIndexType.RANGE_FILTERS, + wait_for_completion=True, + ) + # highlight-end + + print(status.status) # InvertedIndexState.READY + # END AddPropertyIndex + + # Test + assert task.status.value == "STARTED" + assert status.status.value == "ready" + + # Re-issuing the same update is a no-op + repeat = collection.config.update_property_index( + "chunk_number", InvertedIndexType.RANGE_FILTERS, wait_for_completion=False + ) + assert repeat.status.value == "NO_OP" + assert repeat.task_id is None + + # START ChangePropertyIndexTokenization + from weaviate.classes.config import Tokenization, InvertedIndexType + + collection = client.collections.use("Article") + + # highlight-start + # Change the tokenization of the "title" searchable index + status = collection.config.update_property_index( + "title", + InvertedIndexType.SEARCHABLE, + tokenization=Tokenization.FIELD, + wait_for_completion=True, + ) + # highlight-end + + print(status.tokenization) # Tokenization.FIELD + + # The filterable index on the same property reports the new tokenization too + indexes = collection.config.get_property_indexes() + title_property = [p for p in indexes.properties if p.name == "title"][0] + + for index in title_property.indexes: + print(index.type, index.tokenization) + + # filterable Tokenization.FIELD + # searchable Tokenization.FIELD + # END ChangePropertyIndexTokenization + + # Test + assert status.tokenization.value == "field" + assert {i.tokenization.value for i in title_property.indexes} == {"field"} + + # START CheckPropertyIndexes + collection = client.collections.use("Article") + + # highlight-start + indexes = collection.config.get_property_indexes() + # highlight-end + + for property_index in indexes.properties: + for index in property_index.indexes: + print(property_index.name, index.type, index.status, index.progress) + + # title filterable InvertedIndexState.READY None + # title searchable InvertedIndexState.READY None + # END CheckPropertyIndexes + + # Test + assert indexes.collection == "Article" + title_indexes = [p for p in indexes.properties if p.name == "title"][0] + assert {i.type for i in title_indexes.indexes} == {"filterable", "searchable"} + + # START RebuildPropertyIndex + from weaviate.classes.config import InvertedIndexType + + collection = client.collections.use("Article") + + # highlight-start + # Rebuild an index from scratch without changing its configuration + task = collection.config.rebuild_property_index("title", InvertedIndexType.SEARCHABLE) + # highlight-end + + print(task.task_id) # "Article:rebuild-searchable:title:38c6" + + # highlight-start + # Stop a rebuild that is still running + cancelled = collection.config.cancel_property_index_task( + "title", InvertedIndexType.SEARCHABLE + ) + # highlight-end + + print(cancelled.status) # InvertedIndexTaskStatus.CANCELLED + # END RebuildPropertyIndex + + # Test + assert task.task_id is not None + assert cancelled.status.value in ("CANCELLED", "NO_OP") + +# Delete the collection to recreate it +client.collections.delete("Article") + + # =============================================== # ===== CREATE A COLLECTION WITH A RERANKER MODULE ===== # =============================================== diff --git a/docs/weaviate/manage-collections/inverted-index.mdx b/docs/weaviate/manage-collections/inverted-index.mdx index 76cd1155..88efeb23 100644 --- a/docs/weaviate/manage-collections/inverted-index.mdx +++ b/docs/weaviate/manage-collections/inverted-index.mdx @@ -149,6 +149,97 @@ The inverted index in Weaviate can be configured through various parameters at t +## Update inverted indexes at runtime + +:::info Added in `v1.39` +::: + +Add, change, or rebuild an inverted index on a property of an existing collection. Weaviate builds the index in the background, so the collection stays available for reads and writes while the work runs. + +:::note Python client only +Only the Python client supports this API at the moment. If you work in another client library, use the Python client for these operations until support arrives. +::: + +### Add or change an index + +Adding or changing an index is declarative: you describe the index you want, and Weaviate makes the property match that description. Re-running a request that has already taken effect does nothing and returns a `NO_OP` status with no task ID, so it is safe to retry. + +The request returns as soon as Weaviate accepts the task, before the index is built. Use [Check index status](#check-index-status) to follow its progress, or choose to wait until the index is ready so the call returns the final status directly. + +:::note Enabling a searchable index requires a tokenization +You must specify a tokenization when you enable a `searchable` index, even if the property already has one. Without it, the request fails with `enable-searchable requires a tokenization to be set on the request body`. +::: + + + + + + + +To change an index that already exists, describe the configuration you want and Weaviate rebuilds the index to match it. + + + + + + + +:::caution Changing tokenization changes how filters match +Tokenization belongs to the property, not to an individual index. Changing the tokenization of a property's `searchable` index also changes the tokenization reported for its `filterable` index, which changes how filters on that property match. Review both indexes before you change a tokenization. +::: + +For a multi-tenant collection, adding or changing an index always applies to every tenant and cannot be limited to specific tenants. + +### Check index status + +You can list every index on every property of a collection, along with each index's status. + +An index moves through `pending`, then `indexing`, then `ready`. On a small collection, the first two states can pass too quickly to observe. An index can also report `indexing` briefly after the work itself has finished, while Weaviate swaps the new index into place, so wait for `ready` rather than treating the end of `indexing` as completion. + +Progress is reported as a fraction between `0` and `1`, but only while an index is actively being built, and it is often not reported at all. Treat the status as the reliable signal and progress as a hint. + + + + + + + +### Rebuild an index + +Rebuilding regenerates an index from the current object data without changing its configuration. Use it to recover an index you suspect is inconsistent. + +Rebuilding is a background task, and you can cancel one that is still running. Canceling is safe when nothing is in flight, in which case it returns a `NO_OP` status. + + + + + + + +For a multi-tenant collection, you can limit a rebuild to specific tenants. + +While a task is in flight, Weaviate blocks other schema changes to the same property. [Dropping the index](#drop-an-inverted-index) or starting a second rebuild fails until the task finishes or is canceled. + ## Drop an inverted index Drop (delete) an inverted index from a property. This is a destructive operation — the index data is removed from disk. To use the index again, it must be regenerated.