diff --git a/.claude/skills/operating-weaviate-cli/SKILL.md b/.claude/skills/operating-weaviate-cli/SKILL.md index ef90b83..2415b93 100644 --- a/.claude/skills/operating-weaviate-cli/SKILL.md +++ b/.claude/skills/operating-weaviate-cli/SKILL.md @@ -141,7 +141,23 @@ weaviate-cli delete collection --all --json Key create options: `--multitenant`, `--auto_tenant_creation`, `--auto_tenant_activation`, `--shards N`, `--vectorizer `, `--named_vector`, `--replication_deletion_strategy`, `--async_replication_config key=value` (repeatable; requires `--async_enabled` and Weaviate >= v1.36.0), `--object_ttl_type`, `--object_ttl_time`, `--object_ttl_filter_expired`, `--object_ttl_property_name` (only when `object_ttl_type=property`) -Mutable fields: `--async_enabled`, `--replication_factor`, `--vector_index`, `--description`, `--training_limit`, `--auto_tenant_creation`, `--auto_tenant_activation`, `--replication_deletion_strategy`, `--async_replication_config key=value` (repeatable), `--object_ttl_type`, `--object_ttl_time`, `--object_ttl_filter_expired`, `--object_ttl_property_name` (only when `object_ttl_type=property`) +Mutable fields: `--async_enabled`, `--replication_factor`, `--vector_index`, `--description`, `--training_limit`, `--auto_tenant_creation`, `--auto_tenant_activation`, `--replication_deletion_strategy`, `--async_replication_config key=value` (repeatable), `--object_ttl_type`, `--object_ttl_time`, `--object_ttl_filter_expired`, `--object_ttl_property_name` (only when `object_ttl_type=property`), `--drop_vector_index` + +#### Drop a Named Vector Index + +```bash +weaviate-cli update collection --collection Movies --drop_vector_index title_vector --json +``` + +**Destructive.** Deletes the index of one named vector from disk; the vector can no longer be +searched and the stored vectors are stripped by background cleanup. The vector **can be +re-created afterwards** (as a fresh, empty index via the client's `config.add_vector()`) once +the drop has finalized. Named vectors only, cannot be combined with `--vector_index`, and the +drop is applied asynchronously. + +Requires Weaviate >= v1.39.0 started with `ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true` +-- the endpoint is experimental and disabled by default. Dropped vectors show as `none` in the +**Vector Index** column of `get collection` (e.g. `hnsw, none` for a partially dropped collection). #### Async Replication Config diff --git a/.claude/skills/operating-weaviate-cli/references/collections.md b/.claude/skills/operating-weaviate-cli/references/collections.md index d1c916d..5c6f0da 100644 --- a/.claude/skills/operating-weaviate-cli/references/collections.md +++ b/.claude/skills/operating-weaviate-cli/references/collections.md @@ -104,10 +104,42 @@ weaviate-cli update collection \ --json ``` -Mutable fields: `--async_enabled`, `--replication_factor`, `--vector_index`, `--description`, `--training_limit`, `--auto_tenant_creation`, `--auto_tenant_activation`, `--replication_deletion_strategy`, `--async_replication_config`, `--object_ttl_type`, `--object_ttl_time`, `--object_ttl_filter_expired`, `--object_ttl_property_name` (only when `object_ttl_type=property`) +Mutable fields: `--async_enabled`, `--replication_factor`, `--vector_index`, `--description`, `--training_limit`, `--auto_tenant_creation`, `--auto_tenant_activation`, `--replication_deletion_strategy`, `--async_replication_config`, `--object_ttl_type`, `--object_ttl_time`, `--object_ttl_filter_expired`, `--object_ttl_property_name` (only when `object_ttl_type=property`), `--drop_vector_index` **Immutable (cannot change after creation):** multitenant, vectorizer, named_vector, shards +## Drop a Named Vector Index (destructive) +```bash +weaviate-cli update collection --collection Movies --drop_vector_index title_vector --json +``` + +Removes the index of one named vector. The index is deleted from disk and the vector can no +longer be searched; the stored vectors are stripped by background cleanup. **The vector can be +re-created afterwards** as a fresh, empty index (e.g. via the client's `config.add_vector()`) +once the drop has finalized -- you then re-ingest to repopulate it. + +- Named vectors only -- a collection with a single legacy vector is rejected. +- Cannot be combined with `--vector_index` (reconfiguring an index you are deleting is contradictory). +- Requires Weaviate **>= v1.39.0** started with `ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true`. + The endpoint is experimental and disabled by default; without the flag the server answers with a 500. +- The drop is applied **asynchronously**. A success message means Weaviate accepted the request, + not that the index is already gone. Poll `get collection --collection ` to observe completion. + +Once dropped, the vector is reported by the server as `vectorIndexType: "none"` with no +`vectorIndexConfig`. In the collection listing it shows up as `none` in the **Vector Index** +column (e.g. `hnsw, none` when only some of the named vectors were dropped). + +**Re-creation timing.** While the drop is in progress (vector shows `none`), re-creating the +same name is rejected. Once it finalizes (the vector disappears from the schema), the name can +be added again as a brand-new, empty index; the original index and any vectors already stripped +are not restored. + +**Re-triggering a stalled drop.** Re-issuing `--drop_vector_index` on a vector that already +shows `none` is allowed and returns success. While cleanup is still running it is a no-op; +if the background cleanup had FAILED, it re-enqueues a fresh cleanup task. This is the only +way to recover a drop whose marker is stuck at `none`. The command reports that the vector +was already dropped and that it re-triggered cleanup. + **Async replication config examples (update):** ```bash # Update async replication tuning on existing collection diff --git a/.claude/skills/operating-weaviate-cli/references/data.md b/.claude/skills/operating-weaviate-cli/references/data.md index 3c443b5..b9075fd 100644 --- a/.claude/skills/operating-weaviate-cli/references/data.md +++ b/.claude/skills/operating-weaviate-cli/references/data.md @@ -31,6 +31,12 @@ weaviate-cli create data \ - `--tenants "T1,T2"` -- Send data to specific tenants - `--tenant_suffix` -- Prefix for auto-tenant names (default: "Tenant") +**Dropped vector indexes:** with `--randomize` on a named-vector collection, any vector whose +index was dropped (`vectorIndexType: "none"`, see `update collection --drop_vector_index`) is +**skipped automatically** -- generated objects do not carry it, so ingestion keeps working +instead of failing with "writes targeting it are rejected". A one-line note lists the skipped +vectors (suppressed under `--json`). Re-ingest the vector after re-creating its index. + ## Update Data ```bash weaviate-cli update data --collection "Movies" --limit 100 --randomize --json diff --git a/requirements-dev.txt b/requirements-dev.txt index b856f00..c0d6b65 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,9 @@ -weaviate-client>=4.21.0 +# TEMPORARY: pinned to weaviate/weaviate-python-client#1991, which adds +# `collection.config.delete_vector_index()` and `VectorIndexType.NONE`, required by +# `update collection --drop_vector_index`. Swap back to a released `weaviate-client>=X` +# once the PR is merged and shipped. Use `@alter-schema-drop-vector-index` instead of the +# SHA to track new pushes to the PR. +weaviate-client @ git+https://github.com/weaviate/weaviate-python-client.git@2dbfaedf8d232057ce6d5a3b6a3479edcda62f94 click==8.1.7 twine pytest diff --git a/setup.cfg b/setup.cfg index 1169521..6ed15ca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,11 @@ classifiers = include_package_data = True python_requires = >=3.9 install_requires = - weaviate-client>=4.21.0 + # TEMPORARY: pinned to weaviate/weaviate-python-client#1991, which adds + # `collection.config.delete_vector_index()` and `VectorIndexType.NONE`, required by + # `update collection --drop_vector_index`. A direct URL cannot be published to PyPI, + # so this MUST go back to a released `weaviate-client>=X` before the next release. + weaviate-client @ git+https://github.com/weaviate/weaviate-python-client.git@2dbfaedf8d232057ce6d5a3b6a3479edcda62f94 click==8.1.7 semver>=3.0.2 numpy>=1.24.0 diff --git a/test/unittests/test_managers/test_collection_manager.py b/test/unittests/test_managers/test_collection_manager.py index a677b25..cca60fa 100644 --- a/test/unittests/test_managers/test_collection_manager.py +++ b/test/unittests/test_managers/test_collection_manager.py @@ -1,3 +1,4 @@ +import json import pytest from unittest.mock import MagicMock, patch, PropertyMock from weaviate.exceptions import WeaviateConnectionError @@ -1114,3 +1115,273 @@ def test_update_collection_async_replication_config_warns_on_old_version( captured.out + captured.err ) mock_collection.config.update.assert_called_once() + + +def _named_vector(index_type=None, vectorizer="none"): + """Build a mock named vector config. `index_type=None` means the index was dropped.""" + named_vector = MagicMock() + if index_type is None: + named_vector.vector_index_config = None + else: + named_vector.vector_index_config.vector_index_type.return_value = index_type + named_vector.vectorizer.vectorizer.value = vectorizer + return named_vector + + +def _named_vector_schema(vector_config): + """Build a mock collection config that uses named vectors.""" + return MagicMock( + vector_config=vector_config, + vectorizer=None, + vector_index_type=None, + replication_config=MagicMock(factor=1), + multi_tenancy_config=MagicMock( + enabled=False, auto_tenant_creation=False, auto_tenant_activation=False + ), + ) + + +def _drop_ready_collection(mock_client, vector_config=None): + """Wire a mock collection whose named vector index can be dropped.""" + mock_collections = MagicMock() + mock_client.collections = mock_collections + mock_client.collections.exists.side_effect = [True, True] + mock_client.get_meta.return_value = {"version": "1.39.0"} + + mock_collection = MagicMock() + mock_client.collections.get.return_value = mock_collection + mock_collection.config.get.return_value = _named_vector_schema( + vector_config + if vector_config is not None + else {"title_vector": _named_vector("hnsw")} + ) + return mock_collection + + +def test_update_collection_drop_vector_index(mock_client, mock_wvc_object_ttl): + """--drop_vector_index deletes the index of the named vector, after the config update.""" + mock_collection = _drop_ready_collection(mock_client) + + manager = CollectionManager(mock_client) + manager.update_collection( + collection="TestCollection", + drop_vector_index="title_vector", + ) + + mock_collection.config.delete_vector_index.assert_called_once_with( + vector_name="title_vector" + ) + + # `config.update()` reads the whole schema and writes it back, so the drop has to + # come last -- otherwise a `vectorIndexType: "none"` vector is sent back to Weaviate. + call_names = [ + call[0] + for call in mock_collection.config.mock_calls + if call[0] in ("update", "delete_vector_index") + ] + assert call_names == ["update", "delete_vector_index"] + + +def test_update_collection_drop_vector_index_json_output( + mock_client, mock_wvc_object_ttl, capsys +): + """The JSON payload reports the dropped vector and that the removal is async.""" + _drop_ready_collection(mock_client) + + manager = CollectionManager(mock_client) + manager.update_collection( + collection="TestCollection", + drop_vector_index="title_vector", + json_output=True, + ) + + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "success" + assert payload["dropped_vector_index"] == "title_vector" + assert "asynchronously" in payload["message"] + + +def test_update_collection_drop_vector_index_rejects_vector_index_combo(mock_client): + """--drop_vector_index and --vector_index are mutually exclusive.""" + mock_collections = MagicMock() + mock_client.collections = mock_collections + + manager = CollectionManager(mock_client) + with pytest.raises(Exception) as exc_info: + manager.update_collection( + collection="TestCollection", + vector_index="hnsw", + drop_vector_index="title_vector", + ) + + assert "cannot be combined with --vector_index" in str(exc_info.value) + mock_collections.get.assert_not_called() + + +def test_update_collection_drop_vector_index_unknown_vector( + mock_client, mock_wvc_object_ttl +): + """An unknown vector name fails before anything is updated or dropped.""" + mock_collection = _drop_ready_collection( + mock_client, + vector_config={ + "title_vector": _named_vector("hnsw"), + "body_vector": _named_vector("flat"), + }, + ) + + manager = CollectionManager(mock_client) + with pytest.raises(Exception) as exc_info: + manager.update_collection( + collection="TestCollection", + drop_vector_index="missing_vector", + ) + + assert "Named vector 'missing_vector' does not exist" in str(exc_info.value) + assert "body_vector, title_vector" in str(exc_info.value) + mock_collection.config.update.assert_not_called() + mock_collection.config.delete_vector_index.assert_not_called() + + +def test_update_collection_drop_vector_index_without_named_vectors( + mock_client, mock_wvc_object_ttl +): + """Only named vectors can be dropped, a legacy single-vector collection is rejected.""" + mock_collection = _drop_ready_collection(mock_client, vector_config={}) + + manager = CollectionManager(mock_client) + with pytest.raises(Exception) as exc_info: + manager.update_collection( + collection="TestCollection", + drop_vector_index="title_vector", + ) + + assert "has no named vectors" in str(exc_info.value) + mock_collection.config.update.assert_not_called() + mock_collection.config.delete_vector_index.assert_not_called() + + +def test_update_collection_drop_vector_index_already_dropped_re_triggers( + mock_client, mock_wvc_object_ttl, capsys +): + """Re-dropping an already-'none' vector is allowed: it re-triggers cleanup. + + The server returns 200 for a repeat drop (no-op while cleanup runs, or a fresh + cleanup task if the previous one FAILED). The CLI must not block this -- it is the + only way for an operator to recover a stalled drop. + """ + mock_collection = _drop_ready_collection( + mock_client, vector_config={"title_vector": _named_vector(None)} + ) + + manager = CollectionManager(mock_client) + manager.update_collection( + collection="TestCollection", + drop_vector_index="title_vector", + ) + + mock_collection.config.delete_vector_index.assert_called_once_with( + vector_name="title_vector" + ) + assert "already dropped" in capsys.readouterr().out + + +def test_update_collection_drop_vector_index_already_dropped_json_re_trigger( + mock_client, mock_wvc_object_ttl, capsys +): + """JSON output still reports success and the re-trigger note for a repeat drop.""" + _drop_ready_collection( + mock_client, vector_config={"title_vector": _named_vector(None)} + ) + + manager = CollectionManager(mock_client) + manager.update_collection( + collection="TestCollection", + drop_vector_index="title_vector", + json_output=True, + ) + + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "success" + assert payload["dropped_vector_index"] == "title_vector" + assert "re-trigger" in payload["message"] + + +def test_update_collection_drop_vector_index_warns_on_old_version( + mock_client, mock_wvc_object_ttl, capsys +): + """Warn when --drop_vector_index is used against a server older than v1.39.0.""" + mock_collection = _drop_ready_collection(mock_client) + mock_client.get_meta.return_value = {"version": "1.38.0"} + + manager = CollectionManager(mock_client) + manager.update_collection( + collection="TestCollection", + drop_vector_index="title_vector", + ) + + captured = capsys.readouterr() + assert "Warning: --drop_vector_index requires Weaviate >= v1.39.0" in ( + captured.out + captured.err + ) + mock_collection.config.delete_vector_index.assert_called_once() + + +def test_update_collection_drop_vector_index_server_error_hints_at_env_var( + mock_client, mock_wvc_object_ttl +): + """A server rejection points at the experimental feature flag it most likely needs.""" + mock_collection = _drop_ready_collection(mock_client) + mock_collection.config.delete_vector_index.side_effect = Exception( + "endpoint is experimental and disabled by default" + ) + + manager = CollectionManager(mock_client) + with pytest.raises(Exception) as exc_info: + manager.update_collection( + collection="TestCollection", + drop_vector_index="title_vector", + ) + + assert "Failed to drop the index of named vector 'title_vector'" in str( + exc_info.value + ) + assert "ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true" in str( + exc_info.value + ) + + +def test_get_collection_lists_dropped_vector_index_as_none(mock_client, capsys): + """A dropped index surfaces as `none` instead of crashing on a null index config.""" + mock_client.collections = MagicMock() + mock_client.collections.list_all.return_value = ["Movies"] + mock_client.collections.get.return_value.config.get.return_value = ( + _named_vector_schema({"title_vector": _named_vector(None)}) + ) + + manager = CollectionManager(mock_client) + manager.get_collection(collection=None, json_output=True) + + payload = json.loads(capsys.readouterr().out) + assert payload["collections"][0]["vector_index"] == "none" + + +def test_get_collection_lists_all_distinct_vector_index_types(mock_client, capsys): + """Every distinct index type is listed, so a per-vector drop stays visible.""" + mock_client.collections = MagicMock() + mock_client.collections.list_all.return_value = ["Movies"] + mock_client.collections.get.return_value.config.get.return_value = ( + _named_vector_schema( + { + "title_vector": _named_vector("hnsw"), + "body_vector": _named_vector(None), + "extra_vector": _named_vector("hnsw"), + } + ) + ) + + manager = CollectionManager(mock_client) + manager.get_collection(collection=None, json_output=True) + + payload = json.loads(capsys.readouterr().out) + assert payload["collections"][0]["vector_index"] == "hnsw, none" diff --git a/test/unittests/test_managers/test_data_manager.py b/test/unittests/test_managers/test_data_manager.py index a3074bd..a277587 100644 --- a/test/unittests/test_managers/test_data_manager.py +++ b/test/unittests/test_managers/test_data_manager.py @@ -935,3 +935,113 @@ def fake_ingest(collection, *, concurrent_requests, **kwargs): # Single tenant: no reduction assert captured_concurrent == [8] + + +# --------------------------------------------------------------------------- +# create data skips dropped ("none") vector indexes +# --------------------------------------------------------------------------- + + +def _named_vec(dropped, vectorizer_name="none"): + """Mock a named-vector config; dropped => vector_index_config is None.""" + nv = MagicMock() + nv.vector_index_config = None if dropped else MagicMock() + nv.vectorizer.vectorizer = vectorizer_name + return nv + + +def _col_with_named_vectors(vector_config): + col = MagicMock() + col.name = "SkipTest" + col.config.get.return_value = MagicMock( + vectorizer=None, # named-vector collection => empty legacy vectorizer + vector_config=vector_config, + multi_tenancy_config=MagicMock( + enabled=False, auto_tenant_creation=False, auto_tenant_activation=False + ), + ) + col.__len__ = MagicMock(return_value=0) + return col + + +def test_create_data_skips_dropped_vector_index(mock_client, capsys): + """A dropped ("none") named vector is excluded from generated objects.""" + col = _col_with_named_vectors( + {"vec_a": _named_vec(dropped=True), "vec_b": _named_vec(dropped=False)} + ) + _setup_mock_client_with_col(mock_client, col) + + captured = {} + with patch.object( + DataManager, + "_DataManager__producer_consumer_ingest", + return_value=(10, [], MagicMock(total=0)), + ) as prod: + manager = DataManager(mock_client) + manager.create_data(collection="SkipTest", limit=10, randomize=True) + captured = prod.call_args.kwargs + + # vec_a (dropped) must not be generated; only vec_b is passed through + assert captured["named_vectors"] == ["vec_b"] + # informational note surfaced to the user + assert "skipping dropped vector index" in capsys.readouterr().out + + +def test_create_data_all_vectors_dropped_generates_none(mock_client): + """When every named vector is dropped, no vectors are generated (empty list).""" + col = _col_with_named_vectors( + {"vec_a": _named_vec(dropped=True), "vec_b": _named_vec(dropped=True)} + ) + _setup_mock_client_with_col(mock_client, col) + + with patch.object( + DataManager, + "_DataManager__producer_consumer_ingest", + return_value=(10, [], MagicMock(total=0)), + ) as prod: + manager = DataManager(mock_client) + manager.create_data(collection="SkipTest", limit=10, randomize=True) + kwargs = prod.call_args.kwargs + + assert kwargs["named_vectors"] == [] + assert kwargs["vectorizer"] == "none" + + +def test_create_data_no_dropped_vectors_unchanged(mock_client, capsys): + """With no dropped vectors, all named vectors are generated and no note is printed.""" + col = _col_with_named_vectors( + {"vec_a": _named_vec(dropped=False), "vec_b": _named_vec(dropped=False)} + ) + _setup_mock_client_with_col(mock_client, col) + + with patch.object( + DataManager, + "_DataManager__producer_consumer_ingest", + return_value=(10, [], MagicMock(total=0)), + ) as prod: + manager = DataManager(mock_client) + manager.create_data(collection="SkipTest", limit=10, randomize=True) + kwargs = prod.call_args.kwargs + + assert sorted(kwargs["named_vectors"]) == ["vec_a", "vec_b"] + assert "skipping dropped vector index" not in capsys.readouterr().out + + +def test_create_data_skip_note_suppressed_in_json(mock_client, capsys): + """The skip note must not corrupt --json output.""" + col = _col_with_named_vectors( + {"vec_a": _named_vec(dropped=True), "vec_b": _named_vec(dropped=False)} + ) + _setup_mock_client_with_col(mock_client, col) + + with patch.object( + DataManager, + "_DataManager__producer_consumer_ingest", + return_value=(10, [], MagicMock(total=0)), + ): + manager = DataManager(mock_client) + manager.create_data( + collection="SkipTest", limit=10, randomize=True, json_output=True + ) + + assert "skipping dropped vector index" not in capsys.readouterr().out diff --git a/weaviate_cli/commands/update.py b/weaviate_cli/commands/update.py index 741e8a1..8cbf3d6 100644 --- a/weaviate_cli/commands/update.py +++ b/weaviate_cli/commands/update.py @@ -119,6 +119,18 @@ def update() -> None: multiple=True, help=ASYNC_REPLICATION_CONFIG_HELP, ) +@click.option( + "--drop_vector_index", + default=UpdateCollectionDefaults.drop_vector_index, + help=( + "Name of the named vector whose index to drop. Destructive: the index is removed " + "from disk and the vector can no longer be searched, and the stored vectors are " + "stripped by background cleanup. The vector can be re-created afterwards (as a fresh, " + "empty index) once the drop has finalized. Cannot be combined with --vector_index. " + "Requires Weaviate >= v1.39.0 started with " + "ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true." + ), +) @click.pass_context def update_collection_cli( ctx: click.Context, @@ -137,6 +149,7 @@ def update_collection_cli( object_ttl_filter_expired: bool, object_ttl_property_name: Optional[str], async_replication_config: Tuple[str, ...], + drop_vector_index: Optional[str], ) -> None: """Update a collection in Weaviate.""" @@ -173,6 +186,7 @@ def update_collection_cli( async_replication_config=parse_async_replication_config( async_replication_config ), + drop_vector_index=drop_vector_index, ) except Exception as e: click.echo(f"Error: {e}") diff --git a/weaviate_cli/defaults.py b/weaviate_cli/defaults.py index 55e4cc5..dbe2043 100644 --- a/weaviate_cli/defaults.py +++ b/weaviate_cli/defaults.py @@ -263,6 +263,7 @@ class UpdateCollectionDefaults: object_ttl_time: Optional[int] = None object_ttl_filter_expired: Optional[bool] = None object_ttl_property_name: str = "releaseDate" + drop_vector_index: Optional[str] = None @dataclass diff --git a/weaviate_cli/managers/collection_manager.py b/weaviate_cli/managers/collection_manager.py index 72be1a3..6fced37 100644 --- a/weaviate_cli/managers/collection_manager.py +++ b/weaviate_cli/managers/collection_manager.py @@ -31,6 +31,27 @@ def __get_total_objects_with_multitenant(self, col_obj: Collection) -> int: ) return acc + @staticmethod + def __named_vector_index_types(vector_config: Dict) -> str: + """Summarize the index types used by a collection's named vectors. + + Every distinct type is reported, in schema order, so that a per-vector + difference stays visible in the collection listing. A vector whose index was + dropped with `update collection --drop_vector_index` is reported by Weaviate as + `vectorIndexType: "none"` and surfaces here as a `vector_index_config` of `None`. + """ + types: List[str] = [] + for named_vector in vector_config.values(): + index_config = named_vector.vector_index_config + index_type = ( + "none" + if index_config is None + else str(index_config.vector_index_type()) + ) + if index_type not in types: + types.append(index_type) + return ", ".join(types) if types else "None" + def get_collection( self, collection: Optional[str] = GetCollectionDefaults.collection, @@ -67,9 +88,9 @@ def get_collection( list(schema.vector_config.keys())[0] ].vectorizer.vectorizer.value if not schema.vector_index_type: - vector_index_type = schema.vector_config[ - list(schema.vector_config.keys())[0] - ].vector_index_config.vector_index_type() + vector_index_type = self.__named_vector_index_types( + schema.vector_config + ) else: vector_index_type = schema.vector_index_type else: @@ -629,6 +650,44 @@ def create_collection( else: click.echo(f"Collection '{collection}' created successfully in Weaviate.") + @staticmethod + def __check_drop_target(config, collection: str, vector_name: str) -> bool: + """Validate the drop target and report whether its index is already marked dropped. + + Returns True when the vector's index was already dropped (server reports it as + `vectorIndexType: "none"`). Re-issuing the drop is intentionally allowed in that + case: it is a no-op while cleanup is in flight and re-enqueues a fresh cleanup task + if the previous one FAILED — the only way for an operator to recover a stuck drop. + Only the two genuinely invalid states raise. + """ + vector_config = config.vector_config + if not vector_config: + raise Exception( + f"Collection '{collection}' has no named vectors. Only the index " + "of a named vector can be dropped." + ) + if vector_name not in vector_config: + raise Exception( + f"Named vector '{vector_name}' does not exist in collection " + f"'{collection}'. Available named vectors: " + f"{', '.join(sorted(vector_config))}." + ) + return vector_config[vector_name].vector_index_config is None + + @staticmethod + def __drop_vector_index( + col_obj: Collection, collection: str, vector_name: str + ) -> None: + try: + col_obj.config.delete_vector_index(vector_name=vector_name) + except Exception as e: + raise Exception( + f"Failed to drop the index of named vector '{vector_name}' in collection " + f"'{collection}': {e}. This endpoint is experimental, make sure Weaviate " + "is started with " + "ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true." + ) + def update_collection( self, collection: str = UpdateCollectionDefaults.collection, @@ -656,6 +715,7 @@ def update_collection( str ] = UpdateCollectionDefaults.object_ttl_property_name, async_replication_config: Optional[Dict[str, int]] = None, + drop_vector_index: Optional[str] = UpdateCollectionDefaults.drop_vector_index, ) -> None: if ( @@ -670,6 +730,11 @@ def update_collection( raise Exception( "Error: --async_replication_config cannot be used when --async_enabled is False." ) + if drop_vector_index is not None and vector_index is not None: + raise Exception( + "--drop_vector_index cannot be combined with --vector_index. " + "Dropping an index and reconfiguring it in the same call is contradictory." + ) if async_replication_config is not None and older_than_version( self.client, "1.36.0" @@ -679,6 +744,12 @@ def update_collection( "The server may ignore or reject these settings." ) + if drop_vector_index is not None and older_than_version(self.client, "1.39.0"): + click.echo( + "Warning: --drop_vector_index requires Weaviate >= v1.39.0. " + "The server may reject this request." + ) + if not self.client.collections.exists(collection): raise Exception( @@ -728,26 +799,32 @@ def update_collection( } col_obj: Collection = self.client.collections.get(collection) + current_config = col_obj.config.get() + drop_already_marked = False + if drop_vector_index is not None: + drop_already_marked = self.__check_drop_target( + current_config, collection, drop_vector_index + ) rf = ( replication_factor if replication_factor is not None - else col_obj.config.get().replication_config.factor + else current_config.replication_config.factor ) rds_map = { "delete_on_conflict": wvc.ReplicationDeletionStrategy.DELETE_ON_CONFLICT, "no_automated_resolution": wvc.ReplicationDeletionStrategy.NO_AUTOMATED_RESOLUTION, "time_based_resolution": wvc.ReplicationDeletionStrategy.TIME_BASED_RESOLUTION, } - mt = col_obj.config.get().multi_tenancy_config.enabled + mt = current_config.multi_tenancy_config.enabled auto_tenant_creation = ( auto_tenant_creation if auto_tenant_creation is not None - else col_obj.config.get().multi_tenancy_config.auto_tenant_creation + else current_config.multi_tenancy_config.auto_tenant_creation ) auto_tenant_activation = ( auto_tenant_activation if auto_tenant_activation is not None - else col_obj.config.get().multi_tenancy_config.auto_tenant_activation + else current_config.multi_tenancy_config.auto_tenant_activation ) col_obj.config.update( @@ -790,18 +867,32 @@ def update_collection( assert self.client.collections.exists(collection) - if json_output: - click.echo( - json.dumps( - { - "status": "success", - "message": f"Collection '{collection}' modified successfully in Weaviate.", - }, - indent=2, + # Dropped last: `config.update()` reads the whole schema and writes it back, so + # doing this first would send a `vectorIndexType: "none"` vector back to Weaviate. + if drop_vector_index is not None: + self.__drop_vector_index(col_obj, collection, drop_vector_index) + + message = f"Collection '{collection}' modified successfully in Weaviate." + if drop_vector_index is not None: + if drop_already_marked: + message += ( + f" The index of named vector '{drop_vector_index}' was already dropped; " + "the request was re-issued to re-trigger cleanup if it had stalled." ) - ) + else: + message += ( + f" Dropping the index of named vector '{drop_vector_index}' was accepted; " + "the removal runs asynchronously. Once it finalizes the vector can be " + "re-created as a fresh, empty index." + ) + + if json_output: + result: Dict[str, str] = {"status": "success", "message": message} + if drop_vector_index is not None: + result["dropped_vector_index"] = drop_vector_index + click.echo(json.dumps(result, indent=2)) else: - click.echo(f"Collection '{collection}' modified successfully in Weaviate.") + click.echo(message) def delete_collection( self, diff --git a/weaviate_cli/managers/data_manager.py b/weaviate_cli/managers/data_manager.py index d2fcc94..5a3ab63 100644 --- a/weaviate_cli/managers/data_manager.py +++ b/weaviate_cli/managers/data_manager.py @@ -669,10 +669,30 @@ def __ingest_data( config = collection.config.get() if not config.vectorizer and config.vector_config: - named_vectors = list(config.vector_config.keys()) - vectorizer = config.vector_config[ - named_vectors[0] - ].vectorizer.vectorizer + all_named_vectors = list(config.vector_config.keys()) + # Skip vectors whose index was dropped (vectorIndexType "none"): the + # server rejects any object carrying a vector that targets a dropped + # index, which would fail every batch. Such vectors report a + # `vector_index_config` of None. + named_vectors = [ + name + for name in all_named_vectors + if config.vector_config[name].vector_index_config is not None + ] + dropped_vectors = [ + name for name in all_named_vectors if name not in named_vectors + ] + if dropped_vectors and not json_output: + click.echo( + f"Note: skipping dropped vector index(es) " + f"{', '.join(dropped_vectors)} (vectorIndexType 'none'); " + "generated objects will not carry these vectors." + ) + vectorizer = ( + config.vector_config[named_vectors[0]].vectorizer.vectorizer + if named_vectors + else "none" + ) elif config.vectorizer: vectorizer = config.vectorizer named_vectors = None