diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index ddc92d0ed..9c5a86c88 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -33,6 +33,9 @@ services: OBJECTS_TTL_DELETE_SCHEDULE: "@every 12h" # for objectTTL tests to work EXPORT_ENABLED: 'true' EXPORT_DEFAULT_PATH: "/var/lib/weaviate/exports" + # for config.delete_vector_index tests to work. Can be dropped once 1.39.0 is GA, the endpoint + # is enabled by default from then on. Ignored by servers that do not know the flag. + ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT: 'true' contextionary: environment: diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index 786b486cd..fe15593ad 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -1,5 +1,6 @@ import datetime -from typing import Generator, List, Optional, Union +import time +from typing import Any, Dict, Generator, List, Optional, Union import pytest as pytest from _pytest.fixtures import SubRequest @@ -11,6 +12,7 @@ OpenAICollection, _sanitize_collection_name, ) +from weaviate.collections import Collection from weaviate.collections.classes.config import ( _BQConfig, _CollectionConfig, @@ -37,6 +39,7 @@ Rerankers, _RerankerProvider, Tokenization, + _NamedVectorConfig, _NamedVectorConfigCreate, _VectorizerConfigCreate, IndexName, @@ -2694,3 +2697,77 @@ def test_text_analyzer_roundtrip_from_dict( assert config == new assert config.to_dict() == new.to_dict() client.collections.delete(name) + + +def _vector_config_without_index( + collection: Collection[Any, Any], vector_name: str, timeout: float = 30 +) -> Dict[str, _NamedVectorConfig]: + """Poll the collection config until `vector_name` no longer has an index. + + The drop is applied asynchronously, a 200 from the endpoint only means that Weaviate accepted + the request. It then becomes visible in two steps: the vector first stays in the schema with a + `vector_index_config` of `None`, and once the index is gone from disk the entry is removed from + the schema altogether. Both shapes must parse, so accept either. + """ + start = time.time() + while True: + vector_config = collection.config.get().vector_config + assert vector_config is not None + if ( + vector_name not in vector_config + or vector_config[vector_name].vector_index_config is None + ): + return vector_config + if time.time() - start > timeout: + pytest.fail(f"vector index of {vector_name} was not dropped within {timeout}s") + time.sleep(0.2) + + +def test_delete_vector_index(collection_factory: CollectionFactory) -> None: + """Test that dropping the index of a named vector leaves the rest of the collection usable.""" + collection_dummy = collection_factory("dummy") + if collection_dummy._connection._weaviate_version.is_lower_than(1, 39, 0): + pytest.skip("delete vector index not supported before 1.39.0") + + collection = collection_factory( + properties=[Property(name="name", data_type=DataType.TEXT)], + vector_config=[ + Configure.Vectors.self_provided(name="dropped"), + Configure.Vectors.self_provided(name="kept"), + ], + ) + collection.data.insert( + properties={"name": "banana"}, + vector={"dropped": [1, 2], "kept": [3, 4]}, + ) + + config = collection.config.get() + assert config.vector_config is not None + assert config.vector_config["dropped"].vector_index_config is not None + + with pytest.raises(weaviate.exceptions.UnexpectedStatusCodeError): + collection.config.delete_vector_index("does_not_exist") + + assert collection.config.delete_vector_index("dropped") is True + + vector_config = _vector_config_without_index(collection, "dropped") + # vectors that were not dropped keep their index + assert vector_config["kept"].vector_index_config is not None + + # `list_all` parses the schema through the simple config parser, it must cope with the drop too + with weaviate.connect_to_local() as client: + simple = client.collections.list_all()[collection.name] + assert simple.vector_config is not None + assert simple.vector_config["kept"].vector_index_config is not None + + # searching the vector that still has an index keeps working + assert len(collection.query.near_vector([3, 4], target_vector="kept").objects) == 1 + + +def test_delete_vector_index_invalid_input(collection_factory: CollectionFactory) -> None: + """Test that a non-string vector name is rejected before hitting the network.""" + collection = collection_factory( + vector_config=[Configure.Vectors.self_provided(name="vec")], + ) + with pytest.raises(WeaviateInvalidInputError): + collection.config.delete_vector_index(42) # type: ignore[arg-type] diff --git a/mock_tests/test_collection.py b/mock_tests/test_collection.py index 7d325462e..3552a0a98 100644 --- a/mock_tests/test_collection.py +++ b/mock_tests/test_collection.py @@ -484,6 +484,46 @@ def test_collection_exists(weaviate_mock: HTTPServer) -> None: assert e.value.status_code == 500 +def test_delete_vector_index(weaviate_mock: HTTPServer) -> None: + # the collection name is capitalized by the client before it hits the path + weaviate_mock.expect_request( + "/v1/schema/Test/vectors/vec/index", method="DELETE" + ).respond_with_json(response_json={}, status=200) + + with weaviate.connect_to_local( + port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC, skip_init_checks=True + ) as client: + assert client.collections.use("test").config.delete_vector_index("vec") + + with pytest.raises(weaviate.exceptions.WeaviateInvalidInputError): + client.collections.use("test").config.delete_vector_index(42) # type: ignore[arg-type] + + +def test_delete_vector_index_endpoint_disabled(weaviate_mock: HTTPServer) -> None: + # servers without ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true answer 500 + weaviate_mock.expect_request( + "/v1/schema/Test/vectors/vec/index", method="DELETE" + ).respond_with_json( + response_json={ + "error": [ + { + "message": "alter schema drop vector index endpoint is experimental and disabled by default" + } + ] + }, + status=500, + ) + + with weaviate.connect_to_local( + port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC, skip_init_checks=True + ) as client: + with pytest.raises(UnexpectedStatusCodeError) as e: + client.collections.use("Test").config.delete_vector_index("vec") + assert e.value.status_code == 500 + # the error message must not claim the vector is missing, the endpoint is simply off + assert "experimental and disabled by default" in e.value.message + + def test_grpc_client_version_header( metadata_capture_collection: tuple[ weaviate.collections.Collection, MockMetadataCaptureWeaviateService diff --git a/test/collection/test_config_methods.py b/test/collection/test_config_methods.py index 2e40acacc..ad5543daf 100644 --- a/test/collection/test_config_methods.py +++ b/test/collection/test_config_methods.py @@ -1,10 +1,95 @@ +from typing import Any, Dict + +from weaviate.collections.classes.config import VectorIndexType from weaviate.collections.classes.config_methods import ( _collection_config_from_json, + _collection_config_simple_from_json, _collection_configs_simple_from_json, _nested_properties_from_config, _properties_from_config, ) +HNSW_CONFIG = { + "skip": False, + "cleanupIntervalSeconds": 300, + "maxConnections": 64, + "efConstruction": 128, + "ef": -1, + "dynamicEfMin": 100, + "dynamicEfMax": 500, + "dynamicEfFactor": 8, + "vectorCacheMaxObjects": 1000000000000, + "flatSearchCutoff": 40000, + "distance": "cosine", +} + + +def _schema_with_vector_config(vector_config: Dict[str, Any]) -> Dict[str, Any]: + """Build a minimal collection schema, as returned by Weaviate, around the given vectorConfig.""" + return { + "class": "TestCollection", + "vectorConfig": vector_config, + "properties": [], + "invertedIndexConfig": { + "bm25": {"b": 0.75, "k1": 1.2}, + "cleanupIntervalSeconds": 60, + "stopwords": {"preset": "en", "additions": None, "removals": None}, + }, + "multiTenancyConfig": {"enabled": False}, + "replicationConfig": {"factor": 1, "deletionStrategy": "NoAutomatedResolution"}, + "shardingConfig": { + "virtualPerPhysical": 128, + "desiredCount": 1, + "actualCount": 1, + "desiredVirtualCount": 128, + "actualVirtualCount": 128, + "key": "_id", + "strategy": "hash", + "function": "murmur3", + }, + } + + +def test_collection_config_from_json_with_dropped_vector_index() -> None: + """A vector whose index was dropped is returned without a vectorIndexConfig.""" + # Shape returned by Weaviate after `collection.config.delete_vector_index("dropped")`: + # the entry stays in the schema, `vectorIndexType` becomes "none" and `vectorIndexConfig` + # is omitted entirely. + schema = _schema_with_vector_config( + { + "dropped": {"vectorizer": {"none": {}}, "vectorIndexType": "none"}, + "kept": { + "vectorizer": {"none": {}}, + "vectorIndexType": "hnsw", + "vectorIndexConfig": HNSW_CONFIG, + }, + } + ) + + config = _collection_config_from_json(schema) + + assert config.vector_config is not None + assert config.vector_config["dropped"].vector_index_config is None + assert config.vector_config["kept"].vector_index_config is not None + + # The dropped vector must round-trip back to the "none" index type the server reported. + as_dict = config.to_dict() + assert as_dict["vectorConfig"]["dropped"]["vectorIndexType"] == VectorIndexType.NONE.value + assert "vectorIndexConfig" not in as_dict["vectorConfig"]["dropped"] + assert as_dict["vectorConfig"]["kept"]["vectorIndexType"] == VectorIndexType.HNSW.value + + +def test_collection_config_simple_from_json_with_dropped_vector_index() -> None: + """`collections.list_all()` must not choke on a collection with a dropped vector index.""" + schema = _schema_with_vector_config( + {"dropped": {"vectorizer": {"none": {}}, "vectorIndexType": "none"}} + ) + + config = _collection_config_simple_from_json(schema) + + assert config.vector_config is not None + assert config.vector_config["dropped"].vector_index_config is None + def test_collection_config_simple_from_json_with_none_vectorizer_config() -> None: """Test that _collection_configs_simple_from_json handles None vectorizer config.""" diff --git a/test/collection/test_config_update.py b/test/collection/test_config_update.py index 680337291..f336a4faa 100644 --- a/test/collection/test_config_update.py +++ b/test/collection/test_config_update.py @@ -160,3 +160,48 @@ def test_replication_async_config_reset_all_fields() -> None: ) result = update.merge_with_existing(schema) assert result["asyncConfig"] == {} + + +@pytest.mark.parametrize("use_deprecated_syntax", [False, True]) +def test_updating_dropped_vector_index(use_deprecated_syntax: bool) -> None: + """A vector whose index was dropped has no index config to merge into.""" + schema = multi_vector_schema() + # shape reported by Weaviate for a vector dropped via `config.delete_vector_index` + schema["vectorConfig"]["boi"] = {"vectorizer": {"none": {}}, "vectorIndexType": "none"} + + hnsw = Reconfigure.VectorIndex.hnsw(ef=128) + update = ( + _CollectionConfigUpdate( + vectorizer_config=[ + Reconfigure.NamedVectors.update(name="boi", vector_index_config=hnsw) + ] + ) + if use_deprecated_syntax + else _CollectionConfigUpdate( + vector_config=[Reconfigure.Vectors.update(name="boi", vector_index_config=hnsw)] + ) + ) + + with pytest.raises(WeaviateInvalidInputError, match="delete_vector_index"): + update.merge_with_existing(schema) + + +def test_updating_vector_next_to_dropped_vector_index() -> None: + """Vectors that still have an index remain updatable next to a dropped one.""" + schema = multi_vector_schema() + schema["vectorConfig"]["boi"] = {"vectorizer": {"none": {}}, "vectorIndexType": "none"} + + update = _CollectionConfigUpdate( + vector_config=[ + Reconfigure.Vectors.update( + name="yeh", vector_index_config=Reconfigure.VectorIndex.hnsw(ef=128) + ) + ] + ) + new_schema = update.merge_with_existing(schema) + + assert new_schema["vectorConfig"]["yeh"]["vectorIndexConfig"]["ef"] == 128 + assert new_schema["vectorConfig"]["boi"] == { + "vectorizer": {"none": {}}, + "vectorIndexType": "none", + } diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py index 0f6d974c0..ebb9f1133 100644 --- a/weaviate/collections/classes/config.py +++ b/weaviate/collections/classes/config.py @@ -1468,6 +1468,22 @@ def mutual_exclusivity( ) return v + @staticmethod + def __existing_vector_index_config(schema: Dict[str, Any], name: str) -> Dict[str, Any]: + if name not in schema["vectorConfig"]: + raise WeaviateInvalidInputError( + f"Vector config with name {name} does not exist in the existing vector config" + ) + existing = schema["vectorConfig"][name] + if "vectorIndexConfig" not in existing: + # the index was dropped with `collection.config.delete_vector_index`, Weaviate reports + # such a vector as `vectorIndexType: "none"` without any index config to merge into + raise WeaviateInvalidInputError( + f"Vector config with name {name} has no vector index, it was deleted with " + "collection.config.delete_vector_index() and cannot be re-created" + ) + return cast(Dict[str, Any], existing["vectorIndexConfig"]) + def __check_quantizers( self, quantizer: Optional[_QuantizerConfigUpdate], @@ -1580,18 +1596,10 @@ def merge_with_existing(self, schema: Dict[str, Any]) -> Dict[str, Any]: ) else: for vc in self.vectorizerConfig: - if vc.name not in schema["vectorConfig"]: - raise WeaviateInvalidInputError( - f"Vector config with name {vc.name} does not exist in the existing vector config" - ) - self.__check_quantizers( - vc.vectorIndexConfig.quantizer, - schema["vectorConfig"][vc.name]["vectorIndexConfig"], - ) + existing = self.__existing_vector_index_config(schema, vc.name) + self.__check_quantizers(vc.vectorIndexConfig.quantizer, existing) schema["vectorConfig"][vc.name]["vectorIndexConfig"] = ( - vc.vectorIndexConfig.merge_with_existing( - schema["vectorConfig"][vc.name]["vectorIndexConfig"] - ) + vc.vectorIndexConfig.merge_with_existing(existing) ) schema["vectorConfig"][vc.name]["vectorIndexType"] = ( vc.vectorIndexConfig.vector_index_type() @@ -1603,18 +1611,10 @@ def merge_with_existing(self, schema: Dict[str, Any]) -> Dict[str, Any]: else self.vectorConfig ) for vc in vcs: - if vc.name not in schema["vectorConfig"]: - raise WeaviateInvalidInputError( - f"Vector config with name {vc.name} does not exist in the existing vector config" - ) - self.__check_quantizers( - vc.vectorIndexConfig.quantizer, - schema["vectorConfig"][vc.name]["vectorIndexConfig"], - ) + existing = self.__existing_vector_index_config(schema, vc.name) + self.__check_quantizers(vc.vectorIndexConfig.quantizer, existing) schema["vectorConfig"][vc.name]["vectorIndexConfig"] = ( - vc.vectorIndexConfig.merge_with_existing( - schema["vectorConfig"][vc.name]["vectorIndexConfig"] - ) + vc.vectorIndexConfig.merge_with_existing(existing) ) schema["vectorConfig"][vc.name]["vectorIndexType"] = ( vc.vectorIndexConfig.vector_index_type() @@ -2064,16 +2064,23 @@ def to_dict(self) -> Dict[str, Any]: @dataclass class _NamedVectorConfig(_ConfigBase): vectorizer: _NamedVectorizerConfig + # `None` means the index of this vector was dropped with `collection.config.delete_vector_index`. + # The vector data is still stored, but there is no index to configure or search. vector_index_config: Union[ VectorIndexConfigHNSW, VectorIndexConfigFlat, VectorIndexConfigDynamic, VectorIndexConfigHFresh, + None, ] def to_dict(self) -> Dict: ret_dict = super().to_dict() - ret_dict["vectorIndexType"] = self.vector_index_config.vector_index_type() + ret_dict["vectorIndexType"] = ( + VectorIndexType.NONE.value + if self.vector_index_config is None + else self.vector_index_config.vector_index_type() + ) return ret_dict diff --git a/weaviate/collections/classes/config_methods.py b/weaviate/collections/classes/config_methods.py index 691cf208d..a67b62956 100644 --- a/weaviate/collections/classes/config_methods.py +++ b/weaviate/collections/classes/config_methods.py @@ -283,7 +283,12 @@ def __get_vector_config( props = vec_config.pop("properties", None) vector_index_config = __get_vector_index_config(named_vector) - assert vector_index_config is not None + # A vector whose index was dropped with `collection.config.delete_vector_index` is + # returned as `vectorIndexType: "none"` without any `vectorIndexConfig`. + assert ( + vector_index_config is not None + or named_vector.get("vectorIndexType") == VectorIndexType.NONE.value + ) try: vec: Union[str, Vectorizers] = Vectorizers(vectorizer_str) except ValueError: diff --git a/weaviate/collections/classes/config_vector_index.py b/weaviate/collections/classes/config_vector_index.py index ff6a0ba40..fe0166558 100644 --- a/weaviate/collections/classes/config_vector_index.py +++ b/weaviate/collections/classes/config_vector_index.py @@ -36,12 +36,16 @@ class VectorIndexType(str, Enum): FLAT: Flat index. DYNAMIC: Dynamic index. HFRESH: HFRESH index. + NONE: The index of this vector has been dropped, see ``collection.config.delete_vector_index``. + The vector data is still stored, but it cannot be searched. This value is reported by the + server only, it cannot be used to configure a vector. """ HNSW = "hnsw" FLAT = "flat" DYNAMIC = "dynamic" HFRESH = "hfresh" + NONE = "none" class _MultiVectorConfigCreateBase(_ConfigCreateModel): diff --git a/weaviate/collections/config/async_.pyi b/weaviate/collections/config/async_.pyi index 015b70dab..91139dd83 100644 --- a/weaviate/collections/config/async_.pyi +++ b/weaviate/collections/config/async_.pyi @@ -90,3 +90,4 @@ class _ConfigCollectionAsync(_ConfigCollectionExecutor[ConnectionAsync]): self, *, vector_config: Union[_VectorConfigCreate, List[_VectorConfigCreate]] ) -> None: ... async def delete_property_index(self, property_name: str, index_name: IndexName) -> bool: ... + async def delete_vector_index(self, vector_name: str) -> bool: ... diff --git a/weaviate/collections/config/executor.py b/weaviate/collections/config/executor.py index 103ab70ac..b1e7f57bc 100644 --- a/weaviate/collections/config/executor.py +++ b/weaviate/collections/config/executor.py @@ -666,3 +666,47 @@ def resp(res: Response) -> bool: error_msg="Property may not exist", status_codes=_ExpectedStatusCodes(ok_in=[200], error="property exists"), ) + + def delete_vector_index( + self, + vector_name: str, + ) -> executor.Result[bool]: + """Delete the index of a named vector of the collection in Weaviate. + + This is a destructive and irreversible operation. The vectors themselves are kept, but + their index is removed from disk and cannot be re-created afterwards, neither through + this method nor through `collection.config.update()`. Searches and writes targeting the + vector are rejected once the index is gone. + + The drop is applied asynchronously. A successful call means that Weaviate accepted the + request, not that the index is already gone. `collection.config.get()` first reports the + vector with a `vector_index_config` of `None` and drops it from `vector_config` + altogether once the index has been removed from disk. + + Only named vectors can be dropped and the server must be started with + `ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true`, since the endpoint + is experimental and disabled by default. Without it, Weaviate answers with a 500. + + Args: + vector_name: The name of the named vector whose index to delete. + + Raises: + weaviate.exceptions.WeaviateConnectionError: If the network connection to Weaviate fails. + weaviate.exceptions.UnexpectedStatusCodeError: If Weaviate reports a non-OK status, e.g. + if the vector does not exist or if the endpoint is not enabled on the server. + weaviate.exceptions.WeaviateInvalidInputError: If `vector_name` is not a string. + """ + _validate_input([_ValidateArgument(expected=[str], name="vector_name", value=vector_name)]) + + path = f"/schema/{_capitalize_first_letter(self._name)}/vectors/{vector_name}/index" + + def resp(res: Response) -> bool: + return res.status_code == 200 + + return executor.execute( + response_callback=resp, + method=self._connection.delete, + path=path, + error_msg="Vector index may not have been deleted", + status_codes=_ExpectedStatusCodes(ok_in=[200], error="delete vector index"), + ) diff --git a/weaviate/collections/config/sync.pyi b/weaviate/collections/config/sync.pyi index e54d8c8fc..7bd450819 100644 --- a/weaviate/collections/config/sync.pyi +++ b/weaviate/collections/config/sync.pyi @@ -88,3 +88,4 @@ class _ConfigCollection(_ConfigCollectionExecutor[ConnectionSync]): self, *, vector_config: Union[_VectorConfigCreate, List[_VectorConfigCreate]] ) -> None: ... def delete_property_index(self, property_name: str, index_name: IndexName) -> bool: ... + def delete_vector_index(self, vector_name: str) -> bool: ...