feat: add support for alter schema drop vector index

ci/docker-compose.yml

@@ -33,6 +33,9 @@ services:
       OBJECTS_TTL_DELETE_SCHEDULE: "@every 12h" # for objectTTL tests to work
       EXPORT_ENABLED: 'true'
       EXPORT_DEFAULT_PATH: "/var/lib/weaviate/exports"
+      # for config.delete_vector_index tests to work. Can be dropped once 1.39.0 is GA, the endpoint
+      # is enabled by default from then on. Ignored by servers that do not know the flag.
+      ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT: 'true'
 
   contextionary:
     environment:

integration/test_collection_config.py

@@ -1,5 +1,6 @@
 import datetime
-from typing import Generator, List, Optional, Union
+import time
+from typing import Any, Dict, Generator, List, Optional, Union
 
 import pytest as pytest
 from _pytest.fixtures import SubRequest
@@ -11,6 +12,7 @@
     OpenAICollection,
     _sanitize_collection_name,
 )
+from weaviate.collections import Collection
 from weaviate.collections.classes.config import (
     _BQConfig,
     _CollectionConfig,
@@ -37,6 +39,7 @@
     Rerankers,
     _RerankerProvider,
     Tokenization,
+    _NamedVectorConfig,
     _NamedVectorConfigCreate,
     _VectorizerConfigCreate,
     IndexName,
@@ -2694,3 +2697,77 @@ def test_text_analyzer_roundtrip_from_dict(
         assert config == new
         assert config.to_dict() == new.to_dict()
         client.collections.delete(name)
+
+
+def _vector_config_without_index(
+    collection: Collection[Any, Any], vector_name: str, timeout: float = 30
+) -> Dict[str, _NamedVectorConfig]:
+    """Poll the collection config until `vector_name` no longer has an index.
+
+    The drop is applied asynchronously, a 200 from the endpoint only means that Weaviate accepted
+    the request. It then becomes visible in two steps: the vector first stays in the schema with a
+    `vector_index_config` of `None`, and once the index is gone from disk the entry is removed from
+    the schema altogether. Both shapes must parse, so accept either.
+    """
+    start = time.time()
+    while True:
+        vector_config = collection.config.get().vector_config
+        assert vector_config is not None
+        if (
+            vector_name not in vector_config
+            or vector_config[vector_name].vector_index_config is None
+        ):
+            return vector_config
+        if time.time() - start > timeout:
+            pytest.fail(f"vector index of {vector_name} was not dropped within {timeout}s")
+        time.sleep(0.2)
+
+
+def test_delete_vector_index(collection_factory: CollectionFactory) -> None:
+    """Test that dropping the index of a named vector leaves the rest of the collection usable."""
+    collection_dummy = collection_factory("dummy")
+    if collection_dummy._connection._weaviate_version.is_lower_than(1, 39, 0):
+        pytest.skip("delete vector index not supported before 1.39.0")
+
+    collection = collection_factory(
+        properties=[Property(name="name", data_type=DataType.TEXT)],
+        vector_config=[
+            Configure.Vectors.self_provided(name="dropped"),
+            Configure.Vectors.self_provided(name="kept"),
+        ],
+    )
+    collection.data.insert(
+        properties={"name": "banana"},
+        vector={"dropped": [1, 2], "kept": [3, 4]},
+    )
+
+    config = collection.config.get()
+    assert config.vector_config is not None
+    assert config.vector_config["dropped"].vector_index_config is not None
+
+    with pytest.raises(weaviate.exceptions.UnexpectedStatusCodeError):
+        collection.config.delete_vector_index("does_not_exist")
+
+    assert collection.config.delete_vector_index("dropped") is True
+
+    vector_config = _vector_config_without_index(collection, "dropped")
+    # vectors that were not dropped keep their index
+    assert vector_config["kept"].vector_index_config is not None
+
+    # `list_all` parses the schema through the simple config parser, it must cope with the drop too
+    with weaviate.connect_to_local() as client:
+        simple = client.collections.list_all()[collection.name]
+        assert simple.vector_config is not None
+        assert simple.vector_config["kept"].vector_index_config is not None
+
+    # searching the vector that still has an index keeps working
+    assert len(collection.query.near_vector([3, 4], target_vector="kept").objects) == 1
+
+
+def test_delete_vector_index_invalid_input(collection_factory: CollectionFactory) -> None:
+    """Test that a non-string vector name is rejected before hitting the network."""
+    collection = collection_factory(
+        vector_config=[Configure.Vectors.self_provided(name="vec")],
+    )
+    with pytest.raises(WeaviateInvalidInputError):
+        collection.config.delete_vector_index(42)  # type: ignore[arg-type]

mock_tests/test_collection.py

@@ -484,6 +484,46 @@ def test_collection_exists(weaviate_mock: HTTPServer) -> None:
             assert e.value.status_code == 500
 
 
+def test_delete_vector_index(weaviate_mock: HTTPServer) -> None:
+    # the collection name is capitalized by the client before it hits the path
+    weaviate_mock.expect_request(
+        "/v1/schema/Test/vectors/vec/index", method="DELETE"
+    ).respond_with_json(response_json={}, status=200)
+
+    with weaviate.connect_to_local(
+        port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC, skip_init_checks=True
+    ) as client:
+        assert client.collections.use("test").config.delete_vector_index("vec")
+
+        with pytest.raises(weaviate.exceptions.WeaviateInvalidInputError):
+            client.collections.use("test").config.delete_vector_index(42)  # type: ignore[arg-type]
+
+
+def test_delete_vector_index_endpoint_disabled(weaviate_mock: HTTPServer) -> None:
+    # servers without ENABLE_EXPERIMENTAL_ALTER_SCHEMA_DROP_VECTOR_INDEX_ENDPOINT=true answer 500
+    weaviate_mock.expect_request(
+        "/v1/schema/Test/vectors/vec/index", method="DELETE"
+    ).respond_with_json(
+        response_json={
+            "error": [
+                {
+                    "message": "alter schema drop vector index endpoint is experimental and disabled by default"
+                }
+            ]
+        },
+        status=500,
+    )
+
+    with weaviate.connect_to_local(
+        port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC, skip_init_checks=True
+    ) as client:
+        with pytest.raises(UnexpectedStatusCodeError) as e:
+            client.collections.use("Test").config.delete_vector_index("vec")
+        assert e.value.status_code == 500
+        # the error message must not claim the vector is missing, the endpoint is simply off
+        assert "experimental and disabled by default" in e.value.message
+
+
 def test_grpc_client_version_header(
     metadata_capture_collection: tuple[
         weaviate.collections.Collection, MockMetadataCaptureWeaviateService

test/collection/test_config_methods.py

@@ -1,10 +1,95 @@
+from typing import Any, Dict
+
+from weaviate.collections.classes.config import VectorIndexType
 from weaviate.collections.classes.config_methods import (
     _collection_config_from_json,
+    _collection_config_simple_from_json,
     _collection_configs_simple_from_json,
     _nested_properties_from_config,
     _properties_from_config,
 )
 
+HNSW_CONFIG = {
+    "skip": False,
+    "cleanupIntervalSeconds": 300,
+    "maxConnections": 64,
+    "efConstruction": 128,
+    "ef": -1,
+    "dynamicEfMin": 100,
+    "dynamicEfMax": 500,
+    "dynamicEfFactor": 8,
+    "vectorCacheMaxObjects": 1000000000000,
+    "flatSearchCutoff": 40000,
+    "distance": "cosine",
+}
+
+
+def _schema_with_vector_config(vector_config: Dict[str, Any]) -> Dict[str, Any]:
+    """Build a minimal collection schema, as returned by Weaviate, around the given vectorConfig."""
+    return {
+        "class": "TestCollection",
+        "vectorConfig": vector_config,
+        "properties": [],
+        "invertedIndexConfig": {
+            "bm25": {"b": 0.75, "k1": 1.2},
+            "cleanupIntervalSeconds": 60,
+            "stopwords": {"preset": "en", "additions": None, "removals": None},
+        },
+        "multiTenancyConfig": {"enabled": False},
+        "replicationConfig": {"factor": 1, "deletionStrategy": "NoAutomatedResolution"},
+        "shardingConfig": {
+            "virtualPerPhysical": 128,
+            "desiredCount": 1,
+            "actualCount": 1,
+            "desiredVirtualCount": 128,
+            "actualVirtualCount": 128,
+            "key": "_id",
+            "strategy": "hash",
+            "function": "murmur3",
+        },
+    }
+
+
+def test_collection_config_from_json_with_dropped_vector_index() -> None:
+    """A vector whose index was dropped is returned without a vectorIndexConfig."""
+    # Shape returned by Weaviate after `collection.config.delete_vector_index("dropped")`:
+    # the entry stays in the schema, `vectorIndexType` becomes "none" and `vectorIndexConfig`
+    # is omitted entirely.
+    schema = _schema_with_vector_config(
+        {
+            "dropped": {"vectorizer": {"none": {}}, "vectorIndexType": "none"},
+            "kept": {
+                "vectorizer": {"none": {}},
+                "vectorIndexType": "hnsw",
+                "vectorIndexConfig": HNSW_CONFIG,
+            },
+        }
+    )
+
+    config = _collection_config_from_json(schema)
+
+    assert config.vector_config is not None
+    assert config.vector_config["dropped"].vector_index_config is None
+    assert config.vector_config["kept"].vector_index_config is not None
+
+    # The dropped vector must round-trip back to the "none" index type the server reported.
+    as_dict = config.to_dict()
+    assert as_dict["vectorConfig"]["dropped"]["vectorIndexType"] == VectorIndexType.NONE.value
+    assert "vectorIndexConfig" not in as_dict["vectorConfig"]["dropped"]
+    assert as_dict["vectorConfig"]["kept"]["vectorIndexType"] == VectorIndexType.HNSW.value
+
+
+def test_collection_config_simple_from_json_with_dropped_vector_index() -> None:
+    """`collections.list_all()` must not choke on a collection with a dropped vector index."""
+    schema = _schema_with_vector_config(
+        {"dropped": {"vectorizer": {"none": {}}, "vectorIndexType": "none"}}
+    )
+
+    config = _collection_config_simple_from_json(schema)
+
+    assert config.vector_config is not None
+    assert config.vector_config["dropped"].vector_index_config is None
+
 
 def test_collection_config_simple_from_json_with_none_vectorizer_config() -> None:
     """Test that _collection_configs_simple_from_json handles None vectorizer config."""

test/collection/test_config_update.py

@@ -160,3 +160,48 @@ def test_replication_async_config_reset_all_fields() -> None:
     )
     result = update.merge_with_existing(schema)
     assert result["asyncConfig"] == {}
+
+
+@pytest.mark.parametrize("use_deprecated_syntax", [False, True])
+def test_updating_dropped_vector_index(use_deprecated_syntax: bool) -> None:
+    """A vector whose index was dropped has no index config to merge into."""
+    schema = multi_vector_schema()
+    # shape reported by Weaviate for a vector dropped via `config.delete_vector_index`
+    schema["vectorConfig"]["boi"] = {"vectorizer": {"none": {}}, "vectorIndexType": "none"}
+
+    hnsw = Reconfigure.VectorIndex.hnsw(ef=128)
+    update = (
+        _CollectionConfigUpdate(
+            vectorizer_config=[
+                Reconfigure.NamedVectors.update(name="boi", vector_index_config=hnsw)
+            ]
+        )
+        if use_deprecated_syntax
+        else _CollectionConfigUpdate(
+            vector_config=[Reconfigure.Vectors.update(name="boi", vector_index_config=hnsw)]
+        )
+    )
+
+    with pytest.raises(WeaviateInvalidInputError, match="delete_vector_index"):
+        update.merge_with_existing(schema)
+
+
+def test_updating_vector_next_to_dropped_vector_index() -> None:
+    """Vectors that still have an index remain updatable next to a dropped one."""
+    schema = multi_vector_schema()
+    schema["vectorConfig"]["boi"] = {"vectorizer": {"none": {}}, "vectorIndexType": "none"}
+
+    update = _CollectionConfigUpdate(
+        vector_config=[
+            Reconfigure.Vectors.update(
+                name="yeh", vector_index_config=Reconfigure.VectorIndex.hnsw(ef=128)
+            )
+        ]
+    )
+    new_schema = update.merge_with_existing(schema)
+
+    assert new_schema["vectorConfig"]["yeh"]["vectorIndexConfig"]["ef"] == 128
+    assert new_schema["vectorConfig"]["boi"] == {
+        "vectorizer": {"none": {}},
+        "vectorIndexType": "none",
+    }

weaviate/collections/classes/config.py

@@ -1468,6 +1468,22 @@ def mutual_exclusivity(
             )
         return v
 
+    @staticmethod
+    def __existing_vector_index_config(schema: Dict[str, Any], name: str) -> Dict[str, Any]:
+        if name not in schema["vectorConfig"]:
+            raise WeaviateInvalidInputError(
+                f"Vector config with name {name} does not exist in the existing vector config"
+            )
+        existing = schema["vectorConfig"][name]
+        if "vectorIndexConfig" not in existing:
+            # the index was dropped with `collection.config.delete_vector_index`, Weaviate reports
+            # such a vector as `vectorIndexType: "none"` without any index config to merge into
+            raise WeaviateInvalidInputError(
+                f"Vector config with name {name} has no vector index, it was deleted with "
+                "collection.config.delete_vector_index() and cannot be re-created"
+            )
+        return cast(Dict[str, Any], existing["vectorIndexConfig"])
+
     def __check_quantizers(
         self,
         quantizer: Optional[_QuantizerConfigUpdate],
@@ -1580,18 +1596,10 @@ def merge_with_existing(self, schema: Dict[str, Any]) -> Dict[str, Any]:
                 )
             else:
                 for vc in self.vectorizerConfig:
-                    if vc.name not in schema["vectorConfig"]:
-                        raise WeaviateInvalidInputError(
-                            f"Vector config with name {vc.name} does not exist in the existing vector config"
-                        )
-                    self.__check_quantizers(
-                        vc.vectorIndexConfig.quantizer,
-                        schema["vectorConfig"][vc.name]["vectorIndexConfig"],
-                    )
+                    existing = self.__existing_vector_index_config(schema, vc.name)
+                    self.__check_quantizers(vc.vectorIndexConfig.quantizer, existing)
                     schema["vectorConfig"][vc.name]["vectorIndexConfig"] = (
-                        vc.vectorIndexConfig.merge_with_existing(
-                            schema["vectorConfig"][vc.name]["vectorIndexConfig"]
-                        )
+                        vc.vectorIndexConfig.merge_with_existing(existing)
                     )
                     schema["vectorConfig"][vc.name]["vectorIndexType"] = (
                         vc.vectorIndexConfig.vector_index_type()
@@ -1603,18 +1611,10 @@ def merge_with_existing(self, schema: Dict[str, Any]) -> Dict[str, Any]:
                 else self.vectorConfig
             )
             for vc in vcs:
-                if vc.name not in schema["vectorConfig"]:
-                    raise WeaviateInvalidInputError(
-                        f"Vector config with name {vc.name} does not exist in the existing vector config"
-                    )
-                self.__check_quantizers(
-                    vc.vectorIndexConfig.quantizer,
-                    schema["vectorConfig"][vc.name]["vectorIndexConfig"],
-                )
+                existing = self.__existing_vector_index_config(schema, vc.name)
+                self.__check_quantizers(vc.vectorIndexConfig.quantizer, existing)
                 schema["vectorConfig"][vc.name]["vectorIndexConfig"] = (
-                    vc.vectorIndexConfig.merge_with_existing(
-                        schema["vectorConfig"][vc.name]["vectorIndexConfig"]
-                    )
+                    vc.vectorIndexConfig.merge_with_existing(existing)
                 )
                 schema["vectorConfig"][vc.name]["vectorIndexType"] = (
                     vc.vectorIndexConfig.vector_index_type()
@@ -2064,16 +2064,23 @@ def to_dict(self) -> Dict[str, Any]:
 @dataclass
 class _NamedVectorConfig(_ConfigBase):
     vectorizer: _NamedVectorizerConfig
+    # `None` means the index of this vector was dropped with `collection.config.delete_vector_index`.
+    # The vector data is still stored, but there is no index to configure or search.
     vector_index_config: Union[
         VectorIndexConfigHNSW,
         VectorIndexConfigFlat,
         VectorIndexConfigDynamic,
         VectorIndexConfigHFresh,
+        None,
     ]
 
     def to_dict(self) -> Dict:
         ret_dict = super().to_dict()
-        ret_dict["vectorIndexType"] = self.vector_index_config.vector_index_type()
+        ret_dict["vectorIndexType"] = (
+            VectorIndexType.NONE.value
+            if self.vector_index_config is None
+            else self.vector_index_config.vector_index_type()
+        )
         return ret_dict