feat: add delete_old option to add_questions

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "verdikt-sdk"
-version = "0.2.0"
+version = "0.3.0"
 description = "Python SDK for the Verdikt Evaluation API"
 readme = "README.md"
 requires-python = ">=3.13"

tests/test_add_questions.py

@@ -0,0 +1,120 @@
+import json
+
+import httpx
+import pytest
+
+from verdikt_sdk import VerdiktClient
+from verdikt_sdk.models import Question
+
+
+def _build_handler(captured: dict) -> httpx.MockTransport:
+    """Mock transport faking auth + datasets GET/POST/DELETE endpoints.
+
+    Records POSTed body and DELETEd dataset ids in *captured*.
+    """
+    captured.setdefault("deleted_ids", [])
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        url = str(request.url)
+        if url.endswith("/.well-known"):
+            return httpx.Response(200, json={"issuer": "http://issuer.test"})
+        if "/oauth/v2/token" in url:
+            return httpx.Response(
+                200,
+                json={
+                    "access_token": "tok",
+                    "id_token": "id",
+                    "token_type": "Bearer",
+                    "expires_in": 3600,
+                },
+            )
+        if "/v1/app/by-slug/" in url:
+            return httpx.Response(
+                200, json={"id": 7, "slug": "my-app", "name": "My App"}
+            )
+        if url.endswith("/v1/app/7/datasets") and request.method == "GET":
+            return httpx.Response(
+                200,
+                json=[
+                    {"id": 11, "question": "Q1", "human_answer": "A1"},
+                    {"id": 12, "question": "Q2", "human_answer": "A2"},
+                ],
+            )
+        if url.endswith("/v1/app/7/datasets") and request.method == "POST":
+            captured["posted"] = json.loads(request.content)
+            return httpx.Response(201, json=[])
+        if "/v1/app/7/datasets/" in url and request.method == "DELETE":
+            captured["deleted_ids"].append(int(url.rsplit("/", 1)[1]))
+            return httpx.Response(204)
+        return httpx.Response(404)
+
+    return httpx.MockTransport(handler)
+
+
+def _make_client(captured: dict) -> VerdiktClient:
+    client = VerdiktClient(
+        base_url="http://verdikt.test",
+        client_id="cid",
+        client_secret="csec",
+    )
+    client._http = httpx.AsyncClient(transport=_build_handler(captured))
+    client._auth._http = client._http
+    return client
+
+
+@pytest.mark.asyncio
+async def test_add_questions_delete_old_deletes_unprovided_questions():
+    # Arrange
+    captured: dict = {}
+    client = _make_client(captured)
+
+    # Act — existing is [Q1, Q2]; provide Q1 (overlap) + Qnew
+    await client.add_questions(
+        app_slug="my-app",
+        questions=[
+            Question(question="Q1", human_answer="A1"),
+            Question(question="Qnew", human_answer="Anew"),
+        ],
+        delete_old=True,
+    )
+
+    # Assert — only Q2 (id 12) is unprovided, so only it is deleted
+    assert captured["deleted_ids"] == [12]
+
+
+@pytest.mark.asyncio
+async def test_add_questions_delete_old_raises_when_all_would_be_deleted():
+    # Arrange
+    captured: dict = {}
+    client = _make_client(captured)
+
+    # Act / Assert — no overlap with existing [Q1, Q2] => full wipe => refused
+    with pytest.raises(ValueError):
+        await client.add_questions(
+            app_slug="my-app",
+            questions=[Question(question="Qx", human_answer="Ax")],
+            delete_old=True,
+        )
+
+    # Nothing was posted or deleted
+    assert captured["deleted_ids"] == []
+    assert "posted" not in captured
+
+
+@pytest.mark.asyncio
+async def test_add_questions_without_delete_old_does_not_delete():
+    # Arrange
+    captured: dict = {}
+    client = _make_client(captured)
+
+    # Act
+    await client.add_questions(
+        app_slug="my-app",
+        questions=[Question(question="Qx", human_answer="Ax")],
+    )
+
+    # Assert — default leaves existing questions untouched
+    assert captured["deleted_ids"] == []
+    assert captured["posted"]["datasets"] == [
+        {"question": "Qx", "human_answer": "Ax"}
+    ]

verdikt_sdk/client.py

@@ -111,6 +111,7 @@ async def add_questions(
         self,
         app_slug: str,
         questions: list[Question],
+        delete_old: bool = False,
     ) -> None:
         """Idempotent — safe to call on every deploy.
 
@@ -119,11 +120,40 @@ async def add_questions(
         Args:
             app_slug: Slug of the target app.
             questions: List of questions with their expected human answers.
+            delete_old: When ``True``, **DELETES** every existing question whose
+                text is not present in *questions*. Use with care — this removes
+                data on the server.
+
+                As a safeguard, if this would delete *all* existing questions
+                (i.e. none of *questions* overlaps the existing set), a
+                :class:`ValueError` is raised and nothing is deleted or added.
         """
         app_id = await self._resolve_slug(app_slug)
         logger.info("Syncing %d question(s) for app '%s'", len(questions), app_slug)
 
         headers = await self._auth.headers()
+
+        # Determine which existing questions to delete *before* mutating anything,
+        # so a refused full-wipe leaves the server untouched.
+        to_delete: list[DatasetEntry] = []
+        if delete_old:
+            existing_resp = await self._http.get(
+                f"{self.base_url}/v1/app/{app_id}/datasets",
+                headers=headers,
+            )
+            raise_for_status(existing_resp)
+            existing = [
+                DatasetEntry.model_validate(item) for item in existing_resp.json()
+            ]
+            provided = {q.question for q in questions}
+            to_delete = [e for e in existing if e.question not in provided]
+            if existing and len(to_delete) == len(existing):
+                raise ValueError(
+                    f"delete_old would remove all {len(existing)} question(s) for "
+                    f"app '{app_slug}' (provided questions share none of the "
+                    f"existing ones); refusing to wipe the dataset."
+                )
+
         hashes_resp = await self._http.post(
             f"{self.base_url}/v1/app/{app_id}/datasets",
             json={
@@ -136,6 +166,22 @@ async def add_questions(
         )
         raise_for_status(hashes_resp)
 
+        if to_delete:
+            logger.info(
+                "Deleting %d old question(s) for app '%s'", len(to_delete), app_slug
+            )
+            resps = await asyncio.gather(
+                *[
+                    self._http.delete(
+                        f"{self.base_url}/v1/app/{app_id}/datasets/{e.id}",
+                        headers=headers,
+                    )
+                    for e in to_delete
+                ]
+            )
+            for resp in resps:
+                raise_for_status(resp)
+
     async def run_evaluation(
         self,
         app_slug: str,