diff --git a/pyproject.toml b/pyproject.toml index da532c1..fe74734 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "verdikt-sdk" -version = "0.2.0" +version = "0.3.0" description = "Python SDK for the Verdikt Evaluation API" readme = "README.md" requires-python = ">=3.13" diff --git a/tests/test_add_questions.py b/tests/test_add_questions.py new file mode 100644 index 0000000..8091580 --- /dev/null +++ b/tests/test_add_questions.py @@ -0,0 +1,120 @@ +import json + +import httpx +import pytest + +from verdikt_sdk import VerdiktClient +from verdikt_sdk.models import Question + + +def _build_handler(captured: dict) -> httpx.MockTransport: + """Mock transport faking auth + datasets GET/POST/DELETE endpoints. + + Records POSTed body and DELETEd dataset ids in *captured*. + """ + captured.setdefault("deleted_ids", []) + + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("/.well-known"): + return httpx.Response(200, json={"issuer": "http://issuer.test"}) + if "/oauth/v2/token" in url: + return httpx.Response( + 200, + json={ + "access_token": "tok", + "id_token": "id", + "token_type": "Bearer", + "expires_in": 3600, + }, + ) + if "/v1/app/by-slug/" in url: + return httpx.Response( + 200, json={"id": 7, "slug": "my-app", "name": "My App"} + ) + if url.endswith("/v1/app/7/datasets") and request.method == "GET": + return httpx.Response( + 200, + json=[ + {"id": 11, "question": "Q1", "human_answer": "A1"}, + {"id": 12, "question": "Q2", "human_answer": "A2"}, + ], + ) + if url.endswith("/v1/app/7/datasets") and request.method == "POST": + captured["posted"] = json.loads(request.content) + return httpx.Response(201, json=[]) + if "/v1/app/7/datasets/" in url and request.method == "DELETE": + captured["deleted_ids"].append(int(url.rsplit("/", 1)[1])) + return httpx.Response(204) + return httpx.Response(404) + + return httpx.MockTransport(handler) + + +def _make_client(captured: dict) -> VerdiktClient: + client = VerdiktClient( + base_url="http://verdikt.test", + client_id="cid", + client_secret="csec", + ) + client._http = httpx.AsyncClient(transport=_build_handler(captured)) + client._auth._http = client._http + return client + + +@pytest.mark.asyncio +async def test_add_questions_delete_old_deletes_unprovided_questions(): + # Arrange + captured: dict = {} + client = _make_client(captured) + + # Act — existing is [Q1, Q2]; provide Q1 (overlap) + Qnew + await client.add_questions( + app_slug="my-app", + questions=[ + Question(question="Q1", human_answer="A1"), + Question(question="Qnew", human_answer="Anew"), + ], + delete_old=True, + ) + + # Assert — only Q2 (id 12) is unprovided, so only it is deleted + assert captured["deleted_ids"] == [12] + + +@pytest.mark.asyncio +async def test_add_questions_delete_old_raises_when_all_would_be_deleted(): + # Arrange + captured: dict = {} + client = _make_client(captured) + + # Act / Assert — no overlap with existing [Q1, Q2] => full wipe => refused + with pytest.raises(ValueError): + await client.add_questions( + app_slug="my-app", + questions=[Question(question="Qx", human_answer="Ax")], + delete_old=True, + ) + + # Nothing was posted or deleted + assert captured["deleted_ids"] == [] + assert "posted" not in captured + + +@pytest.mark.asyncio +async def test_add_questions_without_delete_old_does_not_delete(): + # Arrange + captured: dict = {} + client = _make_client(captured) + + # Act + await client.add_questions( + app_slug="my-app", + questions=[Question(question="Qx", human_answer="Ax")], + ) + + # Assert — default leaves existing questions untouched + assert captured["deleted_ids"] == [] + assert captured["posted"]["datasets"] == [ + {"question": "Qx", "human_answer": "Ax"} + ] diff --git a/uv.lock b/uv.lock index 828a628..3f62fd2 100644 --- a/uv.lock +++ b/uv.lock @@ -1597,7 +1597,7 @@ wheels = [ [[package]] name = "verdikt-sdk" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/verdikt_sdk/client.py b/verdikt_sdk/client.py index e88395f..0737469 100644 --- a/verdikt_sdk/client.py +++ b/verdikt_sdk/client.py @@ -111,6 +111,7 @@ async def add_questions( self, app_slug: str, questions: list[Question], + delete_old: bool = False, ) -> None: """Idempotent — safe to call on every deploy. @@ -119,11 +120,40 @@ async def add_questions( Args: app_slug: Slug of the target app. questions: List of questions with their expected human answers. + delete_old: When ``True``, **DELETES** every existing question whose + text is not present in *questions*. Use with care — this removes + data on the server. + + As a safeguard, if this would delete *all* existing questions + (i.e. none of *questions* overlaps the existing set), a + :class:`ValueError` is raised and nothing is deleted or added. """ app_id = await self._resolve_slug(app_slug) logger.info("Syncing %d question(s) for app '%s'", len(questions), app_slug) headers = await self._auth.headers() + + # Determine which existing questions to delete *before* mutating anything, + # so a refused full-wipe leaves the server untouched. + to_delete: list[DatasetEntry] = [] + if delete_old: + existing_resp = await self._http.get( + f"{self.base_url}/v1/app/{app_id}/datasets", + headers=headers, + ) + raise_for_status(existing_resp) + existing = [ + DatasetEntry.model_validate(item) for item in existing_resp.json() + ] + provided = {q.question for q in questions} + to_delete = [e for e in existing if e.question not in provided] + if existing and len(to_delete) == len(existing): + raise ValueError( + f"delete_old would remove all {len(existing)} question(s) for " + f"app '{app_slug}' (provided questions share none of the " + f"existing ones); refusing to wipe the dataset." + ) + hashes_resp = await self._http.post( f"{self.base_url}/v1/app/{app_id}/datasets", json={ @@ -136,6 +166,22 @@ async def add_questions( ) raise_for_status(hashes_resp) + if to_delete: + logger.info( + "Deleting %d old question(s) for app '%s'", len(to_delete), app_slug + ) + resps = await asyncio.gather( + *[ + self._http.delete( + f"{self.base_url}/v1/app/{app_id}/datasets/{e.id}", + headers=headers, + ) + for e in to_delete + ] + ) + for resp in resps: + raise_for_status(resp) + async def run_evaluation( self, app_slug: str,