Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion vector_search/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -485,10 +485,18 @@ async def _async_vector_counts(
aggregation_keys = params.get("aggregations") or []

count_result, aggregations = await asyncio.gather(
# This total drives pagination, and Qdrant's approximate count
# overestimates a filtered collection -- which advertises pages
# that return no results. Exact counting fixes that but scales with
# the number of matched points (measured at ~60ms per million), so
# we only set exact=True for the resources collection (~1 point per
# resource). We do not set it for the contentfiles collection due to
# the number (millions) of points; its totals stay approximate,
# which is invisible while nothing paginates them.
client.count(
collection_name=search_collection,
count_filter=search_filter,
exact=False,
exact=search_collection == RESOURCES_COLLECTION_NAME,
),
async_qdrant_aggregations(
aggregation_keys,
Expand Down
42 changes: 42 additions & 0 deletions vector_search/views_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1214,3 +1214,45 @@ def test_content_file_vector_search_partial_invalid_ids_searches_survivors(
id_conditions = [c for c in must if getattr(c, "key", None) == "edx_module_id"]
assert len(id_conditions) == 1
assert list(id_conditions[0].match.any) == [valid_id]


@pytest.mark.django_db(transaction=True)
def test_vector_search_count_is_exact(client, mock_qdrant):
"""The result total must be an exact count.

Qdrant's approximate count overestimates filtered collections, which made
the paginator advertise pages that returned no results.
"""
response = client.get(
reverse("vector_search:v0:vector_learning_resources_search"),
data={
"topic": "Art, Design & Architecture",
"resource_type_group": "learning_material",
},
)

assert response.status_code == 200

mock_qdrant.count.assert_awaited()
assert mock_qdrant.count.await_args.kwargs["exact"] is True
Comment on lines +1219 to +1237

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we also add a test that the content-files endpoint still uses exact=False? Keeping exact counts off the contentfiles collection is the main performance safeguard in this PR (it has 12M+ points in prod), but nothing currently enforces it — if someone later switches this back to a plain exact=True, every test would still pass. A copy of this test that hits the content-files search endpoint and asserts exact is False would cover it.



@pytest.mark.django_db(transaction=True)
def test_content_file_vector_search_count_is_approximate(
client, mock_qdrant, content_file_viewer
):
"""Content-file totals must stay approximate.

Exact counting scales with matched points, and this collection holds
millions of chunks, so an exact count here would cost most of a second.
Nothing paginates content files, so the imprecision is invisible.
"""
response = client.get(
reverse("vector_search:v0:vector_content_files_search"),
data={"q": "test"},
)

assert response.status_code == 200

mock_qdrant.count.assert_awaited()
assert mock_qdrant.count.await_args.kwargs["exact"] is False
Loading