Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions isic/core/tests/test_metadata_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def image_with_metadata(image):
{
"age": 32,
"diagnosis": "Nevus",
"anatom_site": "Scalp",
"patient_id": "supersecretpatientid",
"lesion_id": "supersecretlesionid",
"rcm_case_id": "supersecretrcmcaseid",
Expand All @@ -34,6 +35,9 @@ def test_image_metadata_csv_rows_correct(image_with_metadata):
row = next(rows)
assert row == {
"age_approx": image_with_metadata.accession.age_approx,
"anatom_site_1": "Head and neck",
"anatom_site_2": "Head",
"anatom_site_3": "Scalp",
"attribution": image_with_metadata.accession.attribution,
"copyright_license": image_with_metadata.accession.copyright_license,
"diagnosis_1": "Benign",
Expand All @@ -55,6 +59,9 @@ def test_staff_image_metadata_csv_rows_correct(image_with_metadata):
assert row == {
"age_approx": image_with_metadata.accession.age_approx,
"age": image_with_metadata.accession.age,
"anatom_site_1": "Head and neck",
"anatom_site_2": "Head",
"anatom_site_3": "Scalp",
"attribution": image_with_metadata.accession.attribution,
"cohort_id": image_with_metadata.accession.cohort_id,
"cohort": image_with_metadata.accession.cohort.name,
Expand Down
20 changes: 20 additions & 0 deletions isic/core/tests/test_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,12 @@ def searchable_images(image_factory, _search_index):
image_factory(
public=True,
accession__short_diagnosis="melanoma",
accession__short_anatom_site="scalp",
),
image_factory(
public=False,
accession__short_diagnosis="nevus",
accession__short_anatom_site="forearm",
),
]
for image in images:
Expand Down Expand Up @@ -193,6 +195,14 @@ def test_core_api_image_search(searchable_images, staff_client):
assert r.status_code == 200, r.json()
assert r.json()["count"] == 1, r.json()

r = staff_client.get("/api/v2/images/search/", {"query": "anatom_site_3:Scalp"})
assert r.status_code == 200, r.json()
assert r.json()["count"] == 1, r.json()

r = staff_client.get("/api/v2/images/search/", {"query": 'anatom_site_1:"Upper extremity"'})
assert r.status_code == 200, r.json()
assert r.json()["count"] == 1, r.json()


@pytest.mark.django_db
def test_core_api_image_search_private_image(private_searchable_image, authenticated_client):
Expand Down Expand Up @@ -346,6 +356,16 @@ def test_core_api_image_faceting_structure(searchable_images, client):
"present_count": 1,
}, r.json()

assert len(r.json()["anatom_site_3"]["buckets"]) == 1, r.json()
assert r.json()["anatom_site_3"]["meta"] == {
"missing_count": 0,
"present_count": 1,
}, r.json()
assert r.json()["anatom_site_1"]["meta"] == {
"missing_count": 0,
"present_count": 1,
}, r.json()


@pytest.mark.parametrize(
"client_",
Expand Down
7 changes: 7 additions & 0 deletions isic/core/tests/test_view_image_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ def test_image_list_metadata_download_view(mocker, staff_client, mailoutbox, use
"patient_id": "bar",
"rcm_case_id": "baz",
"diagnosis": "Melanoma Invasive",
"anatom_site": "Scalp",
"image_type": "RCM: macroscopic",
},
ignore_image_check=True,
Expand All @@ -41,6 +42,9 @@ def test_image_list_metadata_download_view(mocker, staff_client, mailoutbox, use
"public",
"age",
"age_approx",
"anatom_site_1",
"anatom_site_2",
"anatom_site_3",
"diagnosis_1",
"diagnosis_2",
"diagnosis_3",
Expand All @@ -67,6 +71,9 @@ def test_image_list_metadata_download_view(mocker, staff_client, mailoutbox, use
image.public,
"57",
"55",
"Head and neck",
"Head",
"Scalp",
"Malignant",
"Malignant melanocytic proliferations (Melanoma)",
"Melanoma Invasive",
Expand Down
39 changes: 39 additions & 0 deletions isic/ingest/migrations/0036_add_anatom_site_hierarchical_fields.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Generated by Django 5.2.3 on 2026-02-19 16:44

from django.conf import settings
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("ingest", "0035_alter_distinctnessmeasure_checksum"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]

operations = [
migrations.AddField(
model_name="accession",
name="anatom_site_1",
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AddField(
model_name="accession",
name="anatom_site_2",
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AddField(
model_name="accession",
name="anatom_site_3",
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AddField(
model_name="accession",
name="anatom_site_4",
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AddField(
model_name="accession",
name="anatom_site_5",
field=models.CharField(blank=True, max_length=255, null=True),
),
]
5 changes: 5 additions & 0 deletions isic/ingest/models/accession.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@ class AccessionMetadata(models.Model):
sex = models.CharField(max_length=6, null=True, blank=True)
anatom_site_general = models.CharField(max_length=255, null=True, blank=True)
anatom_site_special = models.CharField(max_length=255, null=True, blank=True)
anatom_site_1 = models.CharField(max_length=255, null=True, blank=True)
anatom_site_2 = models.CharField(max_length=255, null=True, blank=True)
anatom_site_3 = models.CharField(max_length=255, null=True, blank=True)
anatom_site_4 = models.CharField(max_length=255, null=True, blank=True)
anatom_site_5 = models.CharField(max_length=255, null=True, blank=True)
diagnosis_1 = models.CharField(max_length=255, null=True, blank=True)
diagnosis_2 = models.CharField(max_length=255, null=True, blank=True)
diagnosis_3 = models.CharField(max_length=255, null=True, blank=True)
Expand Down
20 changes: 19 additions & 1 deletion isic/ingest/tests/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import factory
import factory.django
from isic_metadata.fields import DiagnosisEnum
from isic_metadata.fields import AnatomSiteEnum, DiagnosisEnum

from isic.core.models import CopyrightLicense
from isic.factories import UserFactory
Expand Down Expand Up @@ -182,6 +182,24 @@ def short_diagnosis(self, create: bool, extracted: Any, **kwargs: Any) -> None:
if create:
self.save()

@factory.post_generation
def short_anatom_site(self, create: bool, extracted: Any, **kwargs: Any) -> None: # noqa: FBT001
if extracted is None:
return

if extracted == "scalp":
anatom_site = AnatomSiteEnum.head_and_neck_head_scalp
elif extracted == "forearm":
anatom_site = AnatomSiteEnum.upper_extremity_forearm
else:
raise ValueError(f"Unknown short_anatom_site: {extracted}")

for key, value in AnatomSiteEnum.as_dict(anatom_site).items():
setattr(self, key, value)

if create:
self.save()


class AccessionReviewFactory(factory.django.DjangoModelFactory):
class Meta:
Expand Down
22 changes: 22 additions & 0 deletions isic/ingest/tests/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,28 @@ def test_accession_update_metadata_iddx(user, imageless_accession) -> None:
assert imageless_accession.metadata_versions.count() == 1


@pytest.mark.django_db
def test_accession_update_metadata_anatom_site(user, imageless_accession) -> None:
imageless_accession.update_metadata(user, {"anatom_site": "Scalp"})
assert imageless_accession.metadata == {
"anatom_site_1": "Head and neck",
"anatom_site_2": "Head",
"anatom_site_3": "Scalp",
}
assert imageless_accession.metadata_versions.count() == 1


@pytest.mark.django_db
def test_accession_remove_metadata_anatom_site(user, imageless_accession) -> None:
imageless_accession.update_metadata(user, {"anatom_site": "Scalp"})
imageless_accession.remove_metadata(user, ["anatom_site_3"])
assert imageless_accession.metadata == {
"anatom_site_1": "Head and neck",
"anatom_site_2": "Head",
}
assert imageless_accession.metadata_versions.count() == 2


@pytest.mark.django_db
def test_accession_update_metadata_idempotent(user, imageless_accession) -> None:
imageless_accession.update_metadata(user, {"sex": "male", "foo": "bar", "baz": "qux"})
Expand Down
18 changes: 10 additions & 8 deletions isic/ingest/utils/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,15 @@ def validate_internal_consistency(
return _validate_df_consistency(rows)


def validate_archive_consistency( # noqa: C901
def _reassemble_hierarchical_fields(values: dict[str, Any]) -> None:
"""Reassemble level fields (e.g. diagnosis_1..5) into colon-separated parent fields."""
for field_name in ["diagnosis", "anatom_site"]:
level_fields = [f"{field_name}_{i}" for i in range(1, 6) if f"{field_name}_{i}" in values]
if any(values[f] for f in level_fields):
values[field_name] = ":".join(values[f] for f in level_fields if values[f])


def validate_archive_consistency(
rows: csv.DictReader, cohort: Cohort
) -> tuple[ColumnRowErrors, list[Problem]]:
"""
Expand Down Expand Up @@ -173,13 +181,7 @@ def accession_values_to_metadata_dict(accession_values: dict[str, Any]) -> dict[
]
del accession_values[f"{field.relation_name}__{field.internal_id_name}"]

diagnosis_fields = [
f"diagnosis_{i}" for i in range(1, 6) if f"diagnosis_{i}" in accession_values
]
if any(accession_values[field] for field in diagnosis_fields):
accession_values["diagnosis"] = ":".join(
accession_values[field] for field in diagnosis_fields if accession_values[field]
)
_reassemble_hierarchical_fields(accession_values)

return {k: v for (k, v) in accession_values.items() if v is not None}

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ dependencies = [
"gdal",
"google-analytics-data",
"elasticsearch",
"isic-metadata",
"isic-metadata>=4.12.0",
"jaro-winkler",
"numpy",
"pandas",
Expand Down
8 changes: 4 additions & 4 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.