From 2eefc87e0d920a0035ebeb78c9efb417d23243c7 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:20:21 +0100 Subject: [PATCH 01/13] CI/CD testing environment --- .github/workflows/tests.yaml | 33 +++++++++ examples/__init__.py | 0 examples/requirements.txt | 98 +++++++++++++++++++++++--- examples/tests/test_query_assist.py | 103 ++++++++++++++++++++++++++++ 4 files changed, 224 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/tests.yaml create mode 100644 examples/__init__.py create mode 100644 examples/tests/test_query_assist.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..5d750d5 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,33 @@ +name: tests + +on: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + env: + API_KEY: ${{ secrets.DID_API_KEY }} + PYTHONPATH: ${{ github.workspace }}/examples + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install deps + run: | + python -m pip install --upgrade pip + pip install -r examples/requirements.txt + pip install pytest pytest-cov + + - name: Run pytests + run: | + pytest --cov=examples --cov-report=term-missing diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/requirements.txt b/examples/requirements.txt index ae0d4ce..832d5eb 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -1,15 +1,93 @@ +aiohappyeyeballs==2.4.4 +aiohttp==3.11.11 +aiohttp-cors==0.7.0 +aiosignal==1.3.2 +annotated-types==0.7.0 anyio==4.9.0 -Brotli==1.1.0 +attrs==25.1.0 +cachetools==5.5.1 certifi==2025.1.31 -exceptiongroup==1.2.2 -h11==0.14.0 -httpcore==1.0.7 +charset-normalizer==3.4.1 +click==8.1.8 +click-plugins==1.1.1 +cligj==0.7.2 +colorful==0.5.6 +contourpy==1.3.1 +cycler==0.12.1 +distlib==0.3.9 +filelock==3.17.0 +fiona==1.10.1 +fonttools==4.56.0 +frozenlist==1.5.0 +gitdb==4.0.12 +GitPython==3.1.44 +google-api-core==2.24.1 +google-auth==2.38.0 +googleapis-common-protos==1.66.0 +graphviz==0.20.3 +grpcio==1.70.0 +h11==0.16.0 +html5rdf==1.2.1 +httpcore==1.0.9 httpx==0.28.1 idna==3.10 -isodate==0.7.2 -oxrdflib==0.4.0 -pyoxigraph==0.4.9 -pyparsing==3.2.3 -rdflib==7.1.4 +iniconfig==2.1.0 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +kiwisolver==1.4.8 +matplotlib==3.10.0 +msgpack==1.1.0 +multidict==6.1.0 +networkx==3.4.2 +numpy==2.2.2 +opencensus==0.11.4 +opencensus-context==0.1.3 +owlrl==7.1.3 +packaging==24.2 +pandas==2.2.3 +path==17.1.0 +path.py==12.5.0 +pillow==11.1.0 +platformdirs==4.3.6 +pluggy==1.6.0 +prettytable==3.16.0 +prometheus_client==0.21.1 +propcache==0.2.1 +proto-plus==1.26.0 +protobuf==5.29.3 +PuLP==3.1.1 +py-spy==0.4.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 +pydantic==2.10.6 +pydantic_core==2.27.2 +pydotplus==2.0.2 +Pygments==2.19.2 +pyparsing==3.2.1 +pyproj==3.7.1 +pyshacl==0.30.1 +pytest==8.4.1 +python-dateutil==2.9.0.post0 +pytz==2025.1 +PyYAML==6.0.2 +ray==2.44.1 +rdf2dot==0.1.5 +rdflib==7.1.3 +referencing==0.36.2 +requests==2.32.3 +rpds-py==0.22.3 +rsa==4.9 +scipy==1.15.2 +shapely==2.1.1 +six==1.17.0 +smart-open==7.1.0 +smmap==5.0.2 sniffio==1.3.1 -typing_extensions==4.13.2 +tqdm==4.67.1 +typing_extensions==4.12.2 +tzdata==2025.1 +urllib3==2.3.0 +virtualenv==20.29.1 +wcwidth==0.2.13 +wrapt==1.17.2 +yarl==1.18.3 diff --git a/examples/tests/test_query_assist.py b/examples/tests/test_query_assist.py new file mode 100644 index 0000000..382018f --- /dev/null +++ b/examples/tests/test_query_assist.py @@ -0,0 +1,103 @@ +""" +Unit-tests for query_assist.py +Run with: pytest -q +""" +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +import query_assist as qa + +# ---------- pure helpers -------------------------------------------------- # + + +@pytest.mark.parametrize( + "iri, expected", + [ + ("did:lidar-pointcloud-merged", "lidar-pointcloud-merged"), + ("https://w3id.org/dob/voc/did:rgb-image", "rgb-image"), + ("did:ir-false-color-image", "ir-false-color-image"), + ], +) +def test_asset_subdir(iri, expected): + """`asset_subdir` should strip `did:` and leave only safe chars.""" + assert qa.asset_subdir(iri) == expected + + +def test_build_asset_query_filters_are_injected(): + """ + The generated SPARQL must contain the sensor/type filters that + the CLI flags would inject. + """ + args = SimpleNamespace(sensor="bess:OusterLidarSensor", types="did:rgb-image") + query = qa.build_asset_query(["123"], args) + assert "bess:OusterLidarSensor" in query # sensor filter + assert "did:rgb-image" in query # type filter + assert '"123"' in query # UPRN filter + + +# ---------- download path logic ------------------------------------------- # + + +def _dummy_response(tmp_path): + class _Resp: + status_code = 200 + headers = {"Content-Disposition": 'attachment; filename="file.bin"'} + content = b"UNIT-TEST" + + def raise_for_status(self): # noqa: D401 + pass + + return _Resp() + + +def test_download_flow_creates_nested_dirs(tmp_path, monkeypatch): + """ + Integration-style test: simulate `--uprn 42` download with one RGB asset. + Ensures that the target folder becomes `/downloads/42/rgb-image/file.bin`. + """ + + # ---- stub external dependencies -------------------------------------- + class DummyStore: + def __init__(self, *a, **k): + pass + + def query(self, *_): + return [ + { + "uprnValue": "42", + "contentUrl": "https://example.com/file.bin", + "enum": "did:rgb-image", + } + ] + + monkeypatch.setattr(qa, "SPARQLStore", DummyStore) + monkeypatch.setattr(qa.httpx, "get", lambda *a, **k: _dummy_response(tmp_path)) + monkeypatch.setenv("API_KEY", "DUMMY") + + # ---- invoke CLI entry-point ------------------------------------------ + argv_backup, sys.argv = sys.argv, [ + "query_assist", + "--uprn", + "42", + "--download-dir", + str(tmp_path), + ] + try: + qa.main() + finally: + sys.argv = argv_backup + + # ---- assert folder structure ----------------------------------------- + target = Path(tmp_path) / "42" / "rgb-image" / "file.bin" + assert target.is_file(), f"expected {target} to be written" + + +# ---------- CSV helper ---------------------------------------------------- # + + +def test_load_column_from_csv(tmp_path): + csv_path = tmp_path / "sample.csv" + csv_path.write_text("uprn\n1\n2\n\n3\n") + assert qa.load_column_from_csv(csv_path, "uprn") == ["1", "2", "3"] From 0caa5b82b45f2e4f9135047b285f5a6b3ce785ab Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:22:08 +0100 Subject: [PATCH 02/13] remove python 3.9 compatibility --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5d750d5..5244053 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] env: API_KEY: ${{ secrets.DID_API_KEY }} From d4a5e27f1c9083871fba522051aac6b47915dfbe Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:25:07 +0100 Subject: [PATCH 03/13] add status badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2500cec..3c3ccfa 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![tests](https://github.com/abc-rp/asset-api-docs/actions/workflows/tests.yaml/badge.svg)](https://github.com/abc-rp/asset-api-docs/actions/workflows/tests.yaml) [![format](https://github.com/abc-rp/asset-api-docs/actions/workflows/format.yaml/badge.svg)](https://github.com/abc-rp/asset-api-docs/actions/workflows/format.yaml) # xRI Asset API (didapi.io) From d8b3c97d494e3c5c40488a4d37ba29e7fe96cb3d Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:27:42 +0100 Subject: [PATCH 04/13] add python 3.13 testing --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5244053..7709269 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] env: API_KEY: ${{ secrets.DID_API_KEY }} From 6d802929f08bdeb6cef68d30ea74e5e1ec45f456 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:28:43 +0100 Subject: [PATCH 05/13] more advanced testing --- examples/tests/test_query_assist.py | 181 +++++++++++++++++----------- 1 file changed, 110 insertions(+), 71 deletions(-) diff --git a/examples/tests/test_query_assist.py b/examples/tests/test_query_assist.py index 382018f..42b85c2 100644 --- a/examples/tests/test_query_assist.py +++ b/examples/tests/test_query_assist.py @@ -1,103 +1,142 @@ -""" -Unit-tests for query_assist.py -Run with: pytest -q -""" -import sys -from pathlib import Path +# bin/env python3 + from types import SimpleNamespace import pytest -import query_assist as qa -# ---------- pure helpers -------------------------------------------------- # +from examples import query_assist as qa @pytest.mark.parametrize( "iri, expected", [ ("did:lidar-pointcloud-merged", "lidar-pointcloud-merged"), - ("https://w3id.org/dob/voc/did:rgb-image", "rgb-image"), + ("https://w3id.org/foo/did:rgb-image", "rgb-image"), ("did:ir-false-color-image", "ir-false-color-image"), + ("did:weird chars!*£$", "weird_chars____"), # sanitised ], ) def test_asset_subdir(iri, expected): - """`asset_subdir` should strip `did:` and leave only safe chars.""" assert qa.asset_subdir(iri) == expected -def test_build_asset_query_filters_are_injected(): - """ - The generated SPARQL must contain the sensor/type filters that - the CLI flags would inject. - """ - args = SimpleNamespace(sensor="bess:OusterLidarSensor", types="did:rgb-image") - query = qa.build_asset_query(["123"], args) - assert "bess:OusterLidarSensor" in query # sensor filter - assert "did:rgb-image" in query # type filter - assert '"123"' in query # UPRN filter +def test_load_column_from_csv_good(tmp_path): + csv_file = tmp_path / "uprns.csv" + csv_file.write_text("uprn\n1\n2\n\n3\n") + assert qa.load_column_from_csv(csv_file, "uprn") == ["1", "2", "3"] -# ---------- download path logic ------------------------------------------- # +def test_load_column_from_csv_missing_col(tmp_path): + csv_file = tmp_path / "bad.csv" + csv_file.write_text("not_uprn\n123\n") + with pytest.raises(RuntimeError, match="missing required 'uprn'"): + qa.load_column_from_csv(csv_file, "uprn") -def _dummy_response(tmp_path): - class _Resp: - status_code = 200 - headers = {"Content-Disposition": 'attachment; filename="file.bin"'} - content = b"UNIT-TEST" +# --------------------------------------------------------------------------- +# Query builders – make sure the critical bits land in the SPARQL +# --------------------------------------------------------------------------- - def raise_for_status(self): # noqa: D401 - pass - return _Resp() +def test_build_asset_query_injects_everything(): + args = SimpleNamespace(sensor="bess:OusterLidarSensor", types="did:rgb-image") + q = qa.build_asset_query(["123"], args) + assert "bess:OusterLidarSensor" in q + assert "did:rgb-image" in q + assert '"123"' in q + # should ALWAYS bind ?enum + assert "?enum" in q and "dob:typeQualifier" in q -def test_download_flow_creates_nested_dirs(tmp_path, monkeypatch): - """ - Integration-style test: simulate `--uprn 42` download with one RGB asset. - Ensures that the target folder becomes `/downloads/42/rgb-image/file.bin`. - """ +def test_build_output_area_query_formats_values(): + q = qa.build_output_area_query(["sid:E0001"]) + assert "VALUES ?outputArea { sid:E0001 }" in q - # ---- stub external dependencies -------------------------------------- - class DummyStore: - def __init__(self, *a, **k): - pass - def query(self, *_): - return [ - { - "uprnValue": "42", - "contentUrl": "https://example.com/file.bin", - "enum": "did:rgb-image", - } - ] - - monkeypatch.setattr(qa, "SPARQLStore", DummyStore) - monkeypatch.setattr(qa.httpx, "get", lambda *a, **k: _dummy_response(tmp_path)) - monkeypatch.setenv("API_KEY", "DUMMY") - - # ---- invoke CLI entry-point ------------------------------------------ - argv_backup, sys.argv = sys.argv, [ - "query_assist", - "--uprn", - "42", - "--download-dir", - str(tmp_path), - ] - try: - qa.main() - finally: - sys.argv = argv_backup +def test_build_ods_to_uprn_query_values_clause(): + q = qa.build_ods_to_uprn_query(["00ABC", "99XYZ"]) + assert 'VALUES ?odsValue { "00ABC" "99XYZ" }' in q + - # ---- assert folder structure ----------------------------------------- - target = Path(tmp_path) / "42" / "rgb-image" / "file.bin" - assert target.is_file(), f"expected {target} to be written" +# --------------------------------------------------------------------------- +# CLI integration: download path logic and error handling +# --------------------------------------------------------------------------- -# ---------- CSV helper ---------------------------------------------------- # +def _dummy_http_response(): + class _R: + status_code = 200 + headers = {"Content-Disposition": 'attachment; filename="file.bin"'} + content = b"PSEUDO-BINARY" + def raise_for_status(self): + pass -def test_load_column_from_csv(tmp_path): - csv_path = tmp_path / "sample.csv" - csv_path.write_text("uprn\n1\n2\n\n3\n") - assert qa.load_column_from_csv(csv_path, "uprn") == ["1", "2", "3"] + return _R() + + +class _DummyStore: + """Stand-in for rdflib SPARQLStore; returns a synthetic row list.""" + + def __init__(self, *a, **k): + pass + + def query(self, *_): + return [ + { + "uprnValue": "42", + "contentUrl": "https://example.com/file.bin", + "enum": "did:rgb-image", + } + ] + + +def test_cli_download_creates_nested_dir(tmp_path, monkeypatch): + """Full happy-path run – ensures ///file.bin is created.""" + monkeypatch.setattr(qa, "SPARQLStore", _DummyStore) + monkeypatch.setattr(qa.httpx, "get", lambda *a, **k: _dummy_http_response()) + monkeypatch.setenv("API_KEY", "FAKE-KEY") + + argv = ["query_assist", "--uprn", "42", "--download-dir", str(tmp_path)] + monkeypatch.setattr( + qa, + "parse_args", + lambda: qa.argparse.Namespace( + uprn=["42"], + ods=None, + sensor=None, + types=None, + output_area=None, + db_url="http://dummy", + download_dir=str(tmp_path), + api_key_env="API_KEY", + ), + ) + + qa.main() + + expected = tmp_path / "42" / "rgb-image" / "file.bin" + assert expected.is_file(), f"expected {expected} to exist" + + +def test_cli_fails_without_api_key(monkeypatch): + """Main should raise RuntimeError if API_KEY env var is missing.""" + monkeypatch.setattr(qa, "SPARQLStore", _DummyStore) + monkeypatch.delenv("API_KEY", raising=False) + + monkeypatch.setattr( + qa, + "parse_args", + lambda: qa.argparse.Namespace( + uprn=["1"], + ods=None, + sensor=None, + types=None, + output_area=None, + db_url="http://dummy", + download_dir=None, + api_key_env="API_KEY", + ), + ) + with pytest.raises(RuntimeError, match="Env var 'API_KEY' is not set"): + qa.main() From fa26ffee0d9083ea10adf91a0316826bebd5e64e Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:33:02 +0100 Subject: [PATCH 06/13] updated requirements --- examples/requirements.txt | 90 +++------------------------------------ 1 file changed, 7 insertions(+), 83 deletions(-) diff --git a/examples/requirements.txt b/examples/requirements.txt index 832d5eb..7b73293 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -1,93 +1,17 @@ -aiohappyeyeballs==2.4.4 -aiohttp==3.11.11 -aiohttp-cors==0.7.0 -aiosignal==1.3.2 -annotated-types==0.7.0 anyio==4.9.0 -attrs==25.1.0 -cachetools==5.5.1 -certifi==2025.1.31 -charset-normalizer==3.4.1 -click==8.1.8 -click-plugins==1.1.1 -cligj==0.7.2 -colorful==0.5.6 -contourpy==1.3.1 -cycler==0.12.1 -distlib==0.3.9 -filelock==3.17.0 -fiona==1.10.1 -fonttools==4.56.0 -frozenlist==1.5.0 -gitdb==4.0.12 -GitPython==3.1.44 -google-api-core==2.24.1 -google-auth==2.38.0 -googleapis-common-protos==1.66.0 -graphviz==0.20.3 -grpcio==1.70.0 +Brotli==1.1.0 +certifi==2025.6.15 h11==0.16.0 -html5rdf==1.2.1 httpcore==1.0.9 httpx==0.28.1 idna==3.10 iniconfig==2.1.0 -jsonschema==4.23.0 -jsonschema-specifications==2024.10.1 -kiwisolver==1.4.8 -matplotlib==3.10.0 -msgpack==1.1.0 -multidict==6.1.0 -networkx==3.4.2 -numpy==2.2.2 -opencensus==0.11.4 -opencensus-context==0.1.3 -owlrl==7.1.3 -packaging==24.2 -pandas==2.2.3 -path==17.1.0 -path.py==12.5.0 -pillow==11.1.0 -platformdirs==4.3.6 +logging==0.4.9.6 +packaging==25.0 pluggy==1.6.0 -prettytable==3.16.0 -prometheus_client==0.21.1 -propcache==0.2.1 -proto-plus==1.26.0 -protobuf==5.29.3 -PuLP==3.1.1 -py-spy==0.4.0 -pyasn1==0.6.1 -pyasn1_modules==0.4.1 -pydantic==2.10.6 -pydantic_core==2.27.2 -pydotplus==2.0.2 Pygments==2.19.2 -pyparsing==3.2.1 -pyproj==3.7.1 -pyshacl==0.30.1 +pyparsing==3.2.3 pytest==8.4.1 -python-dateutil==2.9.0.post0 -pytz==2025.1 -PyYAML==6.0.2 -ray==2.44.1 -rdf2dot==0.1.5 -rdflib==7.1.3 -referencing==0.36.2 -requests==2.32.3 -rpds-py==0.22.3 -rsa==4.9 -scipy==1.15.2 -shapely==2.1.1 -six==1.17.0 -smart-open==7.1.0 -smmap==5.0.2 +rdflib==7.1.4 sniffio==1.3.1 -tqdm==4.67.1 -typing_extensions==4.12.2 -tzdata==2025.1 -urllib3==2.3.0 -virtualenv==20.29.1 -wcwidth==0.2.13 -wrapt==1.17.2 -yarl==1.18.3 +typing_extensions==4.14.0 From 3f4208783c2b420f1dfdbeb9ebb94fb3fc734cd5 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:33:06 +0100 Subject: [PATCH 07/13] updated requirements --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7709269..561b65c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] env: API_KEY: ${{ secrets.DID_API_KEY }} From e931f84896b2319b9b751d5b9c97952ab05017f7 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:34:04 +0100 Subject: [PATCH 08/13] updated requirements --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 561b65c..7709269 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] env: API_KEY: ${{ secrets.DID_API_KEY }} From 076ddef08a9ef664727107c2db09569e0c5fb9d1 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:36:46 +0100 Subject: [PATCH 09/13] update test query --- examples/tests/test_query_assist.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/tests/test_query_assist.py b/examples/tests/test_query_assist.py index 42b85c2..c870ba0 100644 --- a/examples/tests/test_query_assist.py +++ b/examples/tests/test_query_assist.py @@ -1,10 +1,11 @@ -# bin/env python3 - from types import SimpleNamespace import pytest +import query_assist as qa -from examples import query_assist as qa +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- @pytest.mark.parametrize( @@ -69,7 +70,7 @@ class _R: headers = {"Content-Disposition": 'attachment; filename="file.bin"'} content = b"PSEUDO-BINARY" - def raise_for_status(self): + def raise_for_status(self): # noqa: D401 pass return _R() @@ -113,6 +114,7 @@ def test_cli_download_creates_nested_dir(tmp_path, monkeypatch): ), ) + # run main() qa.main() expected = tmp_path / "42" / "rgb-image" / "file.bin" From 05e6c2092c7503731d8e68b0b5cdebee4fbe919e Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:39:20 +0100 Subject: [PATCH 10/13] additional testing --- examples/tests/test_download_scripts.py | 113 ++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 examples/tests/test_download_scripts.py diff --git a/examples/tests/test_download_scripts.py b/examples/tests/test_download_scripts.py new file mode 100644 index 0000000..f4b0df8 --- /dev/null +++ b/examples/tests/test_download_scripts.py @@ -0,0 +1,113 @@ +import importlib +from pathlib import Path + +import pytest + + +class _DummyRow(dict): + """row['uprnValue'] / row['contentUrl'] lookup just like ResultRow""" + + def __getitem__(self, key): + return super().get(key) + + +class _DummyEndpoint: + """Replaces SPARQLStore instance inside each script.""" + + def __init__(self, rows): + self._rows = rows + + def query(self, *_): + return self._rows + + +def _fake_response(): + class _R: + status_code = 200 + headers = {"Content-Disposition": 'attachment; filename="file.bin"'} + content = b"DUMMY" + + def raise_for_status(self): # noqa: D401 + pass + + return _R() + + +@pytest.mark.parametrize( + "mod_name, expects_uprn_subfolder", + [ + ("examples.get_all_assets_for_a_list_of_uprns", True), + ("examples.get_all_assets_for_a_uprn", False), + ("examples.get_all_assets_for_a_uprn_made_by_a_sensor", True), + ("examples.get_all_assets_of_type_for_list_of_uprns", True), + ], +) +def test_script_downloads(tmp_path, monkeypatch, mod_name, expects_uprn_subfolder): + """Import the script as a module, monkey-patch, run main(), check the file.""" + + # -------------------------------------- + # import AFTER patching httpx so that the module picks up our stub + import httpx # noqa: WPS433 (std-lib) + + monkeypatch.setattr(httpx, "get", lambda *a, **k: _fake_response()) + + # make sure env-var exists + monkeypatch.setenv("API_KEY", "UNIT-TEST-KEY") + + # import module under test + mod = importlib.import_module(mod_name) + + # patch DOWNLOAD_DIR to the temp folder + monkeypatch.setattr(mod, "DOWNLOAD_DIR", str(tmp_path)) + + # patch ResultRow base-class to something simple & fabricate rows + monkeypatch.setattr(mod, "ResultRow", _DummyRow) + + dummy_rows = [_DummyRow({"uprnValue": "999", "contentUrl": "https://x/y.bin"})] + # get the endpoint object name (either global `endpoint` or attr inside module) + if hasattr(mod, "endpoint"): + monkeypatch.setattr(mod, "endpoint", _DummyEndpoint(dummy_rows)) + else: + # safety-net for any differently-named variable + monkeypatch.setattr(mod, "endpoint", _DummyEndpoint(dummy_rows)) + + # -------------------------------------- + # run the script's main() + mod.main() + + # -------------------------------------- + # assert the file has landed where we expect + if expects_uprn_subfolder: + expected = Path(tmp_path) / "999" / "file.bin" + else: + expected = Path(tmp_path) / "file.bin" + + assert expected.is_file(), f"{mod_name}: expected {expected} to exist" + + +# --------------------------------------------------------------------------- +# sanity-check the constant-driven SPARQL strings +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "mod_name, substrings", + [ + ("examples.get_all_assets_for_a_list_of_uprns", ["200003455212", "5045394"]), + ("examples.get_all_assets_for_a_uprn", ["5045394"]), # constant UPRN + ( + "examples.get_all_assets_for_a_uprn_made_by_a_sensor", + ["5045394", "bess:OusterLidarSensor"], + ), + ( + "examples.get_all_assets_of_type_for_list_of_uprns", + ["did:rgb-image", "lidar-pointcloud-merged"], + ), + ], +) +def test_query_contains_expected_literals(mod_name, substrings): + """Make sure the hard-coded constants really appear in the QUERY string.""" + mod = importlib.import_module(mod_name) + q = mod.QUERY + for s in substrings: + assert s in q, f"{mod_name}: missing {s} in QUERY" From 371f09dee512349eef7319bec87ef6d8b7ada7d0 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:41:09 +0100 Subject: [PATCH 11/13] tidy up test --- examples/tests/test_download_scripts.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/examples/tests/test_download_scripts.py b/examples/tests/test_download_scripts.py index f4b0df8..324a8e9 100644 --- a/examples/tests/test_download_scripts.py +++ b/examples/tests/test_download_scripts.py @@ -27,7 +27,7 @@ class _R: headers = {"Content-Disposition": 'attachment; filename="file.bin"'} content = b"DUMMY" - def raise_for_status(self): # noqa: D401 + def raise_for_status(self): pass return _R() @@ -45,38 +45,26 @@ def raise_for_status(self): # noqa: D401 def test_script_downloads(tmp_path, monkeypatch, mod_name, expects_uprn_subfolder): """Import the script as a module, monkey-patch, run main(), check the file.""" - # -------------------------------------- - # import AFTER patching httpx so that the module picks up our stub - import httpx # noqa: WPS433 (std-lib) + import httpx monkeypatch.setattr(httpx, "get", lambda *a, **k: _fake_response()) - # make sure env-var exists monkeypatch.setenv("API_KEY", "UNIT-TEST-KEY") - # import module under test mod = importlib.import_module(mod_name) - # patch DOWNLOAD_DIR to the temp folder monkeypatch.setattr(mod, "DOWNLOAD_DIR", str(tmp_path)) - # patch ResultRow base-class to something simple & fabricate rows monkeypatch.setattr(mod, "ResultRow", _DummyRow) dummy_rows = [_DummyRow({"uprnValue": "999", "contentUrl": "https://x/y.bin"})] - # get the endpoint object name (either global `endpoint` or attr inside module) if hasattr(mod, "endpoint"): monkeypatch.setattr(mod, "endpoint", _DummyEndpoint(dummy_rows)) else: - # safety-net for any differently-named variable monkeypatch.setattr(mod, "endpoint", _DummyEndpoint(dummy_rows)) - # -------------------------------------- - # run the script's main() mod.main() - # -------------------------------------- - # assert the file has landed where we expect if expects_uprn_subfolder: expected = Path(tmp_path) / "999" / "file.bin" else: @@ -85,16 +73,11 @@ def test_script_downloads(tmp_path, monkeypatch, mod_name, expects_uprn_subfolde assert expected.is_file(), f"{mod_name}: expected {expected} to exist" -# --------------------------------------------------------------------------- -# sanity-check the constant-driven SPARQL strings -# --------------------------------------------------------------------------- - - @pytest.mark.parametrize( "mod_name, substrings", [ ("examples.get_all_assets_for_a_list_of_uprns", ["200003455212", "5045394"]), - ("examples.get_all_assets_for_a_uprn", ["5045394"]), # constant UPRN + ("examples.get_all_assets_for_a_uprn", ["5045394"]), ( "examples.get_all_assets_for_a_uprn_made_by_a_sensor", ["5045394", "bess:OusterLidarSensor"], From c09eb624a7f77a2806ff05dbff32677f2e9563c6 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:41:41 +0100 Subject: [PATCH 12/13] tidy up test --- examples/tests/test_query_assist.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/examples/tests/test_query_assist.py b/examples/tests/test_query_assist.py index c870ba0..a431aee 100644 --- a/examples/tests/test_query_assist.py +++ b/examples/tests/test_query_assist.py @@ -3,10 +3,6 @@ import pytest import query_assist as qa -# --------------------------------------------------------------------------- -# Pure helpers -# --------------------------------------------------------------------------- - @pytest.mark.parametrize( "iri, expected", @@ -34,11 +30,6 @@ def test_load_column_from_csv_missing_col(tmp_path): qa.load_column_from_csv(csv_file, "uprn") -# --------------------------------------------------------------------------- -# Query builders – make sure the critical bits land in the SPARQL -# --------------------------------------------------------------------------- - - def test_build_asset_query_injects_everything(): args = SimpleNamespace(sensor="bess:OusterLidarSensor", types="did:rgb-image") q = qa.build_asset_query(["123"], args) @@ -59,18 +50,13 @@ def test_build_ods_to_uprn_query_values_clause(): assert 'VALUES ?odsValue { "00ABC" "99XYZ" }' in q -# --------------------------------------------------------------------------- -# CLI integration: download path logic and error handling -# --------------------------------------------------------------------------- - - def _dummy_http_response(): class _R: status_code = 200 headers = {"Content-Disposition": 'attachment; filename="file.bin"'} content = b"PSEUDO-BINARY" - def raise_for_status(self): # noqa: D401 + def raise_for_status(self): pass return _R() @@ -114,7 +100,6 @@ def test_cli_download_creates_nested_dir(tmp_path, monkeypatch): ), ) - # run main() qa.main() expected = tmp_path / "42" / "rgb-image" / "file.bin" From 3b285f21b2a87c16bd0538caf54f96ae9ff1b4a7 Mon Sep 17 00:00:00 2001 From: gnathoi Date: Wed, 25 Jun 2025 16:48:13 +0100 Subject: [PATCH 13/13] br_decompress test --- examples/tests/test_br_decompression.py | 73 +++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 examples/tests/test_br_decompression.py diff --git a/examples/tests/test_br_decompression.py b/examples/tests/test_br_decompression.py new file mode 100644 index 0000000..6eaf47c --- /dev/null +++ b/examples/tests/test_br_decompression.py @@ -0,0 +1,73 @@ +import br_decompress as br +import brotli +import query_assist as qa + + +def _make_compressed_pair(): + """Return (raw_bytes, brotli_compressed_bytes).""" + raw = b"FOR UNIT TEST ONLY - pretend this is a PCD header\n" + return raw, brotli.compress(raw) + + +class _DummyResponse: + def __init__(self, data): + self.status_code = 200 + self.headers = {"Content-Disposition": 'attachment; filename="cloud.pcd.br"'} + self.content = data + + def raise_for_status(self): + pass + + +class _DummyStore: + """Fake rdflib SPARQLStore that yields exactly one result row.""" + + def __init__(self, *_, **__): + pass + + def query(self, *_): + return [ + { + "uprnValue": "999", + "contentUrl": "https://example.com/cloud.pcd.br", + "enum": "did:lidar-pointcloud-merged", + } + ] + + +def test_download_and_decompress_brotli(tmp_path, monkeypatch): + raw, compressed = _make_compressed_pair() + + monkeypatch.setattr(qa, "SPARQLStore", _DummyStore) + + monkeypatch.setattr(qa.httpx, "get", lambda *a, **k: _DummyResponse(compressed)) + + monkeypatch.setenv("API_KEY", "DUMMY") + + monkeypatch.setattr( + qa, + "parse_args", + lambda: qa.argparse.Namespace( + uprn=["999"], + ods=None, + sensor=None, + types=None, + output_area=None, + db_url="http://dummy", + download_dir=str(tmp_path), + api_key_env="API_KEY", + ), + ) + + qa.main() + + p_br = tmp_path / "999" / "lidar-pointcloud-merged" / "cloud.pcd.br" + assert p_br.is_file(), "compressed asset should have been saved by query_assist" + + br.find_and_replace_pcd_br(str(tmp_path)) + + p_raw = p_br.with_suffix("") + assert p_raw.is_file(), "decompressed .pcd should exist" + assert not p_br.exists(), ".pcd.br should have been removed" + + assert p_raw.read_bytes() == raw, "decompressed bytes should match original"