From 6b851568d4505a708290971a7edec1e69fa5fc1a Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 18 Jul 2025 15:03:12 +0100 Subject: [PATCH 01/16] Add an atlas context --- src/murfey/client/analyser.py | 64 +++++++++++++++-------------- src/murfey/client/contexts/atlas.py | 51 +++++++++++++++++++++++ 2 files changed, 85 insertions(+), 30 deletions(-) create mode 100644 src/murfey/client/contexts/atlas.py diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 18b28b2b4..d3cd92e41 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -15,6 +15,7 @@ from typing import Type from murfey.client.context import Context +from murfey.client.contexts.atlas import AtlasContext from murfey.client.contexts.clem import CLEMContext from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext @@ -135,7 +136,7 @@ def _find_context(self, file_path: Path) -> bool: # Tomography and SPA workflow checks if "atlas" in file_path.parts: - self._context = SPAMetadataContext("epu", self._basepath) + self._context = AtlasContext("epu", self._basepath) return True if "Metadata" in file_path.parts or file_path.name == "EpuSession.dm": @@ -266,7 +267,7 @@ def _analyse(self): ) except Exception as e: logger.error(f"Exception encountered: {e}") - if "atlas" not in transferred_file.parts: + if not isinstance(self._context, AtlasContext): if not dc_metadata: try: dc_metadata = self._context.gather_metadata( @@ -308,6 +309,10 @@ def _analyse(self): ) self.post_transfer(transferred_file) + elif isinstance(self._context, AtlasContext): + logger.debug(f"File {transferred_file.name!r} is part of the atlas") + self.post_transfer(transferred_file) + # Handle files with tomography and SPA context differently elif not self._extension or self._unseen_xml: valid_extension = self._find_extension(transferred_file) @@ -325,36 +330,35 @@ def _analyse(self): ) except Exception as e: logger.error(f"Exception encountered: {e}") - if "atlas" not in transferred_file.parts: - if not dc_metadata: - try: - dc_metadata = self._context.gather_metadata( - mdoc_for_reading - or self._xml_file(transferred_file), - environment=self._environment, - ) - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: {e.args[0]}" - ) - raise e - if not dc_metadata or not self._force_mdoc_metadata: - mdoc_for_reading = None - self._unseen_xml.append(transferred_file) - if dc_metadata: - self._unseen_xml = [] - if dc_metadata.get("file_extension"): - self._extension = dc_metadata["file_extension"] - else: - dc_metadata["file_extension"] = self._extension - dc_metadata["acquisition_software"] = ( - self._context._acquisition_software + if not dc_metadata: + try: + dc_metadata = self._context.gather_metadata( + mdoc_for_reading + or self._xml_file(transferred_file), + environment=self._environment, ) - self.notify( - { - "form": dc_metadata, - } + except KeyError as e: + logger.error( + f"Metadata gathering failed with a key error for key: {e.args[0]}" ) + raise e + if not dc_metadata or not self._force_mdoc_metadata: + mdoc_for_reading = None + self._unseen_xml.append(transferred_file) + if dc_metadata: + self._unseen_xml = [] + if dc_metadata.get("file_extension"): + self._extension = dc_metadata["file_extension"] + else: + dc_metadata["file_extension"] = self._extension + dc_metadata["acquisition_software"] = ( + self._context._acquisition_software + ) + self.notify( + { + "form": dc_metadata, + } + ) elif isinstance( self._context, ( diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py new file mode 100644 index 000000000..9dc11d946 --- /dev/null +++ b/src/murfey/client/contexts/atlas.py @@ -0,0 +1,51 @@ +import logging +from pathlib import Path +from typing import Optional + +import requests + +from murfey.client.context import Context +from murfey.client.contexts.spa import _get_source +from murfey.client.contexts.spa_metadata import _atlas_destination +from murfey.client.instance_environment import MurfeyInstanceEnvironment +from murfey.util.api import url_path_for +from murfey.util.client import authorised_requests, capture_post + +logger = logging.getLogger("murfey.client.contexts.spa_metadata") + +requests.get, requests.post, requests.put, requests.delete = authorised_requests() + + +class AtlasContext(Context): + def __init__(self, acquisition_software: str, basepath: Path): + super().__init__("Atlas", acquisition_software) + self._basepath = basepath + + def post_transfer( + self, + transferred_file: Path, + environment: Optional[MurfeyInstanceEnvironment] = None, + **kwargs, + ): + super().post_transfer( + transferred_file=transferred_file, + environment=environment, + **kwargs, + ) + + if ( + environment + and "Atlas_" in transferred_file.stem + and transferred_file.suffix == ".mrc" + ): + source = _get_source(transferred_file, environment) + if source: + transferred_atlas_name = ( + _atlas_destination(environment, source, transferred_file) + / environment.samples[source].atlas.parent + / transferred_file.name + ) + capture_post( + f"{str(environment.url.geturl())}{url_path_for('session_control.spa_router', 'make_atlas_jpg', session_id=environment.murfey_session)}", + json={"atlas_mrc": transferred_atlas_name}, + ) From b9f836b9624de025bc4859996f45d2c42f8269af Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 18 Jul 2025 15:38:21 +0100 Subject: [PATCH 02/16] Add code to make atlas jpg file --- src/murfey/server/api/session_control.py | 7 +++++ src/murfey/workflows/spa/atlas.py | 35 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 src/murfey/workflows/spa/atlas.py diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 45b282eaf..915b6fe7e 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -54,6 +54,7 @@ SearchMapParameters, Visit, ) +from murfey.workflows.spa.atlas import atlas_jpg_from_mrc from murfey.workflows.spa.flush_spa_preprocess import ( register_foil_hole as _register_foil_hole, ) @@ -319,6 +320,12 @@ def delete_rsyncer(session_id: int, source: Path, db=murfey_db): ) +@spa_router.get("/sessions/{session_id}/make_atlas_jpg") +def make_atlas_jpg(session_id: MurfeySessionID, atlas_mrc: str, db=murfey_db): + session = db.exec(select(Session).where(Session.id == session_id)).one() + return atlas_jpg_from_mrc(session.instrument_name, session.visit, Path(atlas_mrc)) + + @spa_router.get("/sessions/{session_id}/grid_squares") def get_grid_squares(session_id: MurfeySessionID, db=murfey_db): return _get_grid_squares(session_id, db) diff --git a/src/murfey/workflows/spa/atlas.py b/src/murfey/workflows/spa/atlas.py new file mode 100644 index 000000000..75ab28273 --- /dev/null +++ b/src/murfey/workflows/spa/atlas.py @@ -0,0 +1,35 @@ +from pathlib import Path + +import mrcfile +import PIL.Image +from werkzeug.utils import secure_filename + +from murfey.util.config import get_machine_config + + +def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): + with mrcfile.read(atlas_mrc) as mrc: + data = mrc.data + + machine_config = get_machine_config(instrument_name=instrument_name)[ + instrument_name + ] + + parts = [secure_filename(p) for p in atlas_mrc.parts] + visit_idx = parts.index(visit_name) + core = Path("/".join(parts[: visit_idx + 1])) + sample_id = "Sample" + for p in parts: + if "Sample" in p: + sample_id = p + break + atlas_jpg_file = ( + core + / machine_config.processed_directory_name + / "atlas" + / f"{sample_id}_{atlas_mrc.stem}_fullres.jpg" + ) + atlas_jpg_file.parent.mkdir(parents=True, exist_ok=True) + + im = PIL.Image.fromarray(data) + im.save(atlas_jpg_file) From b86058975e9115ab17d961cb1edd60c442f01ed3 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 18 Jul 2025 15:42:52 +0100 Subject: [PATCH 03/16] Update test --- tests/client/test_analyser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 175916aa1..0e24f0b0d 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -3,6 +3,7 @@ import pytest from murfey.client.analyser import Analyser +from murfey.client.contexts.atlas import AtlasContext from murfey.client.contexts.clem import CLEMContext from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext @@ -28,7 +29,7 @@ ["visit/FoilHole_01234_fractions.tiff", SPAModularContext], ["visit/FoilHole_01234_EER.eer", SPAModularContext], # SPA metadata - ["atlas/atlas.mrc", SPAMetadataContext], + ["atlas/atlas.mrc", AtlasContext], ["visit/EpuSession.dm", SPAMetadataContext], ["visit/Metadata/GridSquare.dm", SPAMetadataContext], # CLEM LIF file From 7d912ee9c072d6275e84230328ef2023889ba1fd Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 18 Jul 2025 15:51:54 +0100 Subject: [PATCH 04/16] Add new endpoint to router manifest --- src/murfey/server/api/session_control.py | 12 ++++++------ src/murfey/util/route_manifest.yaml | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 915b6fe7e..a1df73506 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -320,12 +320,6 @@ def delete_rsyncer(session_id: int, source: Path, db=murfey_db): ) -@spa_router.get("/sessions/{session_id}/make_atlas_jpg") -def make_atlas_jpg(session_id: MurfeySessionID, atlas_mrc: str, db=murfey_db): - session = db.exec(select(Session).where(Session.id == session_id)).one() - return atlas_jpg_from_mrc(session.instrument_name, session.visit, Path(atlas_mrc)) - - @spa_router.get("/sessions/{session_id}/grid_squares") def get_grid_squares(session_id: MurfeySessionID, db=murfey_db): return _get_grid_squares(session_id, db) @@ -354,6 +348,12 @@ def get_foil_hole( return _get_foil_hole(session_id, fh_name, db) +@spa_router.post("/sessions/{session_id}/make_atlas_jpg") +def make_atlas_jpg(session_id: MurfeySessionID, atlas_mrc: str, db=murfey_db): + session = db.exec(select(Session).where(Session.id == session_id)).one() + return atlas_jpg_from_mrc(session.instrument_name, session.visit, Path(atlas_mrc)) + + @spa_router.post("/sessions/{session_id}/grid_square/{gsid}") def register_grid_square( session_id: MurfeySessionID, diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index a92821c19..c31695e71 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -814,6 +814,11 @@ murfey.server.api.session_control.spa_router: type: int methods: - GET + - path: /sessions/{session_id}/make_atlas_jpg + function: make_atlas_jpg + path_params: [] + methods: + - POST - path: /session_control/spa/sessions/{session_id}/grid_square/{gsid} function: register_grid_square path_params: From cf2453721b660369a2251987a9614c4432b16493 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 18 Jul 2025 15:53:36 +0100 Subject: [PATCH 05/16] Secure name --- src/murfey/workflows/spa/atlas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/workflows/spa/atlas.py b/src/murfey/workflows/spa/atlas.py index 75ab28273..2746f9559 100644 --- a/src/murfey/workflows/spa/atlas.py +++ b/src/murfey/workflows/spa/atlas.py @@ -27,7 +27,7 @@ def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): core / machine_config.processed_directory_name / "atlas" - / f"{sample_id}_{atlas_mrc.stem}_fullres.jpg" + / secure_filename(f"{sample_id}_{atlas_mrc.stem}_fullres.jpg") ) atlas_jpg_file.parent.mkdir(parents=True, exist_ok=True) From cf59dad1cfb810cef7a6f56cc6e801593b6f194c Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 25 Jul 2025 09:54:56 +0100 Subject: [PATCH 06/16] Try and fix atlas destination determination --- src/murfey/client/contexts/atlas.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py index 9dc11d946..6527c1820 100644 --- a/src/murfey/client/contexts/atlas.py +++ b/src/murfey/client/contexts/atlas.py @@ -11,7 +11,7 @@ from murfey.util.api import url_path_for from murfey.util.client import authorised_requests, capture_post -logger = logging.getLogger("murfey.client.contexts.spa_metadata") +logger = logging.getLogger("murfey.client.contexts.atlas") requests.get, requests.post, requests.put, requests.delete = authorised_requests() @@ -40,11 +40,9 @@ def post_transfer( ): source = _get_source(transferred_file, environment) if source: - transferred_atlas_name = ( - _atlas_destination(environment, source, transferred_file) - / environment.samples[source].atlas.parent - / transferred_file.name - ) + transferred_atlas_name = _atlas_destination( + environment, source, transferred_file + ) / transferred_file.relative_to(source.parent) capture_post( f"{str(environment.url.geturl())}{url_path_for('session_control.spa_router', 'make_atlas_jpg', session_id=environment.murfey_session)}", json={"atlas_mrc": transferred_atlas_name}, From 3914439034e489882db594b14b99c127ef3c84df Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 25 Jul 2025 11:18:33 +0100 Subject: [PATCH 07/16] Fix post to api --- src/murfey/client/contexts/atlas.py | 2 +- src/murfey/util/route_manifest.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py index 6527c1820..1f338cc8e 100644 --- a/src/murfey/client/contexts/atlas.py +++ b/src/murfey/client/contexts/atlas.py @@ -45,5 +45,5 @@ def post_transfer( ) / transferred_file.relative_to(source.parent) capture_post( f"{str(environment.url.geturl())}{url_path_for('session_control.spa_router', 'make_atlas_jpg', session_id=environment.murfey_session)}", - json={"atlas_mrc": transferred_atlas_name}, + json={"atlas_mrc": str(transferred_atlas_name)}, ) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index c31695e71..86db4fad4 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -814,7 +814,7 @@ murfey.server.api.session_control.spa_router: type: int methods: - GET - - path: /sessions/{session_id}/make_atlas_jpg + - path: /session_control/spa/sessions/{session_id}/make_atlas_jpg function: make_atlas_jpg path_params: [] methods: From 574b89b12a97c52c98932911f52eda379dcaff89 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 25 Jul 2025 11:36:01 +0100 Subject: [PATCH 08/16] Atlas post needs to be a model --- src/murfey/client/contexts/atlas.py | 2 +- src/murfey/client/multigrid_control.py | 4 ++-- src/murfey/server/api/session_control.py | 20 ++++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py index 1f338cc8e..d5601eeda 100644 --- a/src/murfey/client/contexts/atlas.py +++ b/src/murfey/client/contexts/atlas.py @@ -45,5 +45,5 @@ def post_transfer( ) / transferred_file.relative_to(source.parent) capture_post( f"{str(environment.url.geturl())}{url_path_for('session_control.spa_router', 'make_atlas_jpg', session_id=environment.murfey_session)}", - json={"atlas_mrc": str(transferred_atlas_name)}, + json={"path": str(transferred_atlas_name)}, ) diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index a016b49f4..44a27ecd9 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -232,7 +232,7 @@ def _rsyncer_stopped(self, source: Path, explicit_stop: bool = False): requests.delete(remove_url) else: stop_url = f"{self.murfey_url}{url_path_for('session_control.router', 'register_stopped_rsyncer', session_id=self.session_id)}" - capture_post(stop_url, json={"source": str(source)}) + capture_post(stop_url, json={"path": str(source)}) def _finalise_rsyncer(self, source: Path): finalise_thread = threading.Thread( @@ -248,7 +248,7 @@ def _finalise_rsyncer(self, source: Path): def _restart_rsyncer(self, source: Path): self.rsync_processes[source].restart() restarted_url = f"{self.murfey_url}{url_path_for('session_control.router', 'register_restarted_rsyncer', session_id=self.session_id)}" - capture_post(restarted_url, json={"source": str(source)}) + capture_post(restarted_url, json={"path": str(source)}) def _request_watcher_stop(self, source: Path): self._environment.watchers[source]._stopping = True diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index a1df73506..b569d6559 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -263,18 +263,18 @@ def get_rsyncers_for_session(session_id: MurfeySessionID, db=murfey_db): return rsync_instances.all() -class RsyncerSource(BaseModel): - source: str +class StringOfPathModel(BaseModel): + path: str @router.post("/sessions/{session_id}/rsyncer_stopped") def register_stopped_rsyncer( - session_id: int, rsyncer_source: RsyncerSource, db=murfey_db + session_id: int, rsyncer_source: StringOfPathModel, db=murfey_db ): rsyncer = db.exec( select(RsyncInstance) .where(RsyncInstance.session_id == session_id) - .where(RsyncInstance.source == rsyncer_source.source) + .where(RsyncInstance.source == rsyncer_source.path) ).one() rsyncer.transferring = False db.add(rsyncer) @@ -283,12 +283,12 @@ def register_stopped_rsyncer( @router.post("/sessions/{session_id}/rsyncer_started") def register_restarted_rsyncer( - session_id: int, rsyncer_source: RsyncerSource, db=murfey_db + session_id: int, rsyncer_source: StringOfPathModel, db=murfey_db ): rsyncer = db.exec( select(RsyncInstance) .where(RsyncInstance.session_id == session_id) - .where(RsyncInstance.source == rsyncer_source.source) + .where(RsyncInstance.source == rsyncer_source.path) ).one() rsyncer.transferring = True db.add(rsyncer) @@ -349,9 +349,13 @@ def get_foil_hole( @spa_router.post("/sessions/{session_id}/make_atlas_jpg") -def make_atlas_jpg(session_id: MurfeySessionID, atlas_mrc: str, db=murfey_db): +def make_atlas_jpg( + session_id: MurfeySessionID, atlas_mrc: StringOfPathModel, db=murfey_db +): session = db.exec(select(Session).where(Session.id == session_id)).one() - return atlas_jpg_from_mrc(session.instrument_name, session.visit, Path(atlas_mrc)) + return atlas_jpg_from_mrc( + session.instrument_name, session.visit, Path(atlas_mrc.path) + ) @spa_router.post("/sessions/{session_id}/grid_square/{gsid}") From 141fddfca67cd10d13198e25cf82abfc346b14dd Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 25 Jul 2025 11:53:47 +0100 Subject: [PATCH 09/16] Convert image to 8-bit --- src/murfey/workflows/spa/atlas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/workflows/spa/atlas.py b/src/murfey/workflows/spa/atlas.py index 2746f9559..784065434 100644 --- a/src/murfey/workflows/spa/atlas.py +++ b/src/murfey/workflows/spa/atlas.py @@ -32,4 +32,4 @@ def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): atlas_jpg_file.parent.mkdir(parents=True, exist_ok=True) im = PIL.Image.fromarray(data) - im.save(atlas_jpg_file) + im.convert(mode="L").save(atlas_jpg_file) From fd3414d2a865143820422ca151e87a9bdb896dce Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 25 Jul 2025 12:35:00 +0100 Subject: [PATCH 10/16] Added logs to keep track of atlas image conversion workflow --- src/murfey/client/contexts/atlas.py | 3 +++ src/murfey/server/api/session_control.py | 1 + src/murfey/workflows/spa/atlas.py | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py index d5601eeda..0dcac02bb 100644 --- a/src/murfey/client/contexts/atlas.py +++ b/src/murfey/client/contexts/atlas.py @@ -47,3 +47,6 @@ def post_transfer( f"{str(environment.url.geturl())}{url_path_for('session_control.spa_router', 'make_atlas_jpg', session_id=environment.murfey_session)}", json={"path": str(transferred_atlas_name)}, ) + logger.info( + f"Submitted request to create JPG image of atlas {str(transferred_atlas_name)!r}" + ) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index b569d6559..b98c1f805 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -352,6 +352,7 @@ def get_foil_hole( def make_atlas_jpg( session_id: MurfeySessionID, atlas_mrc: StringOfPathModel, db=murfey_db ): + logger.debug(f"Received request to create JPG image of atlas {atlas_mrc.path!r}") session = db.exec(select(Session).where(Session.id == session_id)).one() return atlas_jpg_from_mrc( session.instrument_name, session.visit, Path(atlas_mrc.path) diff --git a/src/murfey/workflows/spa/atlas.py b/src/murfey/workflows/spa/atlas.py index 784065434..22f62d89a 100644 --- a/src/murfey/workflows/spa/atlas.py +++ b/src/murfey/workflows/spa/atlas.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path import mrcfile @@ -6,8 +7,11 @@ from murfey.util.config import get_machine_config +logger = logging.getLogger("murfey.workflows.spa.atlas") + def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): + logger.debug(f"Starting workflow to create JPG image of atlas {atlas_mrc}") with mrcfile.read(atlas_mrc) as mrc: data = mrc.data @@ -33,3 +37,4 @@ def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): im = PIL.Image.fromarray(data) im.convert(mode="L").save(atlas_jpg_file) + logger.debug(f"JPG image of atlas saved as {atlas_jpg_file}") From 93537e078cabfb8b113e7ecd324eb88682bc4067 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 25 Jul 2025 12:35:33 +0100 Subject: [PATCH 11/16] Replaced 'mrcfile.read' with 'mrcfile.open' --- src/murfey/workflows/spa/atlas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/workflows/spa/atlas.py b/src/murfey/workflows/spa/atlas.py index 22f62d89a..dce4b83e4 100644 --- a/src/murfey/workflows/spa/atlas.py +++ b/src/murfey/workflows/spa/atlas.py @@ -12,7 +12,7 @@ def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): logger.debug(f"Starting workflow to create JPG image of atlas {atlas_mrc}") - with mrcfile.read(atlas_mrc) as mrc: + with mrcfile.open(atlas_mrc) as mrc: data = mrc.data machine_config = get_machine_config(instrument_name=instrument_name)[ From b9bc8f6f1115f0797280525f3965ff89ee0d4fd0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 25 Jul 2025 13:48:33 +0100 Subject: [PATCH 12/16] Added unit test for the 'atlas_jpg_from_mrc' function --- tests/workflows/spa/test_atlas_workflow.py | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/workflows/spa/test_atlas_workflow.py diff --git a/tests/workflows/spa/test_atlas_workflow.py b/tests/workflows/spa/test_atlas_workflow.py new file mode 100644 index 000000000..1ce765d1d --- /dev/null +++ b/tests/workflows/spa/test_atlas_workflow.py @@ -0,0 +1,39 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import numpy as np +from pytest_mock import MockerFixture + +from murfey.workflows.spa.atlas import atlas_jpg_from_mrc + + +def test_atlas_jpg_from_mrc(mocker: MockerFixture, tmp_path: Path): + visit_name = "test_visit" + instrument_name = "test" + + # Create a 16-bit grayscale image + shape = (64, 64) + test_data = np.ones(shape).astype("uint16") + + # Mock out the data returned from openning the file + mock_mrcfile = mocker.patch("murfey.workflows.spa.atlas.mrcfile") + mock_mrc = MagicMock() + mock_mrc.data = test_data + mock_mrcfile.open.return_value.__enter__.return_value = mock_mrc + + # Mock the return result of 'get_machine_config()' + mock_machine_config = MagicMock() + mock_machine_config.processed_directory_name = "processed" + mocker.patch( + "murfey.workflows.spa.atlas.get_machine_config", + return_value={"test": mock_machine_config}, + ) + + # Create a test file + test_dir = tmp_path / instrument_name / "data" / visit_name / "atlas" + test_dir.mkdir(parents=True, exist_ok=True) + test_file = test_dir / "Atlas1.mrc" + test_file.touch(exist_ok=True) + + # Run the function + atlas_jpg_from_mrc(instrument_name, visit_name, test_file) From 726022ae063b0ad60c7ea9c91aa0370befe4fa9c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 25 Jul 2025 15:42:17 +0100 Subject: [PATCH 13/16] Added a unit test for the atlas image FastAPI endpoint --- tests/server/api/test_session_control.py | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/server/api/test_session_control.py diff --git a/tests/server/api/test_session_control.py b/tests/server/api/test_session_control.py new file mode 100644 index 000000000..abf63285b --- /dev/null +++ b/tests/server/api/test_session_control.py @@ -0,0 +1,76 @@ +from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pytest_mock import MockerFixture + +from murfey.server.api.auth import ( + validate_instrument_server_session_access, + validate_instrument_token, +) +from murfey.server.api.session_control import spa_router +from murfey.server.murfey_db import murfey_db_session +from murfey.util.api import url_path_for + + +def test_make_atlas_jpg(mocker: MockerFixture, tmp_path: Path): + # Set up the objects to mock + instrument_name = "test" + visit_name = "test_visit" + session_id = 1 + + # Override the database session generator + mock_session = MagicMock() + mock_session.instrument_name = instrument_name + mock_session.visit = visit_name + mock_query_result = MagicMock() + mock_query_result.one.return_value = mock_session + mock_db_session = MagicMock() + mock_db_session.exec.return_value = mock_query_result + + def mock_get_db_session(): + yield mock_db_session + + # Mock the instrument server tokens dictionary + mock_tokens = mocker.patch( + "murfey.server.api.instrument.instrument_server_tokens", + {session_id: {"access_token": mock.sentinel}}, + ) + + # Mock the called workflow function + mock_atlas_jpg = mocker.patch( + "murfey.server.api.session_control.atlas_jpg_from_mrc", + return_value=None, + ) + + # Set up the test file + image_dir = tmp_path / instrument_name / "data" / visit_name / "Atlas" + image_dir.mkdir(parents=True, exist_ok=True) + test_file = image_dir / "Atlas1.mrc" + + # Set up the backend server + backend_app = FastAPI() + + # Override validation and database dependencies + backend_app.dependency_overrides[validate_instrument_token] = lambda: None + backend_app.dependency_overrides[validate_instrument_server_session_access] = ( + lambda: session_id + ) + backend_app.dependency_overrides[murfey_db_session] = mock_get_db_session + backend_app.include_router(spa_router) + backend_server = TestClient(backend_app) + + atlas_jpg_url = url_path_for( + "api.session_control.spa_router", "make_atlas_jpg", session_id=session_id + ) + response = backend_server.post( + atlas_jpg_url, + json={"path": str(test_file)}, + headers={"Authorization": f"Bearer {mock_tokens[session_id]['access_token']}"}, + ) + + # Check that the expected calls were made + mock_atlas_jpg.assert_called_once_with(instrument_name, visit_name, test_file) + assert response.status_code == 200 From 144aed609999125a1081e6fd63649127859f7189 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 25 Jul 2025 15:44:15 +0100 Subject: [PATCH 14/16] Sanitised file paths in logs --- src/murfey/server/api/session_control.py | 4 +++- src/murfey/workflows/spa/atlas.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index b98c1f805..f8966a5f2 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -352,7 +352,9 @@ def get_foil_hole( def make_atlas_jpg( session_id: MurfeySessionID, atlas_mrc: StringOfPathModel, db=murfey_db ): - logger.debug(f"Received request to create JPG image of atlas {atlas_mrc.path!r}") + logger.debug( + f"Received request to create JPG image of atlas {sanitise(atlas_mrc.path)!r}" + ) session = db.exec(select(Session).where(Session.id == session_id)).one() return atlas_jpg_from_mrc( session.instrument_name, session.visit, Path(atlas_mrc.path) diff --git a/src/murfey/workflows/spa/atlas.py b/src/murfey/workflows/spa/atlas.py index dce4b83e4..6ea2d5738 100644 --- a/src/murfey/workflows/spa/atlas.py +++ b/src/murfey/workflows/spa/atlas.py @@ -5,13 +5,16 @@ import PIL.Image from werkzeug.utils import secure_filename +from murfey.util import sanitise from murfey.util.config import get_machine_config logger = logging.getLogger("murfey.workflows.spa.atlas") def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): - logger.debug(f"Starting workflow to create JPG image of atlas {atlas_mrc}") + logger.debug( + f"Starting workflow to create JPG image of atlas {sanitise(str(atlas_mrc))}" + ) with mrcfile.open(atlas_mrc) as mrc: data = mrc.data From 88c5bde986229dbc5b56b668c7c0d275e4dfe80a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 25 Jul 2025 16:01:51 +0100 Subject: [PATCH 15/16] Updated log output slightly --- src/murfey/workflows/spa/atlas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/workflows/spa/atlas.py b/src/murfey/workflows/spa/atlas.py index 6ea2d5738..12a761fe7 100644 --- a/src/murfey/workflows/spa/atlas.py +++ b/src/murfey/workflows/spa/atlas.py @@ -13,7 +13,7 @@ def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): logger.debug( - f"Starting workflow to create JPG image of atlas {sanitise(str(atlas_mrc))}" + f"Starting workflow to create JPG image of atlas {sanitise(str(atlas_mrc))!r}" ) with mrcfile.open(atlas_mrc) as mrc: data = mrc.data @@ -40,4 +40,4 @@ def atlas_jpg_from_mrc(instrument_name: str, visit_name: str, atlas_mrc: Path): im = PIL.Image.fromarray(data) im.convert(mode="L").save(atlas_jpg_file) - logger.debug(f"JPG image of atlas saved as {atlas_jpg_file}") + logger.debug(f"JPG image of atlas saved as {str(atlas_jpg_file)!r}") From 2ce7816ca5abc372b76a5fa8c0cc8e97bd3c104f Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 25 Jul 2025 16:03:45 +0100 Subject: [PATCH 16/16] Parametrised test to further improve coverage --- tests/workflows/spa/test_atlas_workflow.py | 42 ++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/tests/workflows/spa/test_atlas_workflow.py b/tests/workflows/spa/test_atlas_workflow.py index 1ce765d1d..1b1ea5e0c 100644 --- a/tests/workflows/spa/test_atlas_workflow.py +++ b/tests/workflows/spa/test_atlas_workflow.py @@ -2,14 +2,29 @@ from unittest.mock import MagicMock import numpy as np +import pytest from pytest_mock import MockerFixture +from werkzeug.utils import secure_filename from murfey.workflows.spa.atlas import atlas_jpg_from_mrc +atlas_jpg_from_mrc_test_matrix = ( + ("Atlas1.mrc",), + ("Sample1/Atlas1.mrc",), +) -def test_atlas_jpg_from_mrc(mocker: MockerFixture, tmp_path: Path): + +@pytest.mark.parametrize("test_params", atlas_jpg_from_mrc_test_matrix) +def test_atlas_jpg_from_mrc( + mocker: MockerFixture, tmp_path: Path, test_params: tuple[str] +): + # Unpack test params + (file_name_stub,) = test_params + + # Set up mock session params visit_name = "test_visit" instrument_name = "test" + processed_dir_name = "processed" # Create a 16-bit grayscale image shape = (64, 64) @@ -23,17 +38,32 @@ def test_atlas_jpg_from_mrc(mocker: MockerFixture, tmp_path: Path): # Mock the return result of 'get_machine_config()' mock_machine_config = MagicMock() - mock_machine_config.processed_directory_name = "processed" + mock_machine_config.processed_directory_name = processed_dir_name mocker.patch( "murfey.workflows.spa.atlas.get_machine_config", return_value={"test": mock_machine_config}, ) # Create a test file - test_dir = tmp_path / instrument_name / "data" / visit_name / "atlas" - test_dir.mkdir(parents=True, exist_ok=True) - test_file = test_dir / "Atlas1.mrc" + test_file = ( + tmp_path / instrument_name / "data" / visit_name / "atlas" / file_name_stub + ) + test_file.parent.mkdir(parents=True, exist_ok=True) test_file.touch(exist_ok=True) - # Run the function + # Create the expected destination directory and file + processed_dir = ( + tmp_path / instrument_name / "data" / visit_name / processed_dir_name / "atlas" + ) + sample_id = "Sample" + for part in file_name_stub.split("/"): + if part.startswith("Sample"): + sample_id = part + break + processed_file_name = processed_dir / secure_filename( + f"{sample_id}_{test_file.stem}_fullres.jpg" + ) + + # Run the function and check that the expected calls are made atlas_jpg_from_mrc(instrument_name, visit_name, test_file) + assert processed_file_name.exists()