From f0dd9e83b2f7c7460ab7270811b0bf2217248a40 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 15 Jul 2025 12:12:31 +0100 Subject: [PATCH 01/11] Add metadata options to _find_context --- src/murfey/client/analyser.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index e43523dc5..1869fe190 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -122,6 +122,17 @@ def _find_context(self, file_path: Path) -> bool: self._context = SPAMetadataContext("epu", self._basepath) return True + if "Metadata" in file_path.parts or file_path.name == "EpuSession.dm": + self._context = SPAMetadataContext("epu", self._basepath) + return True + elif ( + "Batch" in file_path.parts + or "SearchMaps" in file_path.parts + or file_path.name == "Session.dm" + ): + self._context = TomographyMetadataContext("tomo", self._basepath) + return True + # CLEM workflow checks # Look for LIF and XLIF files if file_path.suffix in (".lif", ".xlif"): From be88af8d85ffb81feededb4bab9dcfa450a4bc7c Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 15 Jul 2025 12:16:11 +0100 Subject: [PATCH 02/11] Remove serialem logic --- src/murfey/client/analyser.py | 31 +----------------------- src/murfey/client/contexts/tomo.py | 33 ++------------------------ src/murfey/client/multigrid_control.py | 2 +- src/murfey/util/dummy_setup.py | 2 +- 4 files changed, 5 insertions(+), 63 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 1869fe190..bd0324a19 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -181,30 +181,6 @@ def _find_context(self, file_path: Path) -> bool: self._context = TomographyContext("tomo", self._basepath) self.parameters_model = ProcessingParametersTomo return True - - # Files with these suffixes belong to the serial EM tomography workflow - if file_path.suffix in (".mrc", ".tiff", ".tif", ".eer"): - # Ignore batch files and search maps - if any(p in file_path.parts for p in ("Batch", "SearchMaps")): - return False - # Ignore JPG files - if file_path.with_suffix(".jpg").is_file(): - return False - # Ignore the averaged movies written out by the Falcon - if ( - len( - list( - file_path.parent.glob( - f"{file_path.name}*{file_path.suffix}" - ) - ) - ) - > 1 - ): - return False - self._context = TomographyContext("serialem", self._basepath) - self.parameters_model = ProcessingParametersTomo - return True return False def post_transfer(self, transferred_file: Path): @@ -295,12 +271,7 @@ def _analyse(self): if not dc_metadata: try: dc_metadata = self._context.gather_metadata( - ( - transferred_file.with_suffix(".mdoc") - if self._context._acquisition_software - == "serialem" - else self._xml_file(transferred_file) - ), + self._xml_file(transferred_file), environment=self._environment, ) except NotImplementedError: diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 08fe22c20..a9204259e 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -400,33 +400,6 @@ def _add_tomo_tilt( required_strings=required_strings, ) - def _add_serialem_tilt( - self, file_path: Path, environment: MurfeyInstanceEnvironment | None = None - ) -> List[str]: - delimiters = ("_", "-") - for d in delimiters: - if file_path.name.count(d) > 1: - delimiter = d - break - else: - delimiter = delimiters[0] - - def _extract_tilt_series(p: Path) -> str: - split = p.name.split(delimiter) - for s in split: - if s.isdigit(): - return s - raise ValueError( - f"No digits found in {p.name} after splitting on {delimiter}" - ) - - return self._add_tilt( - file_path, - lambda x: ".".join(x.name.split(delimiter)[-1].split(".")[:-1]), - environment=environment, - required_strings=[], - ) - def post_transfer( self, transferred_file: Path, @@ -464,10 +437,8 @@ def post_transfer( required_strings=kwargs.get("required_strings") or required_strings, ) - elif self._acquisition_software == "serialem": - completed_tilts = self._add_serialem_tilt( - transferred_file, environment=environment - ) + else: + logger.warning(f"Unknown data file {transferred_file}") if transferred_file.suffix == ".mdoc": with open(transferred_file, "r") as md: tilt_series = transferred_file.stem diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index 1ce69e5b2..a016b49f4 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -23,7 +23,7 @@ from murfey.util.api import url_path_for from murfey.util.client import capture_delete, capture_post, get_machine_config_client -log = logging.getLogger("murfey.client.mutligrid_control") +log = logging.getLogger("murfey.client.multigrid_control") @dataclass diff --git a/src/murfey/util/dummy_setup.py b/src/murfey/util/dummy_setup.py index ca89645a6..6ec450398 100644 --- a/src/murfey/util/dummy_setup.py +++ b/src/murfey/util/dummy_setup.py @@ -24,7 +24,7 @@ def initialise(dummy_location: Path) -> Path: yaml.dump( { "m12": { - "acquisition_software": ["epu", "tomo", "serialem"], + "acquisition_software": ["epu", "tomo"], "data_directories": [str(detector_dir)], "rsync_basepath": str(dummy_location), "calibrations": {"dummy": 0}, From cd98ad8cef379acd7b10a9affd8bcf916c1a85b6 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 15 Jul 2025 14:16:50 +0100 Subject: [PATCH 03/11] Tests for finding context --- src/murfey/client/analyser.py | 5 ++++ tests/client/test_analyser.py | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index bd0324a19..3bc1b4bae 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -128,6 +128,7 @@ def _find_context(self, file_path: Path) -> bool: elif ( "Batch" in file_path.parts or "SearchMaps" in file_path.parts + or "Thumbnails" in file_path.parts or file_path.name == "Session.dm" ): self._context = TomographyMetadataContext("tomo", self._basepath) @@ -160,6 +161,9 @@ def _find_context(self, file_path: Path) -> bool: # Tomography and SPA workflow checks split_file_name = file_path.name.split("_") if split_file_name: + if "gain" in split_file_name[-1]: + return False + # Files starting with "FoilHole" belong to the SPA workflow if split_file_name[0].startswith("FoilHole"): if not self._context: @@ -175,6 +179,7 @@ def _find_context(self, file_path: Path) -> bool: or "Fractions" in split_file_name[-1] or "fractions" in split_file_name[-1] or "EER" in split_file_name[-1] + or file_path.suffix == ".mdoc" ): if not self._context: logger.info("Acquisition software: tomo") diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index f9256a0c0..5746c653b 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -1,6 +1,58 @@ from __future__ import annotations +import pytest + from murfey.client.analyser import Analyser +from murfey.client.contexts.spa import SPAModularContext +from murfey.client.contexts.spa_metadata import SPAMetadataContext +from murfey.client.contexts.tomo import TomographyContext +from murfey.client.contexts.tomo_metadata import TomographyMetadataContext +from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo + +example_files = [ + ["visit/Position_1_001_0.0_20250715_012434_fractions.tiff", TomographyContext], + ["visit/Position_1_2_002_3.0_20250715_012434_Fractions.mrc", TomographyContext], + ["visit/Position_1_2_003_6.0_20250715_012434_EER.eer", TomographyContext], + ["visit/name1_004_9.0_20250715_012434_fractions.tiff", TomographyContext], + ["visit/Position_1_[30.0].tiff", TomographyContext], + ["visit/Position_1.mdoc", TomographyContext], + ["visit/name1_2.mdoc", TomographyContext], + ["visit/Session.dm", TomographyMetadataContext], + ["visit/SearchMaps/SearchMap.xml", TomographyMetadataContext], + ["visit/Batch/BatchPositionsList.xml", TomographyMetadataContext], + ["visit/Thumbnails/file.mrc", TomographyMetadataContext], + ["visit/FoilHole_01234_fractions.tiff", SPAModularContext], + ["atlas/atlas.mrc", SPAMetadataContext], + ["visit/EpuSession.dm", SPAMetadataContext], + ["visit/Metadata/GridSquare.dm", SPAMetadataContext], +] + + +@pytest.mark.parametrize("file_and_context", example_files) +def test_find_context(file_and_context, tmp_path): + file_name, context = file_and_context + + analyser = Analyser(tmp_path) + assert analyser._find_context(tmp_path / file_name) + assert isinstance(analyser._context, context) + if isinstance(analyser._context, TomographyContext): + assert analyser.parameters_model == ProcessingParametersTomo + if isinstance(analyser._context, SPAModularContext): + assert analyser.parameters_model == ProcessingParametersSPA + + +contextless_files = [ + "visit/Position_1_gain.tiff", + "visit/FoilHole_01234_gain.tiff", + "visit/file_1.mrc", +] + + +@pytest.mark.parametrize("bad_file", contextless_files) +def test_ignore_contextless_files(bad_file, tmp_path): + analyser = Analyser(tmp_path) + assert not analyser._find_context(tmp_path / bad_file) + assert not analyser._context def test_analyser_setup_and_stopping(tmp_path): From f95b861d3f058b2c687fcb91c2702d81fd4148a6 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 15 Jul 2025 14:24:35 +0100 Subject: [PATCH 04/11] Avoid marking some metadata as SPA --- src/murfey/client/analyser.py | 19 ++++++++++--------- tests/client/test_analyser.py | 7 +++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 3bc1b4bae..e217808a5 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -159,13 +159,17 @@ def _find_context(self, file_path: Path) -> bool: return True # Tomography and SPA workflow checks - split_file_name = file_path.name.split("_") - if split_file_name: - if "gain" in split_file_name[-1]: + split_file_stem = file_path.stem.split("_") + if split_file_stem: + if split_file_stem[-1] == "gain": return False # Files starting with "FoilHole" belong to the SPA workflow - if split_file_name[0].startswith("FoilHole"): + if split_file_stem[0].startswith("FoilHole") and split_file_stem[-1] in [ + "Fractions", + "fractions", + "EER", + ]: if not self._context: logger.info("Acquisition software: EPU") self._context = SPAModularContext("epu", self._basepath) @@ -174,11 +178,8 @@ def _find_context(self, file_path: Path) -> bool: # Files starting with "Position" belong to the standard tomography workflow if ( - split_file_name[0] == "Position" - or "[" in file_path.name - or "Fractions" in split_file_name[-1] - or "fractions" in split_file_name[-1] - or "EER" in split_file_name[-1] + split_file_stem[0] == "Position" + or split_file_stem[-1] in ["Fractions", "fractions", "EER"] or file_path.suffix == ".mdoc" ): if not self._context: diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 5746c653b..8f0be3381 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -45,6 +45,9 @@ def test_find_context(file_and_context, tmp_path): "visit/Position_1_gain.tiff", "visit/FoilHole_01234_gain.tiff", "visit/file_1.mrc", + "visit/FoilHole_01234.mrc", + "visit/FoilHole_01234.jpg", + "visit/FoilHole_01234.xml", ] @@ -66,7 +69,7 @@ def test_analyser_setup_and_stopping(tmp_path): def test_analyser_tomo_determination(tmp_path): - tomo_file = tmp_path / "Position_1_[30.0].tiff" + tomo_file = tmp_path / "Position_1_[30.0]_fractions.tiff" analyser = Analyser(tmp_path) analyser.start() analyser.queue.put(tomo_file) @@ -75,7 +78,7 @@ def test_analyser_tomo_determination(tmp_path): def test_analyser_epu_determination(tmp_path): - tomo_file = tmp_path / "FoilHole_12345_Data_6789.tiff" + tomo_file = tmp_path / "FoilHole_12345_Data_6789_Fractions.tiff" analyser = Analyser(tmp_path) analyser.start() analyser.queue.put(tomo_file) From 07d7eeb48a1a3716b95615972f757d28404075cf Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 17 Jul 2025 18:04:37 +0100 Subject: [PATCH 05/11] Moved CLEM context determination logic to the start of the '_find_context' function, as it conflicts with some of the metadata determindation logic for SPA and Tomo --- src/murfey/client/analyser.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index e217808a5..0a667e385 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -118,21 +118,6 @@ def _find_context(self, file_path: Path) -> bool: in the Context classes themselves. """ logger.debug(f"Finding context using file {str(file_path)!r}") - if "atlas" in file_path.parts: - self._context = SPAMetadataContext("epu", self._basepath) - return True - - if "Metadata" in file_path.parts or file_path.name == "EpuSession.dm": - self._context = SPAMetadataContext("epu", self._basepath) - return True - elif ( - "Batch" in file_path.parts - or "SearchMaps" in file_path.parts - or "Thumbnails" in file_path.parts - or file_path.name == "Session.dm" - ): - self._context = TomographyMetadataContext("tomo", self._basepath) - return True # CLEM workflow checks # Look for LIF and XLIF files @@ -141,9 +126,9 @@ def _find_context(self, file_path: Path) -> bool: return True # Look for TIFF files associated with CLEM workflow # Leica's autosave mode seems to name the TIFFs in the format - # PostionXX--ZXX-CXX.tif + # PostionXX--ZXX--CXX.tif if ( - "--" in file_path.name + all(pattern in file_path.name for pattern in ("--Z", "--C")) and file_path.suffix in (".tiff", ".tif") and self._environment ): @@ -159,6 +144,22 @@ def _find_context(self, file_path: Path) -> bool: return True # Tomography and SPA workflow checks + if "atlas" in file_path.parts: + self._context = SPAMetadataContext("epu", self._basepath) + return True + + if "Metadata" in file_path.parts or file_path.name == "EpuSession.dm": + self._context = SPAMetadataContext("epu", self._basepath) + return True + elif ( + "Batch" in file_path.parts + or "SearchMaps" in file_path.parts + or "Thumbnails" in file_path.parts + or file_path.name == "Session.dm" + ): + self._context = TomographyMetadataContext("tomo", self._basepath) + return True + split_file_stem = file_path.stem.split("_") if split_file_stem: if split_file_stem[-1] == "gain": From 76f52845f7b5f7486f28856c67e2cc701209672a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 17 Jul 2025 18:08:09 +0100 Subject: [PATCH 06/11] Simplified the conditions for determining the context for CLEM TIFF files; the loading of the 'analyse_created_directories' key should not be needed --- src/murfey/client/analyser.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 0a667e385..e1144c92c 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -132,16 +132,8 @@ def _find_context(self, file_path: Path) -> bool: and file_path.suffix in (".tiff", ".tif") and self._environment ): - created_directories = set( - get_machine_config_client( - str(self._environment.url.geturl()), - instrument_name=self._environment.instrument_name, - demo=self._environment.demo, - ).get("analyse_created_directories", []) - ) - if created_directories.intersection(set(file_path.parts)): - self._context = CLEMContext("leica", self._basepath) - return True + self._context = CLEMContext("leica", self._basepath) + return True # Tomography and SPA workflow checks if "atlas" in file_path.parts: From b1a165de30899d0fa2d9cdd38b5095ddd65cb7a3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 17 Jul 2025 18:08:40 +0100 Subject: [PATCH 07/11] Added examples of files from the CLEM workflow that are used to determine the context --- tests/client/test_analyser.py | 71 ++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 8f0be3381..1952751e1 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -1,8 +1,10 @@ from __future__ import annotations import pytest +from pytest_mock import MockerFixture from murfey.client.analyser import Analyser +from murfey.client.contexts.clem import CLEMContext from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.spa_metadata import SPAMetadataContext from murfey.client.contexts.tomo import TomographyContext @@ -10,6 +12,7 @@ from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo example_files = [ + # Tomography ["visit/Position_1_001_0.0_20250715_012434_fractions.tiff", TomographyContext], ["visit/Position_1_2_002_3.0_20250715_012434_Fractions.mrc", TomographyContext], ["visit/Position_1_2_003_6.0_20250715_012434_EER.eer", TomographyContext], @@ -17,24 +20,88 @@ ["visit/Position_1_[30.0].tiff", TomographyContext], ["visit/Position_1.mdoc", TomographyContext], ["visit/name1_2.mdoc", TomographyContext], + # Tomography metadata ["visit/Session.dm", TomographyMetadataContext], ["visit/SearchMaps/SearchMap.xml", TomographyMetadataContext], ["visit/Batch/BatchPositionsList.xml", TomographyMetadataContext], ["visit/Thumbnails/file.mrc", TomographyMetadataContext], + # SPA ["visit/FoilHole_01234_fractions.tiff", SPAModularContext], + # SPA metadata ["atlas/atlas.mrc", SPAMetadataContext], ["visit/EpuSession.dm", SPAMetadataContext], ["visit/Metadata/GridSquare.dm", SPAMetadataContext], + # CLEM LIF file + ["visit/images/test_file.lif", CLEMContext], + # CLEM TIFF files + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12--Z02--C01.tif", + CLEMContext, + ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12_Lng_LVCC--Z02--C01.tif", + CLEMContext, + ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Series001--Z00--C00.tif", + CLEMContext, + ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Series001_Lng_LVCC--Z00--C00.tif", + CLEMContext, + ], + # CLEM TIFF file accompanying metadata + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Position 12.xlif", + CLEMContext, + ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Position 12_Lng_LVCC.xlif", + CLEMContext, + ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Series001.xlif", + CLEMContext, + ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Series001_Lng_LVCC.xlif", + CLEMContext, + ], ] @pytest.mark.parametrize("file_and_context", example_files) -def test_find_context(file_and_context, tmp_path): +def test_find_context(mocker: MockerFixture, file_and_context, tmp_path): + # Unpack parametrised variables file_name, context = file_and_context - analyser = Analyser(tmp_path) + # The CLEM Context requires a MurfeyInstanceEnvironment + if context is CLEMContext: + # Mock the MurfeyInstanceEnvironment + mock_environment = mocker.patch( + "murfey.client.analyser.MurfeyInstanceEnvironment" + ) + mock_environment.url.geturl.return_value = "https://murfey.server.test" + mock_environment.instrument_name = "Test" + mock_environment.demo = False + + # Mock the call to get the machine config + mocker.patch( + "murfey.client.analyser.get_machine_config_client", + return_value={"analyse_created_directories": ["images"]}, + ) + + # Pass the file to the Analyser; add environment as needed + analyser = Analyser( + basepath_local=tmp_path, + environment=(mock_environment if context is CLEMContext else None), + ) + + # Check that the results are as expected assert analyser._find_context(tmp_path / file_name) assert isinstance(analyser._context, context) + + # Checks for the specific workflow contexts if isinstance(analyser._context, TomographyContext): assert analyser.parameters_model == ProcessingParametersTomo if isinstance(analyser._context, SPAModularContext): From 2e6b21b5e39feb08b9bb5f9de119e6f3b86a696b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 17 Jul 2025 18:11:06 +0100 Subject: [PATCH 08/11] Removed the need for an instance environment when determining the context of CLEM files; removed the corresponding mocks in the test --- src/murfey/client/analyser.py | 8 +++----- tests/client/test_analyser.py | 24 ++---------------------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index e1144c92c..de8b1f9b4 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -127,11 +127,9 @@ def _find_context(self, file_path: Path) -> bool: # Look for TIFF files associated with CLEM workflow # Leica's autosave mode seems to name the TIFFs in the format # PostionXX--ZXX--CXX.tif - if ( - all(pattern in file_path.name for pattern in ("--Z", "--C")) - and file_path.suffix in (".tiff", ".tif") - and self._environment - ): + if all( + pattern in file_path.name for pattern in ("--Z", "--C") + ) and file_path.suffix in (".tiff", ".tif"): self._context = CLEMContext("leica", self._basepath) return True diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 1952751e1..a718bd170 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -1,7 +1,6 @@ from __future__ import annotations import pytest -from pytest_mock import MockerFixture from murfey.client.analyser import Analyser from murfey.client.contexts.clem import CLEMContext @@ -71,31 +70,12 @@ @pytest.mark.parametrize("file_and_context", example_files) -def test_find_context(mocker: MockerFixture, file_and_context, tmp_path): +def test_find_context(file_and_context, tmp_path): # Unpack parametrised variables file_name, context = file_and_context - # The CLEM Context requires a MurfeyInstanceEnvironment - if context is CLEMContext: - # Mock the MurfeyInstanceEnvironment - mock_environment = mocker.patch( - "murfey.client.analyser.MurfeyInstanceEnvironment" - ) - mock_environment.url.geturl.return_value = "https://murfey.server.test" - mock_environment.instrument_name = "Test" - mock_environment.demo = False - - # Mock the call to get the machine config - mocker.patch( - "murfey.client.analyser.get_machine_config_client", - return_value={"analyse_created_directories": ["images"]}, - ) - # Pass the file to the Analyser; add environment as needed - analyser = Analyser( - basepath_local=tmp_path, - environment=(mock_environment if context is CLEMContext else None), - ) + analyser = Analyser(basepath_local=tmp_path) # Check that the results are as expected assert analyser._find_context(tmp_path / file_name) From cdbedffff6c3225fe6d347bc8778994227221f18 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 17 Jul 2025 18:20:31 +0100 Subject: [PATCH 09/11] Added example CLEM workflow files that cannot be used to determine context, and more files that can --- tests/client/test_analyser.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index a718bd170..9add7f9bf 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -58,6 +58,14 @@ "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Position 12_Lng_LVCC.xlif", CLEMContext, ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12/Metadata/Position 12_histo.xlif", + CLEMContext, + ], + [ + "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12/Metadata/Position 12_Lng_LVCC_histo.xlif", + CLEMContext, + ], [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Series001.xlif", CLEMContext, @@ -95,6 +103,11 @@ def test_find_context(file_and_context, tmp_path): "visit/FoilHole_01234.mrc", "visit/FoilHole_01234.jpg", "visit/FoilHole_01234.xml", + "visit/images/test_file.lifext", + "visit/images/2024_03_14_12_34_56--Project001/Project001.xlef", + "visit/images/2024_03_14_12_34_56--Project001/Project001.xlef.lock", + "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12/Position 12_histo.lof", + "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12/Series001_histo.lof", ] From 0d363e78ab6407c82b357bd7db179806e7afec40 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 18 Jul 2025 10:44:26 +0100 Subject: [PATCH 10/11] Reinstate the [ form and test one more case --- src/murfey/client/analyser.py | 1 + tests/client/test_analyser.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index de8b1f9b4..246ddb312 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -170,6 +170,7 @@ def _find_context(self, file_path: Path) -> bool: # Files starting with "Position" belong to the standard tomography workflow if ( split_file_stem[0] == "Position" + or "[" in file_path.name or split_file_stem[-1] in ["Fractions", "fractions", "EER"] or file_path.suffix == ".mdoc" ): diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 9add7f9bf..175916aa1 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -26,6 +26,7 @@ ["visit/Thumbnails/file.mrc", TomographyMetadataContext], # SPA ["visit/FoilHole_01234_fractions.tiff", SPAModularContext], + ["visit/FoilHole_01234_EER.eer", SPAModularContext], # SPA metadata ["atlas/atlas.mrc", SPAMetadataContext], ["visit/EpuSession.dm", SPAMetadataContext], @@ -129,7 +130,7 @@ def test_analyser_setup_and_stopping(tmp_path): def test_analyser_tomo_determination(tmp_path): - tomo_file = tmp_path / "Position_1_[30.0]_fractions.tiff" + tomo_file = tmp_path / "Position_1_[30.0].tiff" analyser = Analyser(tmp_path) analyser.start() analyser.queue.put(tomo_file) From 4d926b675e5773dbb4d52cb49cc59df170416300 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 18 Jul 2025 10:48:12 +0100 Subject: [PATCH 11/11] Make note on problem --- src/murfey/client/analyser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 246ddb312..18b28b2b4 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -168,6 +168,7 @@ def _find_context(self, file_path: Path) -> bool: return True # Files starting with "Position" belong to the standard tomography workflow + # NOTE: not completely reliable, mdocs can be in tomography metadata as well if ( split_file_stem[0] == "Position" or "[" in file_path.name