From f4fc46bfc8ed41c6a7f0e8ee13c602d8d19cdf7d Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 10 Jun 2025 16:32:41 +0100 Subject: [PATCH 01/13] Convert find_extension to return a bool --- src/murfey/client/analyser.py | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index e90da8aad..d2ed17cb2 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -85,7 +85,7 @@ def __init__( def __repr__(self) -> str: return f"" - def _find_extension(self, file_path: Path): + def _find_extension(self, file_path: Path) -> bool: """ Identifies the file extension and stores that information in the class. """ @@ -101,22 +101,17 @@ def _find_extension(self, file_path: Path): .get(file_path.suffix) ): if not any(r in file_path.name for r in required_substrings): - return [] + return False - # Checks for MRC, TIFF, TIF, and EER files if no extension has been defined - if ( - file_path.suffix in (".mrc", ".tiff", ".tif", ".eer") - and not self._extension - ): - logger.info(f"File extension determined: {file_path.suffix}") - self._extension = file_path.suffix - # Check for TIFF, TIF, or EER if the file's already been assigned an extension - elif ( - file_path.suffix in (".tiff", ".tif", ".eer") - and self._extension != file_path.suffix - ): - logger.info(f"File extension re-evaluated: {file_path.suffix}") - self._extension = file_path.suffix + # Checks for MRC, TIFF, TIF, and EER files + if file_path.suffix in (".mrc", ".tiff", ".tif", ".eer"): + if not self._extension: + logger.info(f"File extension determined: {file_path.suffix}") + self._extension = file_path.suffix + elif self._extension != file_path.suffix: + logger.info(f"File extension re-evaluated: {file_path.suffix}") + self._extension = file_path.suffix + return True # If we see an .mdoc file first, use that to determine the file extensions elif file_path.suffix == ".mdoc": with open(file_path, "r") as md: @@ -124,9 +119,12 @@ def _find_extension(self, file_path: Path): mdoc_data_block = get_block(md) if subframe_path := mdoc_data_block.get("SubFramePath"): self._extension = Path(subframe_path).suffix + return True # Check for LIF files separately elif file_path.suffix == ".lif": self._extension = file_path.suffix + return True + return False def _find_context(self, file_path: Path) -> bool: """ @@ -269,7 +267,10 @@ def _analyse(self): elif transferred_file.suffix == ".mdoc": mdoc_for_reading = transferred_file if not self._context: - self._find_extension(transferred_file) + valid_extension = self._find_extension(transferred_file) + if not valid_extension: + logger.error(f"No extension found for {transferred_file}") + break found = self._find_context(transferred_file) if not found: logger.debug( @@ -344,7 +345,10 @@ def _analyse(self): # Handle files with tomography and SPA context differently elif not self._extension or self._unseen_xml: - self._find_extension(transferred_file) + valid_extension = self._find_extension(transferred_file) + if not valid_extension: + logger.error(f"No extension found for {transferred_file}") + break if self._extension: logger.info( f"Extension found successfully for {transferred_file}" From 4817ddbd59ff6d5c27b500da3ecc76e001b2609d Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 08:51:38 +0100 Subject: [PATCH 02/13] Migrate parameters from environment to context --- src/murfey/client/context.py | 1 + src/murfey/client/contexts/spa.py | 114 +++++----------------- src/murfey/client/contexts/tomo.py | 74 ++++---------- src/murfey/client/instance_environment.py | 6 +- src/murfey/client/multigrid_control.py | 25 ++--- src/murfey/client/tui/app.py | 29 +++--- src/murfey/client/tui/screens.py | 7 +- 7 files changed, 73 insertions(+), 183 deletions(-) diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 7077ab861..14ec8553f 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -39,6 +39,7 @@ class Context: def __init__(self, name: str, acquisition_software: str): self._acquisition_software = acquisition_software self.name = name + self.data_collection_parameters: dict = {} def post_transfer( self, diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 6b249c8a3..1644b0897 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from datetime import datetime from itertools import count from pathlib import Path from typing import Any, Dict, List, Optional, OrderedDict, Tuple @@ -164,7 +163,6 @@ def gather_metadata( return OrderedDict({}) data = xmltodict.parse(for_parsing) magnification = 0 - num_fractions = 1 metadata: OrderedDict = OrderedDict({}) metadata["experiment_type"] = "SPA" if data.get("Acquisition"): @@ -217,14 +215,6 @@ def gather_metadata( ) # convert e / m^2 to e / A^2 except ValueError: metadata["total_exposed_dose"] = 1 - try: - num_fractions = int( - data["MicroscopeImage"]["microscopeData"]["acquisition"]["camera"][ - "CameraSpecificInput" - ]["a:KeyValueOfstringanyType"][2]["a:Value"]["b:NumberOffractions"] - ) - except (KeyError, IndexError): - pass c2_index = 3 for i, el in enumerate( data["MicroscopeImage"]["CustomData"]["a:KeyValueOfstringanyType"] @@ -293,85 +283,29 @@ def gather_metadata( metadata["image_size_x"] = str(int(metadata["image_size_x"]) * binning_factor) metadata["image_size_y"] = str(int(metadata["image_size_y"]) * binning_factor) metadata["motion_corr_binning"] = 1 if binning_factor_xml == 2 else 2 - if environment: - metadata["gain_ref"] = ( - environment.data_collection_parameters.get("gain_ref") - if environment - and environment.data_collection_parameters.get("gain_ref") - not in (None, "None") - else f"{datetime.now().year}/{environment.visit}/processing/gain.mrc" - ) - metadata["gain_ref_superres"] = ( - environment.data_collection_parameters.get("gain_ref_superres") - if environment - and environment.data_collection_parameters.get("gain_ref_superres") - not in (None, "None") - else f"{datetime.now().year}/{environment.visit}/processing/gain_superres.mrc" - ) - else: - metadata["gain_ref"] = None - metadata["gain_ref_superres"] = None - if metadata.get("total_exposed_dose"): - metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - and environment.data_collection_parameters.get("dose_per_frame") - not in (None, "None") - else round(metadata["total_exposed_dose"] / num_fractions, 3) - ) - else: - metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - else None - ) metadata["use_cryolo"] = ( - environment.data_collection_parameters.get("use_cryolo") - if environment - else None - ) or True - metadata["symmetry"] = ( - environment.data_collection_parameters.get("symmetry") - if environment - else None - ) or "C1" + self.data_collection_parameters.get("use_cryolo") or True + ) + metadata["symmetry"] = self.data_collection_parameters.get("symmetry") or "C1" metadata["mask_diameter"] = ( - environment.data_collection_parameters.get("mask_diameter") - if environment - else None - ) or 190 - metadata["boxsize"] = ( - environment.data_collection_parameters.get("boxsize") - if environment - else None - ) or 256 - metadata["downscale"] = ( - environment.data_collection_parameters.get("downscale") - if environment - else None - ) or True + self.data_collection_parameters.get("mask_diameter") or 190 + ) + metadata["boxsize"] = self.data_collection_parameters.get("boxsize") or 256 + metadata["downscale"] = self.data_collection_parameters.get("downscale") or True metadata["small_boxsize"] = ( - environment.data_collection_parameters.get("small_boxsize") - if environment - else None - ) or 128 + self.data_collection_parameters.get("small_boxsize") or 128 + ) metadata["eer_fractionation"] = ( - environment.data_collection_parameters.get("eer_fractionation") - if environment - else None - ) or 20 + self.data_collection_parameters.get("eer_fractionation") or 20 + ) metadata["source"] = str(self._basepath) metadata["particle_diameter"] = ( - environment.data_collection_parameters.get("particle_diameter") - if environment - else None - ) or 0 + self.data_collection_parameters.get("particle_diameter") or 0 + ) metadata["estimate_particle_diameter"] = ( - environment.data_collection_parameters.get("estimate_particle_diameter") - if environment - else None - ) or True + self.data_collection_parameters.get("estimate_particle_diameter") or True + ) return metadata def _position_analysis( @@ -584,10 +518,10 @@ def post_transfer( f"{str(environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=environment.visit, session_id=environment.murfey_session)}", json={ "eer_path": str(file_transferred_to), - "fractionation": environment.data_collection_parameters[ + "fractionation": self.data_collection_parameters[ "eer_fractionation" ], - "dose_per_frame": environment.data_collection_parameters[ + "dose_per_frame": self.data_collection_parameters[ "dose_per_frame" ], "fractionation_file_name": "eer_fractionation_spa.txt", @@ -619,20 +553,16 @@ def post_transfer( "image_number": environment.movies[ file_transferred_to ].movie_number, - "pixel_size": environment.data_collection_parameters.get( + "pixel_size": self.data_collection_parameters.get( "pixel_size_on_image" ), "autoproc_program_id": None, - "dose_per_frame": environment.data_collection_parameters.get( - "dose_per_frame" - ), - "mc_binning": environment.data_collection_parameters.get( + "dose_per_frame": environment.dose_per_frame, + "mc_binning": self.data_collection_parameters.get( "motion_corr_binning", 1 ), - "gain_ref": environment.data_collection_parameters.get( - "gain_ref" - ), - "extract_downscale": environment.data_collection_parameters.get( + "gain_ref": environment.gain_ref, + "extract_downscale": self.data_collection_parameters.get( "downscale", True ), "eer_fractionation_file": eer_fractionation_file, diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 9562041f7..6219876ce 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from datetime import datetime from pathlib import Path from threading import RLock from typing import Callable, Dict, List, OrderedDict @@ -133,25 +132,23 @@ def register_tomography_data_collections( "tag": tilt_series, } if ( - environment.data_collection_parameters - and environment.data_collection_parameters.get("voltage") + self.data_collection_parameters + and self.data_collection_parameters.get("voltage") ): # Once mdoc parameters are known register processing jobs dc_data.update( { - "voltage": environment.data_collection_parameters[ - "voltage" - ], - "pixel_size_on_image": environment.data_collection_parameters[ + "voltage": self.data_collection_parameters["voltage"], + "pixel_size_on_image": self.data_collection_parameters[ "pixel_size_on_image" ], - "image_size_x": environment.data_collection_parameters[ + "image_size_x": self.data_collection_parameters[ "image_size_x" ], - "image_size_y": environment.data_collection_parameters[ + "image_size_y": self.data_collection_parameters[ "image_size_y" ], - "magnification": environment.data_collection_parameters[ + "magnification": self.data_collection_parameters[ "magnification" ], } @@ -176,7 +173,7 @@ def register_tomography_data_collections( ) except Exception as e: - logger.error(f"ERROR {e}, {environment.data_collection_parameters}") + logger.error(f"ERROR {e}, {self.data_collection_parameters}") def _file_transferred_to( self, environment: MurfeyInstanceEnvironment, source: Path, file_path: Path @@ -315,19 +312,15 @@ def _add_tilt( capture_post(tilt_url, json=tilt_data) eer_fractionation_file = None - if environment.data_collection_parameters.get("num_eer_frames"): + if self.data_collection_parameters.get("num_eer_frames"): response = requests.post( f"{str(environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=environment.visit, session_id=environment.murfey_session)}", json={ - "num_frames": environment.data_collection_parameters[ - "num_eer_frames" - ], - "fractionation": environment.data_collection_parameters[ + "num_frames": self.data_collection_parameters["num_eer_frames"], + "fractionation": self.data_collection_parameters[ "eer_fractionation" ], - "dose_per_frame": environment.data_collection_parameters[ - "dose_per_frame" - ], + "dose_per_frame": environment.dose_per_frame or 0, "fractionation_file_name": "eer_fractionation_tomo.txt", }, ) @@ -337,23 +330,17 @@ def _add_tilt( "path": str(file_transferred_to), "description": "", "image_number": environment.movies[file_transferred_to].movie_number, - "pixel_size": environment.data_collection_parameters.get( + "pixel_size": self.data_collection_parameters.get( "pixel_size_on_image", 0 ), - "dose_per_frame": environment.data_collection_parameters.get( - "dose_per_frame", 0 - ), - "frame_count": environment.data_collection_parameters.get( - "frame_count", 0 - ), - "tilt_axis": environment.data_collection_parameters.get( - "tilt_axis", 85 - ), - "mc_binning": environment.data_collection_parameters.get( + "dose_per_frame": environment.dose_per_frame or 0, + "frame_count": self.data_collection_parameters.get("frame_count", 0), + "tilt_axis": self.data_collection_parameters.get("tilt_axis", 85), + "mc_binning": self.data_collection_parameters.get( "motion_corr_binning", 1 ), - "gain_ref": environment.data_collection_parameters.get("gain_ref"), - "voltage": environment.data_collection_parameters.get("voltage", 300), + "gain_ref": environment.gain_ref, + "voltage": self.data_collection_parameters.get("voltage", 300), "eer_fractionation_file": eer_fractionation_file, "tag": tilt_series, "group_tag": str(self._basepath), @@ -565,9 +552,7 @@ def gather_metadata( metadata["motion_corr_binning"] = 1 metadata["gain_ref"] = None metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - else None + environment.dose_per_frame if environment else None ) metadata["source"] = str(self._basepath) except KeyError: @@ -615,20 +600,6 @@ def gather_metadata( float(mdoc_data["PixelSpacing"]) * 1e-10 ) mdoc_metadata["motion_corr_binning"] = binning_factor - if environment: - mdoc_metadata["gain_ref"] = ( - environment.data_collection_parameters.get("gain_ref") - if environment.data_collection_parameters.get("gain_ref") - not in (None, "None") - else f"{datetime.now().year}/{environment.visit}/processing/gain.mrc" - ) - else: - mdoc_metadata["gain_ref"] = None - mdoc_metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - else None - ) mdoc_metadata["source"] = str(self._basepath) mdoc_metadata["tag"] = str(self._basepath) mdoc_metadata["tilt_series_tag"] = metadata_file.stem @@ -639,11 +610,6 @@ def gather_metadata( mdoc_metadata["file_extension"] = ( f".{mdoc_data_block['SubFramePath'].split('.')[-1]}" ) - mdoc_metadata["eer_fractionation"] = ( - environment.data_collection_parameters.get("eer_fractionation") - if environment - else None - ) or 20 data_file = mdoc_data_block["SubFramePath"].split("\\")[-1] if data_file.split(".")[-1] == "eer": diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index a715a1d1b..743154e02 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -41,14 +41,15 @@ class MurfeyInstanceEnvironment(BaseModel): destination_registry: Dict[str, str] = {} watchers: Dict[Path, DirWatcher] = {} demo: bool = False - data_collection_parameters: dict = {} movies: Dict[Path, MovieTracker] = {} movie_tilt_pair: Dict[Path, str] = {} tilt_angles: Dict[str, List[List[str]]] = {} movie_counters: Dict[str, itertools.count] = {} visit: str = "" processing_only_mode: bool = False + dose_per_frame: Optional[float] = None gain_ref: Optional[Path] = None + symmetry: Optional[str] = None superres: bool = False murfey_session: Optional[int] = None samples: Dict[Path, SampleInfo] = {} @@ -64,9 +65,10 @@ def clear(self): for w in self.watchers.values(): w.stop() self.watchers = {} - self.data_collection_parameters = {} self.movies = {} self.movie_tilt_pair = {} self.tilt_angles = {} self.visit = "" + self.dose_per_frame = None self.gain_ref = None + self.symmetry = None diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index d32aa8747..99a3db2ae 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -401,12 +401,10 @@ def _data_collection_form(self, response: dict): if self._register_dc and response.get("form"): self._form_values = {k: str(v) for k, v in response.get("form", {}).items()} log.info( - f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.data_collection_parameters.get('gain_ref')}" + f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.gain_ref}" ) if self._form_values.get("gain_ref") in (None, "None"): - self._form_values["gain_ref"] = ( - self._environment.data_collection_parameters.get("gain_ref") - ) + self._form_values["gain_ref"] = self._environment.gain_ref self._form_dependencies = response.get("dependencies", {}) self._data_collection_form_complete = True elif self._register_dc is None: @@ -421,11 +419,12 @@ def _start_dc(self, json, from_form: bool = False): json = json.get("form", {}) # Safely convert all entries into strings, but leave None as-is json = {k: str(v) if v is not None else None for k, v in json.items()} - self._environment.data_collection_parameters = { - k: None if v == "None" else v for k, v in json.items() - } source = Path(json["source"]) context = self.analysers[source]._context + if context: + context.data_collection_parameters = { + k: None if v == "None" else v for k, v in json.items() + } if isinstance(context, TomographyContext): source = Path(json["source"]) context.register_tomography_data_collections( @@ -435,19 +434,15 @@ def _start_dc(self, json, from_form: bool = False): ) log.info("Registering tomography processing parameters") - if self._environment.data_collection_parameters.get("num_eer_frames"): + if context.data_collection_parameters.get("num_eer_frames"): eer_response = requests.post( f"{str(self._environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=self._environment.visit, session_id=self._environment.murfey_session)}", json={ - "num_frames": self._environment.data_collection_parameters[ + "num_frames": context.data_collection_parameters[ "num_eer_frames" ], - "fractionation": self._environment.data_collection_parameters[ - "eer_fractionation" - ], - "dose_per_frame": self._environment.data_collection_parameters[ - "dose_per_frame" - ], + "fractionation": self._environment.eer_fractionation, + "dose_per_frame": self._environment.dose_per_frame, "fractionation_file_name": "eer_fractionation_tomo.txt", }, ) diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index b6d73ab5b..e49ad0138 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -442,12 +442,10 @@ def _data_collection_form(self, response: dict): if self._register_dc and response.get("form"): self._form_values = {k: str(v) for k, v in response.get("form", {}).items()} log.info( - f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.data_collection_parameters.get('gain_ref')}" + f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.gain_ref}" ) if self._form_values.get("gain_ref") in (None, "None"): - self._form_values["gain_ref"] = ( - self._environment.data_collection_parameters.get("gain_ref") - ) + self._form_values["gain_ref"] = self._environment.gain_ref self._form_dependencies = response.get("dependencies", {}) self.processing_btn.disabled = False self._data_collection_form_complete = True @@ -472,11 +470,16 @@ def _start_dc(self, json, from_form: bool = False): if from_form: json = json.get("form", {}) json = {k: v if v is None else str(v) for k, v in json.items()} - self._environment.data_collection_parameters = { - k: None if v == "None" else v for k, v in json.items() - } + self._environment.dose_per_frame = json.get("dose_per_frame") + self._environment.gain_ref = json.get("gain_ref") + self._environment.symmetry = json.get("symmetry") + self._environment.eer_fractionation = json.get("eer_fractionation") source = Path(json["source"]) context = self.analysers[source]._context + if context: + context.data_collection_parameters = { + k: None if v == "None" else v for k, v in json.items() + } if isinstance(context, TomographyContext): source = Path(json["source"]) context.register_tomography_data_collections( @@ -486,19 +489,15 @@ def _start_dc(self, json, from_form: bool = False): ) log.info("Registering tomography processing parameters") - if self.app._environment.data_collection_parameters.get("num_eer_frames"): + if context.data_collection_parameters.get("num_eer_frames"): eer_response = requests.post( f"{str(self.app._environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=self.app._environment.visit, session_id=self.app._environment.murfey_session)}", json={ - "num_frames": self.app._environment.data_collection_parameters[ + "num_frames": context.data_collection_parameters[ "num_eer_frames" ], - "fractionation": self.app._environment.data_collection_parameters[ - "eer_fractionation" - ], - "dose_per_frame": self.app._environment.data_collection_parameters[ - "dose_per_frame" - ], + "fractionation": self.app._environment.eer_fractionation, + "dose_per_frame": self.app._environment.dose_per_frame, "fractionation_file_name": "eer_fractionation_tomo.txt", }, ) diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 3ff79d5ae..8e8d4e22a 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -922,12 +922,9 @@ def on_button_pressed(self, event): log.info( f"Gain reference file {process_gain_response.json().get('gain_ref')}" ) - self.app._environment.data_collection_parameters["gain_ref"] = ( - process_gain_response.json().get("gain_ref") + self.app._environment.gain_ref = process_gain_response.json().get( + "gain_ref" ) - self.app._environment.data_collection_parameters[ - "gain_ref_superres" - ] = process_gain_response.json().get("gain_ref_superres") if self._switch_status: self.app.push_screen("directory-select") else: From 5c68c7f2d7158edcf9eee06d8faa7b1163d95607 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 09:02:39 +0100 Subject: [PATCH 03/13] Stop passing so many parameters around --- src/murfey/client/contexts/spa.py | 42 ++--------------------- src/murfey/client/instance_environment.py | 2 ++ src/murfey/server/api/instrument.py | 1 + src/murfey/server/feedback.py | 5 --- 4 files changed, 6 insertions(+), 44 deletions(-) diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 1644b0897..7e22776d6 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -108,27 +108,9 @@ class SPAModularContext(Context): "Dose Per Frame [e- / Angstrom^2 / frame] (after EER grouping if relevant)", default=1, ), - ProcessingParameter( - "estimate_particle_diameter", - "Use crYOLO to Estimate Particle Diameter", - default=True, - ), - ProcessingParameter( - "particle_diameter", "Particle Diameter (Angstroms)", default=None - ), - ProcessingParameter("use_cryolo", "Use crYOLO Autopicking", default=True), ProcessingParameter("symmetry", "Symmetry Group", default="C1"), ProcessingParameter("eer_fractionation", "EER Fractionation", default=20), - ProcessingParameter( - "mask_diameter", "Mask Diameter (2D classification)", default=190 - ), - ProcessingParameter("boxsize", "Box Size", default=256), - ProcessingParameter("downscale", "Downscale Extracted Particles", default=True), - ProcessingParameter( - "small_boxsize", "Downscaled Extracted Particle Size (pixels)", default=128 - ), ProcessingParameter("gain_ref", "Gain Reference"), - ProcessingParameter("gain_ref_superres", "Unbinned Gain Reference"), ] metadata_params = [ ProcessingParameter("voltage", "Voltage"), @@ -283,29 +265,11 @@ def gather_metadata( metadata["image_size_x"] = str(int(metadata["image_size_x"]) * binning_factor) metadata["image_size_y"] = str(int(metadata["image_size_y"]) * binning_factor) metadata["motion_corr_binning"] = 1 if binning_factor_xml == 2 else 2 - - metadata["use_cryolo"] = ( - self.data_collection_parameters.get("use_cryolo") or True - ) - metadata["symmetry"] = self.data_collection_parameters.get("symmetry") or "C1" - metadata["mask_diameter"] = ( - self.data_collection_parameters.get("mask_diameter") or 190 - ) - metadata["boxsize"] = self.data_collection_parameters.get("boxsize") or 256 - metadata["downscale"] = self.data_collection_parameters.get("downscale") or True - metadata["small_boxsize"] = ( - self.data_collection_parameters.get("small_boxsize") or 128 - ) + metadata["symmetry"] = (environment.symmetry if environment else None) or "C1" metadata["eer_fractionation"] = ( - self.data_collection_parameters.get("eer_fractionation") or 20 - ) + environment.eer_fractionation if environment else None + ) or 20 metadata["source"] = str(self._basepath) - metadata["particle_diameter"] = ( - self.data_collection_parameters.get("particle_diameter") or 0 - ) - metadata["estimate_particle_diameter"] = ( - self.data_collection_parameters.get("estimate_particle_diameter") or True - ) return metadata def _position_analysis( diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index 743154e02..6e69a043a 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -50,6 +50,7 @@ class MurfeyInstanceEnvironment(BaseModel): dose_per_frame: Optional[float] = None gain_ref: Optional[Path] = None symmetry: Optional[str] = None + eer_fractionation: Optional[int] = None superres: bool = False murfey_session: Optional[int] = None samples: Dict[Path, SampleInfo] = {} @@ -72,3 +73,4 @@ def clear(self): self.dose_per_frame = None self.gain_ref = None self.symmetry = None + self.eer_fractionation = None diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 34e238208..57fc62937 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -178,6 +178,7 @@ async def pass_proc_params_to_instrument_server( dose_per_frame=proc_params.dose_per_frame, gain_ref=session.current_gain_ref, symmetry=proc_params.symmetry, + eer_fractionation=proc_params.eer_fractionation, ) db.add(session_processing_parameters) db.commit() diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 80ca02cbb..973241473 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -2242,11 +2242,6 @@ def feedback_callback(header: dict, message: dict) -> None: motion_corr_binning=message["motion_corr_binning"], eer_fractionation_file=message["eer_fractionation_file"], symmetry=message["symmetry"], - particle_diameter=message["particle_diameter"], - downscale=message["downscale"], - boxsize=message["boxsize"], - small_boxsize=message["small_boxsize"], - mask_diameter=message["mask_diameter"], ) feedback_params = db.SPAFeedbackParameters( pj_id=collected_ids[2].id, From 135020abf7a76cfe8c5feb41ba9faf28c61ca9d1 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 09:03:03 +0100 Subject: [PATCH 04/13] Fix for eer fractionation from frontend --- src/murfey/server/api/file_io_instrument.py | 1 + src/murfey/util/db.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/murfey/server/api/file_io_instrument.py b/src/murfey/server/api/file_io_instrument.py index ed42ab5ec..99d55e0d5 100644 --- a/src/murfey/server/api/file_io_instrument.py +++ b/src/murfey/server/api/file_io_instrument.py @@ -160,6 +160,7 @@ async def write_eer_fractionation_file( ) ).all() if session_parameters: + fractionation_params.fractionation = session_parameters[0].eer_fractionation session_parameters[0].eer_fractionation_file = str(file_path) db.add(session_parameters[0]) db.commit() diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 167ab91c3..8d43b98c6 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -336,6 +336,7 @@ class SessionProcessingParameters(SQLModel, table=True): # type: ignore session_id: int = Field(foreign_key="session.id", primary_key=True) gain_ref: str dose_per_frame: float + eer_fractionation: int = 20 eer_fractionation_file: str = "" symmetry: str = "C1" session: Optional[Session] = Relationship( @@ -647,7 +648,7 @@ class SPARelionParameters(SQLModel, table=True): # type: ignore eer_fractionation_file: str = "" symmetry: str particle_diameter: Optional[float] - downscale: bool + downscale: bool = True do_icebreaker_jobs: bool = True boxsize: Optional[int] = 256 small_boxsize: Optional[int] = 64 From a391919a1bc3dd78eab71ef7ff69bf823b288959 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 09:13:22 +0100 Subject: [PATCH 05/13] Restore the lost parameters which may be needed --- src/murfey/client/contexts/spa.py | 2 ++ src/murfey/client/contexts/tomo.py | 7 +++++++ src/murfey/client/multigrid_control.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 7e22776d6..004422bf9 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -265,6 +265,8 @@ def gather_metadata( metadata["image_size_x"] = str(int(metadata["image_size_x"]) * binning_factor) metadata["image_size_y"] = str(int(metadata["image_size_y"]) * binning_factor) metadata["motion_corr_binning"] = 1 if binning_factor_xml == 2 else 2 + metadata["gain_ref"] = environment.gain_ref if environment else None + metadata["dose_per_frame"] = environment.dose_per_frame if environment else None metadata["symmetry"] = (environment.symmetry if environment else None) or "C1" metadata["eer_fractionation"] = ( environment.eer_fractionation if environment else None diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 6219876ce..2fec6eb91 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -600,6 +600,10 @@ def gather_metadata( float(mdoc_data["PixelSpacing"]) * 1e-10 ) mdoc_metadata["motion_corr_binning"] = binning_factor + mdoc_metadata["gain_ref"] = environment.gain_ref if environment else None + mdoc_metadata["dose_per_frame"] = ( + environment.dose_per_frame if environment else None + ) mdoc_metadata["source"] = str(self._basepath) mdoc_metadata["tag"] = str(self._basepath) mdoc_metadata["tilt_series_tag"] = metadata_file.stem @@ -610,6 +614,9 @@ def gather_metadata( mdoc_metadata["file_extension"] = ( f".{mdoc_data_block['SubFramePath'].split('.')[-1]}" ) + mdoc_metadata["eer_fractionation"] = ( + environment.eer_fractionation if environment else None + ) or 20 data_file = mdoc_data_block["SubFramePath"].split("\\")[-1] if data_file.split(".")[-1] == "eer": diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index 99a3db2ae..ce10bced9 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -419,6 +419,10 @@ def _start_dc(self, json, from_form: bool = False): json = json.get("form", {}) # Safely convert all entries into strings, but leave None as-is json = {k: str(v) if v is not None else None for k, v in json.items()} + self._environment.dose_per_frame = json.get("dose_per_frame") + self._environment.gain_ref = json.get("gain_ref") + self._environment.symmetry = json.get("symmetry") + self._environment.eer_fractionation = json.get("eer_fractionation") source = Path(json["source"]) context = self.analysers[source]._context if context: From c81d7da3ff63144db3b1acf173fd804129a9292d Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 09:41:05 +0100 Subject: [PATCH 06/13] Need to initialise the environments with the right attributes --- src/murfey/client/multigrid_control.py | 5 ++++- src/murfey/client/tui/screens.py | 6 ++---- src/murfey/server/api/instrument.py | 2 -- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index ce10bced9..eeacdac4b 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -75,7 +75,10 @@ def __post_init__(self): default_destination=f"{datetime.now().year}", demo=self.demo, visit=self.visit, - data_collection_parameters=self.data_collection_parameters, + dose_per_frame=self.data_collection_parameters.get("dose_per_frame"), + gain_ref=self.data_collection_parameters.get("gain_ref"), + symmetry=self.data_collection_parameters.get("symmetry"), + eer_fractionation=self.data_collection_parameters.get("eer_fractionation"), instrument_name=self.instrument_name, # processing_only_mode=server_routing_prefix_found, ) diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 8e8d4e22a..e32cc082d 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -1062,9 +1062,7 @@ def compose(self): if self.app._multigrid and self.app._processing_enabled: for k in self._context.user_params: params_bulk.append(Label(k.label)) - val = self.app._environment.data_collection_parameters.get( - k.name - ) or str(k.default) + val = getattr(self.app._environment, k.name, str(k.default)) self._user_params[k.name] = val if val in ("true", "True", True): i = Switch(value=True, id=k.name, classes="input-destination") @@ -1174,7 +1172,7 @@ def on_button_pressed(self, event): else: self.app._start_rsyncer(s, d) for k, v in self._user_params.items(): - self.app._environment.data_collection_parameters[k] = v + setattr(self.app._environment.data_collection_parameters, k, v) self.app.pop_screen() self.app.push_screen("main") diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 57fc62937..f2e2ef979 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -199,8 +199,6 @@ async def pass_proc_params_to_instrument_server( "label": label, "params": { "dose_per_frame": proc_params.dose_per_frame, - "extract_downscale": proc_params.extract_downscale, - "particle_diameter": proc_params.particle_diameter, "symmetry": proc_params.symmetry, "eer_fractionation": proc_params.eer_fractionation, "gain_ref": session.current_gain_ref, From 7228715d9e5e40bd71fb253612f72c2ea35327e9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 10:56:46 +0100 Subject: [PATCH 07/13] Set attributes of the environment if they exist --- src/murfey/instrument_server/api.py | 3 +++ src/murfey/util/models.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index 95ff5f506..8fcd46756 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -329,6 +329,9 @@ def register_processing_parameters( controllers[session_id].data_collection_parameters.update( data_collection_parameters[proc_param_block.label] ) + for k, v in proc_param_block.params.dict().items(): + if v is not None and hasattr(controllers[session_id]._environment, k): + setattr(controllers[session_id]._environment, k, v) return {"success": True} diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index d51bbda38..a003e4cb6 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -77,7 +77,7 @@ class RsyncerInfo(BaseModel): class ProcessingParametersSPA(BaseModel): tag: str - dose_per_frame: float + dose_per_frame: Optional[float] gain_ref: Optional[str] experiment_type: str voltage: float From 92479821cb0c5823154ce6d6f32a0293f0959a42 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 11:29:36 +0100 Subject: [PATCH 08/13] Gain is better as a string in the client --- src/murfey/client/instance_environment.py | 2 +- src/murfey/client/tui/app.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index 6e69a043a..df30d1da0 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -48,7 +48,7 @@ class MurfeyInstanceEnvironment(BaseModel): visit: str = "" processing_only_mode: bool = False dose_per_frame: Optional[float] = None - gain_ref: Optional[Path] = None + gain_ref: Optional[str] = None symmetry: Optional[str] = None eer_fractionation: Optional[int] = None superres: bool = False diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index e49ad0138..a95aab13d 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -84,7 +84,7 @@ def __init__( self._environment = environment or MurfeyInstanceEnvironment( urlparse("http://localhost:8000") ) - self._environment.gain_ref = gain_ref + self._environment.gain_ref = str(gain_ref) self._sources = self._environment.sources or [Path(".")] self._url = self._environment.url self._default_destinations = self._environment.default_destinations @@ -217,7 +217,7 @@ def _start_rsyncer( # Set up rsync command rsync_cmd = [ "rsync", - f"{posix_path(self._environment.gain_ref)!r}", + f"{posix_path(Path(self._environment.gain_ref))!r}", f"{self._url.hostname}::{self._machine_config.get('rsync_module', 'data')}/{visit_path}/processing", ] # Encase in bash shell @@ -230,7 +230,7 @@ def _start_rsyncer( gain_rsync = subprocess.run(cmd) if gain_rsync.returncode: log.warning( - f"Gain reference file {posix_path(self._environment.gain_ref)!r} was not successfully transferred to {visit_path}/processing" + f"Gain reference file {posix_path(Path(self._environment.gain_ref))!r} was not successfully transferred to {visit_path}/processing" ) if transfer: self.rsync_processes[source] = RSyncer( From 31802b5e25035e6979c3a657e36b1b1503bdb548 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 11:56:24 +0100 Subject: [PATCH 09/13] None of this is now needed --- src/murfey/util/models.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index a003e4cb6..9c80204d3 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -87,14 +87,8 @@ class ProcessingParametersSPA(BaseModel): motion_corr_binning: int file_extension: str acquisition_software: str - use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] - downscale: bool - small_boxsize: Optional[int] eer_fractionation_file: str = "" - particle_diameter: Optional[float] magnification: Optional[int] = None total_exposed_dose: Optional[float] = None c2aperture: Optional[float] = None @@ -105,12 +99,7 @@ class ProcessingParametersSPA(BaseModel): class Base(BaseModel): dose_per_frame: Optional[float] gain_ref: Optional[str] - use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] - downscale: bool - small_boxsize: Optional[int] eer_fractionation: int From 02a777d753a6578567ea9a67be5184055039c978 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 12:16:59 +0100 Subject: [PATCH 10/13] Remove the form dependencies for old parameters --- src/murfey/client/analyser.py | 31 --------------- src/murfey/client/multigrid_control.py | 1 - src/murfey/client/tui/app.py | 6 +-- src/murfey/client/tui/forms.py | 8 ---- src/murfey/client/tui/screens.py | 54 ++------------------------ 5 files changed, 4 insertions(+), 96 deletions(-) delete mode 100644 src/murfey/client/tui/forms.py diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index d2ed17cb2..dcebfa607 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -21,29 +21,12 @@ from murfey.client.contexts.tomo import TomographyContext from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.client.rsync import RSyncerUpdate, TransferResult -from murfey.client.tui.forms import FormDependency from murfey.util.client import Observer, get_machine_config_client from murfey.util.mdoc import get_block from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo logger = logging.getLogger("murfey.client.analyser") -spa_form_dependencies: dict = { - "use_cryolo": FormDependency( - dependencies={"estimate_particle_diameter": False}, trigger_value=False - ), - "estimate_particle_diameter": FormDependency( - dependencies={ - "use_cryolo": True, - "boxsize": "None", - "small_boxsize": "None", - "mask_diameter": "None", - "particle_diameter": "None", - }, - trigger_value=True, - ), -} - class Analyser(Observer): def __init__( @@ -326,13 +309,6 @@ def _analyse(self): self.notify( { "form": dc_metadata, - "dependencies": ( - spa_form_dependencies - if isinstance( - self._context, SPAModularContext - ) - else {} - ), } ) @@ -388,13 +364,6 @@ def _analyse(self): self.notify( { "form": dc_metadata, - "dependencies": ( - spa_form_dependencies - if isinstance( - self._context, SPAModularContext - ) - else {} - ), } ) elif isinstance( diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index eeacdac4b..fdbd606ad 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -408,7 +408,6 @@ def _data_collection_form(self, response: dict): ) if self._form_values.get("gain_ref") in (None, "None"): self._form_values["gain_ref"] = self._environment.gain_ref - self._form_dependencies = response.get("dependencies", {}) self._data_collection_form_complete = True elif self._register_dc is None: self._data_collection_form_complete = True diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index a95aab13d..b5be7a29b 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -63,7 +63,6 @@ class MurfeyTUI(App): rsync_processes: Dict[Path, RSyncer] = {} analysers: Dict[Path, Analyser] = {} _form_values: dict = reactive({}) - _form_dependencies: dict = {} def __init__( self, @@ -446,7 +445,6 @@ def _data_collection_form(self, response: dict): ) if self._form_values.get("gain_ref") in (None, "None"): self._form_values["gain_ref"] = self._environment.gain_ref - self._form_dependencies = response.get("dependencies", {}) self.processing_btn.disabled = False self._data_collection_form_complete = True elif self._register_dc is None: @@ -602,9 +600,7 @@ async def on_load(self, event): ) def _install_processing_form(self): - self.processing_form = ProcessingForm( - self._form_values, dependencies=self._form_dependencies - ) + self.processing_form = ProcessingForm(self._form_values) self.install_screen(self.processing_form, "processing-form") def on_input_submitted(self, event: Input.Submitted): diff --git a/src/murfey/client/tui/forms.py b/src/murfey/client/tui/forms.py deleted file mode 100644 index ebaed1928..000000000 --- a/src/murfey/client/tui/forms.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import annotations - -from typing import Dict, NamedTuple - - -class FormDependency(NamedTuple): - dependencies: Dict[str, bool | str] - trigger_value: bool = True diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index e32cc082d..3aa8f0a5d 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -46,7 +46,7 @@ ) from werkzeug.utils import secure_filename -from murfey.client.analyser import Analyser, spa_form_dependencies +from murfey.client.analyser import Analyser from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.tomo import TomographyContext from murfey.client.gain_ref import determine_gain_ref @@ -55,7 +55,6 @@ global_env_lock, ) from murfey.client.rsync import RSyncer -from murfey.client.tui.forms import FormDependency from murfey.util import posix_path from murfey.util.api import url_path_for from murfey.util.client import capture_post, get_machine_config_client, read_config @@ -346,9 +345,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) transfer_routes[s] = _default self.app.install_screen( - DestinationSelect( - transfer_routes, self._context, dependencies=spa_form_dependencies - ), + DestinationSelect(transfer_routes, self._context), "destination-select-screen", ) self.app.pop_screen() @@ -423,13 +420,11 @@ def __init__( self, form: dict, *args, - dependencies: Dict[str, FormDependency] | None = None, **kwargs, ): super().__init__(*args, **kwargs) self._form = form self._inputs: Dict[Input, str] = {} - self._dependencies = dependencies or {} def compose(self): inputs = [] @@ -447,8 +442,6 @@ def compose(self): i.value = "None" if default is None else default self._inputs[i] = k.name inputs.append(i) - for i, k in self._inputs.items(): - self._check_dependency(k, i.value) confirm_btn = Button("Confirm", id="confirm-btn") if self._form.get("motion_corr_binning") == "2": self._vert = VerticalScroll( @@ -502,23 +495,6 @@ def on_switch_changed(self, event): else: k = self._inputs[event.switch] self._form[k] = event.value - self._check_dependency(k, event.value) - - def _check_dependency(self, key: str, value: Any): - if x := self._dependencies.get(key): - for d, v in x.dependencies.items(): - if value == x.trigger_value: - self._form[d] = v - for i, dk in self._inputs.items(): - if dk == d: - i.value = v - i.disabled = True - break - else: - for i, dk in self._inputs.items(): - if dk == d: - i.disabled = False - break def on_input_changed(self, event): k = self._inputs[event.input] @@ -967,7 +943,6 @@ def __init__( transfer_routes: Dict[Path, str], context: Type[SPAModularContext] | Type[TomographyContext], *args, - dependencies: Dict[str, FormDependency] | None = None, destination_overrides: Optional[Dict[Path, str]] = None, use_transfer_routes: bool = False, **kwargs, @@ -976,7 +951,6 @@ def __init__( self._transfer_routes = transfer_routes self._destination_overrides: Dict[Path, str] = destination_overrides or {} self._user_params: Dict[str, str] = {} - self._dependencies = dependencies or {} self._inputs: Dict[Input, str] = {} self._context = context self._use_transfer_routes = use_transfer_routes @@ -1089,30 +1063,12 @@ def compose(self): ) ) self.app._environment.superres = False - for i, k in self._inputs.items(): - self._check_dependency(k, i.value) yield VerticalScroll( *params_bulk, id="user-params", ) yield Button("Confirm", id="destination-btn") - def _check_dependency(self, key: str, value: Any): - if x := self._dependencies.get(key): - for d, v in x.dependencies.items(): - if value == x.trigger_value: - self._user_params[d] = str(v) - for i, dk in self._inputs.items(): - if dk == d: - i.value = v - i.disabled = True - break - else: - for i, dk in self._inputs.items(): - if dk == d: - i.disabled = False - break - def on_switch_changed(self, event): if event.switch.id == "superres-multigrid": self.app._environment.superres = event.value @@ -1120,7 +1076,6 @@ def on_switch_changed(self, event): for k in self._context.user_params: if event.switch.id == k.name: self._user_params[k.name] = event.value - self._check_dependency(k.name, event.value) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: if event.index == 0: @@ -1133,7 +1088,6 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: DestinationSelect( self._transfer_routes, self._context, - dependencies=spa_form_dependencies, destination_overrides=self._destination_overrides, use_transfer_routes=True, ), @@ -1249,9 +1203,7 @@ def compose(self): if len(self.app.visits) else [Button("No ongoing visits found")] ) - self.app.processing_form = ProcessingForm( - self.app._form_values, dependencies=self.app._form_dependencies - ) + self.app.processing_form = ProcessingForm(self.app._form_values) yield Header() info_widget = RichLog(id="info", markup=True) yield info_widget From 4b739980e6db8358bb5638f6f64df1f1b713b4e0 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 13:55:52 +0100 Subject: [PATCH 11/13] No particle diameter is now provided --- src/murfey/server/feedback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 973241473..e907a04ca 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -2245,7 +2245,7 @@ def feedback_callback(header: dict, message: dict) -> None: ) feedback_params = db.SPAFeedbackParameters( pj_id=collected_ids[2].id, - estimate_particle_diameter=not bool(message["particle_diameter"]), + estimate_particle_diameter=True, hold_class2d=False, hold_class3d=False, class_selection_score=0, From 5f7e5c8ad2529d8228cde4d68078a58282458ef2 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 11 Jun 2025 16:13:41 +0100 Subject: [PATCH 12/13] Remove dc params which was forgotten about --- src/murfey/client/tui/screens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 3aa8f0a5d..f20eb5bb0 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -1126,7 +1126,7 @@ def on_button_pressed(self, event): else: self.app._start_rsyncer(s, d) for k, v in self._user_params.items(): - setattr(self.app._environment.data_collection_parameters, k, v) + setattr(self.app._environment, k, v) self.app.pop_screen() self.app.push_screen("main") From c3e628dc85d24c8939523a41d47165da9dd58645 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 12 Jun 2025 10:49:55 +0100 Subject: [PATCH 13/13] Rename json variable --- src/murfey/client/multigrid_control.py | 61 ++++++++++++++------------ src/murfey/client/tui/app.py | 61 ++++++++++++++------------ 2 files changed, 66 insertions(+), 56 deletions(-) diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index fdbd606ad..91e969bff 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -412,29 +412,31 @@ def _data_collection_form(self, response: dict): elif self._register_dc is None: self._data_collection_form_complete = True - def _start_dc(self, json, from_form: bool = False): + def _start_dc(self, metadata_json, from_form: bool = False): if self.dummy_dc: return # for multigrid the analyser sends the message straight to _start_dc by-passing user input # it is then necessary to extract the data from the message if from_form: - json = json.get("form", {}) + metadata_json = metadata_json.get("form", {}) # Safely convert all entries into strings, but leave None as-is - json = {k: str(v) if v is not None else None for k, v in json.items()} - self._environment.dose_per_frame = json.get("dose_per_frame") - self._environment.gain_ref = json.get("gain_ref") - self._environment.symmetry = json.get("symmetry") - self._environment.eer_fractionation = json.get("eer_fractionation") - source = Path(json["source"]) + metadata_json = { + k: str(v) if v is not None else None for k, v in metadata_json.items() + } + self._environment.dose_per_frame = metadata_json.get("dose_per_frame") + self._environment.gain_ref = metadata_json.get("gain_ref") + self._environment.symmetry = metadata_json.get("symmetry") + self._environment.eer_fractionation = metadata_json.get("eer_fractionation") + source = Path(metadata_json["source"]) context = self.analysers[source]._context if context: context.data_collection_parameters = { - k: None if v == "None" else v for k, v in json.items() + k: None if v == "None" else v for k, v in metadata_json.items() } if isinstance(context, TomographyContext): - source = Path(json["source"]) + source = Path(metadata_json["source"]) context.register_tomography_data_collections( - file_extension=json["file_extension"], + file_extension=metadata_json["file_extension"], image_directory=str(self._environment.default_destinations[source]), environment=self._environment, ) @@ -453,10 +455,10 @@ def _start_dc(self, json, from_form: bool = False): }, ) eer_fractionation_file = eer_response.json()["eer_fractionation_file"] - json.update({"eer_fractionation_file": eer_fractionation_file}) + metadata_json.update({"eer_fractionation_file": eer_fractionation_file}) capture_post( f"{self._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'register_tomo_proc_params', session_id=self._environment.murfey_session)}", - json=json, + json=metadata_json, ) capture_post( f"{self._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'flush_tomography_processing', visit_name=self._environment.visit, session_id=self._environment.murfey_session)}", @@ -484,24 +486,24 @@ def _start_dc(self, json, from_form: bool = False): capture_post(url, json=dcg_data) if from_form: data = { - "voltage": json["voltage"], - "pixel_size_on_image": json["pixel_size_on_image"], - "experiment_type": json["experiment_type"], - "image_size_x": json["image_size_x"], - "image_size_y": json["image_size_y"], - "file_extension": json["file_extension"], - "acquisition_software": json["acquisition_software"], + "voltage": metadata_json["voltage"], + "pixel_size_on_image": metadata_json["pixel_size_on_image"], + "experiment_type": metadata_json["experiment_type"], + "image_size_x": metadata_json["image_size_x"], + "image_size_y": metadata_json["image_size_y"], + "file_extension": metadata_json["file_extension"], + "acquisition_software": metadata_json["acquisition_software"], "image_directory": str( self._environment.default_destinations[source] ), "tag": str(source), "source": str(source), - "magnification": json["magnification"], - "total_exposed_dose": json.get("total_exposed_dose"), - "c2aperture": json.get("c2aperture"), - "exposure_time": json.get("exposure_time"), - "slit_width": json.get("slit_width"), - "phase_plate": json.get("phase_plate", False), + "magnification": metadata_json["magnification"], + "total_exposed_dose": metadata_json.get("total_exposed_dose"), + "c2aperture": metadata_json.get("c2aperture"), + "exposure_time": metadata_json.get("exposure_time"), + "slit_width": metadata_json.get("slit_width"), + "phase_plate": metadata_json.get("phase_plate", False), } capture_post( f"{str(self._environment.url.geturl())}{url_path_for('workflow.router', 'start_dc', visit_name=self._environment.visit, session_id=self.session_id)}", @@ -522,11 +524,14 @@ def _start_dc(self, json, from_form: bool = False): "recipe": recipe, }, ) - log.info(f"Posting SPA processing parameters: {json}") + log.info(f"Posting SPA processing parameters: {metadata_json}") response = capture_post( f"{self._environment.url.geturl()}{url_path_for('workflow.spa_router', 'register_spa_proc_params', session_id=self.session_id)}", json={ - **{k: None if v == "None" else v for k, v in json.items()}, + **{ + k: None if v == "None" else v + for k, v in metadata_json.items() + }, "tag": str(source), }, ) diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index b5be7a29b..7c947dc54 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -460,28 +460,30 @@ def _start_dc_confirm_prompt(self, json: dict): ) ) - def _start_dc(self, json, from_form: bool = False): + def _start_dc(self, metadata_json, from_form: bool = False): if self._dummy_dc: return # for multigrid the analyser sends the message straight to _start_dc by-passing user input # it is then necessary to extract the data from the message if from_form: - json = json.get("form", {}) - json = {k: v if v is None else str(v) for k, v in json.items()} - self._environment.dose_per_frame = json.get("dose_per_frame") - self._environment.gain_ref = json.get("gain_ref") - self._environment.symmetry = json.get("symmetry") - self._environment.eer_fractionation = json.get("eer_fractionation") - source = Path(json["source"]) + metadata_json = metadata_json.get("form", {}) + metadata_json = { + k: v if v is None else str(v) for k, v in metadata_json.items() + } + self._environment.dose_per_frame = metadata_json.get("dose_per_frame") + self._environment.gain_ref = metadata_json.get("gain_ref") + self._environment.symmetry = metadata_json.get("symmetry") + self._environment.eer_fractionation = metadata_json.get("eer_fractionation") + source = Path(metadata_json["source"]) context = self.analysers[source]._context if context: context.data_collection_parameters = { - k: None if v == "None" else v for k, v in json.items() + k: None if v == "None" else v for k, v in metadata_json.items() } if isinstance(context, TomographyContext): - source = Path(json["source"]) + source = Path(metadata_json["source"]) context.register_tomography_data_collections( - file_extension=json["file_extension"], + file_extension=metadata_json["file_extension"], image_directory=str(self._environment.default_destinations[source]), environment=self._environment, ) @@ -500,10 +502,10 @@ def _start_dc(self, json, from_form: bool = False): }, ) eer_fractionation_file = eer_response.json()["eer_fractionation_file"] - json.update({"eer_fractionation_file": eer_fractionation_file}) + metadata_json.update({"eer_fractionation_file": eer_fractionation_file}) requests.post( f"{self.app._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'register_tomo_proc_params', session_id=self.app._environment.murfey_session)}", - json=json, + json=metadata_json, ) capture_post( f"{self.app._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'flush_tomography_processing', visit_name=self._visit, session_id=self.app._environment.murfey_session)}", @@ -530,24 +532,24 @@ def _start_dc(self, json, from_form: bool = False): capture_post(url, json=dcg_data) if from_form: data = { - "voltage": json["voltage"], - "pixel_size_on_image": json["pixel_size_on_image"], - "experiment_type": json["experiment_type"], - "image_size_x": json["image_size_x"], - "image_size_y": json["image_size_y"], - "file_extension": json["file_extension"], - "acquisition_software": json["acquisition_software"], + "voltage": metadata_json["voltage"], + "pixel_size_on_image": metadata_json["pixel_size_on_image"], + "experiment_type": metadata_json["experiment_type"], + "image_size_x": metadata_json["image_size_x"], + "image_size_y": metadata_json["image_size_y"], + "file_extension": metadata_json["file_extension"], + "acquisition_software": metadata_json["acquisition_software"], "image_directory": str( self._environment.default_destinations[source] ), "tag": str(source), "source": str(source), - "magnification": json["magnification"], - "total_exposed_dose": json.get("total_exposed_dose"), - "c2aperture": json.get("c2aperture"), - "exposure_time": json.get("exposure_time"), - "slit_width": json.get("slit_width"), - "phase_plate": json.get("phase_plate", False), + "magnification": metadata_json["magnification"], + "total_exposed_dose": metadata_json.get("total_exposed_dose"), + "c2aperture": metadata_json.get("c2aperture"), + "exposure_time": metadata_json.get("exposure_time"), + "slit_width": metadata_json.get("slit_width"), + "phase_plate": metadata_json.get("phase_plate", False), } capture_post( f"{str(self._url.geturl())}{url_path_for('workflow.router', 'start_dc', visit_name=self._visit, session_id=self._environment.murfey_session)}", @@ -568,11 +570,14 @@ def _start_dc(self, json, from_form: bool = False): "recipe": recipe, }, ) - log.info(f"Posting SPA processing parameters: {json}") + log.info(f"Posting SPA processing parameters: {metadata_json}") response = capture_post( f"{self.app._environment.url.geturl()}{url_path_for('workflow.spa_router', 'register_spa_proc_params', session_id=self.app._environment.murfey_session)}", json={ - **{k: None if v == "None" else v for k, v in json.items()}, + **{ + k: None if v == "None" else v + for k, v in metadata_json.items() + }, "tag": str(source), }, )