diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index e90da8aad..dcebfa607 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -21,29 +21,12 @@ from murfey.client.contexts.tomo import TomographyContext from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.client.rsync import RSyncerUpdate, TransferResult -from murfey.client.tui.forms import FormDependency from murfey.util.client import Observer, get_machine_config_client from murfey.util.mdoc import get_block from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo logger = logging.getLogger("murfey.client.analyser") -spa_form_dependencies: dict = { - "use_cryolo": FormDependency( - dependencies={"estimate_particle_diameter": False}, trigger_value=False - ), - "estimate_particle_diameter": FormDependency( - dependencies={ - "use_cryolo": True, - "boxsize": "None", - "small_boxsize": "None", - "mask_diameter": "None", - "particle_diameter": "None", - }, - trigger_value=True, - ), -} - class Analyser(Observer): def __init__( @@ -85,7 +68,7 @@ def __init__( def __repr__(self) -> str: return f"" - def _find_extension(self, file_path: Path): + def _find_extension(self, file_path: Path) -> bool: """ Identifies the file extension and stores that information in the class. """ @@ -101,22 +84,17 @@ def _find_extension(self, file_path: Path): .get(file_path.suffix) ): if not any(r in file_path.name for r in required_substrings): - return [] + return False - # Checks for MRC, TIFF, TIF, and EER files if no extension has been defined - if ( - file_path.suffix in (".mrc", ".tiff", ".tif", ".eer") - and not self._extension - ): - logger.info(f"File extension determined: {file_path.suffix}") - self._extension = file_path.suffix - # Check for TIFF, TIF, or EER if the file's already been assigned an extension - elif ( - file_path.suffix in (".tiff", ".tif", ".eer") - and self._extension != file_path.suffix - ): - logger.info(f"File extension re-evaluated: {file_path.suffix}") - self._extension = file_path.suffix + # Checks for MRC, TIFF, TIF, and EER files + if file_path.suffix in (".mrc", ".tiff", ".tif", ".eer"): + if not self._extension: + logger.info(f"File extension determined: {file_path.suffix}") + self._extension = file_path.suffix + elif self._extension != file_path.suffix: + logger.info(f"File extension re-evaluated: {file_path.suffix}") + self._extension = file_path.suffix + return True # If we see an .mdoc file first, use that to determine the file extensions elif file_path.suffix == ".mdoc": with open(file_path, "r") as md: @@ -124,9 +102,12 @@ def _find_extension(self, file_path: Path): mdoc_data_block = get_block(md) if subframe_path := mdoc_data_block.get("SubFramePath"): self._extension = Path(subframe_path).suffix + return True # Check for LIF files separately elif file_path.suffix == ".lif": self._extension = file_path.suffix + return True + return False def _find_context(self, file_path: Path) -> bool: """ @@ -269,7 +250,10 @@ def _analyse(self): elif transferred_file.suffix == ".mdoc": mdoc_for_reading = transferred_file if not self._context: - self._find_extension(transferred_file) + valid_extension = self._find_extension(transferred_file) + if not valid_extension: + logger.error(f"No extension found for {transferred_file}") + break found = self._find_context(transferred_file) if not found: logger.debug( @@ -325,13 +309,6 @@ def _analyse(self): self.notify( { "form": dc_metadata, - "dependencies": ( - spa_form_dependencies - if isinstance( - self._context, SPAModularContext - ) - else {} - ), } ) @@ -344,7 +321,10 @@ def _analyse(self): # Handle files with tomography and SPA context differently elif not self._extension or self._unseen_xml: - self._find_extension(transferred_file) + valid_extension = self._find_extension(transferred_file) + if not valid_extension: + logger.error(f"No extension found for {transferred_file}") + break if self._extension: logger.info( f"Extension found successfully for {transferred_file}" @@ -384,13 +364,6 @@ def _analyse(self): self.notify( { "form": dc_metadata, - "dependencies": ( - spa_form_dependencies - if isinstance( - self._context, SPAModularContext - ) - else {} - ), } ) elif isinstance( diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 7077ab861..14ec8553f 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -39,6 +39,7 @@ class Context: def __init__(self, name: str, acquisition_software: str): self._acquisition_software = acquisition_software self.name = name + self.data_collection_parameters: dict = {} def post_transfer( self, diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 6b249c8a3..004422bf9 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from datetime import datetime from itertools import count from pathlib import Path from typing import Any, Dict, List, Optional, OrderedDict, Tuple @@ -109,27 +108,9 @@ class SPAModularContext(Context): "Dose Per Frame [e- / Angstrom^2 / frame] (after EER grouping if relevant)", default=1, ), - ProcessingParameter( - "estimate_particle_diameter", - "Use crYOLO to Estimate Particle Diameter", - default=True, - ), - ProcessingParameter( - "particle_diameter", "Particle Diameter (Angstroms)", default=None - ), - ProcessingParameter("use_cryolo", "Use crYOLO Autopicking", default=True), ProcessingParameter("symmetry", "Symmetry Group", default="C1"), ProcessingParameter("eer_fractionation", "EER Fractionation", default=20), - ProcessingParameter( - "mask_diameter", "Mask Diameter (2D classification)", default=190 - ), - ProcessingParameter("boxsize", "Box Size", default=256), - ProcessingParameter("downscale", "Downscale Extracted Particles", default=True), - ProcessingParameter( - "small_boxsize", "Downscaled Extracted Particle Size (pixels)", default=128 - ), ProcessingParameter("gain_ref", "Gain Reference"), - ProcessingParameter("gain_ref_superres", "Unbinned Gain Reference"), ] metadata_params = [ ProcessingParameter("voltage", "Voltage"), @@ -164,7 +145,6 @@ def gather_metadata( return OrderedDict({}) data = xmltodict.parse(for_parsing) magnification = 0 - num_fractions = 1 metadata: OrderedDict = OrderedDict({}) metadata["experiment_type"] = "SPA" if data.get("Acquisition"): @@ -217,14 +197,6 @@ def gather_metadata( ) # convert e / m^2 to e / A^2 except ValueError: metadata["total_exposed_dose"] = 1 - try: - num_fractions = int( - data["MicroscopeImage"]["microscopeData"]["acquisition"]["camera"][ - "CameraSpecificInput" - ]["a:KeyValueOfstringanyType"][2]["a:Value"]["b:NumberOffractions"] - ) - except (KeyError, IndexError): - pass c2_index = 3 for i, el in enumerate( data["MicroscopeImage"]["CustomData"]["a:KeyValueOfstringanyType"] @@ -293,85 +265,13 @@ def gather_metadata( metadata["image_size_x"] = str(int(metadata["image_size_x"]) * binning_factor) metadata["image_size_y"] = str(int(metadata["image_size_y"]) * binning_factor) metadata["motion_corr_binning"] = 1 if binning_factor_xml == 2 else 2 - if environment: - metadata["gain_ref"] = ( - environment.data_collection_parameters.get("gain_ref") - if environment - and environment.data_collection_parameters.get("gain_ref") - not in (None, "None") - else f"{datetime.now().year}/{environment.visit}/processing/gain.mrc" - ) - metadata["gain_ref_superres"] = ( - environment.data_collection_parameters.get("gain_ref_superres") - if environment - and environment.data_collection_parameters.get("gain_ref_superres") - not in (None, "None") - else f"{datetime.now().year}/{environment.visit}/processing/gain_superres.mrc" - ) - else: - metadata["gain_ref"] = None - metadata["gain_ref_superres"] = None - if metadata.get("total_exposed_dose"): - metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - and environment.data_collection_parameters.get("dose_per_frame") - not in (None, "None") - else round(metadata["total_exposed_dose"] / num_fractions, 3) - ) - else: - metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - else None - ) - - metadata["use_cryolo"] = ( - environment.data_collection_parameters.get("use_cryolo") - if environment - else None - ) or True - metadata["symmetry"] = ( - environment.data_collection_parameters.get("symmetry") - if environment - else None - ) or "C1" - metadata["mask_diameter"] = ( - environment.data_collection_parameters.get("mask_diameter") - if environment - else None - ) or 190 - metadata["boxsize"] = ( - environment.data_collection_parameters.get("boxsize") - if environment - else None - ) or 256 - metadata["downscale"] = ( - environment.data_collection_parameters.get("downscale") - if environment - else None - ) or True - metadata["small_boxsize"] = ( - environment.data_collection_parameters.get("small_boxsize") - if environment - else None - ) or 128 + metadata["gain_ref"] = environment.gain_ref if environment else None + metadata["dose_per_frame"] = environment.dose_per_frame if environment else None + metadata["symmetry"] = (environment.symmetry if environment else None) or "C1" metadata["eer_fractionation"] = ( - environment.data_collection_parameters.get("eer_fractionation") - if environment - else None + environment.eer_fractionation if environment else None ) or 20 metadata["source"] = str(self._basepath) - metadata["particle_diameter"] = ( - environment.data_collection_parameters.get("particle_diameter") - if environment - else None - ) or 0 - metadata["estimate_particle_diameter"] = ( - environment.data_collection_parameters.get("estimate_particle_diameter") - if environment - else None - ) or True return metadata def _position_analysis( @@ -584,10 +484,10 @@ def post_transfer( f"{str(environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=environment.visit, session_id=environment.murfey_session)}", json={ "eer_path": str(file_transferred_to), - "fractionation": environment.data_collection_parameters[ + "fractionation": self.data_collection_parameters[ "eer_fractionation" ], - "dose_per_frame": environment.data_collection_parameters[ + "dose_per_frame": self.data_collection_parameters[ "dose_per_frame" ], "fractionation_file_name": "eer_fractionation_spa.txt", @@ -619,20 +519,16 @@ def post_transfer( "image_number": environment.movies[ file_transferred_to ].movie_number, - "pixel_size": environment.data_collection_parameters.get( + "pixel_size": self.data_collection_parameters.get( "pixel_size_on_image" ), "autoproc_program_id": None, - "dose_per_frame": environment.data_collection_parameters.get( - "dose_per_frame" - ), - "mc_binning": environment.data_collection_parameters.get( + "dose_per_frame": environment.dose_per_frame, + "mc_binning": self.data_collection_parameters.get( "motion_corr_binning", 1 ), - "gain_ref": environment.data_collection_parameters.get( - "gain_ref" - ), - "extract_downscale": environment.data_collection_parameters.get( + "gain_ref": environment.gain_ref, + "extract_downscale": self.data_collection_parameters.get( "downscale", True ), "eer_fractionation_file": eer_fractionation_file, diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 9562041f7..2fec6eb91 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from datetime import datetime from pathlib import Path from threading import RLock from typing import Callable, Dict, List, OrderedDict @@ -133,25 +132,23 @@ def register_tomography_data_collections( "tag": tilt_series, } if ( - environment.data_collection_parameters - and environment.data_collection_parameters.get("voltage") + self.data_collection_parameters + and self.data_collection_parameters.get("voltage") ): # Once mdoc parameters are known register processing jobs dc_data.update( { - "voltage": environment.data_collection_parameters[ - "voltage" - ], - "pixel_size_on_image": environment.data_collection_parameters[ + "voltage": self.data_collection_parameters["voltage"], + "pixel_size_on_image": self.data_collection_parameters[ "pixel_size_on_image" ], - "image_size_x": environment.data_collection_parameters[ + "image_size_x": self.data_collection_parameters[ "image_size_x" ], - "image_size_y": environment.data_collection_parameters[ + "image_size_y": self.data_collection_parameters[ "image_size_y" ], - "magnification": environment.data_collection_parameters[ + "magnification": self.data_collection_parameters[ "magnification" ], } @@ -176,7 +173,7 @@ def register_tomography_data_collections( ) except Exception as e: - logger.error(f"ERROR {e}, {environment.data_collection_parameters}") + logger.error(f"ERROR {e}, {self.data_collection_parameters}") def _file_transferred_to( self, environment: MurfeyInstanceEnvironment, source: Path, file_path: Path @@ -315,19 +312,15 @@ def _add_tilt( capture_post(tilt_url, json=tilt_data) eer_fractionation_file = None - if environment.data_collection_parameters.get("num_eer_frames"): + if self.data_collection_parameters.get("num_eer_frames"): response = requests.post( f"{str(environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=environment.visit, session_id=environment.murfey_session)}", json={ - "num_frames": environment.data_collection_parameters[ - "num_eer_frames" - ], - "fractionation": environment.data_collection_parameters[ + "num_frames": self.data_collection_parameters["num_eer_frames"], + "fractionation": self.data_collection_parameters[ "eer_fractionation" ], - "dose_per_frame": environment.data_collection_parameters[ - "dose_per_frame" - ], + "dose_per_frame": environment.dose_per_frame or 0, "fractionation_file_name": "eer_fractionation_tomo.txt", }, ) @@ -337,23 +330,17 @@ def _add_tilt( "path": str(file_transferred_to), "description": "", "image_number": environment.movies[file_transferred_to].movie_number, - "pixel_size": environment.data_collection_parameters.get( + "pixel_size": self.data_collection_parameters.get( "pixel_size_on_image", 0 ), - "dose_per_frame": environment.data_collection_parameters.get( - "dose_per_frame", 0 - ), - "frame_count": environment.data_collection_parameters.get( - "frame_count", 0 - ), - "tilt_axis": environment.data_collection_parameters.get( - "tilt_axis", 85 - ), - "mc_binning": environment.data_collection_parameters.get( + "dose_per_frame": environment.dose_per_frame or 0, + "frame_count": self.data_collection_parameters.get("frame_count", 0), + "tilt_axis": self.data_collection_parameters.get("tilt_axis", 85), + "mc_binning": self.data_collection_parameters.get( "motion_corr_binning", 1 ), - "gain_ref": environment.data_collection_parameters.get("gain_ref"), - "voltage": environment.data_collection_parameters.get("voltage", 300), + "gain_ref": environment.gain_ref, + "voltage": self.data_collection_parameters.get("voltage", 300), "eer_fractionation_file": eer_fractionation_file, "tag": tilt_series, "group_tag": str(self._basepath), @@ -565,9 +552,7 @@ def gather_metadata( metadata["motion_corr_binning"] = 1 metadata["gain_ref"] = None metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - else None + environment.dose_per_frame if environment else None ) metadata["source"] = str(self._basepath) except KeyError: @@ -615,19 +600,9 @@ def gather_metadata( float(mdoc_data["PixelSpacing"]) * 1e-10 ) mdoc_metadata["motion_corr_binning"] = binning_factor - if environment: - mdoc_metadata["gain_ref"] = ( - environment.data_collection_parameters.get("gain_ref") - if environment.data_collection_parameters.get("gain_ref") - not in (None, "None") - else f"{datetime.now().year}/{environment.visit}/processing/gain.mrc" - ) - else: - mdoc_metadata["gain_ref"] = None + mdoc_metadata["gain_ref"] = environment.gain_ref if environment else None mdoc_metadata["dose_per_frame"] = ( - environment.data_collection_parameters.get("dose_per_frame") - if environment - else None + environment.dose_per_frame if environment else None ) mdoc_metadata["source"] = str(self._basepath) mdoc_metadata["tag"] = str(self._basepath) @@ -640,9 +615,7 @@ def gather_metadata( f".{mdoc_data_block['SubFramePath'].split('.')[-1]}" ) mdoc_metadata["eer_fractionation"] = ( - environment.data_collection_parameters.get("eer_fractionation") - if environment - else None + environment.eer_fractionation if environment else None ) or 20 data_file = mdoc_data_block["SubFramePath"].split("\\")[-1] diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index a715a1d1b..df30d1da0 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -41,14 +41,16 @@ class MurfeyInstanceEnvironment(BaseModel): destination_registry: Dict[str, str] = {} watchers: Dict[Path, DirWatcher] = {} demo: bool = False - data_collection_parameters: dict = {} movies: Dict[Path, MovieTracker] = {} movie_tilt_pair: Dict[Path, str] = {} tilt_angles: Dict[str, List[List[str]]] = {} movie_counters: Dict[str, itertools.count] = {} visit: str = "" processing_only_mode: bool = False - gain_ref: Optional[Path] = None + dose_per_frame: Optional[float] = None + gain_ref: Optional[str] = None + symmetry: Optional[str] = None + eer_fractionation: Optional[int] = None superres: bool = False murfey_session: Optional[int] = None samples: Dict[Path, SampleInfo] = {} @@ -64,9 +66,11 @@ def clear(self): for w in self.watchers.values(): w.stop() self.watchers = {} - self.data_collection_parameters = {} self.movies = {} self.movie_tilt_pair = {} self.tilt_angles = {} self.visit = "" + self.dose_per_frame = None self.gain_ref = None + self.symmetry = None + self.eer_fractionation = None diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index d32aa8747..91e969bff 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -75,7 +75,10 @@ def __post_init__(self): default_destination=f"{datetime.now().year}", demo=self.demo, visit=self.visit, - data_collection_parameters=self.data_collection_parameters, + dose_per_frame=self.data_collection_parameters.get("dose_per_frame"), + gain_ref=self.data_collection_parameters.get("gain_ref"), + symmetry=self.data_collection_parameters.get("symmetry"), + eer_fractionation=self.data_collection_parameters.get("eer_fractionation"), instrument_name=self.instrument_name, # processing_only_mode=server_routing_prefix_found, ) @@ -401,61 +404,61 @@ def _data_collection_form(self, response: dict): if self._register_dc and response.get("form"): self._form_values = {k: str(v) for k, v in response.get("form", {}).items()} log.info( - f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.data_collection_parameters.get('gain_ref')}" + f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.gain_ref}" ) if self._form_values.get("gain_ref") in (None, "None"): - self._form_values["gain_ref"] = ( - self._environment.data_collection_parameters.get("gain_ref") - ) - self._form_dependencies = response.get("dependencies", {}) + self._form_values["gain_ref"] = self._environment.gain_ref self._data_collection_form_complete = True elif self._register_dc is None: self._data_collection_form_complete = True - def _start_dc(self, json, from_form: bool = False): + def _start_dc(self, metadata_json, from_form: bool = False): if self.dummy_dc: return # for multigrid the analyser sends the message straight to _start_dc by-passing user input # it is then necessary to extract the data from the message if from_form: - json = json.get("form", {}) + metadata_json = metadata_json.get("form", {}) # Safely convert all entries into strings, but leave None as-is - json = {k: str(v) if v is not None else None for k, v in json.items()} - self._environment.data_collection_parameters = { - k: None if v == "None" else v for k, v in json.items() - } - source = Path(json["source"]) + metadata_json = { + k: str(v) if v is not None else None for k, v in metadata_json.items() + } + self._environment.dose_per_frame = metadata_json.get("dose_per_frame") + self._environment.gain_ref = metadata_json.get("gain_ref") + self._environment.symmetry = metadata_json.get("symmetry") + self._environment.eer_fractionation = metadata_json.get("eer_fractionation") + source = Path(metadata_json["source"]) context = self.analysers[source]._context + if context: + context.data_collection_parameters = { + k: None if v == "None" else v for k, v in metadata_json.items() + } if isinstance(context, TomographyContext): - source = Path(json["source"]) + source = Path(metadata_json["source"]) context.register_tomography_data_collections( - file_extension=json["file_extension"], + file_extension=metadata_json["file_extension"], image_directory=str(self._environment.default_destinations[source]), environment=self._environment, ) log.info("Registering tomography processing parameters") - if self._environment.data_collection_parameters.get("num_eer_frames"): + if context.data_collection_parameters.get("num_eer_frames"): eer_response = requests.post( f"{str(self._environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=self._environment.visit, session_id=self._environment.murfey_session)}", json={ - "num_frames": self._environment.data_collection_parameters[ + "num_frames": context.data_collection_parameters[ "num_eer_frames" ], - "fractionation": self._environment.data_collection_parameters[ - "eer_fractionation" - ], - "dose_per_frame": self._environment.data_collection_parameters[ - "dose_per_frame" - ], + "fractionation": self._environment.eer_fractionation, + "dose_per_frame": self._environment.dose_per_frame, "fractionation_file_name": "eer_fractionation_tomo.txt", }, ) eer_fractionation_file = eer_response.json()["eer_fractionation_file"] - json.update({"eer_fractionation_file": eer_fractionation_file}) + metadata_json.update({"eer_fractionation_file": eer_fractionation_file}) capture_post( f"{self._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'register_tomo_proc_params', session_id=self._environment.murfey_session)}", - json=json, + json=metadata_json, ) capture_post( f"{self._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'flush_tomography_processing', visit_name=self._environment.visit, session_id=self._environment.murfey_session)}", @@ -483,24 +486,24 @@ def _start_dc(self, json, from_form: bool = False): capture_post(url, json=dcg_data) if from_form: data = { - "voltage": json["voltage"], - "pixel_size_on_image": json["pixel_size_on_image"], - "experiment_type": json["experiment_type"], - "image_size_x": json["image_size_x"], - "image_size_y": json["image_size_y"], - "file_extension": json["file_extension"], - "acquisition_software": json["acquisition_software"], + "voltage": metadata_json["voltage"], + "pixel_size_on_image": metadata_json["pixel_size_on_image"], + "experiment_type": metadata_json["experiment_type"], + "image_size_x": metadata_json["image_size_x"], + "image_size_y": metadata_json["image_size_y"], + "file_extension": metadata_json["file_extension"], + "acquisition_software": metadata_json["acquisition_software"], "image_directory": str( self._environment.default_destinations[source] ), "tag": str(source), "source": str(source), - "magnification": json["magnification"], - "total_exposed_dose": json.get("total_exposed_dose"), - "c2aperture": json.get("c2aperture"), - "exposure_time": json.get("exposure_time"), - "slit_width": json.get("slit_width"), - "phase_plate": json.get("phase_plate", False), + "magnification": metadata_json["magnification"], + "total_exposed_dose": metadata_json.get("total_exposed_dose"), + "c2aperture": metadata_json.get("c2aperture"), + "exposure_time": metadata_json.get("exposure_time"), + "slit_width": metadata_json.get("slit_width"), + "phase_plate": metadata_json.get("phase_plate", False), } capture_post( f"{str(self._environment.url.geturl())}{url_path_for('workflow.router', 'start_dc', visit_name=self._environment.visit, session_id=self.session_id)}", @@ -521,11 +524,14 @@ def _start_dc(self, json, from_form: bool = False): "recipe": recipe, }, ) - log.info(f"Posting SPA processing parameters: {json}") + log.info(f"Posting SPA processing parameters: {metadata_json}") response = capture_post( f"{self._environment.url.geturl()}{url_path_for('workflow.spa_router', 'register_spa_proc_params', session_id=self.session_id)}", json={ - **{k: None if v == "None" else v for k, v in json.items()}, + **{ + k: None if v == "None" else v + for k, v in metadata_json.items() + }, "tag": str(source), }, ) diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index b6d73ab5b..7c947dc54 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -63,7 +63,6 @@ class MurfeyTUI(App): rsync_processes: Dict[Path, RSyncer] = {} analysers: Dict[Path, Analyser] = {} _form_values: dict = reactive({}) - _form_dependencies: dict = {} def __init__( self, @@ -84,7 +83,7 @@ def __init__( self._environment = environment or MurfeyInstanceEnvironment( urlparse("http://localhost:8000") ) - self._environment.gain_ref = gain_ref + self._environment.gain_ref = str(gain_ref) self._sources = self._environment.sources or [Path(".")] self._url = self._environment.url self._default_destinations = self._environment.default_destinations @@ -217,7 +216,7 @@ def _start_rsyncer( # Set up rsync command rsync_cmd = [ "rsync", - f"{posix_path(self._environment.gain_ref)!r}", + f"{posix_path(Path(self._environment.gain_ref))!r}", f"{self._url.hostname}::{self._machine_config.get('rsync_module', 'data')}/{visit_path}/processing", ] # Encase in bash shell @@ -230,7 +229,7 @@ def _start_rsyncer( gain_rsync = subprocess.run(cmd) if gain_rsync.returncode: log.warning( - f"Gain reference file {posix_path(self._environment.gain_ref)!r} was not successfully transferred to {visit_path}/processing" + f"Gain reference file {posix_path(Path(self._environment.gain_ref))!r} was not successfully transferred to {visit_path}/processing" ) if transfer: self.rsync_processes[source] = RSyncer( @@ -442,13 +441,10 @@ def _data_collection_form(self, response: dict): if self._register_dc and response.get("form"): self._form_values = {k: str(v) for k, v in response.get("form", {}).items()} log.info( - f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.data_collection_parameters.get('gain_ref')}" + f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.gain_ref}" ) if self._form_values.get("gain_ref") in (None, "None"): - self._form_values["gain_ref"] = ( - self._environment.data_collection_parameters.get("gain_ref") - ) - self._form_dependencies = response.get("dependencies", {}) + self._form_values["gain_ref"] = self._environment.gain_ref self.processing_btn.disabled = False self._data_collection_form_complete = True elif self._register_dc is None: @@ -464,49 +460,52 @@ def _start_dc_confirm_prompt(self, json: dict): ) ) - def _start_dc(self, json, from_form: bool = False): + def _start_dc(self, metadata_json, from_form: bool = False): if self._dummy_dc: return # for multigrid the analyser sends the message straight to _start_dc by-passing user input # it is then necessary to extract the data from the message if from_form: - json = json.get("form", {}) - json = {k: v if v is None else str(v) for k, v in json.items()} - self._environment.data_collection_parameters = { - k: None if v == "None" else v for k, v in json.items() - } - source = Path(json["source"]) + metadata_json = metadata_json.get("form", {}) + metadata_json = { + k: v if v is None else str(v) for k, v in metadata_json.items() + } + self._environment.dose_per_frame = metadata_json.get("dose_per_frame") + self._environment.gain_ref = metadata_json.get("gain_ref") + self._environment.symmetry = metadata_json.get("symmetry") + self._environment.eer_fractionation = metadata_json.get("eer_fractionation") + source = Path(metadata_json["source"]) context = self.analysers[source]._context + if context: + context.data_collection_parameters = { + k: None if v == "None" else v for k, v in metadata_json.items() + } if isinstance(context, TomographyContext): - source = Path(json["source"]) + source = Path(metadata_json["source"]) context.register_tomography_data_collections( - file_extension=json["file_extension"], + file_extension=metadata_json["file_extension"], image_directory=str(self._environment.default_destinations[source]), environment=self._environment, ) log.info("Registering tomography processing parameters") - if self.app._environment.data_collection_parameters.get("num_eer_frames"): + if context.data_collection_parameters.get("num_eer_frames"): eer_response = requests.post( f"{str(self.app._environment.url.geturl())}{url_path_for('file_io_instrument.router', 'write_eer_fractionation_file', visit_name=self.app._environment.visit, session_id=self.app._environment.murfey_session)}", json={ - "num_frames": self.app._environment.data_collection_parameters[ + "num_frames": context.data_collection_parameters[ "num_eer_frames" ], - "fractionation": self.app._environment.data_collection_parameters[ - "eer_fractionation" - ], - "dose_per_frame": self.app._environment.data_collection_parameters[ - "dose_per_frame" - ], + "fractionation": self.app._environment.eer_fractionation, + "dose_per_frame": self.app._environment.dose_per_frame, "fractionation_file_name": "eer_fractionation_tomo.txt", }, ) eer_fractionation_file = eer_response.json()["eer_fractionation_file"] - json.update({"eer_fractionation_file": eer_fractionation_file}) + metadata_json.update({"eer_fractionation_file": eer_fractionation_file}) requests.post( f"{self.app._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'register_tomo_proc_params', session_id=self.app._environment.murfey_session)}", - json=json, + json=metadata_json, ) capture_post( f"{self.app._environment.url.geturl()}{url_path_for('workflow.tomo_router', 'flush_tomography_processing', visit_name=self._visit, session_id=self.app._environment.murfey_session)}", @@ -533,24 +532,24 @@ def _start_dc(self, json, from_form: bool = False): capture_post(url, json=dcg_data) if from_form: data = { - "voltage": json["voltage"], - "pixel_size_on_image": json["pixel_size_on_image"], - "experiment_type": json["experiment_type"], - "image_size_x": json["image_size_x"], - "image_size_y": json["image_size_y"], - "file_extension": json["file_extension"], - "acquisition_software": json["acquisition_software"], + "voltage": metadata_json["voltage"], + "pixel_size_on_image": metadata_json["pixel_size_on_image"], + "experiment_type": metadata_json["experiment_type"], + "image_size_x": metadata_json["image_size_x"], + "image_size_y": metadata_json["image_size_y"], + "file_extension": metadata_json["file_extension"], + "acquisition_software": metadata_json["acquisition_software"], "image_directory": str( self._environment.default_destinations[source] ), "tag": str(source), "source": str(source), - "magnification": json["magnification"], - "total_exposed_dose": json.get("total_exposed_dose"), - "c2aperture": json.get("c2aperture"), - "exposure_time": json.get("exposure_time"), - "slit_width": json.get("slit_width"), - "phase_plate": json.get("phase_plate", False), + "magnification": metadata_json["magnification"], + "total_exposed_dose": metadata_json.get("total_exposed_dose"), + "c2aperture": metadata_json.get("c2aperture"), + "exposure_time": metadata_json.get("exposure_time"), + "slit_width": metadata_json.get("slit_width"), + "phase_plate": metadata_json.get("phase_plate", False), } capture_post( f"{str(self._url.geturl())}{url_path_for('workflow.router', 'start_dc', visit_name=self._visit, session_id=self._environment.murfey_session)}", @@ -571,11 +570,14 @@ def _start_dc(self, json, from_form: bool = False): "recipe": recipe, }, ) - log.info(f"Posting SPA processing parameters: {json}") + log.info(f"Posting SPA processing parameters: {metadata_json}") response = capture_post( f"{self.app._environment.url.geturl()}{url_path_for('workflow.spa_router', 'register_spa_proc_params', session_id=self.app._environment.murfey_session)}", json={ - **{k: None if v == "None" else v for k, v in json.items()}, + **{ + k: None if v == "None" else v + for k, v in metadata_json.items() + }, "tag": str(source), }, ) @@ -603,9 +605,7 @@ async def on_load(self, event): ) def _install_processing_form(self): - self.processing_form = ProcessingForm( - self._form_values, dependencies=self._form_dependencies - ) + self.processing_form = ProcessingForm(self._form_values) self.install_screen(self.processing_form, "processing-form") def on_input_submitted(self, event: Input.Submitted): diff --git a/src/murfey/client/tui/forms.py b/src/murfey/client/tui/forms.py deleted file mode 100644 index ebaed1928..000000000 --- a/src/murfey/client/tui/forms.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import annotations - -from typing import Dict, NamedTuple - - -class FormDependency(NamedTuple): - dependencies: Dict[str, bool | str] - trigger_value: bool = True diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 3ff79d5ae..f20eb5bb0 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -46,7 +46,7 @@ ) from werkzeug.utils import secure_filename -from murfey.client.analyser import Analyser, spa_form_dependencies +from murfey.client.analyser import Analyser from murfey.client.contexts.spa import SPAModularContext from murfey.client.contexts.tomo import TomographyContext from murfey.client.gain_ref import determine_gain_ref @@ -55,7 +55,6 @@ global_env_lock, ) from murfey.client.rsync import RSyncer -from murfey.client.tui.forms import FormDependency from murfey.util import posix_path from murfey.util.api import url_path_for from murfey.util.client import capture_post, get_machine_config_client, read_config @@ -346,9 +345,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) transfer_routes[s] = _default self.app.install_screen( - DestinationSelect( - transfer_routes, self._context, dependencies=spa_form_dependencies - ), + DestinationSelect(transfer_routes, self._context), "destination-select-screen", ) self.app.pop_screen() @@ -423,13 +420,11 @@ def __init__( self, form: dict, *args, - dependencies: Dict[str, FormDependency] | None = None, **kwargs, ): super().__init__(*args, **kwargs) self._form = form self._inputs: Dict[Input, str] = {} - self._dependencies = dependencies or {} def compose(self): inputs = [] @@ -447,8 +442,6 @@ def compose(self): i.value = "None" if default is None else default self._inputs[i] = k.name inputs.append(i) - for i, k in self._inputs.items(): - self._check_dependency(k, i.value) confirm_btn = Button("Confirm", id="confirm-btn") if self._form.get("motion_corr_binning") == "2": self._vert = VerticalScroll( @@ -502,23 +495,6 @@ def on_switch_changed(self, event): else: k = self._inputs[event.switch] self._form[k] = event.value - self._check_dependency(k, event.value) - - def _check_dependency(self, key: str, value: Any): - if x := self._dependencies.get(key): - for d, v in x.dependencies.items(): - if value == x.trigger_value: - self._form[d] = v - for i, dk in self._inputs.items(): - if dk == d: - i.value = v - i.disabled = True - break - else: - for i, dk in self._inputs.items(): - if dk == d: - i.disabled = False - break def on_input_changed(self, event): k = self._inputs[event.input] @@ -922,12 +898,9 @@ def on_button_pressed(self, event): log.info( f"Gain reference file {process_gain_response.json().get('gain_ref')}" ) - self.app._environment.data_collection_parameters["gain_ref"] = ( - process_gain_response.json().get("gain_ref") + self.app._environment.gain_ref = process_gain_response.json().get( + "gain_ref" ) - self.app._environment.data_collection_parameters[ - "gain_ref_superres" - ] = process_gain_response.json().get("gain_ref_superres") if self._switch_status: self.app.push_screen("directory-select") else: @@ -970,7 +943,6 @@ def __init__( transfer_routes: Dict[Path, str], context: Type[SPAModularContext] | Type[TomographyContext], *args, - dependencies: Dict[str, FormDependency] | None = None, destination_overrides: Optional[Dict[Path, str]] = None, use_transfer_routes: bool = False, **kwargs, @@ -979,7 +951,6 @@ def __init__( self._transfer_routes = transfer_routes self._destination_overrides: Dict[Path, str] = destination_overrides or {} self._user_params: Dict[str, str] = {} - self._dependencies = dependencies or {} self._inputs: Dict[Input, str] = {} self._context = context self._use_transfer_routes = use_transfer_routes @@ -1065,9 +1036,7 @@ def compose(self): if self.app._multigrid and self.app._processing_enabled: for k in self._context.user_params: params_bulk.append(Label(k.label)) - val = self.app._environment.data_collection_parameters.get( - k.name - ) or str(k.default) + val = getattr(self.app._environment, k.name, str(k.default)) self._user_params[k.name] = val if val in ("true", "True", True): i = Switch(value=True, id=k.name, classes="input-destination") @@ -1094,30 +1063,12 @@ def compose(self): ) ) self.app._environment.superres = False - for i, k in self._inputs.items(): - self._check_dependency(k, i.value) yield VerticalScroll( *params_bulk, id="user-params", ) yield Button("Confirm", id="destination-btn") - def _check_dependency(self, key: str, value: Any): - if x := self._dependencies.get(key): - for d, v in x.dependencies.items(): - if value == x.trigger_value: - self._user_params[d] = str(v) - for i, dk in self._inputs.items(): - if dk == d: - i.value = v - i.disabled = True - break - else: - for i, dk in self._inputs.items(): - if dk == d: - i.disabled = False - break - def on_switch_changed(self, event): if event.switch.id == "superres-multigrid": self.app._environment.superres = event.value @@ -1125,7 +1076,6 @@ def on_switch_changed(self, event): for k in self._context.user_params: if event.switch.id == k.name: self._user_params[k.name] = event.value - self._check_dependency(k.name, event.value) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: if event.index == 0: @@ -1138,7 +1088,6 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: DestinationSelect( self._transfer_routes, self._context, - dependencies=spa_form_dependencies, destination_overrides=self._destination_overrides, use_transfer_routes=True, ), @@ -1177,7 +1126,7 @@ def on_button_pressed(self, event): else: self.app._start_rsyncer(s, d) for k, v in self._user_params.items(): - self.app._environment.data_collection_parameters[k] = v + setattr(self.app._environment, k, v) self.app.pop_screen() self.app.push_screen("main") @@ -1254,9 +1203,7 @@ def compose(self): if len(self.app.visits) else [Button("No ongoing visits found")] ) - self.app.processing_form = ProcessingForm( - self.app._form_values, dependencies=self.app._form_dependencies - ) + self.app.processing_form = ProcessingForm(self.app._form_values) yield Header() info_widget = RichLog(id="info", markup=True) yield info_widget diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index 95ff5f506..8fcd46756 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -329,6 +329,9 @@ def register_processing_parameters( controllers[session_id].data_collection_parameters.update( data_collection_parameters[proc_param_block.label] ) + for k, v in proc_param_block.params.dict().items(): + if v is not None and hasattr(controllers[session_id]._environment, k): + setattr(controllers[session_id]._environment, k, v) return {"success": True} diff --git a/src/murfey/server/api/file_io_instrument.py b/src/murfey/server/api/file_io_instrument.py index ed42ab5ec..99d55e0d5 100644 --- a/src/murfey/server/api/file_io_instrument.py +++ b/src/murfey/server/api/file_io_instrument.py @@ -160,6 +160,7 @@ async def write_eer_fractionation_file( ) ).all() if session_parameters: + fractionation_params.fractionation = session_parameters[0].eer_fractionation session_parameters[0].eer_fractionation_file = str(file_path) db.add(session_parameters[0]) db.commit() diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 34e238208..f2e2ef979 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -178,6 +178,7 @@ async def pass_proc_params_to_instrument_server( dose_per_frame=proc_params.dose_per_frame, gain_ref=session.current_gain_ref, symmetry=proc_params.symmetry, + eer_fractionation=proc_params.eer_fractionation, ) db.add(session_processing_parameters) db.commit() @@ -198,8 +199,6 @@ async def pass_proc_params_to_instrument_server( "label": label, "params": { "dose_per_frame": proc_params.dose_per_frame, - "extract_downscale": proc_params.extract_downscale, - "particle_diameter": proc_params.particle_diameter, "symmetry": proc_params.symmetry, "eer_fractionation": proc_params.eer_fractionation, "gain_ref": session.current_gain_ref, diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 80ca02cbb..e907a04ca 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -2242,15 +2242,10 @@ def feedback_callback(header: dict, message: dict) -> None: motion_corr_binning=message["motion_corr_binning"], eer_fractionation_file=message["eer_fractionation_file"], symmetry=message["symmetry"], - particle_diameter=message["particle_diameter"], - downscale=message["downscale"], - boxsize=message["boxsize"], - small_boxsize=message["small_boxsize"], - mask_diameter=message["mask_diameter"], ) feedback_params = db.SPAFeedbackParameters( pj_id=collected_ids[2].id, - estimate_particle_diameter=not bool(message["particle_diameter"]), + estimate_particle_diameter=True, hold_class2d=False, hold_class3d=False, class_selection_score=0, diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 167ab91c3..8d43b98c6 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -336,6 +336,7 @@ class SessionProcessingParameters(SQLModel, table=True): # type: ignore session_id: int = Field(foreign_key="session.id", primary_key=True) gain_ref: str dose_per_frame: float + eer_fractionation: int = 20 eer_fractionation_file: str = "" symmetry: str = "C1" session: Optional[Session] = Relationship( @@ -647,7 +648,7 @@ class SPARelionParameters(SQLModel, table=True): # type: ignore eer_fractionation_file: str = "" symmetry: str particle_diameter: Optional[float] - downscale: bool + downscale: bool = True do_icebreaker_jobs: bool = True boxsize: Optional[int] = 256 small_boxsize: Optional[int] = 64 diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index d51bbda38..9c80204d3 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -77,7 +77,7 @@ class RsyncerInfo(BaseModel): class ProcessingParametersSPA(BaseModel): tag: str - dose_per_frame: float + dose_per_frame: Optional[float] gain_ref: Optional[str] experiment_type: str voltage: float @@ -87,14 +87,8 @@ class ProcessingParametersSPA(BaseModel): motion_corr_binning: int file_extension: str acquisition_software: str - use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] - downscale: bool - small_boxsize: Optional[int] eer_fractionation_file: str = "" - particle_diameter: Optional[float] magnification: Optional[int] = None total_exposed_dose: Optional[float] = None c2aperture: Optional[float] = None @@ -105,12 +99,7 @@ class ProcessingParametersSPA(BaseModel): class Base(BaseModel): dose_per_frame: Optional[float] gain_ref: Optional[str] - use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] - downscale: bool - small_boxsize: Optional[int] eer_fractionation: int