From e2188f73ecbb6fcb612936bfae3ae99ab78f0775 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 27 Jan 2025 16:38:27 +0000 Subject: [PATCH 1/3] Need frame counts for tomography dose weighting --- src/murfey/client/contexts/tomo.py | 11 +++++++++++ src/murfey/server/__init__.py | 5 +++++ src/murfey/server/api/__init__.py | 11 +++++++++-- src/murfey/util/db.py | 1 + src/murfey/util/models.py | 1 + 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index f09dad951..a3f3b8d64 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -89,6 +89,7 @@ class TomographyContext(Context): ProcessingParameter("image_size_y", "Image Size Y"), ProcessingParameter("pixel_size_on_image", "Pixel Size"), ProcessingParameter("motion_corr_binning", "Motion Correction Binning"), + ProcessingParameter("frame_count", "Number of image frames"), ProcessingParameter("num_eer_frames", "Number of EER Frames"), ] @@ -190,6 +191,9 @@ def _complete_process_file( "dose_per_frame": environment.data_collection_parameters.get( "dose_per_frame" ), + "frame_count": environment.data_collection_parameters.get( + "frame_count", 0 + ), "mc_binning": environment.data_collection_parameters.get( "motion_corr_binning", 1 ), @@ -452,6 +456,9 @@ def _add_tilt( "dose_per_frame": environment.data_collection_parameters.get( "dose_per_frame", 0 ), + "frame_count": environment.data_collection_parameters.get( + "frame_count", 0 + ), "mc_binning": environment.data_collection_parameters.get( "motion_corr_binning", 1 ), @@ -682,6 +689,7 @@ def gather_metadata( mdoc_metadata: OrderedDict = OrderedDict({}) mdoc_metadata["experiment_type"] = "tomography" mdoc_metadata["voltage"] = float(mdoc_data["Voltage"]) + mdoc_metadata["frame_count"] = float(mdoc_data_block["NumSubFrames"]) mdoc_metadata["image_size_x"] = int(mdoc_data["ImageSize"][0]) mdoc_metadata["image_size_y"] = int(mdoc_data["ImageSize"][1]) mdoc_metadata["magnification"] = int(mdoc_data_block["Magnification"]) @@ -749,6 +757,9 @@ def gather_metadata( mdoc_metadata["num_eer_frames"] = murfey.util.eer.num_frames( metadata_file.parent / data_file ) + mdoc_metadata["frame_count"] = int( + mdoc_metadata["eer_fractionation"] / mdoc_metadata["num_eer_frames"] + ) except Exception as e: logger.error(f"Exception encountered in metadata gathering: {str(e)}") return OrderedDict({}) diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 11c4b3b35..1d6c0ec77 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -2012,6 +2012,7 @@ def _flush_tomography_preprocessing(message: dict): "mc_uuid": murfey_ids[0], "ft_bin": proc_params.motion_corr_binning, "fm_dose": proc_params.dose_per_frame, + "frame_count": proc_params.frame_count, "gain_ref": ( str(machine_config.rsync_basepath / proc_params.gain_ref) if proc_params.gain_ref @@ -2474,6 +2475,9 @@ def feedback_callback(header: dict, message: dict) -> None: "dcid": ids.dcid, "appid": ids.appid, "stack_file": str(stack_file), + "dose_per_frame": preproc_params.dose_per_frame, + "frame_count": preproc_params.frame_count, + "kv": preproc_params.voltage, "pixel_size": preproc_params.pixel_size, "manual_tilt_offset": -tilt_offset, "node_creator_queue": machine_config.node_creator_queue, @@ -2831,6 +2835,7 @@ def feedback_callback(header: dict, message: dict) -> None: pixel_size=float(message["pixel_size_on_image"]) * 10**10, voltage=message["voltage"], dose_per_frame=message["dose_per_frame"], + frame_count=message["frame_count"], motion_corr_binning=message["motion_corr_binning"], gain_ref=message["gain_ref"], eer_fractionation_file=message["eer_fractionation_file"], diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py index 7fa7048d1..7be0133a3 100644 --- a/src/murfey/server/api/__init__.py +++ b/src/murfey/server/api/__init__.py @@ -730,6 +730,9 @@ def register_completed_tilt_series( "dcid": ids.dcid, "appid": ids.appid, "stack_file": str(stack_file), + "dose_per_frame": preproc_params.dose_per_frame, + "frame_count": preproc_params.frame_count, + "kv": preproc_params.voltage, "pixel_size": preproc_params.pixel_size, "manual_tilt_offset": -tilt_offset, "node_creator_queue": machine_config.node_creator_queue, @@ -1131,6 +1134,9 @@ async def request_tomography_preprocessing( / str(ppath.stem + "_motion_corrected.mrc") ) mrc_out = Path("/".join(secure_filename(p) for p in mrc_out.parts)) + + recipe_name = machine_config.recipes.get("em-tomo-preprocess", "em-tomo-preprocess") + data_collection = db.exec( select(DataCollectionGroup, DataCollection, ProcessingJob, AutoProcProgram) .where(DataCollectionGroup.session_id == session_id) @@ -1139,7 +1145,7 @@ async def request_tomography_preprocessing( .where(DataCollection.dcg_id == DataCollectionGroup.id) .where(ProcessingJob.dc_id == DataCollection.id) .where(AutoProcProgram.pj_id == ProcessingJob.id) - .where(ProcessingJob.recipe == "em-tomo-preprocess") + .where(ProcessingJob.recipe == recipe_name) ).all() if data_collection: if registered_tilts := db.exec( @@ -1154,7 +1160,7 @@ async def request_tomography_preprocessing( if not mrc_out.parent.exists(): mrc_out.parent.mkdir(parents=True, exist_ok=True) zocalo_message: dict = { - "recipes": ["em-tomo-preprocess"], + "recipes": [recipe_name], "parameters": { "node_creator_queue": machine_config.node_creator_queue, "dcid": dcid, @@ -1169,6 +1175,7 @@ async def request_tomography_preprocessing( "mc_uuid": murfey_ids[0], "ft_bin": proc_file.mc_binning, "fm_dose": proc_file.dose_per_frame, + "frame_count": proc_file.frame_count, "gain_ref": ( str(machine_config.rsync_basepath / proc_file.gain_ref) if proc_file.gain_ref and machine_config.data_transfer_enabled diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index c546c1529..4ba1f90fe 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -458,6 +458,7 @@ class TomographyPreprocessingParameters(SQLModel, table=True): # type: ignore dcg_id: int = Field(primary_key=True, foreign_key="datacollectiongroup.id") pixel_size: float dose_per_frame: float + frame_count: int voltage: int eer_fractionation_file: Optional[str] = None motion_corr_binning: int = 1 diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index e8dd8cac3..9e9673186 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -356,6 +356,7 @@ class CompletedTiltSeries(BaseModel): class PreprocessingParametersTomo(BaseModel): dose_per_frame: float + frame_count: int gain_ref: Optional[str] experiment_type: str voltage: float From ec089a7cb0b7cb0fe58b4d197a7739af3febfcdf Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 28 Jan 2025 09:07:16 +0000 Subject: [PATCH 2/3] Frame count should be int --- src/murfey/client/contexts/tomo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index a3f3b8d64..21507c27b 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -689,7 +689,7 @@ def gather_metadata( mdoc_metadata: OrderedDict = OrderedDict({}) mdoc_metadata["experiment_type"] = "tomography" mdoc_metadata["voltage"] = float(mdoc_data["Voltage"]) - mdoc_metadata["frame_count"] = float(mdoc_data_block["NumSubFrames"]) + mdoc_metadata["frame_count"] = int(mdoc_data_block["NumSubFrames"]) mdoc_metadata["image_size_x"] = int(mdoc_data["ImageSize"][0]) mdoc_metadata["image_size_y"] = int(mdoc_data["ImageSize"][1]) mdoc_metadata["magnification"] = int(mdoc_data_block["Magnification"]) From 17dd60844b78afe647924ad0200c497ec7de77e0 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 30 Jan 2025 16:41:29 +0000 Subject: [PATCH 3/3] Can use existing function to get preproc params --- src/murfey/server/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 1d6c0ec77..b7fdbd763 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -1954,11 +1954,7 @@ def _flush_tomography_preprocessing(message: dict): .where(db.DataCollectionGroup.session_id == session_id) .where(db.DataCollectionGroup.tag == message["data_collection_group_tag"]) ).first() - proc_params = murfey_db.exec( - select(db.TomographyPreprocessingParameters).where( - db.TomographyPreprocessingParameters.dcg_id == collected_ids.id - ) - ).one() + proc_params = get_tomo_preproc_params(collected_ids.id) if not proc_params: visit_name = message["visit_name"].replace("\r\n", "").replace("\n", "") logger.warning(