From 8183266a0a72e0ea27492cfd7c2fd25a540b093e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 5 Mar 2025 14:59:03 +0000 Subject: [PATCH 01/16] Added Helm charts to create an instrument server for TEM workflow --- Helm/Chart.yaml | 1 + .../murfey-instrument-server-tem/Chart.yaml | 4 ++ .../templates/deployment.yaml | 68 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 Helm/charts/murfey-instrument-server-tem/Chart.yaml create mode 100644 Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml index 4fe0f7793..56fa2e596 100644 --- a/Helm/Chart.yaml +++ b/Helm/Chart.yaml @@ -3,6 +3,7 @@ name: murfey-services description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data version: 0.16.12 dependencies: + - name: murfey-instrument-server-clem - name: murfey-instrument-server-clem - name: murfey-server - name: murfey-rsync diff --git a/Helm/charts/murfey-instrument-server-tem/Chart.yaml b/Helm/charts/murfey-instrument-server-tem/Chart.yaml new file mode 100644 index 000000000..f0e3818d1 --- /dev/null +++ b/Helm/charts/murfey-instrument-server-tem/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +name: murfey-instrument-server-tem +description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files +version: 0.16.12 diff --git a/Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml b/Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml new file mode 100644 index 000000000..69fae72ef --- /dev/null +++ b/Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.appName }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.appName }} +spec: + type: LoadBalancer + externalTrafficPolicy: Cluster + ports: + {{- toYaml .Values.servicePorts | nindent 2 }} + selector: + app: {{ .Values.appName }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.appName }} + namespace: {{ .Values.global.namespace }} + labels: + app: {{ .Values.appName }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: {{ .Values.appName }} + template: + metadata: + labels: + app: {{ .Values.appName }} + spec: + securityContext: + runAsUser: {{ .Values.global.runAsUser }} + runAsGroup: {{ .Values.global.runAsGroup }} + volumes: + # Mount config files from secrets + - name: murfey-client-config + secret: + secretName: {{ .Values.global.murfeyClientConfigCLEMSecretName }} + items: + - key: {{ .Values.global.murfeyClientConfigCLEMFileName }} + path: .murfey + # Mount data directories + {{- toYaml .Values.extraVolumes | nindent 8 }} + containers: + - name: {{ .Values.appName }} + image: {{ .Values.image }} + imagePullPolicy: Always + securityContext: + privileged: false + volumeMounts: + # Mount Murfey client config + - name: murfey-client-config + mountPath: /murfey/config/.murfey + subPath: .murfey + readOnly: false + # Mount data directories + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + env: + - name: MURFEY_CLIENT_CONFIG_HOME + value: "/tmp" + ports: + - containerPort: {{ .Values.containerPort }} + command: + {{- toYaml .Values.command | nindent 12 }} + args: + {{- toYaml .Values.args | nindent 12 }} From e511c22bb303378a6066fe9353dbf7183d55176e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 5 Mar 2025 15:03:16 +0000 Subject: [PATCH 02/16] Updated client secret to the TEM one --- .../murfey-instrument-server-tem/templates/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml b/Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml index 69fae72ef..8e023ef32 100644 --- a/Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml +++ b/Helm/charts/murfey-instrument-server-tem/templates/deployment.yaml @@ -37,9 +37,9 @@ spec: # Mount config files from secrets - name: murfey-client-config secret: - secretName: {{ .Values.global.murfeyClientConfigCLEMSecretName }} + secretName: {{ .Values.global.murfeyClientConfigTEMSecretName }} items: - - key: {{ .Values.global.murfeyClientConfigCLEMFileName }} + - key: {{ .Values.global.murfeyClientConfigTEMFileName }} path: .murfey # Mount data directories {{- toYaml .Values.extraVolumes | nindent 8 }} From e7bb542038ec18378c218b61907a64c980ecbae5 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 5 Mar 2025 15:11:37 +0000 Subject: [PATCH 03/16] Added TEM instrument server to master Helm chart --- Helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Helm/Chart.yaml b/Helm/Chart.yaml index 56fa2e596..2a4fd6424 100644 --- a/Helm/Chart.yaml +++ b/Helm/Chart.yaml @@ -4,6 +4,6 @@ description: Umbrella Helm chart for deploying the servers and daemons needed to version: 0.16.12 dependencies: - name: murfey-instrument-server-clem - - name: murfey-instrument-server-clem + - name: murfey-instrument-server-tem - name: murfey-server - name: murfey-rsync From 13526d1af3a3a460c7e5826777a0d60845ef3813 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 6 Mar 2025 17:55:58 +0000 Subject: [PATCH 04/16] Added 'rsync_url' as an attribute to 'MurfeyInstanceEnvironment' class --- src/murfey/client/__init__.py | 5 +++++ src/murfey/client/instance_environment.py | 1 + src/murfey/client/tui/screens.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/__init__.py b/src/murfey/client/__init__.py index 14693da75..ce194ea04 100644 --- a/src/murfey/client/__init__.py +++ b/src/murfey/client/__init__.py @@ -322,6 +322,11 @@ def run(): default_destination=args.destination or str(datetime.now().year), demo=args.demo, processing_only_mode=server_routing_prefix_found, + rsync_url=( + urlparse(machine_data["rsync_url"]).hostname + if machine_data.get("rsync_url") + else "" + ), ) ws.environment = instance_environment diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index 1f8a642e9..cdddba00c 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -52,6 +52,7 @@ class MurfeyInstanceEnvironment(BaseModel): superres: bool = True murfey_session: Optional[int] = None samples: Dict[Path, SampleInfo] = {} + rsync_url: str = "" class Config: validate_assignment: bool = True diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index a9e6499d4..419ec0b10 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -886,7 +886,7 @@ def on_button_pressed(self, event): rsync_cmd = [ "rsync", f"{posix_path(self._dir_tree._gain_reference)!r}", - f"{self.app._environment.url.hostname}::{self.app._machine_config.get('rsync_module', 'data')}/{visit_path}/processing/{secure_filename(self._dir_tree._gain_reference.name)}", + f"{self.app._environment.rsync_url or self.app._environment.url.hostname}::{self.app._machine_config.get('rsync_module', 'data')}/{visit_path}/processing/{secure_filename(self._dir_tree._gain_reference.name)}", ] # Encase in bash shell cmd = ["bash", "-c", " ".join(rsync_cmd)] From 0823ea68fe3c122cf5ea9c877c53805fa9ded375 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 6 Mar 2025 18:17:53 +0000 Subject: [PATCH 05/16] Get instrument name from database lookup instead of using the 'get_microscope()' function --- src/murfey/server/api/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py index 94083b4a2..e4692bd2d 100644 --- a/src/murfey/server/api/__init__.py +++ b/src/murfey/server/api/__init__.py @@ -1055,7 +1055,7 @@ async def request_spa_preprocessing( "mrc_out": str(mrc_out), "pixel_size": proc_params["angpix"], "image_number": proc_file.image_number, - "microscope": get_microscope(), + "microscope": instrument_name, "mc_uuid": murfey_ids[0], "foil_hole_id": foil_hole_id, "ft_bin": proc_params["motion_corr_binning"], @@ -1163,7 +1163,7 @@ async def request_tomography_preprocessing( "pixel_size": (proc_file.pixel_size) * 10**10, "image_number": proc_file.image_number, "kv": int(proc_file.voltage), - "microscope": get_microscope(), + "microscope": instrument_name, "mc_uuid": murfey_ids[0], "ft_bin": proc_file.mc_binning, "fm_dose": proc_file.dose_per_frame, @@ -1374,7 +1374,7 @@ def start_dc( instrument_name ] log.info( - f"Starting data collection on microscope {get_microscope(machine_config=machine_config)} " + f"Starting data collection on microscope {instrument_name!r} " f"with basepath {sanitise(str(machine_config.rsync_basepath))} and directory {sanitise(dc_params.image_directory)}" ) dc_parameters = { @@ -1407,7 +1407,7 @@ def start_dc( { "register": "data_collection", **dc_parameters, - "microscope": get_microscope(machine_config=machine_config), + "microscope": instrument_name, "proposal_code": ispyb_proposal_code, "proposal_number": ispyb_proposal_number, "visit_number": ispyb_visit_number, From 663a6ae9aed3316e2361c8709fe07a068f5bfebc Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 7 Mar 2025 10:39:35 +0000 Subject: [PATCH 06/16] Use MachineConfig 'gain_directory_name' key to set the save path for the EER fractionation file --- src/murfey/server/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py index e4692bd2d..1cb6c5f16 100644 --- a/src/murfey/server/api/__init__.py +++ b/src/murfey/server/api/__init__.py @@ -1551,7 +1551,7 @@ async def write_eer_fractionation_file( Path(machine_config.rsync_basepath) / str(datetime.datetime.now().year) / secure_filename(visit_name) - / "processing" + / machine_config.gain_directory_name / secure_filename(fractionation_params.fractionation_file_name) ) if file_path.is_file(): From b544aee7aa277633b837441ca2c9c085c0e24e67 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 7 Mar 2025 10:42:45 +0000 Subject: [PATCH 07/16] Replaced hard-coded 'processing' with 'gain_directory_name' value from MachineConfig in the demo API --- src/murfey/server/demo_api.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index 18f08797d..6f3bff0e3 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -1576,9 +1576,17 @@ async def process_gain( else: return {"gain_ref": None} gain_ref_out = ( - (filepath / "processing" / f"gain_{gain_reference_params.tag}.mrc") + ( + filepath + / machine_config.get("gain_directory_name", "processing") + / f"gain_{gain_reference_params.tag}.mrc" + ) if gain_reference_params.tag - else (filepath / "processing" / "gain.mrc") + else ( + filepath + / machine_config.get("gain_directory_name", "processing") + / "gain.mrc" + ) ) return { "gain_ref": gain_ref_out.relative_to(Path(machine_config["rsync_basepath"])) From 71e44999c66c106e520699d16fd9dfed57e4d14e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 7 Mar 2025 10:44:21 +0000 Subject: [PATCH 08/16] Stored 'gain_directory_name' as a variable to shorten file line count --- src/murfey/server/demo_api.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index 6f3bff0e3..29c35d9af 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -1575,18 +1575,11 @@ async def process_gain( ) else: return {"gain_ref": None} + gain_ref_folder = machine_config.get("gain_directory_name", "processing") gain_ref_out = ( - ( - filepath - / machine_config.get("gain_directory_name", "processing") - / f"gain_{gain_reference_params.tag}.mrc" - ) + (filepath / gain_ref_folder / f"gain_{gain_reference_params.tag}.mrc") if gain_reference_params.tag - else ( - filepath - / machine_config.get("gain_directory_name", "processing") - / "gain.mrc" - ) + else (filepath / gain_ref_folder / "gain.mrc") ) return { "gain_ref": gain_ref_out.relative_to(Path(machine_config["rsync_basepath"])) From 3e87fdc53b357a38f29996b10ad760b61adf7b63 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 7 Mar 2025 11:44:03 +0000 Subject: [PATCH 09/16] Added logs to report when the 'prepare_(eer_)_gain' functions fail --- src/murfey/server/gain.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/murfey/server/gain.py b/src/murfey/server/gain.py index f026be9a2..5f78b70a0 100644 --- a/src/murfey/server/gain.py +++ b/src/murfey/server/gain.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging import os from enum import Enum from pathlib import Path @@ -8,6 +9,8 @@ from murfey.util import secure_path +logger = logging.getLogger("murfey.server.gain") + class Camera(Enum): K3_FLIPX = 1 @@ -30,8 +33,10 @@ async def prepare_gain( tag: str = "", ) -> Tuple[Path | None, Path | None]: if not all(executables.get(s) for s in ("dm2mrc", "clip", "newstack")): + logger.error("No executables were provided to prepare the gain reference with") return None, None if camera == Camera.FALCON: + logger.info("Gain reference preparation not needed for Falcon detector") return None, None if gain_path.suffix == ".dm4": gain_out = ( @@ -63,6 +68,10 @@ async def prepare_gain( ) stdout, stderr = await dm4_proc.communicate() if dm4_proc.returncode: + logger.error( + "Error encountered while trying to process the gain reference with 'dm2mrc': \n" + f"{stderr.decode('utf-8').strip()}" + ) return None, None clip_proc = await asyncio.create_subprocess_shell( f"{executables['clip']} {flip} {secure_path(gain_path_mrc)} {secure_path(gain_path_superres) if rescale else secure_path(gain_out)}", @@ -71,6 +80,10 @@ async def prepare_gain( ) stdout, stderr = await clip_proc.communicate() if clip_proc.returncode: + logger.error( + "Error encountered while trying to process the gain reference with 'clip': \n" + f"{stderr.decode('utf-8').strip()}" + ) return None, None if rescale: newstack_proc = await asyncio.create_subprocess_shell( @@ -80,6 +93,10 @@ async def prepare_gain( ) await newstack_proc.communicate() if newstack_proc.returncode: + logger.error( + "Error encountered while trying to process the gain reference with 'newstack': \n" + f"{stderr.decode('utf-8').strip()}" + ) return None, None if rescale: secure_path(gain_out_superres).symlink_to(secure_path(gain_path_superres)) @@ -91,6 +108,9 @@ async def prepare_eer_gain( gain_path: Path, executables: Dict[str, str], env: Dict[str, str], tag: str = "" ) -> Tuple[Path | None, Path | None]: if not executables.get("tif2mrc"): + logger.error( + "No executables were provided to prepare the EER gain reference with" + ) return None, None gain_out = ( gain_path.parent / f"gain_{tag}.mrc" if tag else gain_path.parent / "gain.mrc" @@ -100,7 +120,11 @@ async def prepare_eer_gain( mrc_convert = await asyncio.create_subprocess_shell( f"{executables['tif2mrc']} {secure_path(gain_path)} {secure_path(gain_out)}" ) - await mrc_convert.communicate() + stdout, stderr = await mrc_convert.communicate() if mrc_convert.returncode: + logger.error( + "Error encountered while trying to process the EER gain reference: \n" + f"{stderr.decode('utf-8').strip()}" + ) return None, None return gain_out, None From 98f02b2c165ce0f7a16a21aacbfd14bd26d10e63 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 7 Mar 2025 17:00:40 +0000 Subject: [PATCH 10/16] Added logic to use .mdoc files to work out the file extension to use with the Analyser; added debug logs to track progress of Analyser --- src/murfey/client/analyser.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 7910ca170..9e7a37d1d 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -23,6 +23,7 @@ from murfey.client.rsync import RSyncerUpdate, TransferResult from murfey.client.tui.forms import FormDependency from murfey.util import Observer, get_machine_config_client +from murfey.util.mdoc import get_block from murfey.util.models import PreprocessingParametersTomo, ProcessingParametersSPA logger = logging.getLogger("murfey.client.analyser") @@ -113,6 +114,13 @@ def _find_extension(self, file_path: Path): ): logger.info(f"File extension re-evaluated: {file_path.suffix}") self._extension = file_path.suffix + # If we see an .mdoc file first, use that to determine the file extensions + elif file_path.suffix == ".mdoc": + with open(file_path, "r") as md: + md.seek(0) + mdoc_data_block = get_block(md) + if subframe_path := mdoc_data_block.get("SubFramePath"): + self._extension = Path(subframe_path).suffix # Check for LIF files separately elif file_path.suffix == ".lif": self._extension = file_path.suffix @@ -124,6 +132,7 @@ def _find_context(self, file_path: Path) -> bool: stages of processing. Actions to take for individual files will be determined in the Context classes themselves. """ + logger.debug(f"Finding context using file {str(file_path)!r}") if "atlas" in file_path.parts: self._context = SPAMetadataContext("epu", self._basepath) return True @@ -258,9 +267,9 @@ def _analyse(self): self._find_extension(transferred_file) found = self._find_context(transferred_file) if not found: - # logger.warning( - # f"Context not understood for {transferred_file}, stopping analysis" - # ) + logger.debug( + f"Couldn't find context for {str(transferred_file)!r}" + ) self.queue.task_done() continue elif self._extension: @@ -383,6 +392,10 @@ def _analyse(self): SPAMetadataContext, ), ): + context = str(self._context).split(" ")[0].split(".")[-1] + logger.debug( + f"Transferring file {str(transferred_file)} with context {context!r}" + ) self.post_transfer(transferred_file) self.queue.task_done() From e24cd752d597d9665c69e8a7470284d3009fbd64 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 7 Mar 2025 17:12:48 +0000 Subject: [PATCH 11/16] Added logs to track the gradual population of tilt series in the tomogrpahy workflow --- src/murfey/client/contexts/tomo.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 331b9a728..7c027d390 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -359,6 +359,7 @@ def _check_tilt_series( newly_completed_series: List[str] = [] mdoc_tilt_series_size = self._tilt_series_sizes.get(tilt_series, 0) if not self._tilt_series or not mdoc_tilt_series_size: + logger.debug(f"Tilt series size not yet set for {tilt_series!r}") return newly_completed_series counted_tilts = len(self._tilt_series.get(tilt_series, [])) @@ -366,6 +367,11 @@ def _check_tilt_series( if tilt_series_size_check and tilt_series not in self._completed_tilt_series: self._completed_tilt_series.append(tilt_series) newly_completed_series.append(tilt_series) + else: + logger.debug( + f"{tilt_series!r} not complete yet. Counted {counted_tilts} tilts. " + f"Expected number of tilts was {mdoc_tilt_series_size}" + ) return newly_completed_series def _add_tomo_tilt( From 62fd675657949b67b80448c99e3b8c3825a76e62 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 10 Mar 2025 10:20:17 +0000 Subject: [PATCH 12/16] Added logic to properly handle case where workflow search turns up an empty list --- src/murfey/server/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 2ace38da3..9f56e592c 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -8,6 +8,7 @@ import time from datetime import datetime from functools import partial, singledispatch +from importlib.metadata import EntryPoint # For type hinting only from importlib.resources import files from pathlib import Path from threading import Thread @@ -20,7 +21,6 @@ from backports.entry_points_selectable import entry_points from fastapi import Request from fastapi.templating import Jinja2Templates -from importlib_metadata import EntryPoint # For type hinting only from ispyb.sqlalchemy._auto_db_schema import ( Atlas, AutoProcProgram, @@ -2942,12 +2942,17 @@ def feedback_callback(header: dict, message: dict) -> None: elif ( message["register"] in entry_points().select(group="murfey.workflows").names ): - # Run the workflow if a match is found - workflow: EntryPoint = list( # Returns a list of either 1 or 0 + # Search for corresponding workflow + workflows: list[EntryPoint] = list( entry_points().select( group="murfey.workflows", name=message["register"] ) - )[0] + ) # Returns either 1 item or empty list + if not workflows: + logger.error(f"No workflow found for {sanitise(message['register'])}") + return None + # Run the workflow if a match is found + workflow: EntryPoint = workflows[0] result = workflow.load()( message=message, murfey_db=murfey_db, From 834941bd8fa305ef14c1ee2eea83314020a0826b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 10 Mar 2025 11:54:07 +0000 Subject: [PATCH 13/16] Applied sentence case to logs --- src/murfey/client/multigrid_control.py | 2 +- src/murfey/client/tui/app.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index 4a720de74..e12d95400 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -410,7 +410,7 @@ def _start_dc(self, json, from_form: bool = False): f"{self._environment.url.geturl()}/visits/{self._environment.visit}/{self._environment.murfey_session}/flush_tomography_processing", json={"rsync_source": str(source)}, ) - log.info("tomography processing flushed") + log.info("Tomography processing flushed") elif isinstance(context, SPAModularContext): url = f"{str(self._environment.url.geturl())}/visits/{str(self._environment.visit)}/{self.session_id}/register_data_collection_group" diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index 9bf229008..83468704c 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -511,7 +511,7 @@ def _start_dc(self, json, from_form: bool = False): f"{self.app._environment.url.geturl()}/visits/{self._visit}/{self.app._environment.murfey_session}/flush_tomography_processing", json={"rsync_source": str(source)}, ) - log.info("tomography processing flushed") + log.info("Tomography processing flushed") elif isinstance(context, SPAModularContext): url = f"{str(self._url.geturl())}/visits/{str(self._visit)}/{self._environment.murfey_session}/register_data_collection_group" dcg_data = { From 16d8a9337bc1a5bc074366a9ed86d0ebb3a22569 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 10 Mar 2025 16:24:06 +0000 Subject: [PATCH 14/16] Optimised '_register_picked_particles_use_diameter' function --- src/murfey/server/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 9f56e592c..7b3a3364f 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -620,8 +620,6 @@ def _register_picked_particles_use_diameter( ) ).one() - particle_diameter = relion_params.particle_diameter - if feedback_params.picker_ispyb_id is None: if demo or not _transport_object: feedback_params.picker_ispyb_id = 1000 @@ -649,15 +647,15 @@ def _register_picked_particles_use_diameter( _db.delete(s) _db.commit() - if not particle_diameter: + # Calculate diameter if it wasn't provided + if not relion_params.particle_diameter: # If the diameter has not been calculated then find it picking_db = _db.exec( select(db.ParticleSizes.particle_size).where( db.ParticleSizes.pj_id == pj_id ) ).all() - particle_diameter = np.quantile(list(picking_db), 0.75) - relion_params.particle_diameter = particle_diameter + relion_params.particle_diameter = np.quantile(list(picking_db), 0.75) _db.add(relion_params) _db.commit() @@ -682,7 +680,7 @@ def _register_picked_particles_use_diameter( "defocus_u": saved_message.defocus_u, "defocus_v": saved_message.defocus_v, "defocus_angle": saved_message.defocus_angle, - "particle_diameter": particle_diameter, + "particle_diameter": relion_params.particle_diameter, "downscale": relion_options["downscale"], "kv": relion_options["voltage"], "node_creator_queue": machine_config.node_creator_queue, @@ -702,9 +700,9 @@ def _register_picked_particles_use_diameter( _transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) + # Use provided diameter for next step else: # If the diameter is known then just send the new message - particle_diameter = relion_params.particle_diameter zocalo_message = { "parameters": { "micrographs_file": params_to_forward["micrographs_file"], @@ -723,7 +721,7 @@ def _register_picked_particles_use_diameter( "defocus_u": params_to_forward["ctf_values"]["DefocusU"], "defocus_v": params_to_forward["ctf_values"]["DefocusV"], "defocus_angle": params_to_forward["ctf_values"]["DefocusAngle"], - "particle_diameter": particle_diameter, + "particle_diameter": relion_params.particle_diameter, "downscale": relion_options["downscale"], "kv": relion_options["voltage"], "node_creator_queue": machine_config.node_creator_queue, From 13f52e621c952a74db992041bc24d3dbc553d3dc Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 10 Mar 2025 16:24:31 +0000 Subject: [PATCH 15/16] Locked Numpy to <2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0daf313f8..f32da376f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ server = [ "ispyb", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; v10.0.0: sqlalchemy <2, mysql-connector-python >=8.0.32 "jinja2", "mrcfile", - "numpy", + "numpy<2", "packaging", "passlib", "pillow", From c99618e28bce6c8f2132af9652e6e09b54f54c77 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 11 Mar 2025 15:36:40 +0000 Subject: [PATCH 16/16] Nack message if no workflow entry point is found --- src/murfey/server/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 7b3a3364f..9c268e07b 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -2948,6 +2948,8 @@ def feedback_callback(header: dict, message: dict) -> None: ) # Returns either 1 item or empty list if not workflows: logger.error(f"No workflow found for {sanitise(message['register'])}") + if _transport_object: + _transport_object.transport.nack(header, requeue=False) return None # Run the workflow if a match is found workflow: EntryPoint = workflows[0]