From b1d39ba9802e4d77cf415f19dba632af22de8dbc Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Mar 2025 09:23:46 +0000 Subject: [PATCH 01/11] Convert weird MSys2 path before rsyncing the gain reference --- src/murfey/instrument_server/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index a094fce39..2dad52893 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -21,7 +21,7 @@ from murfey.client.multigrid_control import MultigridController from murfey.client.rsync import RSyncer from murfey.client.watchdir_multigrid import MultigridDirWatcher -from murfey.util import sanitise, sanitise_nonpath, secure_path +from murfey.util import posix_path, sanitise, sanitise_nonpath, secure_path from murfey.util.instrument_models import MultigridWatcherSpec from murfey.util.models import File, Token @@ -291,7 +291,7 @@ def upload_gain_reference( ).json() cmd = [ "rsync", - safe_gain_path, + posix_path(Path(safe_gain_path)), f"{urlparse(_get_murfey_url(), allow_fragments=False).hostname}::{machine_config.get('rsync_module', 'data')}/{safe_visit_path}/{safe_destination_dir}/{secure_filename(gain_reference.gain_path.name)}", ] gain_rsync = subprocess.run(cmd) From b75b288a8df6cd07a7ebf3d88da0fdb7e813a60a Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Mar 2025 09:26:43 +0000 Subject: [PATCH 02/11] Need to check for colons that need putting back in to path names when there are spaces as well --- src/murfey/util/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/__init__.py b/src/murfey/util/__init__.py index c1610c98c..450fa9858 100644 --- a/src/murfey/util/__init__.py +++ b/src/murfey/util/__init__.py @@ -74,9 +74,14 @@ def sanitise_nonpath(in_string: str) -> str: def secure_path(in_path: Path, keep_spaces: bool = False) -> Path: if keep_spaces: - secured_parts = [ - secure_filename(p) if " " not in p else p for p in in_path.parts - ] + secured_parts = [] + for p, part in enumerate(in_path.parts): + if " " in part: + secured_parts.append(part) + elif ":" in part and not p: + secured_parts.append(secure_filename(part) + ":") + else: + secured_parts.append(secure_filename(part)) else: secured_parts = [ ( From c27bcf4ac67052f5b5363ae713ede8b41aaefcea Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Mar 2025 15:22:04 +0000 Subject: [PATCH 03/11] Full MSys2 paths do no start at root so there is no need to drop the leading forward slash --- src/murfey/client/contexts/spa.py | 13 ++++++++----- src/murfey/client/contexts/spa_metadata.py | 15 ++++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 6f9f48993..5684dacf6 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -431,12 +431,15 @@ def _position_analysis( grid_square_metadata_file, grid_square, ) + metadata_source_as_str = ( + "/".join(source.parts[:-2]) + + f"/{environment.visit}/" + + source.parts[-2] + ) metadata_source = Path( - ( - "/".join(source.parts[:-2]) - + f"/{environment.visit}/" - + source.parts[-2] - )[1:] + metadata_source_as_str[1:] + if metadata_source_as_str.startswith("//") + else metadata_source_as_str ) image_path = ( _file_transferred_to(environment, metadata_source, Path(gs.image)) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index c4d3e978d..120020348 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -241,6 +241,8 @@ def post_transfer( ) fh_positions = _foil_hole_positions(transferred_file, int(gs_name)) source = _get_source(transferred_file, environment=environment) + if source is None: + return None visitless_source_search_dir = str(source).replace( f"/{environment.visit}", "" ) @@ -261,12 +263,15 @@ def post_transfer( transferred_file, int(gs_name), ) + metadata_source_as_str = ( + "/".join(source.parts[:-2]) + + f"/{environment.visit}/" + + source.parts[-2] + ) metadata_source = Path( - ( - "/".join(Path(visitless_source).parts[:-2]) - + f"/{environment.visit}/" - + Path(visitless_source).parts[-2] - )[1:] + metadata_source_as_str[1:] + if metadata_source_as_str.startswith("//") + else metadata_source_as_str ) image_path = ( _file_transferred_to( From 5486fd61a84cd30624154fa6aceac458f6396fe8 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Mar 2025 15:36:54 +0000 Subject: [PATCH 04/11] Make multigrid controller match TUI version --- src/murfey/client/multigrid_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index e12d95400..01ef1d635 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -143,7 +143,7 @@ def _start_rsyncer_multigrid( source, destination, force_metadata=self.processing_enabled, - analyse=not extra_directory and use_suggested_path and analyse, + analyse=analyse, remove_files=remove_files, tag=tag, limited=limited, From a48496812bb408ace7946962227b4406570956d4 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Mar 2025 15:44:17 +0000 Subject: [PATCH 05/11] Another fix for code trying to lose an extra / --- src/murfey/client/contexts/spa_metadata.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 120020348..ad931f30d 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -215,11 +215,16 @@ def post_transfer( ): # Make sure we have a data collection group before trying to register grid square url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group" - dcg_search_dir = "/" + "/".join( + dcg_search_dir = "/".join( p - for p in transferred_file.parent.parent.parts[1:] + for p in transferred_file.parent.parent.parts if p != environment.visit ) + dcg_search_dir = ( + dcg_search_dir[1:] + if dcg_search_dir.startswith("//") + else dcg_search_dir + ) dcg_images_dirs = sorted( Path(dcg_search_dir).glob("Images-Disc*"), key=lambda x: x.stat().st_ctime, From 4c59c2e6a3a2402698f2d1dd5d90cdf8af15dc20 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Mar 2025 15:47:09 +0000 Subject: [PATCH 06/11] Downgrade log severity --- src/murfey/client/contexts/spa_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index ad931f30d..116b50cdb 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -44,7 +44,7 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole required_key = key break if not required_key: - logger.warning(f"Required key not found for {str(xml_path)}") + logger.info(f"Required key not found for {str(xml_path)}") return {} foil_holes = {} for fh_block in serialization_array[required_key]: From c09445caa2d0a5acefe5bf0da3d160e5cc826265 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Wed, 19 Mar 2025 09:12:42 +0000 Subject: [PATCH 07/11] Another fix for not starting path with / --- src/murfey/client/contexts/spa_metadata.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 116b50cdb..2d08570a7 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -163,11 +163,16 @@ def post_transfer( atlas=Path(partial_path), sample=sample ) url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group" - dcg_search_dir = "/" + "/".join( + dcg_search_dir = "/".join( p - for p in transferred_file.parent.parts[1:] + for p in transferred_file.parent.parent.parts if p != environment.visit ) + dcg_search_dir = ( + dcg_search_dir[1:] + if dcg_search_dir.startswith("//") + else dcg_search_dir + ) dcg_images_dirs = sorted( Path(dcg_search_dir).glob("Images-Disc*"), key=lambda x: x.stat().st_ctime, From 94ab227f7a09a35e60e887b10d9e03dfc86df90b Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Wed, 19 Mar 2025 09:14:50 +0000 Subject: [PATCH 08/11] Yet another not starting from root properly fix --- src/murfey/client/contexts/spa.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 5684dacf6..17db68027 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -474,12 +474,15 @@ def _position_analysis( foil_hole, grid_square, ) + metadata_source_as_str = ( + "/".join(source.parts[:-2]) + + f"/{environment.visit}/" + + source.parts[-2] + ) metadata_source = Path( - ( - "/".join(source.parts[:-2]) - + f"/{environment.visit}/" - + source.parts[-2] - )[1:] + metadata_source_as_str[1:] + if metadata_source_as_str.startswith("//") + else metadata_source_as_str ) image_path = ( _file_transferred_to(environment, metadata_source, Path(fh.image)) From 64e44674b1fce6387b8ffc1a09ce16b63cf87000 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Wed, 19 Mar 2025 10:33:11 +0000 Subject: [PATCH 09/11] Setup logging to graylog for instrument server via main server --- src/murfey/instrument_server/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/murfey/instrument_server/__init__.py b/src/murfey/instrument_server/__init__.py index bb17695a8..1334e954f 100644 --- a/src/murfey/instrument_server/__init__.py +++ b/src/murfey/instrument_server/__init__.py @@ -5,6 +5,8 @@ from rich.logging import RichHandler import murfey +from murfey.client import read_config +from murfey.client.customlogging import CustomHandler from murfey.util import LogFilter logger = logging.getLogger("murfey.instrument_server") @@ -33,6 +35,13 @@ def run(): logging.getLogger("fastapi").addHandler(rich_handler) logging.getLogger("uvicorn").addHandler(rich_handler) + ws = murfey.client.websocket.WSApp( + server=read_config()["Murfey"].get("server"), + ) + + handler = CustomHandler(ws.send) + logging.getLogger().addHandler(handler) + logger.info( f"Starting Murfey server version {murfey.__version__}, listening on {args.host}:{args.port}" ) From 71d026a36594afb3a19625da30ef4b7e396a0bf3 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Wed, 19 Mar 2025 10:34:15 +0000 Subject: [PATCH 10/11] Need to set ID fro websocket connection --- src/murfey/instrument_server/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/murfey/instrument_server/__init__.py b/src/murfey/instrument_server/__init__.py index 1334e954f..1bf239827 100644 --- a/src/murfey/instrument_server/__init__.py +++ b/src/murfey/instrument_server/__init__.py @@ -37,6 +37,7 @@ def run(): ws = murfey.client.websocket.WSApp( server=read_config()["Murfey"].get("server"), + id=0, ) handler = CustomHandler(ws.send) From 7fe2fb0f4e435b369db6c5c5561b3f7cf00e6ea7 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Wed, 19 Mar 2025 13:14:17 +0000 Subject: [PATCH 11/11] Too many parents --- src/murfey/client/contexts/spa_metadata.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 2d08570a7..cb388c035 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -164,9 +164,7 @@ def post_transfer( ) url = f"{str(environment.url.geturl())}/visits/{environment.visit}/{environment.murfey_session}/register_data_collection_group" dcg_search_dir = "/".join( - p - for p in transferred_file.parent.parent.parts - if p != environment.visit + p for p in transferred_file.parent.parts if p != environment.visit ) dcg_search_dir = ( dcg_search_dir[1:]