From 10a8a08e53cc82ec735452d0e179ff5753d592bc Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Wed, 22 Jan 2025 14:37:01 +0000 Subject: [PATCH 1/6] Simple path construction fixes --- src/murfey/client/contexts/spa_metadata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 9a53c1a52..9e5e00924 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -128,10 +128,12 @@ def post_transfer( source_visit_dir = source.parent logger.info( - f"Looking for atlas XML file in metadata directory {str((source_visit_dir / partial_path).parent)}" + f"Looking for atlas XML file in metadata directory {str((source_visit_dir / environment.visit / partial_path).parent)}" ) atlas_xml_path = list( - (source_visit_dir / partial_path).parent.glob("Atlas_*.xml") + (source_visit_dir / environment.visit / partial_path).parent.glob( + "Atlas_*.xml" + ) )[0] logger.info(f"Atlas XML path {str(atlas_xml_path)} found") with open(atlas_xml_path, "rb") as atlas_xml: @@ -190,7 +192,7 @@ def post_transfer( ) elif transferred_file.suffix == ".dm" and environment: - gs_name = transferred_file.name.split("_")[1] + gs_name = transferred_file.stem.split("_")[1] fh_positions = _foil_hole_positions(transferred_file, int(gs_name)) source = _get_source(transferred_file, environment=environment) visitless_source = str(source).replace(f"/{environment.visit}", "") From 2f732e38e4021ded306b65340e7cb1f4838b7b74 Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Wed, 22 Jan 2025 14:43:00 +0000 Subject: [PATCH 2/6] Convert pixel size from string to float --- src/murfey/client/contexts/spa_metadata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 9e5e00924..422d9e30e 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -138,9 +138,11 @@ def post_transfer( logger.info(f"Atlas XML path {str(atlas_xml_path)} found") with open(atlas_xml_path, "rb") as atlas_xml: atlas_xml_data = xmltodict.parse(atlas_xml) - atlas_original_pixel_size = atlas_xml_data["MicroscopeImage"][ - "SpatialScale" - ]["pixelSize"]["x"]["numericValue"] + atlas_original_pixel_size = float( + atlas_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][ + "numericValue" + ] + ) # need to calculate the pixel size of the downscaled image atlas_pixel_size = atlas_original_pixel_size * 7.8 From 331a2d2b0aa4fa2413b7ea85ba1685f9e58f52e7 Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Wed, 22 Jan 2025 14:47:51 +0000 Subject: [PATCH 3/6] Remove data only present in XML not .dm files --- src/murfey/client/contexts/spa_metadata.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 422d9e30e..d049c2a7a 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -21,13 +21,6 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole for_parsing = xml.read() data = xmltodict.parse(for_parsing) data = data["GridSquareXml"] - readout_area = data["MicroscopeImage"]["microscopeData"]["acquisition"]["camera"][ - "ReadoutArea" - ] - pixel_size = data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][ - "numericValue" - ] - full_size = (int(readout_area["a:width"]), int(readout_area["a:height"])) serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][ "a:m_serializationArray" ] @@ -58,11 +51,6 @@ def _foil_hole_positions(xml_path: Path, grid_square: int) -> Dict[str, FoilHole y_location=int(float(pix_loc["c:y"])), x_stage_position=float(stage["c:X"]), y_stage_position=float(stage["c:Y"]), - readout_area_x=full_size[0] if image_path else None, - readout_area_y=full_size[1] if image_path else None, - thumbnail_size_x=None, - thumbnail_size_y=None, - pixel_size=float(pixel_size) if image_path else None, image=str(image_path), diameter=int(float(diameter)), ) From 2b5d793238d15704aa7439ff6a198004c1560ebb Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Wed, 22 Jan 2025 15:33:09 +0000 Subject: [PATCH 4/6] Cover the case where the .dm file is seen before the grid square XML so that the foil hole positions can't be determined until the XML is seen, at which point there needs to be an update --- src/murfey/client/contexts/spa.py | 5 +- src/murfey/server/ispyb.py | 33 ++++----- src/murfey/util/spa_metadata.py | 69 +++++++++++-------- .../workflows/spa/flush_spa_preprocess.py | 39 +++++++---- 4 files changed, 86 insertions(+), 60 deletions(-) diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 0bec9e984..b798816d3 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -456,10 +456,7 @@ def _position_analysis( foil_hole = foil_hole_from_file(transferred_file) if foil_hole not in self._foil_holes[grid_square]: fh_url = f"{str(environment.url.geturl())}/sessions/{environment.murfey_session}/grid_square/{grid_square}/foil_hole" - if ( - grid_square_metadata_file.is_file() - and environment.murfey_session is not None - ): + if environment.murfey_session is not None: fh = foil_hole_data( grid_square_metadata_file, foil_hole, diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index ff9f09b7d..d07342ce9 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -253,7 +253,7 @@ def do_update_grid_square( def do_insert_foil_hole( self, grid_square_id: int, - scale_factor: float, + scale_factor: Optional[float], foil_hole_parameters: FoilHoleParameters, ): if ( @@ -265,21 +265,22 @@ def do_insert_foil_hole( foil_hole_parameters.readout_area_x / foil_hole_parameters.thumbnail_size_x ) - foil_hole_parameters.diameter = ( - int(foil_hole_parameters.diameter * scale_factor) - if foil_hole_parameters.diameter - else None - ) - foil_hole_parameters.x_location = ( - int(foil_hole_parameters.x_location * scale_factor) - if foil_hole_parameters.x_location - else None - ) - foil_hole_parameters.y_location = ( - int(foil_hole_parameters.y_location * scale_factor) - if foil_hole_parameters.y_location - else None - ) + if scale_factor: + foil_hole_parameters.diameter = ( + int(foil_hole_parameters.diameter * scale_factor) + if foil_hole_parameters.diameter + else None + ) + foil_hole_parameters.x_location = ( + int(foil_hole_parameters.x_location * scale_factor) + if foil_hole_parameters.x_location + else None + ) + foil_hole_parameters.y_location = ( + int(foil_hole_parameters.y_location * scale_factor) + if foil_hole_parameters.y_location + else None + ) record = FoilHole( gridSquareId=grid_square_id, foilHoleLabel=foil_hole_parameters.name, diff --git a/src/murfey/util/spa_metadata.py b/src/murfey/util/spa_metadata.py index fe82d1e0e..5af671b80 100644 --- a/src/murfey/util/spa_metadata.py +++ b/src/murfey/util/spa_metadata.py @@ -148,36 +148,38 @@ def grid_square_data(xml_path: Path, grid_square: int) -> GridSquareInfo: def foil_hole_data(xml_path: Path, foil_hole: int, grid_square: int) -> FoilHoleInfo: - with open(xml_path, "r") as xml: - for_parsing = xml.read() - data = xmltodict.parse(for_parsing) - data = data["GridSquareXml"] - serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][ - "a:m_serializationArray" - ] required_key = "" - for key in serialization_array.keys(): - if key.startswith("b:KeyValuePairOfintTargetLocation"): - required_key = key - break - if required_key: - image_paths = list( - (xml_path.parent.parent).glob( - f"Images-Disc*/GridSquare_{grid_square}/FoilHoles/FoilHole_{foil_hole}_*.jpg" - ) + if xml_path.is_file(): + with open(xml_path, "r") as xml: + for_parsing = xml.read() + data = xmltodict.parse(for_parsing) + data = data["GridSquareXml"] + serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][ + "a:m_serializationArray" + ] + required_key = "" + for key in serialization_array.keys(): + if key.startswith("b:KeyValuePairOfintTargetLocation"): + required_key = key + break + image_paths = list( + (xml_path.parent.parent).glob( + f"Images-Disc*/GridSquare_{grid_square}/FoilHoles/FoilHole_{foil_hole}_*.jpg" ) - image_paths.sort(key=lambda x: x.stat().st_ctime) - image_path: Union[Path, str] = image_paths[-1] if image_paths else "" - if image_path: - with open(Path(image_path).with_suffix(".xml")) as fh_xml: - fh_xml_data = xmltodict.parse(fh_xml.read()) - readout_area = fh_xml_data["MicroscopeImage"]["microscopeData"][ - "acquisition" - ]["camera"]["ReadoutArea"] - pixel_size = fh_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"][ - "x" - ]["numericValue"] - full_size = (int(readout_area["a:width"]), int(readout_area["a:height"])) + ) + image_paths.sort(key=lambda x: x.stat().st_ctime) + image_path: Union[Path, str] = image_paths[-1] if image_paths else "" + if image_path: + with open(Path(image_path).with_suffix(".xml")) as fh_xml: + fh_xml_data = xmltodict.parse(fh_xml.read()) + readout_area = fh_xml_data["MicroscopeImage"]["microscopeData"]["acquisition"][ + "camera" + ]["ReadoutArea"] + pixel_size = fh_xml_data["MicroscopeImage"]["SpatialScale"]["pixelSize"]["x"][ + "numericValue" + ] + full_size = (int(readout_area["a:width"]), int(readout_area["a:height"])) + if required_key: for fh_block in serialization_array[required_key]: pix = fh_block["b:value"]["PixelCenter"] stage = fh_block["b:value"]["StagePosition"] @@ -198,6 +200,17 @@ def foil_hole_data(xml_path: Path, foil_hole: int, grid_square: int) -> FoilHole image=str(image_path), diameter=diameter, ) + elif image_path: + return FoilHoleInfo( + id=foil_hole, + grid_square_id=grid_square, + readout_area_x=full_size[0] if image_path else None, + readout_area_y=full_size[1] if image_path else None, + thumbnail_size_x=None, + thumbnail_size_y=None, + pixel_size=float(pixel_size) if image_path else None, + image=str(image_path), + ) logger.warning( f"Foil hole positions could not be determined from metadata file {xml_path} for foil hole {foil_hole}" ) diff --git a/src/murfey/workflows/spa/flush_spa_preprocess.py b/src/murfey/workflows/spa/flush_spa_preprocess.py index 1f8cbc70e..8f65158e1 100644 --- a/src/murfey/workflows/spa/flush_spa_preprocess.py +++ b/src/murfey/workflows/spa/flush_spa_preprocess.py @@ -122,23 +122,37 @@ def register_foil_hole( .where(FoilHole.grid_square_id == gsid) .where(FoilHole.session_id == session_id) ).one() - foil_hole.x_location = foil_hole_params.x_location - foil_hole.y_location = foil_hole_params.y_location - foil_hole.x_stage_position = foil_hole_params.x_stage_position - foil_hole.y_stage_position = foil_hole_params.y_stage_position - foil_hole.readout_area_x = foil_hole_params.readout_area_x - foil_hole.readout_area_y = foil_hole_params.readout_area_y - foil_hole.thumbnail_size_x = foil_hole_params.thumbnail_size_x or jpeg_size[0] - foil_hole.thumbnail_size_y = foil_hole_params.thumbnail_size_y or jpeg_size[1] - foil_hole.pixel_size = foil_hole_params.pixel_size - if _transport_object: + foil_hole.x_location = foil_hole_params.x_location or foil_hole.x_location + foil_hole.y_location = foil_hole_params.y_location or foil_hole.y_location + foil_hole.x_stage_position = ( + foil_hole_params.x_stage_position or foil_hole.x_stage_position + ) + foil_hole.y_stage_position = ( + foil_hole_params.y_stage_position or foil_hole.y_stage_position + ) + foil_hole.readout_area_x = ( + foil_hole_params.readout_area_x or foil_hole.readout_area_x + ) + foil_hole.readout_area_y = ( + foil_hole_params.readout_area_y or foil_hole.readout_area_y + ) + foil_hole.thumbnail_size_x = ( + foil_hole_params.thumbnail_size_x or foil_hole.thumbnail_size_x + ) or jpeg_size[0] + foil_hole.thumbnail_size_y = ( + foil_hole_params.thumbnail_size_y or foil_hole.thumbnail_size_y + ) or jpeg_size[1] + foil_hole.pixel_size = foil_hole_params.pixel_size or foil_hole.pixel_size + if _transport_object and gs.readout_area_x: _transport_object.do_update_foil_hole( foil_hole.id, gs.thumbnail_size_x / gs.readout_area_x, foil_hole_params ) except Exception: if _transport_object: fh_ispyb_response = _transport_object.do_insert_foil_hole( - gs.id, gs.thumbnail_size_x / gs.readout_area_x, foil_hole_params + gs.id, + gs.thumbnail_size_x / gs.readout_area_x if gs.readout_area_x else None, + foil_hole_params, ) else: fh_ispyb_response = {"success": False, "return_value": None} @@ -334,7 +348,8 @@ def flush_spa_preprocessing(message: dict, db: Session, demo: bool = False): ) except Exception as e: logger.error( - f"Flushing position analysis for {f.file_path} caused exception {e}", exc_info=True + f"Flushing position analysis for {f.file_path} caused exception {e}", + exc_info=True, ) foil_hole_id = None From 80fcd515a2e04d7171852d23a49c305d7532abea Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Wed, 22 Jan 2025 15:39:16 +0000 Subject: [PATCH 5/6] Don't update stuff as None --- src/murfey/workflows/spa/flush_spa_preprocess.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/murfey/workflows/spa/flush_spa_preprocess.py b/src/murfey/workflows/spa/flush_spa_preprocess.py index 8f65158e1..f2d68b36c 100644 --- a/src/murfey/workflows/spa/flush_spa_preprocess.py +++ b/src/murfey/workflows/spa/flush_spa_preprocess.py @@ -39,10 +39,14 @@ def register_grid_square( .where(GridSquare.tag == grid_square_params.tag) .where(GridSquare.session_id == session_id) ).one() - grid_square.x_location = grid_square_params.x_location - grid_square.y_location = grid_square_params.y_location - grid_square.x_stage_position = grid_square_params.x_stage_position - grid_square.y_stage_position = grid_square_params.y_stage_position + grid_square.x_location = grid_square_params.x_location or grid_square.x_location + grid_square.y_location = grid_square_params.y_location or grid_square.y_location + grid_square.x_stage_position = ( + grid_square_params.x_stage_position or grid_square.x_stage_position + ) + grid_square.y_stage_position = ( + grid_square_params.y_stage_position or grid_square.y_stage_position + ) if _transport_object: _transport_object.do_update_grid_square(grid_square.id, grid_square_params) except Exception: From b56d8ff7ed4fd52ff55c3b5db53cf0af005b8415 Mon Sep 17 00:00:00 2001 From: Dan Hatton Date: Wed, 22 Jan 2025 15:43:22 +0000 Subject: [PATCH 6/6] Remove redefinition --- src/murfey/util/spa_metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/murfey/util/spa_metadata.py b/src/murfey/util/spa_metadata.py index 5af671b80..6dea16ac1 100644 --- a/src/murfey/util/spa_metadata.py +++ b/src/murfey/util/spa_metadata.py @@ -157,7 +157,6 @@ def foil_hole_data(xml_path: Path, foil_hole: int, grid_square: int) -> FoilHole serialization_array = data["TargetLocations"]["TargetLocationsEfficient"][ "a:m_serializationArray" ] - required_key = "" for key in serialization_array.keys(): if key.startswith("b:KeyValuePairOfintTargetLocation"): required_key = key