Convert xrm images to tiffs (#837)

This adds functionality to read xrm images and request these are converted to tiffs.

Currently this identifies atlas and roi images by finding out if they are montages, and splits them based on pixel size. Better methods may be possible but this should work for now.

Rather than doing the xrm to tiff conversion here it puts a message onto rabbitmq. See DiamondLightSource/cryoem-services#278

Also contains a stub for registering rois in the database.

Author: stephen-riggs

src/murfey/client/contexts/sxt.py

@@ -58,6 +58,10 @@ def _find_reference(txrm_file: Path) -> Path | None:
     return None
 
 
+def _get_ole_header_value(ole_file, title: str, dtype: np.dtype):
+    return np.frombuffer(ole_file.openstream(title).getvalue(), dtype)
+
+
 class SXTContext(Context):
     def __init__(
         self,
@@ -156,69 +160,221 @@ def post_transfer(
             environment=environment,
             **kwargs,
         )
+        metadata: dict[str, Any] = {}
+
+        if transferred_file.suffix == ".xrm" and environment:
+            # Make sure we have a dcg for this grid
+            dcg_tag = ensure_dcg_exists(
+                collection_type="sxt",
+                metadata_source=self._basepath,
+                environment=environment,
+                machine_config=self._machine_config,
+                token=self._token,
+            )
 
-        data_suffixes = [".txrm"]
+            with OleFileIO(str(transferred_file)) as xrm_ole:
+                if xrm_ole.exists("ImageInfo/XPosition") and xrm_ole.exists(
+                    "ImageInfo/YPosition"
+                ):
+                    x_tiles = _get_ole_header_value(
+                        xrm_ole, "ImageInfo/XPosition", np.float32
+                    ).tolist()
+                    y_tiles = _get_ole_header_value(
+                        xrm_ole, "ImageInfo/YPosition", np.float32
+                    ).tolist()
+                    metadata["x_position"] = x_tiles[int(len(x_tiles) / 2)]
+                    metadata["y_position"] = y_tiles[int(len(y_tiles) / 2)]
 
-        if transferred_file.suffix in data_suffixes and environment:
+                if xrm_ole.exists("ImageInfo/PixelSize"):
+                    metadata["pixel_size"] = _get_ole_header_value(
+                        xrm_ole, "ImageInfo/PixelSize", np.float32
+                    ).tolist()[0]
+
+                if xrm_ole.exists("ImageInfo/ImageHeight"):
+                    metadata["height"] = _get_ole_header_value(
+                        xrm_ole, "ImageInfo/ImageHeight", np.int32
+                    ).tolist()[0]
+
+                if xrm_ole.exists("ImageInfo/ImageWidth"):
+                    metadata["width"] = _get_ole_header_value(
+                        xrm_ole, "ImageInfo/ImageWidth", np.int32
+                    ).tolist()[0]
+
+                # Find images which are not mosaics (txrm spec typos this as mosiac)
+                if xrm_ole.exists("ImageInfo/MosiacRows") and xrm_ole.exists(
+                    "ImageInfo/MosiacColumns"
+                ):
+                    metadata["mosaic_rows"] = _get_ole_header_value(
+                        xrm_ole, "ImageInfo/MosiacRows", np.int32
+                    )[0]
+                    metadata["mosaic_columns"] = _get_ole_header_value(
+                        xrm_ole, "ImageInfo/MosiacColumns", np.int32
+                    )[0]
+                    metadata["mosaic_size"] = int(
+                        metadata["mosaic_rows"] * metadata["mosaic_columns"]
+                    )
+
+            source = _get_source(transferred_file, environment=environment)
+            if source:
+                image_path = _file_transferred_to(
+                    environment,
+                    source,
+                    transferred_file,
+                    Path(self._machine_config.get("rsync_basepath", "")),
+                )
+                if (
+                    environment.visit
+                    in Path(environment.default_destinations[source]).parts
+                ):
+                    # Split either side of the raw directory
+                    visit_idx = Path(
+                        environment.default_destinations[source]
+                    ).parts.index(environment.visit)
+                    destination_base = "/".join(
+                        Path(environment.default_destinations[source]).parts[
+                            : visit_idx + 1
+                        ]
+                    )
+                    destination_extra = "/".join(
+                        Path(environment.default_destinations[source]).parts[
+                            visit_idx + 2 :
+                        ]
+                    )
+                else:
+                    destination_base = str(
+                        Path(environment.default_destinations[source])
+                        / environment.visit
+                    )
+                    destination_extra = ""
+                converted_file_path = (
+                    Path(self._machine_config.get("rsync_basepath", ""))
+                    / destination_base
+                    / self._machine_config.get("processed_directory_name", "")
+                    / self._machine_config.get("processed_extra_directory", "")
+                    / destination_extra
+                    / f"{transferred_file.relative_to(source).stem}_Annotated.tiff"
+                )
+                capture_post(
+                    base_url=str(environment.url.geturl()),
+                    router_name="workflow_sxt.router",
+                    function_name="convert_xrm_to_tiff",
+                    token=self._token,
+                    instrument_name=environment.instrument_name,
+                    data={
+                        "xrm_path": str(image_path),
+                        "tiff_path": str(converted_file_path),
+                    },
+                )
+
+                if (
+                    metadata.get("mosaic_size", 1) > 0
+                    and metadata.get("pixel_size", 0) > 0.1
+                ):
+                    # Large pixel size, this is an atlas
+                    dcg_data = {
+                        "experiment_type_id": 44,  # Atlas
+                        "tag": dcg_tag,
+                        "atlas": str(converted_file_path),
+                        "atlas_pixel_size": round(metadata.get("pixel_size", 0), 2),
+                        "atlas_x_stage_position": metadata.get("x_position", None),
+                        "atlas_y_stage_position": metadata.get("y_position", None),
+                        "atlas_height": int(
+                            metadata.get("height", 0) * metadata["mosaic_rows"]
+                        ),
+                        "atlas_width": int(
+                            metadata.get("width", 0) * metadata["mosaic_columns"]
+                        ),
+                    }
+                    capture_post(
+                        base_url=str(environment.url.geturl()),
+                        router_name="workflow.router",
+                        function_name="register_dc_group",
+                        token=self._token,
+                        instrument_name=environment.instrument_name,
+                        visit_name=environment.visit,
+                        session_id=environment.murfey_session,
+                        data=dcg_data,
+                    )
+                elif metadata.get("mosaic_size", 1) > 0:
+                    # Other mosaic images are of grid squares
+                    capture_post(
+                        base_url=str(environment.url.geturl()),
+                        router_name="workflow_sxt.router",
+                        function_name="register_sxt_roi",
+                        token=self._token,
+                        instrument_name=environment.instrument_name,
+                        visit_name=environment.visit,
+                        session_id=environment.murfey_session,
+                        data={
+                            "tag": dcg_tag,
+                            "name": transferred_file.stem,
+                            "x_stage_position": metadata.get("x_position", None),
+                            "y_stage_position": metadata.get("y_position", None),
+                            "pixel_size": round(metadata.get("pixel_size", 0), 2),
+                            "height": int(
+                                metadata.get("height", 0) * metadata["mosaic_rows"]
+                            ),
+                            "width": int(
+                                metadata.get("width", 0) * metadata["mosaic_columns"]
+                            ),
+                            "image": str(converted_file_path),
+                        },
+                    )
+
+        elif transferred_file.suffix == ".txrm" and environment:
             source = _get_source(transferred_file, environment)
             if not source:
                 logger.warning(f"No source found for file {transferred_file}")
                 return False
 
             # Read the tilt angles and pixel size from the txrm
             angles: list = []
-            metadata: dict[str, Any] = {
-                "source": str(self._basepath),
-                "tilt_series_tag": transferred_file.stem,
-            }
+            metadata["source"] = str(self._basepath)
+            metadata["tilt_series_tag"] = transferred_file.stem
             with OleFileIO(str(transferred_file)) as txrm_ole:
                 if txrm_ole.exists("ReferenceData/Image"):
                     metadata["has_reference"] = True
 
                 if txrm_ole.exists("ImageInfo/Angles"):
-                    angles = np.frombuffer(
-                        txrm_ole.openstream("ImageInfo/Angles").getvalue(), np.float32
+                    angles = _get_ole_header_value(
+                        txrm_ole, "ImageInfo/Angles", np.float32
                     ).tolist()
                     metadata["minimum_angle"] = min(angles)
                     metadata["maximum_angle"] = max(angles)
 
                 if txrm_ole.exists("ImageInfo/PixelSize"):
-                    pixel_size_txrm = np.frombuffer(
-                        txrm_ole.openstream("ImageInfo/PixelSize").getvalue(),
-                        np.float32,
+                    pixel_size_txrm = _get_ole_header_value(
+                        txrm_ole, "ImageInfo/PixelSize", np.float32
                     ).tolist()
                     metadata["pixel_size"] = pixel_size_txrm[0] * 1e4
 
                 if txrm_ole.exists("ImageInfo/ImageWidth"):
-                    image_width_txrm = np.frombuffer(
-                        txrm_ole.openstream("ImageInfo/ImageWidth").getvalue(), np.int32
+                    image_width_txrm = _get_ole_header_value(
+                        txrm_ole, "ImageInfo/ImageWidth", np.int32
                     ).tolist()
                     metadata["image_size_x"] = image_width_txrm[0]
 
                 if txrm_ole.exists("ImageInfo/ImageHeight"):
-                    image_height_txrm = np.frombuffer(
-                        txrm_ole.openstream("ImageInfo/ImageHeight").getvalue(),
-                        np.int32,
+                    image_height_txrm = _get_ole_header_value(
+                        txrm_ole, "ImageInfo/ImageHeight", np.int32
                     ).tolist()
                     metadata["image_size_y"] = image_height_txrm[0]
 
                 if txrm_ole.exists("ImageInfo/ExpTimes"):
-                    exposure_time_txrm = np.frombuffer(
-                        txrm_ole.openstream("ImageInfo/ExpTimes").getvalue(), np.float32
+                    exposure_time_txrm = _get_ole_header_value(
+                        txrm_ole, "ImageInfo/ExpTimes", np.float32
                     ).tolist()
                     metadata["exposure_time"] = exposure_time_txrm[0]
 
                 if txrm_ole.exists("ImageInfo/XrayMagnification"):
-                    magnification_txrm = np.frombuffer(
-                        txrm_ole.openstream("ImageInfo/XrayMagnification").getvalue(),
-                        np.float32,
+                    magnification_txrm = _get_ole_header_value(
+                        txrm_ole, "ImageInfo/XrayMagnification", np.float32
                     ).tolist()
                     metadata["magnification"] = magnification_txrm[0]
 
                 if txrm_ole.exists("ImageInfo/ImagesTaken"):
-                    tilt_count_txrm = np.frombuffer(
-                        txrm_ole.openstream("ImageInfo/ImagesTaken").getvalue(),
-                        np.int32,
+                    tilt_count_txrm = _get_ole_header_value(
+                        txrm_ole, "ImageInfo/ImagesTaken", np.int32
                     ).tolist()
                     metadata["tilt_series_length"] = tilt_count_txrm[0]
 
@@ -235,9 +391,8 @@ def post_transfer(
                         .split("\x00")
                         if i
                     ]
-                    axis_values = np.frombuffer(
-                        txrm_ole.openstream("PositionInfo/MotorPositions").getvalue(),
-                        np.float32,
+                    axis_values = _get_ole_header_value(
+                        txrm_ole, "PositionInfo/MotorPositions", np.float32
                     )
                     if "Energy" in axis_names:
                         energy_index = list(np.array(axis_names) == "Energy").index(
@@ -302,7 +457,7 @@ def post_transfer(
                 reference_file_transferred_to = None
             capture_post(
                 base_url=str(environment.url.geturl()),
-                router_name="workflow.sxt_router",
+                router_name="workflow_sxt.router",
                 function_name="process_sxt_tilt_series",
                 token=self._token,
                 instrument_name=environment.instrument_name,

src/murfey/server/api/workflow.py

@@ -78,9 +78,6 @@
     motion_corrected_mrc,
 )
 from murfey.util.tomo import midpoint
-from murfey.workflows.sxt.process_sxt_tilt_series import (
-    SXTTiltSeriesInfo,
-)
 from murfey.workflows.tomo.tomo_metadata import register_search_map_in_database
 
 logger = getLogger("murfey.server.api.workflow")
@@ -100,6 +97,10 @@ class DCGroupParameters(BaseModel):
     atlas: str = ""
     sample: Optional[int] = None
     atlas_pixel_size: float = 0
+    atlas_x_stage_position: float | None = None
+    atlas_y_stage_position: float | None = None
+    atlas_width: int | None = None
+    atlas_height: int | None = None
     create_smartem_grid: bool = False
     acquisition_uuid: Optional[str] = None
 
@@ -1071,32 +1072,6 @@ def _add_tilt():
         _add_tilt()
 
 
-sxt_router = APIRouter(
-    prefix="/workflow/sxt",
-    dependencies=[Depends(validate_instrument_token)],
-    tags=["Workflows: Soft x-ray tomography"],
-)
-
-
-@sxt_router.post("/visits/{visit_name}/sessions/{session_id}/sxt_tilt_series")
-def process_sxt_tilt_series(
-    visit_name: str,
-    session_id: MurfeySessionID,
-    tilt_series_info: SXTTiltSeriesInfo,
-    db=murfey_db,
-):
-    if _transport_object:
-        _transport_object.send(
-            _transport_object.feedback_queue,
-            {
-                "register": "sxt.process_tilt_series",
-                "session_id": session_id,
-                "visit_name": visit_name,
-                "tilt_series_info": tilt_series_info.model_dump(mode="json"),
-            },
-        )
-
-
 correlative_router = APIRouter(
     prefix="/workflow/correlative",
     dependencies=[Depends(validate_instrument_token)],