diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0667d1fda..f43f010d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,53 +45,15 @@ repos: name: Formatting the pyproject.toml file additional_dependencies: ["tox>=4.9"] - # Automatic source code formatting - - repo: https://github.com/psf/black - rev: 24.3.0 # Released 2024-03-15 + # Format and lint using Ruff + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.0 hooks: - - id: black - name: Formatting Python scripts according to Black - args: [--safe, --quiet] - - # Automatically sort imports - - repo: https://github.com/PyCQA/isort - rev: 5.13.2 # Released 2023-12-13 - hooks: - - id: isort - name: Tidying Python imports via Isort - args: [ - #'-a', 'from __future__ import annotations', # 3.7-3.11 - "--rm", - "from __future__ import absolute_import", # -3.0 - "--rm", - "from __future__ import division", # -3.0 - "--rm", - "from __future__ import generator_stop", # -3.7 - "--rm", - "from __future__ import generators", # -2.3 - "--rm", - "from __future__ import nested_scopes", # -2.2 - "--rm", - "from __future__ import print_function", # -3.0 - "--rm", - "from __future__ import unicode_literals", # -3.0 - "--rm", - "from __future__ import with_statement", # -2.6 - ] - - # Linting - - repo: https://github.com/PyCQA/flake8 - # Release history: https://flake8.pycqa.org/en/latest/release-notes/index.html - rev: 7.0.0 # Released 2024-01-05 - hooks: - - id: flake8 - name: Running Flake8 linter on Python files - additional_dependencies: [ - # Added Flake8-pyproject plugin for .toml compatibility - "Flake8-pyproject==1.2.3", # Released 2023-03-21 - # flake8-comprehensions URL: https://github.com/adamchainz/flake8-comprehensions - "flake8-comprehensions==3.14.0", # Released 2023-07-10 - ] + - id: ruff + name: Running Ruff linter + args: ["--fix"] + - id: ruff-format + name: Running Ruff formatter # Type checking - repo: https://github.com/pre-commit/mirrors-mypy diff --git a/pyproject.toml b/pyproject.toml index db18f1c71..835d0b5e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,53 +121,26 @@ zip-safe = false [tool.setuptools.packages.find] where = ["src", "tests"] -[tool.isort] -profile = "black" - -[tool.flake8] -# Flake8-pyproject allows TOML file settings to be read into Flake8 -# URL: https://pypi.org/project/Flake8-pyproject/ -select = [ +[tool.ruff] +line-length = 88 +lint.extend-ignore = ["E203", "E266", "E501", "E731", "E741"] +lint.select = [ "C4", - "E401", - "E711", - "E712", - "E713", - "E714", - "E721", - "E722", - "E901", - "F401", - "F402", - "F403", - "F405", - "F541", - "F631", - "F632", - "F633", - "F811", - "F812", - "F821", - "F822", - "F841", - "F901", - "W191", - "W291", - "W292", - "W293", - "W602", - "W603", - "W604", - "W605", - "W606", -] -ignore = [ - "E203", - "E266", - "E501", - "W503", + "E4", "E7", "E9", + "F", + "I", + "W6", ] -max-line-length = "88" +fix = true + +[tool.ruff.lint.isort] +known-first-party = ["murfey"] +combine-as-imports = true + +[tool.ruff.format] +quote-style="double" +indent-style="space" +line-ending="auto" [tool.pyproject-fmt] inputs = "pyproject.toml" diff --git a/src/murfey/cli/generate_route_manifest.py b/src/murfey/cli/generate_route_manifest.py index bbea95e3d..d0ee1e464 100644 --- a/src/murfey/cli/generate_route_manifest.py +++ b/src/murfey/cli/generate_route_manifest.py @@ -68,7 +68,6 @@ def is_implicitly_resolved(value: str) -> bool: def find_routers(name: str) -> dict[str, APIRouter]: - def _extract_routers_from_module(module: ModuleType): routers = {} for name, obj in inspect.getmembers(module): @@ -115,7 +114,6 @@ def _extract_routers_from_module(module: ModuleType): def get_route_manifest(routers: dict[str, APIRouter]): - manifest = {} for router_name, router in routers.items(): diff --git a/src/murfey/cli/inject_spa_processing.py b/src/murfey/cli/inject_spa_processing.py index 2294835a5..7a7f66bfa 100644 --- a/src/murfey/cli/inject_spa_processing.py +++ b/src/murfey/cli/inject_spa_processing.py @@ -173,7 +173,6 @@ def run(): / str(ppath.stem + "_motion_corrected.mrc") ) if proc_params: - detached_ids = [c.id for c in collected_ids] if not mrc_out.parent.exists(): diff --git a/src/murfey/cli/spa_ispyb_messages.py b/src/murfey/cli/spa_ispyb_messages.py index f78388008..640b585fd 100644 --- a/src/murfey/cli/spa_ispyb_messages.py +++ b/src/murfey/cli/spa_ispyb_messages.py @@ -14,8 +14,7 @@ DataCollectionGroup, ProcessingJob, ) -from sqlmodel import Session as MurfeySession -from sqlmodel import create_engine, select +from sqlmodel import Session as MurfeySession, create_engine, select from murfey.client.contexts.spa import _get_xml_list_index from murfey.server.feedback import _murfey_id, _register diff --git a/src/murfey/cli/transfer.py b/src/murfey/cli/transfer.py deleted file mode 100644 index d5bd7a503..000000000 --- a/src/murfey/cli/transfer.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import argparse -import subprocess -from pathlib import Path -from urllib.parse import ParseResult, urlparse - -import requests -from rich.console import Console -from rich.prompt import Confirm - -from murfey.client import read_config -from murfey.util.api import url_path_for -from murfey.util.config import MachineConfig - - -def run(): - config = read_config() - known_server = config["Murfey"].get("server", "") - instrument_name = config["Murfey"].get("instrument_name", "") - - parser = argparse.ArgumentParser(description="Transfer using a remote rsync daemon") - - parser.add_argument("--source", type=str, default="") - parser.add_argument("--destination", "-d", type=str) - parser.add_argument("--destination-prefix", type=str, default="data") - parser.add_argument("--delete", action="store_true") - parser.add_argument( - "--server", - metavar="HOST:PORT", - type=str, - help=f"Murfey server to connect to ({known_server})", - default=known_server, - ) - - args = parser.parse_args() - - console = Console() - murfey_url: ParseResult = urlparse(args.server, allow_fragments=False) - - machine_data = MachineConfig( - requests.get( - f"{murfey_url.geturl()}{url_path_for('session_control.router', 'machine_info_by_instrument', instrument_name=instrument_name)}" - ).json() - ) - if Path(args.source or ".").resolve() in machine_data.data_directories: - console.print("[red]Source directory is the base directory, exiting") - return - - cmd = [ - "rsync", - "-iiv", - "--times", - "--progress", - "-o", # preserve ownership - "-p", # preserve permissions - ] - if args.delete: - cmd.append("--remove-source-files") - if Path(args.source or ".").is_file(): - num_files = 1 - else: - num_files = len( - [f for f in Path(args.source or ".").glob("**/*") if f.is_file()] - ) - delete_prompt = Confirm.ask( - f"Do you want to remove {num_files} from {args.source or Path('.').resolve()}?" - ) - if not delete_prompt: - return - console.print( - f"Copying {args.source} -> {murfey_url.hostname}::{args.destination_prefix}/{args.destination}" - ) - if Path(args.source or ".").is_file(): - cmd.extend( - [ - args.source or ".", - f"{murfey_url.hostname}::{args.destination_prefix}/{args.destination}", - ] - ) - else: - cmd.append("-r") - cmd.extend(list(Path(args.source or ".").glob("*"))) - cmd.append(f"{murfey_url.hostname}::{args.destination}") - - result = subprocess.run(cmd) - if result.returncode: - console.print(f"[red]rsync failed returning code {result.returncode}") diff --git a/src/murfey/client/contexts/clem.py b/src/murfey/client/contexts/clem.py index 0c3aa36b2..772c72959 100644 --- a/src/murfey/client/contexts/clem.py +++ b/src/murfey/client/contexts/clem.py @@ -65,7 +65,6 @@ def _get_image_elements(root: ET.Element) -> List[ET.Element]: def _find_elements_recursively( node: ET.Element, ) -> Generator[ET.Element, None, None]: - # Find items labelled "Element" under current node elem_list = node.findall("./Children/Element") if len(elem_list) < 1: # Try alternative path for top-level of XML tree @@ -103,7 +102,6 @@ def post_transfer( environment: Optional[MurfeyInstanceEnvironment] = None, **kwargs, ) -> bool: - super().post_transfer(transferred_file, environment=environment, **kwargs) # Process files generated by "auto-save" acquisition mode @@ -188,7 +186,6 @@ def post_transfer( # Process XLIF files if transferred_file.suffix == ".xlif": - # Skip processing of "_histo" histogram XLIF files if transferred_file.stem.endswith("_histo"): logger.debug( diff --git a/src/murfey/client/tui/progress.py b/src/murfey/client/tui/progress.py index b1e787632..f23155a49 100644 --- a/src/murfey/client/tui/progress.py +++ b/src/murfey/client/tui/progress.py @@ -71,7 +71,6 @@ def _get_pulse_segments( def __rich_console__( self, console: Console, options: ConsoleOptions ) -> RenderResult: - width = min(self.width or options.max_width, options.max_width) ascii = options.ascii_only if self.pulse: diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 780eadf9d..21af41dd0 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -193,7 +193,6 @@ def __init__( self._context = SPAModularContext def compose(self): - machine_data = capture_get( base_url=str(self.app._environment.url.geturl()), router_name="session_control.router", @@ -1013,7 +1012,7 @@ def compose(self): name_root += st if dest_num: dest = str( - dest_path.parent / f"{name_root}{dest_num+1}" + dest_path.parent / f"{name_root}{dest_num + 1}" ) else: dest = str(dest_path.parent / f"{name_root}2") diff --git a/src/murfey/server/api/auth.py b/src/murfey/server/api/auth.py index add0f8342..a09c7eeac 100644 --- a/src/murfey/server/api/auth.py +++ b/src/murfey/server/api/auth.py @@ -23,8 +23,7 @@ from murfey.server.murfey_db import murfey_db, url from murfey.util.api import url_path_for from murfey.util.config import get_security_config -from murfey.util.db import MurfeyUser as User -from murfey.util.db import Session as MurfeySession +from murfey.util.db import MurfeyUser as User, Session as MurfeySession # Set up logger logger = getLogger("murfey.server.api.auth") @@ -147,7 +146,7 @@ def validate_session_against_visit(session_id: int, visit: str): async def validate_instrument_token( - token: Annotated[str, Depends(instrument_oauth2_scheme)] + token: Annotated[str, Depends(instrument_oauth2_scheme)], ): """ Used by the backend routers to check the incoming instrument server token. @@ -334,7 +333,6 @@ def validate_user(username: str, password: str) -> bool: def create_access_token(data: dict, token: str = "") -> str: - # If authenticating with password, auth URL needs a 'mint_session_token' endpoint if security_config.auth_type == "password": if auth_url and data.get("session"): @@ -420,6 +418,6 @@ async def mint_session_token(session_id: MurfeySessionIDFrontend, db=murfey_db): @router.get("/validate_token") async def simple_token_validation( - token: Annotated[str, Depends(validate_instrument_token)] + token: Annotated[str, Depends(validate_instrument_token)], ): return {"valid": True} diff --git a/src/murfey/server/api/bootstrap.py b/src/murfey/server/api/bootstrap.py index da74d5bd7..42980ba8b 100644 --- a/src/murfey/server/api/bootstrap.py +++ b/src/murfey/server/api/bootstrap.py @@ -308,7 +308,7 @@ def parse_cygwin_request( raise ValueError(f"{request_path!r} is not a valid request path") try: - url = f'{find_cygwin_mirror()}{quote(request_path, safe="/")}' + url = f"{find_cygwin_mirror()}{quote(request_path, safe='/')}" except Exception: raise HTTPException( status_code=503, detail="Could not identify a suitable Cygwin mirror" @@ -533,7 +533,7 @@ def get_msys2_environment_index( raise ValueError(f"{system!r} is not a valid msys2 environment") # Construct URL to main MSYS repo and get response - arch_url = f'{msys2_url}/{quote(system, safe="/")}' + arch_url = f"{msys2_url}/{quote(system, safe='/')}" response = http_session.get(arch_url) # Parse and rewrite package index content @@ -579,7 +579,7 @@ def get_msys2_package_index( # Construct URL to main MSYS repo and get response package_list_url = ( - f'{msys2_url}/{quote(system, safe="/")}/{quote(environment, safe="/")}' + f"{msys2_url}/{quote(system, safe='/')}/{quote(environment, safe='/')}" ) response = http_session.get(package_list_url) return Response( @@ -617,7 +617,7 @@ def get_msys2_package_file( raise ValueError(f"{package!r} is not a valid package name") # Construct URL to main MSYS repo and get response - package_url = f'{msys2_url}/{quote(system, safe="/")}/{quote(environment, safe="/")}/{quote(package, safe="/")}' + package_url = f"{msys2_url}/{quote(system, safe='/')}/{quote(environment, safe='/')}/{quote(package, safe='/')}" response = http_session.get(package_url) if response.status_code != 200: raise HTTPException(status_code=response.status_code) diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py index f4f2060e3..e5fdbe988 100644 --- a/src/murfey/server/api/clem.py +++ b/src/murfey/server/api/clem.py @@ -24,8 +24,8 @@ CLEMImageStack, CLEMLIFFile, CLEMTIFFFile, + Session as MurfeySession, ) -from murfey.util.db import Session as MurfeySession # Set up logger logger = getLogger("murfey.server.api.clem") @@ -409,7 +409,6 @@ def register_clem_metadata( associated_stacks: list[Path] = [], db: Session = murfey_db, ): - # Return database entry if it already exists try: clem_metadata: CLEMImageMetadata = get_db_entry( diff --git a/src/murfey/server/api/file_io_frontend.py b/src/murfey/server/api/file_io_frontend.py index 531fa81c8..0356859b3 100644 --- a/src/murfey/server/api/file_io_frontend.py +++ b/src/murfey/server/api/file_io_frontend.py @@ -2,10 +2,14 @@ from fastapi import APIRouter, Depends -from murfey.server.api.auth import MurfeySessionIDFrontend as MurfeySessionID -from murfey.server.api.auth import validate_token -from murfey.server.api.file_io_shared import GainReference -from murfey.server.api.file_io_shared import process_gain as _process_gain +from murfey.server.api.auth import ( + MurfeySessionIDFrontend as MurfeySessionID, + validate_token, +) +from murfey.server.api.file_io_shared import ( + GainReference, + process_gain as _process_gain, +) from murfey.server.murfey_db import murfey_db logger = getLogger("murfey.server.api.file_io_frontend") diff --git a/src/murfey/server/api/file_io_instrument.py b/src/murfey/server/api/file_io_instrument.py index 4ee48dc40..2644b7f89 100644 --- a/src/murfey/server/api/file_io_instrument.py +++ b/src/murfey/server/api/file_io_instrument.py @@ -8,10 +8,14 @@ from sqlmodel import select from werkzeug.utils import secure_filename -from murfey.server.api.auth import MurfeySessionIDInstrument as MurfeySessionID -from murfey.server.api.auth import validate_instrument_token -from murfey.server.api.file_io_shared import GainReference -from murfey.server.api.file_io_shared import process_gain as _process_gain +from murfey.server.api.auth import ( + MurfeySessionIDInstrument as MurfeySessionID, + validate_instrument_token, +) +from murfey.server.api.file_io_shared import ( + GainReference, + process_gain as _process_gain, +) from murfey.server.murfey_db import murfey_db from murfey.util import sanitise, secure_path from murfey.util.config import get_machine_config diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 699dc01c6..018e62490 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -14,9 +14,9 @@ from werkzeug.utils import secure_filename import murfey.server.prometheus as prom -from murfey.server.api.auth import MurfeyInstrumentNameFrontend as MurfeyInstrumentName -from murfey.server.api.auth import MurfeySessionIDFrontend as MurfeySessionID from murfey.server.api.auth import ( + MurfeyInstrumentNameFrontend as MurfeyInstrumentName, + MurfeySessionIDFrontend as MurfeySessionID, create_access_token, instrument_server_tokens, oauth2_scheme, diff --git a/src/murfey/server/api/processing_parameters.py b/src/murfey/server/api/processing_parameters.py index 80061a234..bbc7ab4cc 100644 --- a/src/murfey/server/api/processing_parameters.py +++ b/src/murfey/server/api/processing_parameters.py @@ -6,8 +6,10 @@ from pydantic import BaseModel from sqlmodel import Session, select -from murfey.server.api.auth import MurfeySessionIDFrontend as MurfeySessionID -from murfey.server.api.auth import validate_token +from murfey.server.api.auth import ( + MurfeySessionIDFrontend as MurfeySessionID, + validate_token, +) from murfey.server.murfey_db import murfey_db from murfey.util.db import SessionProcessingParameters diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 255cc0246..6316cbcb3 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -13,23 +13,20 @@ import murfey.server.prometheus as prom from murfey.server import _transport_object -from murfey.server.api.auth import MurfeySessionIDInstrument as MurfeySessionID -from murfey.server.api.auth import validate_instrument_token -from murfey.server.api.shared import get_foil_hole as _get_foil_hole -from murfey.server.api.shared import ( - get_foil_holes_from_grid_square as _get_foil_holes_from_grid_square, +from murfey.server.api.auth import ( + MurfeySessionIDInstrument as MurfeySessionID, + validate_instrument_token, ) -from murfey.server.api.shared import get_grid_squares as _get_grid_squares from murfey.server.api.shared import ( + get_foil_hole as _get_foil_hole, + get_foil_holes_from_grid_square as _get_foil_holes_from_grid_square, + get_grid_squares as _get_grid_squares, get_grid_squares_from_dcg as _get_grid_squares_from_dcg, -) -from murfey.server.api.shared import ( get_machine_config_for_instrument, get_upstream_tiff_dirs, remove_session_by_id, ) -from murfey.server.ispyb import DB as ispyb_db -from murfey.server.ispyb import get_all_ongoing_visits +from murfey.server.ispyb import DB as ispyb_db, get_all_ongoing_visits from murfey.server.murfey_db import murfey_db from murfey.util import sanitise from murfey.util.config import MachineConfig, get_machine_config @@ -57,8 +54,6 @@ from murfey.workflows.spa.atlas import atlas_jpg_from_mrc from murfey.workflows.spa.flush_spa_preprocess import ( register_foil_hole as _register_foil_hole, -) -from murfey.workflows.spa.flush_spa_preprocess import ( register_grid_square as _register_grid_square, ) from murfey.workflows.tomo.tomo_metadata import ( diff --git a/src/murfey/server/api/session_info.py b/src/murfey/server/api/session_info.py index bc6014347..62f3be1f3 100644 --- a/src/murfey/server/api/session_info.py +++ b/src/murfey/server/api/session_info.py @@ -12,24 +12,21 @@ import murfey.server.api.websocket as ws from murfey.server import _transport_object from murfey.server.api import templates -from murfey.server.api.auth import MurfeyInstrumentNameFrontend as MurfeyInstrumentName -from murfey.server.api.auth import MurfeySessionIDFrontend as MurfeySessionID -from murfey.server.api.auth import validate_token -from murfey.server.api.shared import get_foil_hole as _get_foil_hole -from murfey.server.api.shared import ( - get_foil_holes_from_grid_square as _get_foil_holes_from_grid_square, +from murfey.server.api.auth import ( + MurfeyInstrumentNameFrontend as MurfeyInstrumentName, + MurfeySessionIDFrontend as MurfeySessionID, + validate_token, ) -from murfey.server.api.shared import get_grid_squares as _get_grid_squares from murfey.server.api.shared import ( + get_foil_hole as _get_foil_hole, + get_foil_holes_from_grid_square as _get_foil_holes_from_grid_square, + get_grid_squares as _get_grid_squares, get_grid_squares_from_dcg as _get_grid_squares_from_dcg, -) -from murfey.server.api.shared import ( get_machine_config_for_instrument, get_upstream_tiff_dirs, remove_session_by_id, ) -from murfey.server.ispyb import DB as ispyb_db -from murfey.server.ispyb import get_all_ongoing_visits +from murfey.server.ispyb import DB as ispyb_db, get_all_ongoing_visits from murfey.server.murfey_db import murfey_db from murfey.util import sanitise from murfey.util.config import MachineConfig, get_machine_config diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index dcfbea914..8fec45f5d 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -26,8 +26,10 @@ import murfey.server.prometheus as prom from murfey.server import _transport_object -from murfey.server.api.auth import MurfeySessionIDInstrument as MurfeySessionID -from murfey.server.api.auth import validate_instrument_token +from murfey.server.api.auth import ( + MurfeySessionIDInstrument as MurfeySessionID, + validate_instrument_token, +) from murfey.server.feedback import ( _murfey_id, check_tilt_series_mc, @@ -36,8 +38,7 @@ get_job_ids, get_tomo_proc_params, ) -from murfey.server.ispyb import DB as ispyb_db -from murfey.server.ispyb import get_proposal_id +from murfey.server.ispyb import DB as ispyb_db, get_proposal_id from murfey.server.murfey_db import murfey_db from murfey.util import sanitise from murfey.util.config import get_machine_config @@ -435,7 +436,6 @@ async def request_spa_preprocessing( ) foil_hole_id = None if proc_params: - detached_ids = [c.id for c in collected_ids] murfey_ids = _murfey_id(detached_ids[3], db, number=2, close=False) @@ -494,9 +494,9 @@ async def request_spa_preprocessing( } # log.info(f"Sending Zocalo message {zocalo_message}") if _transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = _transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + _transport_object.feedback_queue + ) _transport_object.send("processing_recipe", zocalo_message) else: logger.error( @@ -723,9 +723,9 @@ async def request_tomography_preprocessing( }, } if _transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = _transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + _transport_object.feedback_queue + ) _transport_object.send("processing_recipe", zocalo_message) else: logger.error( diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index 24280d5de..c59cde2c3 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -22,8 +22,10 @@ import murfey.server.api.bootstrap import murfey.server.prometheus as prom from murfey.server.api import templates -from murfey.server.api.auth import MurfeySessionIDFrontend as MurfeySessionID -from murfey.server.api.auth import validate_token +from murfey.server.api.auth import ( + MurfeySessionIDFrontend as MurfeySessionID, + validate_token, +) from murfey.server.api.session_info import Visit from murfey.server.api.workflow import ( DCGroupParameters, diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index fd5a68207..288f3926f 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -390,9 +390,9 @@ def _release_2d_hold(message: dict, _db): _db.commit() _db.close() if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -466,9 +466,9 @@ def _release_3d_hold(message: dict, _db): "recipes": ["em-spa-class3d"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -547,9 +547,9 @@ def _release_refine_hold(message: dict, _db): "recipes": ["em-spa-refine"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -662,9 +662,9 @@ def _register_incomplete_2d_batch(message: dict, _db, demo: bool = False): "recipes": ["em-spa-class2d"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -825,9 +825,9 @@ def _register_complete_2d_batch(message: dict, _db, demo: bool = False): "recipes": ["em-spa-class2d"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -894,9 +894,9 @@ def _register_complete_2d_batch(message: dict, _db, demo: bool = False): "recipes": ["em-spa-class2d"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -984,9 +984,9 @@ def _flush_class2d( "recipes": ["em-spa-class2d"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -1266,7 +1266,7 @@ def _register_3d_batch(message: dict, _db, demo: bool = False): other_options["initial_model"] = str(rescaled_initial_model_path) next_job = feedback_params.next_job class3d_dir = ( - f"{class3d_message['class3d_dir']}{(feedback_params.next_job+1):03}" + f"{class3d_message['class3d_dir']}{(feedback_params.next_job + 1):03}" ) feedback_params.next_job += 1 _db.add(feedback_params) @@ -1305,7 +1305,7 @@ def _register_3d_batch(message: dict, _db, demo: bool = False): # For the first batch, start a container and set the database to wait next_job = feedback_params.next_job class3d_dir = ( - f"{class3d_message['class3d_dir']}{(feedback_params.next_job+1):03}" + f"{class3d_message['class3d_dir']}{(feedback_params.next_job + 1):03}" ) class3d_grp_uuid = _murfey_id(message["program_id"], _db)[0] class_uuids = _murfey_id(message["program_id"], _db, number=4) @@ -1351,9 +1351,9 @@ def _register_3d_batch(message: dict, _db, demo: bool = False): "recipes": ["em-spa-class3d"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -1394,9 +1394,9 @@ def _register_3d_batch(message: dict, _db, demo: bool = False): "recipes": ["em-spa-class3d"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -1512,9 +1512,9 @@ def _flush_tomography_preprocessing(message: dict, _db): f"Launching tomography preprocessing with Zocalo message: {zocalo_message}" ) if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -1695,9 +1695,9 @@ def _register_refinement(message: dict, _db, demo: bool = False): "recipes": ["em-spa-refine"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -1818,9 +1818,9 @@ def _register_bfactors(message: dict, _db, demo: bool = False): "recipes": ["em-spa-bfactor"], } if murfey.server._transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = murfey.server._transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + murfey.server._transport_object.feedback_queue + ) murfey.server._transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index a09220dcc..70dc58e8c 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -706,7 +706,6 @@ def get_session_id( visit_number: str, db: Session | None, ) -> int | None: - # Log received lookup parameters log.debug( "Looking up ISPyB BLSession ID using the following values:\n" diff --git a/src/murfey/util/client.py b/src/murfey/util/client.py index 1b1119aea..c7d897a79 100644 --- a/src/murfey/util/client.py +++ b/src/murfey/util/client.py @@ -232,7 +232,9 @@ async def anotify( listeners = ( self._secondary_listeners if secondary - else self._final_listeners if final else self._listeners + else self._final_listeners + if final + else self._listeners ) for notify_function in listeners: result = notify_function(*args, **kwargs) @@ -253,7 +255,9 @@ def notify( listeners = ( self._secondary_listeners if secondary - else self._final_listeners if final else self._listeners + else self._final_listeners + if final + else self._listeners ) for notify_function in listeners: result = notify_function(*args, **kwargs) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 8dee78fcf..ef1606130 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -195,7 +195,6 @@ class CLEMTIFFFile(SQLModel, table=True): # type: ignore class CLEMImageMetadata(SQLModel, table=True): # type: ignore - id: Optional[int] = Field(default=None, primary_key=True) file_path: str = Field(index=True) # Full path to metadata file diff --git a/src/murfey/util/spa_metadata.py b/src/murfey/util/spa_metadata.py index e471aa93d..2a11ffebf 100644 --- a/src/murfey/util/spa_metadata.py +++ b/src/murfey/util/spa_metadata.py @@ -49,7 +49,9 @@ def foil_hole_from_file(f: Path) -> int: return int(f.name.split("_")[1]) -def get_grid_square_atlas_positions(xml_path: Path, grid_square: str = "") -> Dict[ +def get_grid_square_atlas_positions( + xml_path: Path, grid_square: str = "" +) -> Dict[ str, Tuple[ Optional[int], diff --git a/src/murfey/workflows/clem/__init__.py b/src/murfey/workflows/clem/__init__.py index 16ff4e4f7..b40d48cb3 100644 --- a/src/murfey/workflows/clem/__init__.py +++ b/src/murfey/workflows/clem/__init__.py @@ -15,8 +15,8 @@ CLEMImageStack, CLEMLIFFile, CLEMTIFFFile, + Session as MurfeySession, ) -from murfey.util.db import Session as MurfeySession logger = logging.getLogger("murfey.workflows.clem") diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 191512c1c..f52ccb471 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -24,8 +24,8 @@ CLEMImageStack, CLEMLIFFile, CLEMTIFFFile, + Session as MurfeySession, ) -from murfey.util.db import Session as MurfeySession from murfey.util.processing_params import ( default_clem_align_and_merge_parameters as processing_params, ) @@ -57,7 +57,6 @@ class CLEMPreprocessingResult(BaseModel): def run(message: dict, murfey_db: Session, demo: bool = False) -> bool: - session_id: int = ( int(message["session_id"]) if not isinstance(message["session_id"], int) diff --git a/src/murfey/workflows/spa/flush_spa_preprocess.py b/src/murfey/workflows/spa/flush_spa_preprocess.py index b225dfb69..98dc83096 100644 --- a/src/murfey/workflows/spa/flush_spa_preprocess.py +++ b/src/murfey/workflows/spa/flush_spa_preprocess.py @@ -20,9 +20,10 @@ Movie, PreprocessStash, ProcessingJob, + Session as MurfeySession, + SPAFeedbackParameters, + SPARelionParameters, ) -from murfey.util.db import Session as MurfeySession -from murfey.util.db import SPAFeedbackParameters, SPARelionParameters from murfey.util.models import FoilHoleParameters, GridSquareParameters from murfey.util.processing_params import cryolo_model_path, default_spa_parameters from murfey.util.spa_metadata import ( @@ -430,9 +431,9 @@ def flush_spa_preprocess(message: dict, murfey_db: Session, demo: bool = False) }, } if _transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = _transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + _transport_object.feedback_queue + ) _transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) diff --git a/src/murfey/workflows/spa/picking.py b/src/murfey/workflows/spa/picking.py index e383f438f..72a1e3ba2 100644 --- a/src/murfey/workflows/spa/picking.py +++ b/src/murfey/workflows/spa/picking.py @@ -25,9 +25,10 @@ ParticleSizes, ProcessingJob, SelectionStash, + Session as MurfeySession, + SPAFeedbackParameters, + SPARelionParameters, ) -from murfey.util.db import Session as MurfeySession -from murfey.util.db import SPAFeedbackParameters, SPARelionParameters from murfey.util.processing_params import default_spa_parameters logger = getLogger("murfey.workflows.spa.picking") @@ -154,9 +155,9 @@ def _register_picked_particles_use_diameter( "recipes": ["em-spa-extract"], } if _transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = _transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + _transport_object.feedback_queue + ) _transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -194,9 +195,9 @@ def _register_picked_particles_use_diameter( "recipes": ["em-spa-extract"], } if _transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = _transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + _transport_object.feedback_queue + ) _transport_object.send( "processing_recipe", zocalo_message, new_connection=True ) @@ -316,9 +317,9 @@ def _register_picked_particles_use_boxsize(message: dict, _db: Session): "recipes": ["em-spa-extract"], } if _transport_object: - zocalo_message["parameters"][ - "feedback_queue" - ] = _transport_object.feedback_queue + zocalo_message["parameters"]["feedback_queue"] = ( + _transport_object.feedback_queue + ) _transport_object.send("processing_recipe", zocalo_message, new_connection=True) _db.close() diff --git a/src/murfey/workflows/tomo/tomo_metadata.py b/src/murfey/workflows/tomo/tomo_metadata.py index 9c145ab6a..2dbc25cb7 100644 --- a/src/murfey/workflows/tomo/tomo_metadata.py +++ b/src/murfey/workflows/tomo/tomo_metadata.py @@ -8,9 +8,12 @@ from murfey.server.gain import Camera from murfey.util import sanitise from murfey.util.config import get_machine_config -from murfey.util.db import DataCollectionGroup, SearchMap -from murfey.util.db import Session as MurfeySession -from murfey.util.db import TiltSeries +from murfey.util.db import ( + DataCollectionGroup, + SearchMap, + Session as MurfeySession, + TiltSeries, +) from murfey.util.models import BatchPositionParameters, SearchMapParameters logger = logging.getLogger("murfey.client.util.tomo_metadata") diff --git a/tests/conftest.py b/tests/conftest.py index 247744512..4429e6074 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,10 +12,8 @@ from sqlalchemy import Engine, RootTransaction, and_, create_engine, event, select from sqlalchemy.exc import InterfaceError from sqlalchemy.ext.declarative import DeclarativeMeta -from sqlalchemy.orm import Session as SQLAlchemySession -from sqlalchemy.orm import sessionmaker -from sqlmodel import Session as SQLModelSession -from sqlmodel import SQLModel +from sqlalchemy.orm import Session as SQLAlchemySession, sessionmaker +from sqlmodel import Session as SQLModelSession, SQLModel from murfey.util.db import Session as MurfeySession diff --git a/tests/instrument_server/test_api.py b/tests/instrument_server/test_api.py index 54ed53563..cf48b3ba7 100644 --- a/tests/instrument_server/test_api.py +++ b/tests/instrument_server/test_api.py @@ -8,9 +8,11 @@ from fastapi.testclient import TestClient from pytest_mock import MockerFixture -from murfey.instrument_server.api import _get_murfey_url -from murfey.instrument_server.api import router as client_router -from murfey.instrument_server.api import validate_session_token +from murfey.instrument_server.api import ( + _get_murfey_url, + router as client_router, + validate_session_token, +) from murfey.util import posix_path from murfey.util.api import url_path_for diff --git a/tests/instrument_server/test_init.py b/tests/instrument_server/test_init.py index 30e218b67..e575ecea0 100644 --- a/tests/instrument_server/test_init.py +++ b/tests/instrument_server/test_init.py @@ -12,8 +12,7 @@ import murfey from murfey.client.update import UPDATE_SUCCESS from murfey.instrument_server import check_for_updates, start_instrument_server -from murfey.server.api.bootstrap import pypi as pypi_router -from murfey.server.api.bootstrap import version as version_router +from murfey.server.api.bootstrap import pypi as pypi_router, version as version_router from murfey.util.api import url_path_for # Set up a test router with only the essential endpoints @@ -37,7 +36,6 @@ def test_check_for_updates( test_params: tuple[str], mocker: MockerFixture, ): - # Unpack test params (bump_client_version,) = test_params @@ -136,7 +134,6 @@ def test_check_for_updates( def test_start_instrument_server( mocker: MockerFixture, test_params: tuple[Optional[str], Optional[int]] ): - # Unpack test params host, port = test_params diff --git a/tests/server/api/test_movies.py b/tests/server/api/test_movies.py index 00cc4f464..a3c8abe89 100644 --- a/tests/server/api/test_movies.py +++ b/tests/server/api/test_movies.py @@ -15,7 +15,6 @@ def test_movie_count( murfey_db_session: Session, # From conftest.py ): - # Insert table dependencies dcg_entry: DataCollectionGroup = get_or_create_db_entry( murfey_db_session, diff --git a/tests/workflows/clem/test_align_and_merge.py b/tests/workflows/clem/test_align_and_merge.py index 96e4cc430..b7d5d8ec3 100644 --- a/tests/workflows/clem/test_align_and_merge.py +++ b/tests/workflows/clem/test_align_and_merge.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Literal from unittest.mock import MagicMock, patch import pytest @@ -24,9 +25,9 @@ # Align and merge settings crop_to_n_frames = 20 -align_self = "enabled" -flatten = "max" -align_across = "enabled" +align_self: Literal["enabled", ""] = "enabled" +flatten: Literal["mean", "min", "max", ""] = "max" +align_across: Literal["enabled", ""] = "enabled" @pytest.fixture @@ -38,7 +39,6 @@ def processed_dir(tmp_path: Path): @pytest.fixture def image_stacks(processed_dir: Path): - image_dir = processed_dir / area_name / series_name image_dir.mkdir(parents=True, exist_ok=True) @@ -52,7 +52,6 @@ def image_stacks(processed_dir: Path): @pytest.fixture def metadata(processed_dir: Path): - metadata_dir = processed_dir / area_name / series_name / "metadata" metadata_dir.mkdir(parents=True, exist_ok=True) @@ -70,7 +69,6 @@ def test_submit_cluster_request( metadata: Path, processed_dir: Path, ): - # Construct the long series name series_name_long = "--".join( image_stacks[0].parent.relative_to(processed_dir).parts diff --git a/tests/workflows/clem/test_process_raw_lifs.py b/tests/workflows/clem/test_process_raw_lifs.py index 3399a80df..32b573fc8 100644 --- a/tests/workflows/clem/test_process_raw_lifs.py +++ b/tests/workflows/clem/test_process_raw_lifs.py @@ -33,7 +33,6 @@ def test_zocalo_cluster_request( lif_file: Path, raw_dir: Path, ): - # Create a mock tranpsort object mock_transport = MagicMock(spec=TransportManager) mock_transport.feedback_queue = feedback_queue diff --git a/tests/workflows/clem/test_process_raw_tiffs.py b/tests/workflows/clem/test_process_raw_tiffs.py index f784a154f..6cc92bcf2 100644 --- a/tests/workflows/clem/test_process_raw_tiffs.py +++ b/tests/workflows/clem/test_process_raw_tiffs.py @@ -54,7 +54,6 @@ def test_zocalo_cluster_request( metadata: Path, raw_dir: Path, ): - # Create a mock tranpsort object mock_transport = MagicMock(spec=TransportManager) mock_transport.feedback_queue = feedback_queue