diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bb24e03a..a2a490928 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Use Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -67,13 +68,13 @@ jobs: docker run --detach --name rabbitmq -p 127.0.0.1:5672:5672 -p 127.0.0.1:15672:15672 test-rabbitmq docker container list -a - - name: Get ispyb database + - name: Get ISPyB database uses: actions/download-artifact@v4 with: name: database path: database/ - - name: Install package + - name: Install Murfey run: | set -eux pip install --disable-pip-version-check -e "."[cicd,client,server,developer] @@ -84,7 +85,7 @@ jobs: mysql-version: "11.3" auto-start: false - - name: Set up test ipsyb database + - name: Set up test ISPyB database run: | set -eu cp ".github/workflows/config/my.cnf" .my.cnf @@ -101,9 +102,14 @@ jobs: schemas/ispyb/routines.sql \ grants/ispyb_processing.sql \ grants/ispyb_import.sql; do - echo Importing ${f}... + + echo "Patching ${f} in SQL files to fix CLI escape issues..." + sed -i 's/\\-/-/g' "$f" + + echo "Importing ${f}..." mariadb --defaults-file=.my.cnf < $f done + mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api@'%' IDENTIFIED BY 'password_1234'; GRANT ispyb_processing to ispyb_api@'%'; GRANT ispyb_import to ispyb_api@'%'; SET DEFAULT ROLE ispyb_processing FOR ispyb_api@'%';" mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api_future@'%' IDENTIFIED BY 'password_4321'; GRANT SELECT ON ispybtest.* to ispyb_api_future@'%';" mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api_sqlalchemy@'%' IDENTIFIED BY 'password_5678'; GRANT SELECT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT INSERT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT UPDATE ON ispybtest.* to ispyb_api_sqlalchemy@'%';" @@ -112,7 +118,7 @@ jobs: - name: Check RabbitMQ is alive run: wget -t 10 -w 1 http://127.0.0.1:15672 -O - - - name: Run tests + - name: Run Murfey tests env: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 @@ -120,10 +126,9 @@ jobs: POSTGRES_PASSWORD: psql_pwd POSTGRES_USER: psql_user run: | - export ISPYB_CREDENTIALS=".github/workflows/config/ispyb.cfg" PYTHONDEVMODE=1 pytest -v -ra --cov=murfey --cov-report=xml --cov-branch - - name: Upload to Codecov + - name: Upload test results to Codecov uses: codecov/codecov-action@v5 with: name: ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1bbe2e327..a3a09032d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,11 +3,12 @@ name: Build and test on: [push, pull_request] env: - DATABASE_SCHEMA: 4.2.1 # released 2024-08-19 + ISPYB_DATABASE_SCHEMA: 4.6.0 # Installs from GitHub # Versions: https://github.com/DiamondLightSource/ispyb-database/tags # Previous version(s): - # 4.1.0 + # 4.2.1 # released 2024-08-19 + # 4.1.0 # released 2024-03-26 permissions: contents: read @@ -53,10 +54,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Download ISPyB DB schema v${{ env.DATABASE_SCHEMA }} for tests + - name: Download ISPyB DB schema v${{ env.ISPYB_DATABASE_SCHEMA }} for tests run: | mkdir database - wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v${{ env.DATABASE_SCHEMA }}/ispyb-database-${{ env.DATABASE_SCHEMA }}.tar.gz -O database/ispyb-database.tar.gz + wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v${{ env.ISPYB_DATABASE_SCHEMA }}/ispyb-database-${{ env.ISPYB_DATABASE_SCHEMA }}.tar.gz -O database/ispyb-database.tar.gz - name: Store database artifact uses: actions/upload-artifact@v4 with: diff --git a/src/murfey/cli/spa_ispyb_messages.py b/src/murfey/cli/spa_ispyb_messages.py index 0cca27dca..87264366d 100644 --- a/src/murfey/cli/spa_ispyb_messages.py +++ b/src/murfey/cli/spa_ispyb_messages.py @@ -19,7 +19,7 @@ from murfey.client.contexts.spa import _get_xml_list_index from murfey.server import _murfey_id, _register -from murfey.server.ispyb import Session, TransportManager, get_session_id +from murfey.server.ispyb import ISPyBSession, TransportManager, get_session_id from murfey.server.murfey_db import url from murfey.util import db from murfey.util.config import get_machine_config, get_microscope, get_security_config @@ -256,7 +256,7 @@ def run(): proposal_code=args.visit[:2], proposal_number=args.visit[2:].split("-")[0], visit_number=args.visit.split("-")[1], - db=Session(), + db=ISPyBSession(), ), ) diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 92dfc44b3..334c5dcbe 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -46,7 +46,7 @@ import murfey import murfey.server.prometheus as prom import murfey.util.db as db -from murfey.server.ispyb import get_session_id +from murfey.server.ispyb import ISPyBSession, get_session_id from murfey.server.murfey_db import url # murfey_db from murfey.util import LogFilter from murfey.util.config import ( @@ -2203,7 +2203,7 @@ def feedback_callback(header: dict, message: dict) -> None: proposal_code=message["proposal_code"], proposal_number=message["proposal_number"], visit_number=message["visit_number"], - db=murfey.server.ispyb.Session(), + db=ISPyBSession(), ) if dcg_murfey := murfey_db.exec( select(db.DataCollectionGroup) @@ -2281,7 +2281,7 @@ def feedback_callback(header: dict, message: dict) -> None: proposal_code=message["proposal_code"], proposal_number=message["proposal_number"], visit_number=message["visit_number"], - db=murfey.server.ispyb.Session(), + db=ISPyBSession(), ) dcg = murfey_db.exec( select(db.DataCollectionGroup) @@ -2380,7 +2380,7 @@ def feedback_callback(header: dict, message: dict) -> None: ).all(): pid = pj_murfey[0].id else: - if murfey.server.ispyb.Session() is None: + if ISPyBSession() is None: murfey_pj = db.ProcessingJob(recipe=message["recipe"], dc_id=_dcid) else: record = ProcessingJob( @@ -2410,7 +2410,7 @@ def feedback_callback(header: dict, message: dict) -> None: if not murfey_db.exec( select(db.AutoProcProgram).where(db.AutoProcProgram.pj_id == pid) ).all(): - if murfey.server.ispyb.Session() is None: + if ISPyBSession() is None: murfey_app = db.AutoProcProgram(pj_id=pid) else: record = AutoProcProgram( diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py index f63ca27aa..a833118da 100644 --- a/src/murfey/server/api/__init__.py +++ b/src/murfey/server/api/__init__.py @@ -892,6 +892,9 @@ def _add_tilt(): @router.get("/instruments/{instrument_name}/visits_raw", response_model=List[Visit]) def get_current_visits(instrument_name: str, db=murfey.server.ispyb.DB): + log.debug( + f"Received request to look up ongoing visits for {sanitise(instrument_name)}" + ) return murfey.server.ispyb.get_all_ongoing_visits(instrument_name, db) diff --git a/src/murfey/server/ispyb.py b/src/murfey/server/ispyb.py index c44f19ccf..c12bc3939 100644 --- a/src/murfey/server/ispyb.py +++ b/src/murfey/server/ispyb.py @@ -5,9 +5,6 @@ from typing import Callable, Generator, List, Literal, Optional import ispyb - -# import ispyb.sqlalchemy -import sqlalchemy.orm import workflows.transport from fastapi import Depends from ispyb.sqlalchemy import ( @@ -29,6 +26,8 @@ ZcZocaloBuffer, url, ) +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker from murfey.util import sanitise from murfey.util.config import get_security_config @@ -38,11 +37,16 @@ security_config = get_security_config() try: - Session = sqlalchemy.orm.sessionmaker( - bind=sqlalchemy.create_engine(url(), connect_args={"use_pure": True}) + ISPyBSession = sessionmaker( + bind=create_engine( + url(credentials=security_config.ispyb_credentials), + connect_args={"use_pure": True}, + ) ) + log.info("Loaded ISPyB database session") except AttributeError: - Session = lambda: None + log.error("Error loading ISPyB session", exc_info=True) + ISPyBSession = lambda: None def _send_using_new_connection(transport_type: str, queue: str, message: dict) -> None: @@ -67,6 +71,8 @@ def __init__(self, transport_type: Literal["PikaTransport"]): if security_config.ispyb_credentials else None ) + if self.ispyb is not None: + print("Loaded ISPyB databse") self._connection_callback: Callable | None = None def reconnect(self): @@ -87,7 +93,7 @@ def do_insert_data_collection_group( **kwargs, ): try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created DataCollectionGroup {record.dataCollectionGroupId}") @@ -102,7 +108,7 @@ def do_insert_data_collection_group( def do_insert_atlas(self, record: Atlas): try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created Atlas {record.atlasId}") @@ -119,7 +125,7 @@ def do_update_atlas( self, atlas_id: int, atlas_image: str, pixel_size: float, slot: int ): try: - with Session() as db: + with ISPyBSession() as db: atlas = db.query(Atlas).filter(Atlas.atlasId == atlas_id).one() atlas.atlasImage = atlas_image or atlas.atlasImage atlas.pixelSize = pixel_size or atlas.pixelSize @@ -187,7 +193,7 @@ def do_insert_grid_square( pixelSize=grid_square_parameters.pixel_size, ) try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created GridSquare {record.gridSquareId}") @@ -204,7 +210,7 @@ def do_update_grid_square( self, grid_square_id: int, grid_square_parameters: GridSquareParameters ): try: - with Session() as db: + with ISPyBSession() as db: grid_square = ( db.query(GridSquare) .filter(GridSquare.gridSquareId == grid_square_id) @@ -295,7 +301,7 @@ def do_insert_foil_hole( pixelSize=foil_hole_parameters.pixel_size, ) try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created FoilHole {record.foilHoleId}") @@ -315,7 +321,7 @@ def do_update_foil_hole( foil_hole_parameters: FoilHoleParameters, ): try: - with Session() as db: + with ISPyBSession() as db: foil_hole = ( db.query(FoilHole).filter(FoilHole.foilHoleId == foil_hole_id).one() ) @@ -373,7 +379,7 @@ def do_insert_data_collection(self, record: DataCollection, message=None, **kwar else "Created for Murfey" ) try: - with Session() as db: + with ISPyBSession() as db: record.comments = comment db.add(record) db.commit() @@ -389,7 +395,7 @@ def do_insert_data_collection(self, record: DataCollection, message=None, **kwar def do_insert_sample_group(self, record: BLSampleGroup) -> dict: try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created BLSampleGroup {record.blSampleGroupId}") @@ -404,7 +410,7 @@ def do_insert_sample_group(self, record: BLSampleGroup) -> dict: def do_insert_sample(self, record: BLSample, sample_group_id: int) -> dict: try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created BLSample {record.blSampleId}") @@ -427,7 +433,7 @@ def do_insert_sample(self, record: BLSample, sample_group_id: int) -> dict: def do_insert_subsample(self, record: BLSubSample) -> dict: try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created BLSubSample {record.blSubSampleId}") @@ -442,7 +448,7 @@ def do_insert_subsample(self, record: BLSubSample) -> dict: def do_insert_sample_image(self, record: BLSampleImage) -> dict: try: - with Session() as db: + with ISPyBSession() as db: db.add(record) db.commit() log.info(f"Created BLSampleImage {record.blSampleImageId}") @@ -525,7 +531,7 @@ def do_update_processing_status(self, record: AutoProcProgram, **kwargs): return {"success": False, "return_value": None} def do_buffer_lookup(self, app_id: int, uuid: int) -> Optional[int]: - with Session() as db: + with ISPyBSession() as db: buffer_objects = ( db.query(ZcZocaloBuffer) .filter_by(AutoProcProgramID=app_id, UUID=uuid) @@ -536,8 +542,8 @@ def do_buffer_lookup(self, app_id: int, uuid: int) -> Optional[int]: return reference -def _get_session() -> Generator[Optional[sqlalchemy.orm.Session], None, None]: - db = Session() +def _get_session() -> Generator[Optional[Session], None, None]: + db = ISPyBSession() if db is None: yield None return @@ -556,7 +562,7 @@ def get_session_id( proposal_code: str, proposal_number: str, visit_number: str, - db: sqlalchemy.orm.Session | None, + db: Session | None, ) -> int | None: # Log received lookup parameters @@ -589,9 +595,7 @@ def get_session_id( return res -def get_proposal_id( - proposal_code: str, proposal_number: str, db: sqlalchemy.orm.Session -) -> int: +def get_proposal_id(proposal_code: str, proposal_number: str, db: Session) -> int: query = ( db.query(Proposal) .filter( @@ -603,7 +607,7 @@ def get_proposal_id( return query[0].proposalId -def get_sub_samples_from_visit(visit: str, db: sqlalchemy.orm.Session) -> List[Sample]: +def get_sub_samples_from_visit(visit: str, db: Session) -> List[Sample]: proposal_id = get_proposal_id(visit[:2], visit.split("-")[0][2:], db) samples = ( db.query(BLSampleGroup, BLSampleGroupHasBLSample, BLSample, BLSubSample) @@ -628,10 +632,9 @@ def get_sub_samples_from_visit(visit: str, db: sqlalchemy.orm.Session) -> List[S return res -def get_all_ongoing_visits( - microscope: str, db: sqlalchemy.orm.Session | None -) -> list[Visit]: +def get_all_ongoing_visits(microscope: str, db: Session | None) -> list[Visit]: if db is None: + print("No database found") return [] query = ( db.query(BLSession) @@ -664,16 +667,3 @@ def get_all_ongoing_visits( ) for row in query ] - - -def get_data_collection_group_ids(session_id): - query = ( - Session() - .query(DataCollectionGroup) - .filter( - DataCollectionGroup.sessionId == session_id, - ) - .all() - ) - dcgids = [row.dataCollectionGroupId for row in query] - return dcgids diff --git a/tests/__init__.py b/tests/__init__.py index 92085ab06..e69de29bb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,10 +0,0 @@ -import os - -from sqlmodel import create_engine - -url = ( - f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}" - f"@{os.environ['POSTGRES_HOST']}:{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_DB']}" -) - -engine = create_engine(url) diff --git a/tests/cli/test_repost_failed_calls.py b/tests/cli/test_repost_failed_calls.py index 9a018fb4f..6ef2c7e81 100644 --- a/tests/cli/test_repost_failed_calls.py +++ b/tests/cli/test_repost_failed_calls.py @@ -6,7 +6,6 @@ from unittest import mock from murfey.cli import repost_failed_calls -from tests.conftest import mock_security_config_name @mock.patch("murfey.cli.repost_failed_calls.PikaTransport") @@ -167,12 +166,11 @@ def test_run_repost_failed_calls( mock_repost, mock_purge, mock_security_configuration, - tmp_path, ): mock_jwt.encode.return_value = "dummy_token" mock_purge.return_value = ["/path/to/msg1"] - config_file = tmp_path / mock_security_config_name + config_file = mock_security_configuration with open(config_file) as f: security_config = json.load(f) diff --git a/tests/conftest.py b/tests/conftest.py index 9176d86a1..bb1d86f37 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,28 +1,33 @@ +from __future__ import annotations + import json +import os from configparser import ConfigParser +from pathlib import Path +from typing import Any, Generator, Type, TypeVar +import ispyb import pytest -from sqlmodel import Session +from ispyb.sqlalchemy import BLSession, ExperimentType, Person, Proposal, url +from sqlalchemy import Engine, RootTransaction, and_, create_engine, event, select +from sqlalchemy.ext.declarative import DeclarativeMeta +from sqlalchemy.orm import Session as SQLAlchemySession +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session as SQLModelSession +from sqlmodel import SQLModel from murfey.util.db import Session as MurfeySession -from murfey.util.db import clear, setup -from tests import engine, url - -mock_security_config_name = "security_config.yaml" -@pytest.fixture -def start_postgres(): - clear(url) - setup(url) - - murfey_session = MurfeySession(id=2, name="cm12345-6") - with Session(engine) as murfey_db: - murfey_db.add(murfey_session) - murfey_db.commit() +@pytest.fixture(scope="session") +def session_tmp_path(tmp_path_factory) -> Path: + """ + Creates a temporary path that persists for the entire test session + """ + return tmp_path_factory.mktemp("session_tmp") -@pytest.fixture() +@pytest.fixture(scope="session") def mock_client_configuration() -> ConfigParser: """ Returns the client-side configuration file as a pre-loaded ConfigParser object. @@ -36,16 +41,306 @@ def mock_client_configuration() -> ConfigParser: return config -@pytest.fixture() -def mock_security_configuration(tmp_path): - config_file = tmp_path / mock_security_config_name +@pytest.fixture(scope="session") +def mock_ispyb_credentials(session_tmp_path: Path) -> Path: + creds_file = session_tmp_path / "ispyb_creds.cfg" + ispyb_config = ConfigParser() + # Use values from the GitHub workflow ISPyB config file + ispyb_config["ispyb_sqlalchemy"] = { + "username": "ispyb_api_sqlalchemy", + "password": "password_5678", + "host": "localhost", + "port": "3306", + "database": "ispybtest", + } + with open(creds_file, "w") as file: + ispyb_config.write(file) + return creds_file + + +@pytest.fixture(scope="session") +def mock_security_configuration( + session_tmp_path: Path, + mock_ispyb_credentials: Path, +) -> Path: + config_file = session_tmp_path / "security_config.yaml" security_config = { + "murfey_db_credentials": "/path/to/murfey_db_credentials", + "crypto_key": "crypto_key", "auth_key": "auth_key", "auth_algorithm": "auth_algorithm", - "feedback_queue": "murfey_feedback", "rabbitmq_credentials": "/path/to/rabbitmq.yaml", - "murfey_db_credentials": "/path/to/murfey_db_credentials", - "crypto_key": "crypto_key", + "feedback_queue": "murfey_feedback", + "ispyb_credentials": str(mock_ispyb_credentials), } with open(config_file, "w") as f: json.dump(security_config, f) + return config_file + + +""" +======================================================================================= +Database-related helper functions and classes +======================================================================================= +""" + + +class ExampleVisit: + """ + This class stores information that will be common to all database entries for a + particular Murfey session, to enable ease of replication when creating database + fixtures. + """ + + # Visit-related (ISPyB & Murfey) + instrument_name = "i01" + proposal_code = "cm" + proposal_number = 12345 + visit_number = 6 + + # Murfey-specific + murfey_session_id = 1 + + # Person (ISPyB) + given_name = "Eliza" + family_name = "Murfey" + login = "murfey123" + + +class ISPyBTableValues: + """ + Visit-independent default values for ISPyB tables + """ + + # ExperimentType (ISPyB) + experiment_types = { + "Tomography": 36, + "Single Particle": 37, + "Atlas": 44, + } + + +SQLAlchemyTable = TypeVar("SQLAlchemyTable", bound=DeclarativeMeta) + + +def get_or_create_db_entry( + session: SQLAlchemySession | SQLModelSession, + table: Type[SQLAlchemyTable], + lookup_kwargs: dict[str, Any] = {}, + insert_kwargs: dict[str, Any] = {}, +) -> SQLAlchemyTable: + """ + Helper function to facilitate looking up or creating SQLAlchemy table entries. + Returns the entry if a match based on the lookup criteria is found, otherwise + creates and returns a new entry. + """ + + # if lookup kwargs are provided, check if entry exists + if lookup_kwargs: + conditions = [ + getattr(table, key) == value for key, value in lookup_kwargs.items() + ] + # Use 'exec()' for SQLModel sessions + if isinstance(session, SQLModelSession): + entry = session.exec(select(table).where(and_(*conditions))).first() + # Use 'execute()' for SQLAlchemy sessions + elif isinstance(session, SQLAlchemySession): + entry = ( + session.execute(select(table).where(and_(*conditions))) + .scalars() + .first() + ) + else: + raise TypeError("Unexpected Session type") + if entry: + return entry + + # If not present, create and return new entry + # Use new kwargs if provided; otherwise, use lookup kwargs + insert_kwargs = insert_kwargs or lookup_kwargs + entry = table(**insert_kwargs) + session.add(entry) + session.commit() + return entry + + +def restart_savepoint( + session: SQLAlchemySession | SQLModelSession, transaction: RootTransaction +): + """ + Re-establish a SAVEPOINT after a nested transaction is committed or rolled back. + This helps to maintain isolation across different test cases. + """ + if transaction.nested and not transaction._parent.nested: + session.begin_nested() + + +""" +======================================================================================= +Fixtures for setting up test ISPyB database +======================================================================================= +These were adapted from the tests found at: +https://github.com/DiamondLightSource/ispyb-api/blob/main/tests/conftest.py +""" + + +@pytest.fixture(scope="session") +def ispyb_db_connection(mock_ispyb_credentials): + with ispyb.open(mock_ispyb_credentials) as connection: + yield connection + + +@pytest.fixture(scope="session") +def ispyb_engine(mock_ispyb_credentials): + engine = create_engine( + url=url(mock_ispyb_credentials), connect_args={"use_pure": True} + ) + yield engine + engine.dispose() + + +@pytest.fixture(scope="session") +def ispyb_db_session_factory(ispyb_engine): + return sessionmaker(bind=ispyb_engine, expire_on_commit=False) + + +@pytest.fixture(scope="session") +def seed_ispyb_db(ispyb_db_session_factory): + + # Populate the ISPyB table with some initial values + # Return existing table entry if already present + ispyb_db_session: SQLAlchemySession = ispyb_db_session_factory() + person_db_entry = get_or_create_db_entry( + session=ispyb_db_session, + table=Person, + lookup_kwargs={ + "givenName": ExampleVisit.given_name, + "familyName": ExampleVisit.family_name, + "login": ExampleVisit.login, + }, + ) + proposal_db_entry = get_or_create_db_entry( + session=ispyb_db_session, + table=Proposal, + lookup_kwargs={ + "personId": person_db_entry.personId, + "proposalCode": ExampleVisit.proposal_code, + "proposalNumber": str(ExampleVisit.proposal_number), + }, + ) + _ = get_or_create_db_entry( + session=ispyb_db_session, + table=BLSession, + lookup_kwargs={ + "proposalId": proposal_db_entry.proposalId, + "beamLineName": ExampleVisit.instrument_name, + "visit_number": ExampleVisit.visit_number, + }, + ) + _ = [ + get_or_create_db_entry( + session=ispyb_db_session, + table=ExperimentType, + lookup_kwargs={ + "experimentTypeId": id, + "name": name, + "proposalType": "em", + "active": 1, + }, + ) + for name, id in ISPyBTableValues.experiment_types.items() + ] + ispyb_db_session.close() + + +@pytest.fixture +def ispyb_db_session( + ispyb_db_session_factory, + ispyb_engine: Engine, + seed_ispyb_db, +) -> Generator[SQLAlchemySession, None, None]: + """ + Returns a test-safe session that wraps each test in a rollback-safe save point. + """ + connection = ispyb_engine.connect() + transaction = connection.begin() # Outer transaction + + session: SQLAlchemySession = ispyb_db_session_factory(bind=connection) + session.begin_nested() # Save point for test + + # Trigger the restart_savepoint function after the end of the transaction + event.listen(session, "after_transaction_end", restart_savepoint) + + try: + yield session + finally: + session.close() + transaction.rollback() + connection.close() + + +""" +======================================================================================= +Fixtures for setting up test Murfey database +======================================================================================= +""" + + +@pytest.fixture(scope="session") +def murfey_db_engine(): + url = ( + f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}" + f"@{os.environ['POSTGRES_HOST']}:{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_DB']}" + ) + engine = create_engine(url) + SQLModel.metadata.create_all(engine) + yield engine + engine.dispose() + + +@pytest.fixture(scope="session") +def murfey_db_session_factory(murfey_db_engine): + return sessionmaker( + bind=murfey_db_engine, expire_on_commit=False, class_=SQLModelSession + ) + + +@pytest.fixture(scope="session") +def seed_murfey_db(murfey_db_session_factory): + # Populate Murfey database with initial values + session: SQLModelSession = murfey_db_session_factory() + _ = get_or_create_db_entry( + session=session, + table=MurfeySession, + lookup_kwargs={ + "id": ExampleVisit.murfey_session_id, + "name": f"{ExampleVisit.proposal_code}{ExampleVisit.proposal_number}-{ExampleVisit.visit_number}", + }, + ) + session.close() + + +@pytest.fixture +def murfey_db_session( + murfey_db_session_factory, + murfey_db_engine: Engine, + seed_murfey_db, +) -> Generator[SQLModelSession, None, None]: + """ + Returns a test-safe session that wraps each test in a rollback-safe save point + """ + connection = murfey_db_engine.connect() + transaction = connection.begin() + + session: SQLModelSession = murfey_db_session_factory(bind=connection) + session.begin_nested() # Save point for test + + # Trigger the restart_savepoint function after the end of the transaction + event.listen(session, "after_transaction_end", restart_savepoint) + + try: + yield session + finally: + session.close() + transaction.rollback() + connection.close() diff --git a/tests/server/test_ispyb.py b/tests/server/test_ispyb.py new file mode 100644 index 000000000..d60ada7bc --- /dev/null +++ b/tests/server/test_ispyb.py @@ -0,0 +1,69 @@ +from ispyb.sqlalchemy import BLSession, Proposal +from pytest import mark +from sqlalchemy import select +from sqlalchemy.orm import Session + +from murfey.server.ispyb import get_proposal_id, get_session_id +from tests.conftest import ExampleVisit + + +def test_get_session_id( + ispyb_db_session: Session, +): + # Manually get the BLSession ID for comparison + bl_session_id = ( + ispyb_db_session.execute( + select(BLSession) + .join(Proposal) + .where(BLSession.proposalId == Proposal.proposalId) + .where(BLSession.beamLineName == ExampleVisit.instrument_name) + .where(Proposal.proposalCode == ExampleVisit.proposal_code) + .where(Proposal.proposalNumber == str(ExampleVisit.proposal_number)) + .where(BLSession.visit_number == ExampleVisit.visit_number) + ) + .scalar_one() + .sessionId + ) + + # Test function + result = get_session_id( + microscope=ExampleVisit.instrument_name, + proposal_code=ExampleVisit.proposal_code, + proposal_number=str(ExampleVisit.proposal_number), + visit_number=str(ExampleVisit.visit_number), + db=ispyb_db_session, + ) + assert bl_session_id == result + + +def test_get_proposal_id( + ispyb_db_session: Session, +): + # Manually query the Proposal ID + proposal_id = ( + ispyb_db_session.execute( + select(Proposal) + .where(Proposal.proposalCode == ExampleVisit.proposal_code) + .where(Proposal.proposalNumber == str(ExampleVisit.proposal_number)) + ) + .scalar_one() + .proposalId + ) + + # Test function + result = get_proposal_id( + proposal_code=ExampleVisit.proposal_code, + proposal_number=str(ExampleVisit.proposal_number), + db=ispyb_db_session, + ) + assert proposal_id == result + + +@mark.skip +def test_get_sub_samples_from_visit(): + pass + + +@mark.skip +def test_get_all_ongoing_visits(): + pass diff --git a/tests/workflows/spa/test_flush_spa_preprocess.py b/tests/workflows/spa/test_flush_spa_preprocess.py index f55f9838c..04c03a786 100644 --- a/tests/workflows/spa/test_flush_spa_preprocess.py +++ b/tests/workflows/spa/test_flush_spa_preprocess.py @@ -5,22 +5,23 @@ from murfey.util.db import DataCollectionGroup, GridSquare from murfey.util.models import GridSquareParameters from murfey.workflows.spa import flush_spa_preprocess -from tests import engine +from tests.conftest import ExampleVisit @mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object") -def test_register_grid_square_update_add_locations(mock_transport, start_postgres): +def test_register_grid_square_update_add_locations( + mock_transport, murfey_db_session: Session +): """Test the updating of an existing grid square""" # Create a grid square to update grid_square = GridSquare( id=1, name=101, - session_id=2, + session_id=ExampleVisit.murfey_session_id, tag="session_tag", ) - with Session(engine) as murfey_db: - murfey_db.add(grid_square) - murfey_db.commit() + murfey_db_session.add(grid_square) + murfey_db_session.commit() # Parameters to update with new_parameters = GridSquareParameters( @@ -32,15 +33,15 @@ def test_register_grid_square_update_add_locations(mock_transport, start_postgre ) # Run the registration - with Session(engine) as murfey_db: - flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db) + flush_spa_preprocess.register_grid_square( + ExampleVisit.murfey_session_id, 101, new_parameters, murfey_db_session + ) # Check this would have updated ispyb mock_transport.do_update_grid_square.assert_called_with(1, new_parameters) # Confirm the database was updated - with Session(engine) as murfey_db: - grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one() + grid_square_final_parameters = murfey_db_session.exec(select(GridSquare)).one() assert grid_square_final_parameters.x_location == new_parameters.x_location assert grid_square_final_parameters.y_location == new_parameters.y_location assert ( @@ -52,36 +53,37 @@ def test_register_grid_square_update_add_locations(mock_transport, start_postgre @mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object") -def test_register_grid_square_update_add_nothing(mock_transport, start_postgres): +def test_register_grid_square_update_add_nothing( + mock_transport, murfey_db_session: Session +): """Test the updating of an existing grid square, but with nothing to update with""" # Create a grid square to update grid_square = GridSquare( id=1, name=101, - session_id=2, + session_id=ExampleVisit.murfey_session_id, tag="session_tag", x_location=0.1, y_location=0.2, x_stage_position=0.3, y_stage_position=0.4, ) - with Session(engine) as murfey_db: - murfey_db.add(grid_square) - murfey_db.commit() + murfey_db_session.add(grid_square) + murfey_db_session.commit() # Parameters to update with new_parameters = GridSquareParameters(tag="session_tag") # Run the registration - with Session(engine) as murfey_db: - flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db) + flush_spa_preprocess.register_grid_square( + ExampleVisit.murfey_session_id, 101, new_parameters, murfey_db_session + ) # Check this would have updated ispyb mock_transport.do_update_grid_square.assert_called_with(1, new_parameters) # Confirm the database was not updated - with Session(engine) as murfey_db: - grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one() + grid_square_final_parameters = murfey_db_session.exec(select(GridSquare)).one() assert grid_square_final_parameters.x_location == 0.1 assert grid_square_final_parameters.y_location == 0.2 assert grid_square_final_parameters.x_stage_position == 0.3 @@ -90,18 +92,17 @@ def test_register_grid_square_update_add_nothing(mock_transport, start_postgres) @mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object") def test_register_grid_square_insert_with_ispyb( - mock_transport, start_postgres, tmp_path + mock_transport, murfey_db_session: Session, tmp_path ): # Create a data collection group for lookups grid_square = DataCollectionGroup( id=1, - session_id=2, + session_id=ExampleVisit.murfey_session_id, tag="session_tag", atlas_id=90, ) - with Session(engine) as murfey_db: - murfey_db.add(grid_square) - murfey_db.commit() + murfey_db_session.add(grid_square) + murfey_db_session.commit() # Set the ispyb return mock_transport.do_insert_grid_square.return_value = { @@ -126,18 +127,18 @@ def test_register_grid_square_insert_with_ispyb( ) # Run the registration - with Session(engine) as murfey_db: - flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db) + flush_spa_preprocess.register_grid_square( + ExampleVisit.murfey_session_id, 101, new_parameters, murfey_db_session + ) # Check this would have updated ispyb mock_transport.do_insert_grid_square.assert_called_with(90, 101, new_parameters) # Confirm the database entry was made - with Session(engine) as murfey_db: - grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one() + grid_square_final_parameters = murfey_db_session.exec(select(GridSquare)).one() assert grid_square_final_parameters.id == 1 assert grid_square_final_parameters.name == 101 - assert grid_square_final_parameters.session_id == 2 + assert grid_square_final_parameters.session_id == ExampleVisit.murfey_session_id assert grid_square_final_parameters.tag == "session_tag" assert grid_square_final_parameters.x_location == 1.1 assert grid_square_final_parameters.y_location == 1.2