From 263417b67eac7d649430268ec56553686da18b8f Mon Sep 17 00:00:00 2001 From: Felix Evers Date: Fri, 30 Jan 2026 14:42:05 +0100 Subject: [PATCH 1/2] add scaffolding strategy --- backend/config.py | 14 ++ .../versions/add_scaffold_import_state.py | 29 +++ .../make_patients_clinic_id_nullable.py | 34 +++ .../make_patients_clinic_id_required.py | 34 +++ backend/database/models/__init__.py | 1 + backend/database/models/scaffold.py | 10 + backend/scaffold.py | 237 +++++++++++++++--- scaffold/README.md | 56 +++++ 8 files changed, 375 insertions(+), 40 deletions(-) create mode 100644 backend/database/migrations/versions/add_scaffold_import_state.py create mode 100644 backend/database/migrations/versions/make_patients_clinic_id_nullable.py create mode 100644 backend/database/migrations/versions/make_patients_clinic_id_required.py create mode 100644 backend/database/models/scaffold.py create mode 100644 scaffold/README.md diff --git a/backend/config.py b/backend/config.py index 26abbe63..e2ed44ed 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,9 +1,17 @@ import os +from enum import Enum from dotenv import load_dotenv load_dotenv() + +class ScaffoldStrategy(str, Enum): + CHECK = "CHECK" + MERGE = "MERGE" + FORCE = "FORCE" + + _db_hostname = os.getenv("DATABASE_HOSTNAME", "postgres") _db_name = os.getenv("DATABASE_NAME", "postgres") _db_username = os.getenv("DATABASE_USERNAME", "postgres") @@ -52,6 +60,12 @@ SCAFFOLD_DIRECTORY = os.getenv("SCAFFOLD_DIRECTORY", None) +_raw_scaffold_strategy = os.getenv("SCAFFOLD_STRATEGY", "CHECK").strip().upper() +try: + SCAFFOLD_STRATEGY = ScaffoldStrategy(_raw_scaffold_strategy) +except ValueError: + SCAFFOLD_STRATEGY = ScaffoldStrategy.CHECK + INFLUXDB_URL = os.getenv("INFLUXDB_URL", "http://localhost:8086") INFLUXDB_TOKEN = os.getenv("INFLUXDB_TOKEN", None) INFLUXDB_ORG = os.getenv("INFLUXDB_ORG", "tasks") diff --git a/backend/database/migrations/versions/add_scaffold_import_state.py b/backend/database/migrations/versions/add_scaffold_import_state.py new file mode 100644 index 00000000..f669a87b --- /dev/null +++ b/backend/database/migrations/versions/add_scaffold_import_state.py @@ -0,0 +1,29 @@ +"""Add scaffold_import_state table for directory hash + +Revision ID: add_scaffold_import_state +Revises: make_patients_clinic_required +Create Date: 2026-01-30 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "add_scaffold_import_state" +down_revision: Union[str, Sequence[str], None] = "make_patients_clinic_required" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scaffold_import_state", + sa.Column("key", sa.String(), nullable=False), + sa.Column("value", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("key"), + ) + + +def downgrade() -> None: + op.drop_table("scaffold_import_state") diff --git a/backend/database/migrations/versions/make_patients_clinic_id_nullable.py b/backend/database/migrations/versions/make_patients_clinic_id_nullable.py new file mode 100644 index 00000000..94265c24 --- /dev/null +++ b/backend/database/migrations/versions/make_patients_clinic_id_nullable.py @@ -0,0 +1,34 @@ +"""Make patients.clinic_id nullable for scaffold FORCE unassign + +Revision ID: make_patients_clinic_nullable +Revises: 655294b10318 +Create Date: 2026-01-30 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "make_patients_clinic_nullable" +down_revision: Union[str, Sequence[str], None] = "655294b10318" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=True, + ) + + +def downgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=False, + ) diff --git a/backend/database/migrations/versions/make_patients_clinic_id_required.py b/backend/database/migrations/versions/make_patients_clinic_id_required.py new file mode 100644 index 00000000..9b6aba9b --- /dev/null +++ b/backend/database/migrations/versions/make_patients_clinic_id_required.py @@ -0,0 +1,34 @@ +"""Make patients.clinic_id required again + +Revision ID: make_patients_clinic_required +Revises: make_patients_clinic_nullable +Create Date: 2026-01-30 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "make_patients_clinic_required" +down_revision: Union[str, Sequence[str], None] = "make_patients_clinic_nullable" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=False, + ) + + +def downgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=True, + ) diff --git a/backend/database/models/__init__.py b/backend/database/models/__init__.py index 5de948a3..8108c8c7 100644 --- a/backend/database/models/__init__.py +++ b/backend/database/models/__init__.py @@ -3,3 +3,4 @@ from .patient import Patient, patient_locations, patient_teams # noqa: F401 from .task import Task, task_dependencies # noqa: F401 from .property import PropertyDefinition, PropertyValue # noqa: F401 +from .scaffold import ScaffoldImportState # noqa: F401 diff --git a/backend/database/models/scaffold.py b/backend/database/models/scaffold.py new file mode 100644 index 00000000..34368a07 --- /dev/null +++ b/backend/database/models/scaffold.py @@ -0,0 +1,10 @@ +from database.models.base import Base +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy import String + + +class ScaffoldImportState(Base): + __tablename__ = "scaffold_import_state" + + key: Mapped[str] = mapped_column(String, primary_key=True) + value: Mapped[str] = mapped_column(String) diff --git a/backend/scaffold.py b/backend/scaffold.py index 0016bf88..c6c6dc32 100644 --- a/backend/scaffold.py +++ b/backend/scaffold.py @@ -1,16 +1,63 @@ +import hashlib import json import logging +import time from pathlib import Path from typing import Any from api.inputs import LocationType -from config import LOGGER, SCAFFOLD_DIRECTORY +from config import ( + LOGGER, + SCAFFOLD_DIRECTORY, + SCAFFOLD_STRATEGY, + ScaffoldStrategy, +) from database.models.location import LocationNode, location_organizations +from database.models.patient import Patient, patient_locations, patient_teams +from database.models.scaffold import ScaffoldImportState +from database.models.task import Task +from database.models.user import user_root_locations from database.session import async_session -from sqlalchemy import select +from sqlalchemy import delete, select, update logger = logging.getLogger(LOGGER) +REINITIALIZATION_WAIT_SECONDS = 120 + + +def _compute_scaffold_directory_hash(scaffold_path: Path) -> str: + parts: list[str] = [] + for path in sorted(scaffold_path.glob("*.json")): + parts.append(path.name) + parts.append(path.stat().st_mtime_ns.__str__()) + parts.append(path.stat().st_size.__str__()) + with open(path, "rb") as f: + parts.append(hashlib.sha256(f.read()).hexdigest()) + return hashlib.sha256("".join(parts).encode()).hexdigest() + + +SCAFFOLD_STATE_KEY = "directory_hash" + + +def _load_and_merge_json_payload(scaffold_path: Path) -> list[dict[str, Any]]: + json_files = sorted(scaffold_path.glob("*.json")) + merged: list[dict[str, Any]] = [] + for json_file in json_files: + try: + with open(json_file, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, list): + merged.extend(data) + elif isinstance(data, dict): + merged.append(data) + else: + logger.warning( + f"Invalid JSON structure in {json_file}, expected list or object" + ) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSON file {json_file}: {e}") + return merged + async def load_scaffold_data() -> None: if not SCAFFOLD_DIRECTORY: @@ -30,55 +77,165 @@ async def load_scaffold_data() -> None: ) return - async with async_session() as session: - result = await session.execute(select(LocationNode).limit(1)) - existing_location = result.scalar_one_or_none() + payload = _load_and_merge_json_payload(scaffold_path) + if not payload: + logger.info( + f"No valid JSON root items in {SCAFFOLD_DIRECTORY}, skipping scaffold loading" + ) + return - if existing_location: - logger.info( - "Location nodes already exist in database, skipping scaffold loading" - ) - return + json_files = list(scaffold_path.glob("*.json")) + logger.info( + f"Loading scaffold data from {len(json_files)} JSON file(s) in {SCAFFOLD_DIRECTORY}, " + f"merged into {len(payload)} root item(s), strategy={SCAFFOLD_STRATEGY.value}" + ) - json_files = list(scaffold_path.glob("*.json")) + current_hash: str | None = None + if SCAFFOLD_STRATEGY in (ScaffoldStrategy.MERGE, ScaffoldStrategy.FORCE): + current_hash = _compute_scaffold_directory_hash(scaffold_path) - if not json_files: - logger.info( - f"No JSON files found in {SCAFFOLD_DIRECTORY}, skipping scaffold loading" + async with async_session() as session: + if SCAFFOLD_STRATEGY in (ScaffoldStrategy.MERGE, ScaffoldStrategy.FORCE): + result = await session.execute( + select(ScaffoldImportState.value).where( + ScaffoldImportState.key == SCAFFOLD_STATE_KEY + ) ) - return + last_hash: str | None = result.scalar_one_or_none() + if last_hash == current_hash: + logger.info( + "Scaffold directory unchanged (DB state), " + "skipping reinitialization wait" + ) + else: + logger.warning( + "Scaffold directory changed or first run. " + "Waiting %d seconds for reinitialization...", + REINITIALIZATION_WAIT_SECONDS, + ) + time.sleep(REINITIALIZATION_WAIT_SECONDS) - logger.info( - f"Loading scaffold data from {len(json_files)} JSON file(s) in {SCAFFOLD_DIRECTORY}" - ) + if SCAFFOLD_STRATEGY == ScaffoldStrategy.CHECK: + result = await session.execute(select(LocationNode).limit(1)) + if result.scalar_one_or_none(): + logger.info( + "Location nodes already exist (CHECK strategy), skipping scaffold loading" + ) + return - for json_file in json_files: - try: - with open(json_file, "r", encoding="utf-8") as f: - data = json.load(f) + if SCAFFOLD_STRATEGY == ScaffoldStrategy.FORCE: + personal_ids_result = await session.execute( + select(LocationNode.id).where( + LocationNode.parent_id.is_(None), + LocationNode.title.like("%'s Organization"), + ~LocationNode.id.in_( + select(location_organizations.c.location_id) + ), + ) + ) + personal_location_ids = { + row[0] for row in personal_ids_result.all() + } - if isinstance(data, list): - for item in data: - await _create_location_tree(session, item, None) - elif isinstance(data, dict): - await _create_location_tree(session, data, None) - else: - logger.warning( - f"Invalid JSON structure in {json_file}, expected list or object" + result = await session.execute( + select(Patient.id).where(Patient.clinic_id.isnot(None)).limit(1) + ) + has_patients_with_clinic = result.scalar_one_or_none() is not None + backup_clinic_id: str | None = None + if has_patients_with_clinic: + backup_clinic = LocationNode( + title="Scaffold backup clinic", + kind="CLINIC", + parent_id=None, + ) + session.add(backup_clinic) + await session.flush() + backup_clinic_id = backup_clinic.id + await session.execute( + location_organizations.insert().values( + location_id=backup_clinic_id, + organization_id="global", ) + ) + await session.execute( + update(Patient) + .where(Patient.clinic_id.isnot(None)) + .values(clinic_id=backup_clinic_id) + ) - await session.commit() - logger.info( - f"Successfully loaded scaffold data from {json_file}" + await session.execute(delete(patient_locations)) + await session.execute(delete(patient_teams)) + await session.execute( + delete(user_root_locations).where( + ~user_root_locations.c.location_id.in_(personal_location_ids) + ) + ) + await session.execute(update(Task).values(assignee_team_id=None)) + + ids_to_keep = set(personal_location_ids) + if backup_clinic_id is not None: + ids_to_keep.add(backup_clinic_id) + all_ids_result = await session.execute(select(LocationNode.id)) + all_ids = {row[0] for row in all_ids_result.all()} + ids_to_delete = all_ids - ids_to_keep + + if ids_to_delete: + await session.execute( + update(Patient) + .where( + Patient.assigned_location_id.in_(ids_to_delete) + ) + .values(assigned_location_id=None) ) - except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON file {json_file}: {e}") - await session.rollback() - except Exception as e: - logger.error( - f"Error loading scaffold data from {json_file}: {e}" + await session.execute( + update(Patient) + .where(Patient.position_id.in_(ids_to_delete)) + .values(position_id=None) ) - await session.rollback() + await session.execute( + delete(location_organizations).where( + location_organizations.c.location_id.in_(ids_to_delete) + ) + ) + await session.execute( + update(LocationNode) + .where(LocationNode.id.in_(ids_to_delete)) + .values(parent_id=None) + ) + await session.execute( + delete(LocationNode).where( + LocationNode.id.in_(ids_to_delete) + ) + ) + await session.flush() + + try: + for item in payload: + await _create_location_tree(session, item, None) + if ( + current_hash is not None + and SCAFFOLD_STRATEGY in (ScaffoldStrategy.MERGE, ScaffoldStrategy.FORCE) + ): + existing = await session.execute( + select(ScaffoldImportState).where( + ScaffoldImportState.key == SCAFFOLD_STATE_KEY + ) + ) + row = existing.scalar_one_or_none() + if row: + row.value = current_hash + else: + session.add( + ScaffoldImportState( + key=SCAFFOLD_STATE_KEY, + value=current_hash, + ) + ) + await session.commit() + logger.info("Successfully loaded scaffold data (single import)") + except Exception as e: + logger.error(f"Error loading scaffold data: {e}") + await session.rollback() async def _create_location_tree( diff --git a/scaffold/README.md b/scaffold/README.md new file mode 100644 index 00000000..cf1182c5 --- /dev/null +++ b/scaffold/README.md @@ -0,0 +1,56 @@ +# Scaffold + +Location-tree data loaded at backend startup from JSON files in a configurable directory. + +## Environment + +- **SCAFFOLD_DIRECTORY** – Path to a directory containing `*.json` files. If unset, scaffold loading is skipped. +- **SCAFFOLD_STRATEGY** – One of `CHECK`, `MERGE`, `FORCE`. Parsed from env (case-insensitive); invalid values fall back to `CHECK`. + +## JSON format + +Each file may be a single root object or a list of root objects. Root objects have: + +- `name`, `type` (e.g. `HOSPITAL`, `CLINIC`, `WARD`, `ROOM`, `BED`, `TEAM`, `PRACTICE`, `OTHER`) +- optional `children` (array of same shape) +- optional `organization_ids` (for HOSPITAL, CLINIC, PRACTICE, TEAM) + +All `*.json` files in the directory are loaded, merged into one list of root items, and imported in a single transaction. + +## Strategies + +### CHECK (default) + +- If any location node already exists in the database, scaffold loading is **skipped**. +- No wait, no overwrite. + +### MERGE + +- All JSON files are loaded and merged into one payload; that payload is imported **once**. +- New location nodes are created **beside** existing ones (existing nodes are matched by `title`, `kind`, `parent_id` and reused). +- **Reinitialization wait:** If the scaffold directory content (file set and contents) has **changed** since the last run, the backend logs a **warning** and waits **120 seconds** before importing. If the content is **unchanged** (same hash in the state file), the wait is skipped. + +### FORCE + +- All JSON files are loaded and merged; then the backend **replaces** scaffold location data with that payload. +- **Reinitialization wait:** Same 120-second warning and hash check as MERGE; no wait when directory content is unchanged. + +**Before deleting locations:** + +1. **Backup clinic for existing patients** + If any patient has a `clinic_id`, a **“Scaffold backup clinic”** is created (root CLINIC) and linked to the organization **`global`**. All such patients are moved to this backup clinic so they are not left without a clinic. + +2. **Personal location nodes are not deleted** + Location nodes created for **users without an attached organization** (title `"{username}'s Organization"`, root, no row in `location_organizations`) are **preserved**. Only other scaffold/organization locations are removed. + +3. User root-location links (`user_root_locations`) are removed only for locations that are being deleted; links to preserved personal locations stay. +4. Patient assignments (locations, teams), task assignee team, and then `location_organizations` and the non-preserved location nodes are cleared. Preserved nodes and the backup clinic (if created) are kept. +5. The merged JSON payload is imported as in MERGE. + +## Personal location nodes + +When a user has **no** organization attached, the backend creates a single root location for them: title `"{username}'s Organization"`, kind CLINIC, with no `location_organizations` row. These nodes are considered **personal** and are **never** deleted by scaffold FORCE, so user access is preserved. + +## Reinitialization check (MERGE / FORCE) + +A hash of the scaffold directory (file names, mtimes, sizes, and contents) is computed in memory and stored in the **database** after a successful import (table `scaffold_import_state`, key `directory_hash`). No files are written to the scaffold directory. On the next run, the backend compares the current directory hash with the stored value; if they match, the 120-second reinitialization wait is skipped. From 0d408e6546da9ef1dc7c76a78a44abc5add710b9 Mon Sep 17 00:00:00 2001 From: Felix Evers Date: Fri, 30 Jan 2026 14:53:40 +0100 Subject: [PATCH 2/2] add fallback clinic --- backend/scaffold.py | 67 +++++++++++++++++++++++++++++++++++++++------ scaffold/README.md | 6 ++-- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/backend/scaffold.py b/backend/scaffold.py index c6c6dc32..2df53ea0 100644 --- a/backend/scaffold.py +++ b/backend/scaffold.py @@ -37,6 +37,7 @@ def _compute_scaffold_directory_hash(scaffold_path: Path) -> str: SCAFFOLD_STATE_KEY = "directory_hash" +FALLBACK_CLINIC_TITLE = "FALLBACK_CLINIC" def _load_and_merge_json_payload(scaffold_path: Path) -> list[dict[str, Any]]: @@ -95,6 +96,54 @@ async def load_scaffold_data() -> None: current_hash = _compute_scaffold_directory_hash(scaffold_path) async with async_session() as session: + fallback_result = await session.execute( + select(LocationNode.id).where( + LocationNode.title == FALLBACK_CLINIC_TITLE, + LocationNode.parent_id.is_(None), + LocationNode.kind == "CLINIC", + ) + ) + fallback_id = fallback_result.scalar_one_or_none() + if fallback_id is not None: + patient_ref = await session.execute( + select(Patient.id).where( + Patient.clinic_id == fallback_id + ).limit(1) + ) + user_root_ref = await session.execute( + select(user_root_locations.c.location_id).where( + user_root_locations.c.location_id == fallback_id + ).limit(1) + ) + task_ref = await session.execute( + select(Task.id).where( + Task.assignee_team_id == fallback_id + ).limit(1) + ) + team_ref = await session.execute( + select(patient_teams.c.location_id).where( + patient_teams.c.location_id == fallback_id + ).limit(1) + ) + if ( + patient_ref.scalar_one_or_none() is None + and user_root_ref.scalar_one_or_none() is None + and task_ref.scalar_one_or_none() is None + and team_ref.scalar_one_or_none() is None + ): + await session.execute( + delete(location_organizations).where( + location_organizations.c.location_id == fallback_id + ) + ) + await session.execute( + delete(LocationNode).where(LocationNode.id == fallback_id) + ) + await session.flush() + logger.info( + "Removed unused FALLBACK_CLINIC (no references)" + ) + if SCAFFOLD_STRATEGY in (ScaffoldStrategy.MERGE, ScaffoldStrategy.FORCE): result = await session.execute( select(ScaffoldImportState.value).where( @@ -141,26 +190,26 @@ async def load_scaffold_data() -> None: select(Patient.id).where(Patient.clinic_id.isnot(None)).limit(1) ) has_patients_with_clinic = result.scalar_one_or_none() is not None - backup_clinic_id: str | None = None + fallback_clinic_id: str | None = None if has_patients_with_clinic: - backup_clinic = LocationNode( - title="Scaffold backup clinic", + fallback_clinic = LocationNode( + title=FALLBACK_CLINIC_TITLE, kind="CLINIC", parent_id=None, ) - session.add(backup_clinic) + session.add(fallback_clinic) await session.flush() - backup_clinic_id = backup_clinic.id + fallback_clinic_id = fallback_clinic.id await session.execute( location_organizations.insert().values( - location_id=backup_clinic_id, + location_id=fallback_clinic_id, organization_id="global", ) ) await session.execute( update(Patient) .where(Patient.clinic_id.isnot(None)) - .values(clinic_id=backup_clinic_id) + .values(clinic_id=fallback_clinic_id) ) await session.execute(delete(patient_locations)) @@ -173,8 +222,8 @@ async def load_scaffold_data() -> None: await session.execute(update(Task).values(assignee_team_id=None)) ids_to_keep = set(personal_location_ids) - if backup_clinic_id is not None: - ids_to_keep.add(backup_clinic_id) + if fallback_clinic_id is not None: + ids_to_keep.add(fallback_clinic_id) all_ids_result = await session.execute(select(LocationNode.id)) all_ids = {row[0] for row in all_ids_result.all()} ids_to_delete = all_ids - ids_to_keep diff --git a/scaffold/README.md b/scaffold/README.md index cf1182c5..3bef0e1b 100644 --- a/scaffold/README.md +++ b/scaffold/README.md @@ -37,14 +37,14 @@ All `*.json` files in the directory are loaded, merged into one list of root ite **Before deleting locations:** -1. **Backup clinic for existing patients** - If any patient has a `clinic_id`, a **“Scaffold backup clinic”** is created (root CLINIC) and linked to the organization **`global`**. All such patients are moved to this backup clinic so they are not left without a clinic. +1. **Fallback clinic for existing patients** + If any patient has a `clinic_id`, a **“FALLBACK_CLINIC”** is created (root CLINIC) and linked to the organization **`global`**. All such patients are moved to this fallback clinic so they are not left without a clinic. On every scaffold load (restart), if that fallback node still exists but no patient has it as clinic, and no user has it as root location, and no task has it as assignee team, and no patient has it in teams, the fallback node is **deleted**. 2. **Personal location nodes are not deleted** Location nodes created for **users without an attached organization** (title `"{username}'s Organization"`, root, no row in `location_organizations`) are **preserved**. Only other scaffold/organization locations are removed. 3. User root-location links (`user_root_locations`) are removed only for locations that are being deleted; links to preserved personal locations stay. -4. Patient assignments (locations, teams), task assignee team, and then `location_organizations` and the non-preserved location nodes are cleared. Preserved nodes and the backup clinic (if created) are kept. +4. Patient assignments (locations, teams), task assignee team, and then `location_organizations` and the non-preserved location nodes are cleared. Preserved nodes and the fallback clinic (if created) are kept. 5. The merged JSON payload is imported as in MERGE. ## Personal location nodes