diff --git a/backend/config.py b/backend/config.py index 26abbe63..e2ed44ed 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,9 +1,17 @@ import os +from enum import Enum from dotenv import load_dotenv load_dotenv() + +class ScaffoldStrategy(str, Enum): + CHECK = "CHECK" + MERGE = "MERGE" + FORCE = "FORCE" + + _db_hostname = os.getenv("DATABASE_HOSTNAME", "postgres") _db_name = os.getenv("DATABASE_NAME", "postgres") _db_username = os.getenv("DATABASE_USERNAME", "postgres") @@ -52,6 +60,12 @@ SCAFFOLD_DIRECTORY = os.getenv("SCAFFOLD_DIRECTORY", None) +_raw_scaffold_strategy = os.getenv("SCAFFOLD_STRATEGY", "CHECK").strip().upper() +try: + SCAFFOLD_STRATEGY = ScaffoldStrategy(_raw_scaffold_strategy) +except ValueError: + SCAFFOLD_STRATEGY = ScaffoldStrategy.CHECK + INFLUXDB_URL = os.getenv("INFLUXDB_URL", "http://localhost:8086") INFLUXDB_TOKEN = os.getenv("INFLUXDB_TOKEN", None) INFLUXDB_ORG = os.getenv("INFLUXDB_ORG", "tasks") diff --git a/backend/database/migrations/versions/add_scaffold_import_state.py b/backend/database/migrations/versions/add_scaffold_import_state.py new file mode 100644 index 00000000..f669a87b --- /dev/null +++ b/backend/database/migrations/versions/add_scaffold_import_state.py @@ -0,0 +1,29 @@ +"""Add scaffold_import_state table for directory hash + +Revision ID: add_scaffold_import_state +Revises: make_patients_clinic_required +Create Date: 2026-01-30 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "add_scaffold_import_state" +down_revision: Union[str, Sequence[str], None] = "make_patients_clinic_required" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "scaffold_import_state", + sa.Column("key", sa.String(), nullable=False), + sa.Column("value", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("key"), + ) + + +def downgrade() -> None: + op.drop_table("scaffold_import_state") diff --git a/backend/database/migrations/versions/make_patients_clinic_id_nullable.py b/backend/database/migrations/versions/make_patients_clinic_id_nullable.py new file mode 100644 index 00000000..94265c24 --- /dev/null +++ b/backend/database/migrations/versions/make_patients_clinic_id_nullable.py @@ -0,0 +1,34 @@ +"""Make patients.clinic_id nullable for scaffold FORCE unassign + +Revision ID: make_patients_clinic_nullable +Revises: 655294b10318 +Create Date: 2026-01-30 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "make_patients_clinic_nullable" +down_revision: Union[str, Sequence[str], None] = "655294b10318" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=True, + ) + + +def downgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=False, + ) diff --git a/backend/database/migrations/versions/make_patients_clinic_id_required.py b/backend/database/migrations/versions/make_patients_clinic_id_required.py new file mode 100644 index 00000000..9b6aba9b --- /dev/null +++ b/backend/database/migrations/versions/make_patients_clinic_id_required.py @@ -0,0 +1,34 @@ +"""Make patients.clinic_id required again + +Revision ID: make_patients_clinic_required +Revises: make_patients_clinic_nullable +Create Date: 2026-01-30 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "make_patients_clinic_required" +down_revision: Union[str, Sequence[str], None] = "make_patients_clinic_nullable" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=False, + ) + + +def downgrade() -> None: + op.alter_column( + "patients", + "clinic_id", + existing_type=sa.String(), + nullable=True, + ) diff --git a/backend/database/models/__init__.py b/backend/database/models/__init__.py index 5de948a3..8108c8c7 100644 --- a/backend/database/models/__init__.py +++ b/backend/database/models/__init__.py @@ -3,3 +3,4 @@ from .patient import Patient, patient_locations, patient_teams # noqa: F401 from .task import Task, task_dependencies # noqa: F401 from .property import PropertyDefinition, PropertyValue # noqa: F401 +from .scaffold import ScaffoldImportState # noqa: F401 diff --git a/backend/database/models/scaffold.py b/backend/database/models/scaffold.py new file mode 100644 index 00000000..34368a07 --- /dev/null +++ b/backend/database/models/scaffold.py @@ -0,0 +1,10 @@ +from database.models.base import Base +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy import String + + +class ScaffoldImportState(Base): + __tablename__ = "scaffold_import_state" + + key: Mapped[str] = mapped_column(String, primary_key=True) + value: Mapped[str] = mapped_column(String) diff --git a/backend/scaffold.py b/backend/scaffold.py index 0016bf88..2df53ea0 100644 --- a/backend/scaffold.py +++ b/backend/scaffold.py @@ -1,16 +1,64 @@ +import hashlib import json import logging +import time from pathlib import Path from typing import Any from api.inputs import LocationType -from config import LOGGER, SCAFFOLD_DIRECTORY +from config import ( + LOGGER, + SCAFFOLD_DIRECTORY, + SCAFFOLD_STRATEGY, + ScaffoldStrategy, +) from database.models.location import LocationNode, location_organizations +from database.models.patient import Patient, patient_locations, patient_teams +from database.models.scaffold import ScaffoldImportState +from database.models.task import Task +from database.models.user import user_root_locations from database.session import async_session -from sqlalchemy import select +from sqlalchemy import delete, select, update logger = logging.getLogger(LOGGER) +REINITIALIZATION_WAIT_SECONDS = 120 + + +def _compute_scaffold_directory_hash(scaffold_path: Path) -> str: + parts: list[str] = [] + for path in sorted(scaffold_path.glob("*.json")): + parts.append(path.name) + parts.append(path.stat().st_mtime_ns.__str__()) + parts.append(path.stat().st_size.__str__()) + with open(path, "rb") as f: + parts.append(hashlib.sha256(f.read()).hexdigest()) + return hashlib.sha256("".join(parts).encode()).hexdigest() + + +SCAFFOLD_STATE_KEY = "directory_hash" +FALLBACK_CLINIC_TITLE = "FALLBACK_CLINIC" + + +def _load_and_merge_json_payload(scaffold_path: Path) -> list[dict[str, Any]]: + json_files = sorted(scaffold_path.glob("*.json")) + merged: list[dict[str, Any]] = [] + for json_file in json_files: + try: + with open(json_file, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, list): + merged.extend(data) + elif isinstance(data, dict): + merged.append(data) + else: + logger.warning( + f"Invalid JSON structure in {json_file}, expected list or object" + ) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse JSON file {json_file}: {e}") + return merged + async def load_scaffold_data() -> None: if not SCAFFOLD_DIRECTORY: @@ -30,55 +78,213 @@ async def load_scaffold_data() -> None: ) return + payload = _load_and_merge_json_payload(scaffold_path) + if not payload: + logger.info( + f"No valid JSON root items in {SCAFFOLD_DIRECTORY}, skipping scaffold loading" + ) + return + + json_files = list(scaffold_path.glob("*.json")) + logger.info( + f"Loading scaffold data from {len(json_files)} JSON file(s) in {SCAFFOLD_DIRECTORY}, " + f"merged into {len(payload)} root item(s), strategy={SCAFFOLD_STRATEGY.value}" + ) + + current_hash: str | None = None + if SCAFFOLD_STRATEGY in (ScaffoldStrategy.MERGE, ScaffoldStrategy.FORCE): + current_hash = _compute_scaffold_directory_hash(scaffold_path) + async with async_session() as session: - result = await session.execute(select(LocationNode).limit(1)) - existing_location = result.scalar_one_or_none() + fallback_result = await session.execute( + select(LocationNode.id).where( + LocationNode.title == FALLBACK_CLINIC_TITLE, + LocationNode.parent_id.is_(None), + LocationNode.kind == "CLINIC", + ) + ) + fallback_id = fallback_result.scalar_one_or_none() + if fallback_id is not None: + patient_ref = await session.execute( + select(Patient.id).where( + Patient.clinic_id == fallback_id + ).limit(1) + ) + user_root_ref = await session.execute( + select(user_root_locations.c.location_id).where( + user_root_locations.c.location_id == fallback_id + ).limit(1) + ) + task_ref = await session.execute( + select(Task.id).where( + Task.assignee_team_id == fallback_id + ).limit(1) + ) + team_ref = await session.execute( + select(patient_teams.c.location_id).where( + patient_teams.c.location_id == fallback_id + ).limit(1) + ) + if ( + patient_ref.scalar_one_or_none() is None + and user_root_ref.scalar_one_or_none() is None + and task_ref.scalar_one_or_none() is None + and team_ref.scalar_one_or_none() is None + ): + await session.execute( + delete(location_organizations).where( + location_organizations.c.location_id == fallback_id + ) + ) + await session.execute( + delete(LocationNode).where(LocationNode.id == fallback_id) + ) + await session.flush() + logger.info( + "Removed unused FALLBACK_CLINIC (no references)" + ) - if existing_location: - logger.info( - "Location nodes already exist in database, skipping scaffold loading" + if SCAFFOLD_STRATEGY in (ScaffoldStrategy.MERGE, ScaffoldStrategy.FORCE): + result = await session.execute( + select(ScaffoldImportState.value).where( + ScaffoldImportState.key == SCAFFOLD_STATE_KEY + ) ) - return + last_hash: str | None = result.scalar_one_or_none() + if last_hash == current_hash: + logger.info( + "Scaffold directory unchanged (DB state), " + "skipping reinitialization wait" + ) + else: + logger.warning( + "Scaffold directory changed or first run. " + "Waiting %d seconds for reinitialization...", + REINITIALIZATION_WAIT_SECONDS, + ) + time.sleep(REINITIALIZATION_WAIT_SECONDS) - json_files = list(scaffold_path.glob("*.json")) + if SCAFFOLD_STRATEGY == ScaffoldStrategy.CHECK: + result = await session.execute(select(LocationNode).limit(1)) + if result.scalar_one_or_none(): + logger.info( + "Location nodes already exist (CHECK strategy), skipping scaffold loading" + ) + return - if not json_files: - logger.info( - f"No JSON files found in {SCAFFOLD_DIRECTORY}, skipping scaffold loading" + if SCAFFOLD_STRATEGY == ScaffoldStrategy.FORCE: + personal_ids_result = await session.execute( + select(LocationNode.id).where( + LocationNode.parent_id.is_(None), + LocationNode.title.like("%'s Organization"), + ~LocationNode.id.in_( + select(location_organizations.c.location_id) + ), + ) ) - return + personal_location_ids = { + row[0] for row in personal_ids_result.all() + } - logger.info( - f"Loading scaffold data from {len(json_files)} JSON file(s) in {SCAFFOLD_DIRECTORY}" - ) + result = await session.execute( + select(Patient.id).where(Patient.clinic_id.isnot(None)).limit(1) + ) + has_patients_with_clinic = result.scalar_one_or_none() is not None + fallback_clinic_id: str | None = None + if has_patients_with_clinic: + fallback_clinic = LocationNode( + title=FALLBACK_CLINIC_TITLE, + kind="CLINIC", + parent_id=None, + ) + session.add(fallback_clinic) + await session.flush() + fallback_clinic_id = fallback_clinic.id + await session.execute( + location_organizations.insert().values( + location_id=fallback_clinic_id, + organization_id="global", + ) + ) + await session.execute( + update(Patient) + .where(Patient.clinic_id.isnot(None)) + .values(clinic_id=fallback_clinic_id) + ) - for json_file in json_files: - try: - with open(json_file, "r", encoding="utf-8") as f: - data = json.load(f) + await session.execute(delete(patient_locations)) + await session.execute(delete(patient_teams)) + await session.execute( + delete(user_root_locations).where( + ~user_root_locations.c.location_id.in_(personal_location_ids) + ) + ) + await session.execute(update(Task).values(assignee_team_id=None)) - if isinstance(data, list): - for item in data: - await _create_location_tree(session, item, None) - elif isinstance(data, dict): - await _create_location_tree(session, data, None) - else: - logger.warning( - f"Invalid JSON structure in {json_file}, expected list or object" - ) + ids_to_keep = set(personal_location_ids) + if fallback_clinic_id is not None: + ids_to_keep.add(fallback_clinic_id) + all_ids_result = await session.execute(select(LocationNode.id)) + all_ids = {row[0] for row in all_ids_result.all()} + ids_to_delete = all_ids - ids_to_keep - await session.commit() - logger.info( - f"Successfully loaded scaffold data from {json_file}" + if ids_to_delete: + await session.execute( + update(Patient) + .where( + Patient.assigned_location_id.in_(ids_to_delete) + ) + .values(assigned_location_id=None) + ) + await session.execute( + update(Patient) + .where(Patient.position_id.in_(ids_to_delete)) + .values(position_id=None) + ) + await session.execute( + delete(location_organizations).where( + location_organizations.c.location_id.in_(ids_to_delete) + ) + ) + await session.execute( + update(LocationNode) + .where(LocationNode.id.in_(ids_to_delete)) + .values(parent_id=None) ) - except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON file {json_file}: {e}") - await session.rollback() - except Exception as e: - logger.error( - f"Error loading scaffold data from {json_file}: {e}" + await session.execute( + delete(LocationNode).where( + LocationNode.id.in_(ids_to_delete) + ) + ) + await session.flush() + + try: + for item in payload: + await _create_location_tree(session, item, None) + if ( + current_hash is not None + and SCAFFOLD_STRATEGY in (ScaffoldStrategy.MERGE, ScaffoldStrategy.FORCE) + ): + existing = await session.execute( + select(ScaffoldImportState).where( + ScaffoldImportState.key == SCAFFOLD_STATE_KEY + ) ) - await session.rollback() + row = existing.scalar_one_or_none() + if row: + row.value = current_hash + else: + session.add( + ScaffoldImportState( + key=SCAFFOLD_STATE_KEY, + value=current_hash, + ) + ) + await session.commit() + logger.info("Successfully loaded scaffold data (single import)") + except Exception as e: + logger.error(f"Error loading scaffold data: {e}") + await session.rollback() async def _create_location_tree( diff --git a/scaffold/README.md b/scaffold/README.md new file mode 100644 index 00000000..3bef0e1b --- /dev/null +++ b/scaffold/README.md @@ -0,0 +1,56 @@ +# Scaffold + +Location-tree data loaded at backend startup from JSON files in a configurable directory. + +## Environment + +- **SCAFFOLD_DIRECTORY** – Path to a directory containing `*.json` files. If unset, scaffold loading is skipped. +- **SCAFFOLD_STRATEGY** – One of `CHECK`, `MERGE`, `FORCE`. Parsed from env (case-insensitive); invalid values fall back to `CHECK`. + +## JSON format + +Each file may be a single root object or a list of root objects. Root objects have: + +- `name`, `type` (e.g. `HOSPITAL`, `CLINIC`, `WARD`, `ROOM`, `BED`, `TEAM`, `PRACTICE`, `OTHER`) +- optional `children` (array of same shape) +- optional `organization_ids` (for HOSPITAL, CLINIC, PRACTICE, TEAM) + +All `*.json` files in the directory are loaded, merged into one list of root items, and imported in a single transaction. + +## Strategies + +### CHECK (default) + +- If any location node already exists in the database, scaffold loading is **skipped**. +- No wait, no overwrite. + +### MERGE + +- All JSON files are loaded and merged into one payload; that payload is imported **once**. +- New location nodes are created **beside** existing ones (existing nodes are matched by `title`, `kind`, `parent_id` and reused). +- **Reinitialization wait:** If the scaffold directory content (file set and contents) has **changed** since the last run, the backend logs a **warning** and waits **120 seconds** before importing. If the content is **unchanged** (same hash in the state file), the wait is skipped. + +### FORCE + +- All JSON files are loaded and merged; then the backend **replaces** scaffold location data with that payload. +- **Reinitialization wait:** Same 120-second warning and hash check as MERGE; no wait when directory content is unchanged. + +**Before deleting locations:** + +1. **Fallback clinic for existing patients** + If any patient has a `clinic_id`, a **“FALLBACK_CLINIC”** is created (root CLINIC) and linked to the organization **`global`**. All such patients are moved to this fallback clinic so they are not left without a clinic. On every scaffold load (restart), if that fallback node still exists but no patient has it as clinic, and no user has it as root location, and no task has it as assignee team, and no patient has it in teams, the fallback node is **deleted**. + +2. **Personal location nodes are not deleted** + Location nodes created for **users without an attached organization** (title `"{username}'s Organization"`, root, no row in `location_organizations`) are **preserved**. Only other scaffold/organization locations are removed. + +3. User root-location links (`user_root_locations`) are removed only for locations that are being deleted; links to preserved personal locations stay. +4. Patient assignments (locations, teams), task assignee team, and then `location_organizations` and the non-preserved location nodes are cleared. Preserved nodes and the fallback clinic (if created) are kept. +5. The merged JSON payload is imported as in MERGE. + +## Personal location nodes + +When a user has **no** organization attached, the backend creates a single root location for them: title `"{username}'s Organization"`, kind CLINIC, with no `location_organizations` row. These nodes are considered **personal** and are **never** deleted by scaffold FORCE, so user access is preserved. + +## Reinitialization check (MERGE / FORCE) + +A hash of the scaffold directory (file names, mtimes, sizes, and contents) is computed in memory and stored in the **database** after a successful import (table `scaffold_import_state`, key `directory_hash`). No files are written to the scaffold directory. On the next run, the backend compares the current directory hash with the stored value; if they match, the 120-second reinitialization wait is skipped.