From 96f8679a73150514bbf2f7d86830697ca0ecf0f7 Mon Sep 17 00:00:00 2001 From: Mateo Date: Fri, 1 May 2026 07:10:15 +0200 Subject: [PATCH 1/5] chore: scaffold alembic migrations directory --- annotation_api/Dockerfile | 1 + annotation_api/docker-compose-dev.yml | 1 + annotation_api/src/migrations/README.md | 47 ++++++++++++ annotation_api/src/migrations/env.py | 78 ++++++++++++++++++++ annotation_api/src/migrations/script.py.mako | 26 +++++++ 5 files changed, 153 insertions(+) create mode 100644 annotation_api/src/migrations/README.md create mode 100644 annotation_api/src/migrations/env.py create mode 100644 annotation_api/src/migrations/script.py.mako diff --git a/annotation_api/Dockerfile b/annotation_api/Dockerfile index 9d28ce2..b57315e 100644 --- a/annotation_api/Dockerfile +++ b/annotation_api/Dockerfile @@ -33,6 +33,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Copy project files COPY src/alembic.ini /app/alembic.ini +COPY src/migrations /app/migrations COPY src/app /app/app # Install project diff --git a/annotation_api/docker-compose-dev.yml b/annotation_api/docker-compose-dev.yml index 581b148..64e4673 100644 --- a/annotation_api/docker-compose-dev.yml +++ b/annotation_api/docker-compose-dev.yml @@ -70,6 +70,7 @@ services: volumes: - ./src/app:/app/app - ./src/alembic.ini:/app/alembic.ini + - ./src/migrations:/app/migrations - ./src/tests:/app/tests command: "sh -c 'python app/db.py && uvicorn app.main:app --reload --host 0.0.0.0 --port 5050 --proxy-headers'" restart: always diff --git a/annotation_api/src/migrations/README.md b/annotation_api/src/migrations/README.md new file mode 100644 index 0000000..f8eb0dd --- /dev/null +++ b/annotation_api/src/migrations/README.md @@ -0,0 +1,47 @@ +# Alembic migrations + +Alembic is the sole owner of the database schema. Every table, column, +index and constraint is created and evolved through revision files in +`src/migrations/versions/`. + +All commands assume the dev stack is running: + +```shell +make start +``` + +## Create a new revision + +After modifying a SQLModel in `src/app/models.py`, autogenerate a +revision file from the diff between your models and the live DB: + +```shell +make migrate m="add active column to detections" +``` + +This is a thin wrapper around: + +```shell +docker compose exec -T backend alembic revision --autogenerate -m "" +``` + +The new file lands in `src/migrations/versions/`. **Always review it +before committing** — `--autogenerate` does not detect renames (it +emits drop+create), enum value additions, `server_default` changes, or +some `CHECK` constraints. Adjust by hand when needed. + +## Apply pending revisions + +Migrations are applied automatically when the backend container starts +(`alembic upgrade head` is the first thing in the compose command). To +apply without restarting: + +```shell +make migrate-up +``` + +Equivalent to: + +```shell +docker compose exec backend alembic upgrade head +``` diff --git a/annotation_api/src/migrations/env.py b/annotation_api/src/migrations/env.py new file mode 100644 index 0000000..1d559b6 --- /dev/null +++ b/annotation_api/src/migrations/env.py @@ -0,0 +1,78 @@ +# Copyright (C) 2024-2026, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import create_async_engine +from sqlmodel import SQLModel +from sqlmodel.sql.sqltypes import AutoString + +from app.core.config import settings +from app.models import * # noqa: F401,F403 + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here for 'autogenerate' support. +target_metadata = SQLModel.metadata + + +def render_item(type_, obj, autogen_context): # noqa: ARG001 + """Render sqlmodel's AutoString as plain sa.String so migrations don't import sqlmodel.""" + if type_ == "type" and isinstance(obj, AutoString): + length = getattr(obj.impl, "length", None) + return f"sa.String(length={length})" if length else "sa.String()" + return False + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode.""" + context.configure( + url=settings.POSTGRES_URL, + target_metadata=target_metadata, + literal_binds=True, + compare_type=True, + dialect_opts={"paramstyle": "named"}, + render_item=render_item, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_item=render_item, + ) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + connectable = create_async_engine(settings.POSTGRES_URL, echo=False, future=True) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + asyncio.run(run_migrations_online()) diff --git a/annotation_api/src/migrations/script.py.mako b/annotation_api/src/migrations/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/annotation_api/src/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} From f9c0ab9ac6ec20e241384393aa617314b1218b73 Mon Sep 17 00:00:00 2001 From: Mateo Date: Fri, 1 May 2026 07:13:24 +0200 Subject: [PATCH 2/5] feat: add initial alembic migration --- .../2026_05_01_0512-063dc76c9846_initial.py | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py diff --git a/annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py b/annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py new file mode 100644 index 0000000..bc5ab70 --- /dev/null +++ b/annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py @@ -0,0 +1,200 @@ +"""initial + +Revision ID: 063dc76c9846 +Revises: +Create Date: 2026-05-01 05:12:09.152220 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '063dc76c9846' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('sequences', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('source_api', sa.Enum('PYRONEAR_FRENCH_API', 'ALERT_WILDFIRE', 'CENIA', name='sourceapi'), nullable=False), + sa.Column('alert_api_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('recorded_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_seen_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('camera_name', sa.String(), nullable=False), + sa.Column('camera_id', sa.Integer(), nullable=False), + sa.Column('lat', sa.Float(), nullable=False), + sa.Column('lon', sa.Float(), nullable=False), + sa.Column('azimuth', sa.Integer(), nullable=True), + sa.Column('is_wildfire_alertapi', sa.Enum('WILDFIRE_SMOKE', 'OTHER_SMOKE', 'OTHER', name='annotationtype'), nullable=True), + sa.Column('organisation_name', sa.String(), nullable=False), + sa.Column('organisation_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('alert_api_id', 'source_api', name='uq_sequence_alert_source') + ) + op.create_index('ix_sequence_camera_id', 'sequences', ['camera_id'], unique=False) + op.create_index('ix_sequence_camera_name', 'sequences', ['camera_name'], unique=False) + op.create_index('ix_sequence_camera_org', 'sequences', ['camera_name', 'organisation_name'], unique=False) + op.create_index('ix_sequence_created_at', 'sequences', ['created_at'], unique=False) + op.create_index('ix_sequence_full_filter', 'sequences', ['source_api', 'camera_name', 'organisation_name', 'is_wildfire_alertapi'], unique=False) + op.create_index('ix_sequence_is_wildfire', 'sequences', ['is_wildfire_alertapi'], unique=False) + op.create_index('ix_sequence_last_seen_at', 'sequences', ['last_seen_at'], unique=False) + op.create_index('ix_sequence_organisation_id', 'sequences', ['organisation_id'], unique=False) + op.create_index('ix_sequence_organisation_name', 'sequences', ['organisation_name'], unique=False) + op.create_index('ix_sequence_recorded_at', 'sequences', ['recorded_at'], unique=False) + op.create_index('ix_sequence_source_api', 'sequences', ['source_api'], unique=False) + op.create_index('ix_sequence_source_camera', 'sequences', ['source_api', 'camera_name'], unique=False) + op.create_index('ix_sequence_source_camera_org', 'sequences', ['source_api', 'camera_name', 'organisation_name'], unique=False) + op.create_index('ix_sequence_source_camera_wildfire', 'sequences', ['source_api', 'camera_name', 'is_wildfire_alertapi'], unique=False) + op.create_index('ix_sequence_source_org', 'sequences', ['source_api', 'organisation_name'], unique=False) + op.create_index('ix_sequence_source_org_wildfire', 'sequences', ['source_api', 'organisation_name', 'is_wildfire_alertapi'], unique=False) + op.create_index('ix_sequence_source_wildfire', 'sequences', ['source_api', 'is_wildfire_alertapi'], unique=False) + op.create_table('users', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('username', sa.String(length=50), nullable=False), + sa.Column('hashed_password', sa.String(), nullable=False), + sa.Column('is_active', sa.Boolean(), nullable=False), + sa.Column('is_superuser', sa.Boolean(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('username', name='uq_user_username') + ) + op.create_index('ix_user_username', 'users', ['username'], unique=False) + op.create_table('detections', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('recorded_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('alert_api_id', sa.Integer(), nullable=False), + sa.Column('sequence_id', sa.Integer(), nullable=True), + sa.Column('bucket_key', sa.String(), nullable=False), + sa.Column('algo_predictions', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.ForeignKeyConstraint(['sequence_id'], ['sequences.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('alert_api_id', 'id', name='uq_detection_alert_id') + ) + op.create_index('ix_detection_created_at', 'detections', ['created_at'], unique=False) + op.create_index('ix_detection_recorded_at', 'detections', ['recorded_at'], unique=False) + op.create_index('ix_detection_sequence_created', 'detections', ['sequence_id', 'created_at'], unique=False) + op.create_index('ix_detection_sequence_id', 'detections', ['sequence_id'], unique=False) + op.create_table('sequences_annotations', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('sequence_id', sa.Integer(), nullable=True), + sa.Column('has_smoke', sa.Boolean(), nullable=False), + sa.Column('has_false_positives', sa.Boolean(), nullable=False), + sa.Column('false_positive_types', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('smoke_types', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('has_missed_smoke', sa.Boolean(), nullable=False), + sa.Column('is_unsure', sa.Boolean(), nullable=True), + sa.Column('annotation', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('processing_stage', sa.Enum('IMPORTED', 'READY_TO_ANNOTATE', 'UNDER_ANNOTATION', 'SEQ_ANNOTATION_DONE', 'IN_REVIEW', 'NEEDS_MANUAL', 'ANNOTATED', name='sequenceannotationprocessingstage'), nullable=False), + sa.ForeignKeyConstraint(['sequence_id'], ['sequences.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('sequence_id', name='uq_sequence_annotation_sequence_id') + ) + op.create_index('ix_sequence_annotation_created_at', 'sequences_annotations', ['created_at'], unique=False) + op.create_index('ix_sequence_annotation_fp_types', 'sequences_annotations', ['false_positive_types'], unique=False, postgresql_using='gin') + op.create_index('ix_sequence_annotation_has_false_positives', 'sequences_annotations', ['has_false_positives'], unique=False) + op.create_index('ix_sequence_annotation_has_missed_smoke', 'sequences_annotations', ['has_missed_smoke'], unique=False) + op.create_index('ix_sequence_annotation_has_smoke', 'sequences_annotations', ['has_smoke'], unique=False) + op.create_index('ix_sequence_annotation_is_unsure', 'sequences_annotations', ['is_unsure'], unique=False) + op.create_index('ix_sequence_annotation_processing_stage', 'sequences_annotations', ['processing_stage'], unique=False) + op.create_index('ix_sequence_annotation_smoke_types', 'sequences_annotations', ['smoke_types'], unique=False, postgresql_using='gin') + op.create_index('ix_sequence_annotation_stage_date', 'sequences_annotations', ['processing_stage', 'created_at'], unique=False) + op.create_table('detections_annotations', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('detection_id', sa.Integer(), nullable=True), + sa.Column('annotation', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('processing_stage', sa.Enum('IMPORTED', 'VISUAL_CHECK', 'BBOX_ANNOTATION', 'ANNOTATED', name='detectionannotationprocessingstage'), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['detection_id'], ['detections.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('detection_id', name='uq_detection_annotation_detection_id') + ) + op.create_index('ix_detection_annotation_created_at', 'detections_annotations', ['created_at'], unique=False) + op.create_index('ix_detection_annotation_detection_date', 'detections_annotations', ['detection_id', 'created_at'], unique=False) + op.create_index('ix_detection_annotation_processing_stage', 'detections_annotations', ['processing_stage'], unique=False) + op.create_index('ix_detection_annotation_stage_date', 'detections_annotations', ['processing_stage', 'created_at'], unique=False) + op.create_table('sequence_annotation_contributions', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('sequence_annotation_id', sa.Integer(), nullable=True), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('contributed_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['sequence_annotation_id'], ['sequences_annotations.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('ix_seq_contrib_annotation_user', 'sequence_annotation_contributions', ['sequence_annotation_id', 'user_id'], unique=False) + op.create_index('ix_seq_contrib_user_time', 'sequence_annotation_contributions', ['user_id', 'contributed_at'], unique=False) + op.create_table('detection_annotation_contributions', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('detection_annotation_id', sa.Integer(), nullable=True), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('contributed_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['detection_annotation_id'], ['detections_annotations.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('ix_det_contrib_annotation_user', 'detection_annotation_contributions', ['detection_annotation_id', 'user_id'], unique=False) + op.create_index('ix_det_contrib_user_time', 'detection_annotation_contributions', ['user_id', 'contributed_at'], unique=False) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('ix_det_contrib_user_time', table_name='detection_annotation_contributions') + op.drop_index('ix_det_contrib_annotation_user', table_name='detection_annotation_contributions') + op.drop_table('detection_annotation_contributions') + op.drop_index('ix_seq_contrib_user_time', table_name='sequence_annotation_contributions') + op.drop_index('ix_seq_contrib_annotation_user', table_name='sequence_annotation_contributions') + op.drop_table('sequence_annotation_contributions') + op.drop_index('ix_detection_annotation_stage_date', table_name='detections_annotations') + op.drop_index('ix_detection_annotation_processing_stage', table_name='detections_annotations') + op.drop_index('ix_detection_annotation_detection_date', table_name='detections_annotations') + op.drop_index('ix_detection_annotation_created_at', table_name='detections_annotations') + op.drop_table('detections_annotations') + op.drop_index('ix_sequence_annotation_stage_date', table_name='sequences_annotations') + op.drop_index('ix_sequence_annotation_smoke_types', table_name='sequences_annotations', postgresql_using='gin') + op.drop_index('ix_sequence_annotation_processing_stage', table_name='sequences_annotations') + op.drop_index('ix_sequence_annotation_is_unsure', table_name='sequences_annotations') + op.drop_index('ix_sequence_annotation_has_smoke', table_name='sequences_annotations') + op.drop_index('ix_sequence_annotation_has_missed_smoke', table_name='sequences_annotations') + op.drop_index('ix_sequence_annotation_has_false_positives', table_name='sequences_annotations') + op.drop_index('ix_sequence_annotation_fp_types', table_name='sequences_annotations', postgresql_using='gin') + op.drop_index('ix_sequence_annotation_created_at', table_name='sequences_annotations') + op.drop_table('sequences_annotations') + op.drop_index('ix_detection_sequence_id', table_name='detections') + op.drop_index('ix_detection_sequence_created', table_name='detections') + op.drop_index('ix_detection_recorded_at', table_name='detections') + op.drop_index('ix_detection_created_at', table_name='detections') + op.drop_table('detections') + op.drop_index('ix_user_username', table_name='users') + op.drop_table('users') + op.drop_index('ix_sequence_source_wildfire', table_name='sequences') + op.drop_index('ix_sequence_source_org_wildfire', table_name='sequences') + op.drop_index('ix_sequence_source_org', table_name='sequences') + op.drop_index('ix_sequence_source_camera_wildfire', table_name='sequences') + op.drop_index('ix_sequence_source_camera_org', table_name='sequences') + op.drop_index('ix_sequence_source_camera', table_name='sequences') + op.drop_index('ix_sequence_source_api', table_name='sequences') + op.drop_index('ix_sequence_recorded_at', table_name='sequences') + op.drop_index('ix_sequence_organisation_name', table_name='sequences') + op.drop_index('ix_sequence_organisation_id', table_name='sequences') + op.drop_index('ix_sequence_last_seen_at', table_name='sequences') + op.drop_index('ix_sequence_is_wildfire', table_name='sequences') + op.drop_index('ix_sequence_full_filter', table_name='sequences') + op.drop_index('ix_sequence_created_at', table_name='sequences') + op.drop_index('ix_sequence_camera_org', table_name='sequences') + op.drop_index('ix_sequence_camera_name', table_name='sequences') + op.drop_index('ix_sequence_camera_id', table_name='sequences') + op.drop_table('sequences') + # ### end Alembic commands ### From 680965fe64bbc6cd4000755c5eb20f7130e190e7 Mon Sep 17 00:00:00 2001 From: Mateo Date: Fri, 1 May 2026 07:14:44 +0200 Subject: [PATCH 3/5] feat: run alembic migrations on container startup Replace SQLModel.metadata.create_all with alembic upgrade head in all compose entrypoints. Drop init_db from app.db and from main.py lifespan. Add make migrate / migrate-up wrappers. --- annotation_api/Makefile | 13 +++++++++++++ annotation_api/docker-compose-dev.yml | 2 +- annotation_api/docker-compose.yml | 2 +- annotation_api/src/app/db.py | 20 +++----------------- annotation_api/src/app/main.py | 10 +++++----- docker-compose.yml | 2 +- 6 files changed, 24 insertions(+), 25 deletions(-) diff --git a/annotation_api/Makefile b/annotation_api/Makefile index 193bed2..5a39d8c 100644 --- a/annotation_api/Makefile +++ b/annotation_api/Makefile @@ -10,6 +10,8 @@ help: @echo " clean - Remove containers and volumes (fresh start)" @echo " test - Run comprehensive test suite with coverage" @echo " test-specific - Run specific test (TEST=test_file.py::test_method)" + @echo " migrate - Autogenerate alembic revision (m=\"description\")" + @echo " migrate-up - Apply pending alembic revisions" @echo "" @echo "Sequence annotation workflow (A):" @echo " pull-sequences - Duplicate N sequences from remote API to local API" @@ -103,6 +105,16 @@ test-specific: docker compose -f docker-compose-dev.yml down --volumes; \ exit $$TEST_EXIT_CODE +# Generate a new alembic migration from the current models. +# Requires the dev stack to be running (make start). Usage: make migrate m="add foo column" +migrate: + @if [ -z "$(m)" ]; then echo "Usage: make migrate m=\"short description\""; exit 1; fi + docker compose -f docker-compose.yml exec -T backend alembic revision --autogenerate -m "$(m)" + +# Apply pending alembic migrations to the running DB. +migrate-up: + docker compose -f docker-compose.yml exec -T backend alembic upgrade head + # ========================================================================= # Data workflow targets # All targets requiring remote auth expect MAIN_ANNOTATION_LOGIN / MAIN_ANNOTATION_PASSWORD @@ -309,6 +321,7 @@ import-platform: --loglevel $(LOGLEVEL) .PHONY: help lint fix setup docker-build start stop clean test test-specific \ + migrate migrate-up \ pull-sequences push-annotations \ pull-seq-annotations auto-annotate visual-check apply-review \ pull-fp visual-check-fp apply-review-fp \ diff --git a/annotation_api/docker-compose-dev.yml b/annotation_api/docker-compose-dev.yml index 64e4673..57a4804 100644 --- a/annotation_api/docker-compose-dev.yml +++ b/annotation_api/docker-compose-dev.yml @@ -72,7 +72,7 @@ services: - ./src/alembic.ini:/app/alembic.ini - ./src/migrations:/app/migrations - ./src/tests:/app/tests - command: "sh -c 'python app/db.py && uvicorn app.main:app --reload --host 0.0.0.0 --port 5050 --proxy-headers'" + command: "sh -c 'alembic upgrade head && uvicorn app.main:app --reload --host 0.0.0.0 --port 5050 --proxy-headers'" restart: always healthcheck: test: ["CMD-SHELL", "curl http://localhost:5050/status"] diff --git a/annotation_api/docker-compose.yml b/annotation_api/docker-compose.yml index 6a4602e..4131d84 100644 --- a/annotation_api/docker-compose.yml +++ b/annotation_api/docker-compose.yml @@ -61,7 +61,7 @@ services: - JWT_SECRET=your_jwt_secret_key_change_in_production_please - ACCESS_TOKEN_EXPIRE_HOURS=24 restart: always - command: "sh -c 'python app/db.py && uvicorn app.main:app --host 0.0.0.0 --port 5050 --proxy-headers'" + command: "sh -c 'alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 5050 --proxy-headers'" healthcheck: test: ["CMD-SHELL", "curl http://localhost:5050/status"] interval: 10s diff --git a/annotation_api/src/app/db.py b/annotation_api/src/app/db.py index 4fa19fe..1bd4af1 100644 --- a/annotation_api/src/app/db.py +++ b/annotation_api/src/app/db.py @@ -1,20 +1,19 @@ -# Copyright (C) 2024, Pyronear. +# Copyright (C) 2024-2026, Pyronear. # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. -import asyncio import logging from sqlalchemy.ext.asyncio.engine import AsyncEngine from sqlalchemy.orm import sessionmaker -from sqlmodel import SQLModel, create_engine +from sqlmodel import create_engine from sqlmodel.ext.asyncio.session import AsyncSession from app.core.config import settings from app.models import * # noqa -__all__ = ["get_session", "init_db"] +__all__ = ["get_session"] logger = logging.getLogger("uvicorn.error") @@ -52,16 +51,3 @@ async def get_session() -> AsyncSession: # type: ignore[misc] ) async with async_session() as session: yield session - - -async def init_db() -> None: - async with engine.begin() as conn: - await conn.run_sync(SQLModel.metadata.create_all) - - -async def main() -> None: - await init_db() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/annotation_api/src/app/main.py b/annotation_api/src/app/main.py index f227679..bf50e29 100644 --- a/annotation_api/src/app/main.py +++ b/annotation_api/src/app/main.py @@ -22,7 +22,7 @@ from app.api.api_v1.router import api_router from app.core.config import settings from app.crud import UserCRUD -from app.db import get_session, init_db +from app.db import get_session from app.schemas.base import Status from app.schemas.user import UserCreate @@ -31,11 +31,11 @@ @asynccontextmanager async def lifespan(app: FastAPI): - """Application lifespan manager for startup and shutdown events.""" - # Startup - logger.info("Initializing database...") - await init_db() + """Application lifespan manager for startup and shutdown events. + Database schema is managed by Alembic (see ``src/migrations``); migrations + are applied by the container entrypoint before uvicorn starts. + """ # Create admin user from environment variables if not exists async for session in get_session(): user_crud = UserCRUD(session) diff --git a/docker-compose.yml b/docker-compose.yml index 980f261..34b6f91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,7 +65,7 @@ services: - JWT_SECRET=your_jwt_secret_key_change_in_production_please - ACCESS_TOKEN_EXPIRE_HOURS=24 restart: unless-stopped - command: "sh -c 'python app/db.py && uvicorn app.main:app --host 0.0.0.0 --port 5050 --proxy-headers'" + command: "sh -c 'alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 5050 --proxy-headers'" healthcheck: test: ["CMD-SHELL", "curl http://localhost:5050/status"] interval: 10s From 2e25806ae592a1ac4bcfc7f5cdaa5b086abcd344 Mon Sep 17 00:00:00 2001 From: Mateo Date: Fri, 1 May 2026 07:19:37 +0200 Subject: [PATCH 4/5] test: reset schema via alembic upgrade in conftest --- annotation_api/src/tests/conftest.py | 32 +++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/annotation_api/src/tests/conftest.py b/annotation_api/src/tests/conftest.py index 631598a..12c8729 100644 --- a/annotation_api/src/tests/conftest.py +++ b/annotation_api/src/tests/conftest.py @@ -1,10 +1,14 @@ +import asyncio import io from datetime import datetime, timedelta, UTC +from pathlib import Path from typing import AsyncGenerator import pytest import pytest_asyncio import requests +from alembic import command +from alembic.config import Config from botocore.exceptions import ClientError from httpx import AsyncClient from sqlalchemy.orm import sessionmaker @@ -20,6 +24,24 @@ from app.schemas.user import UserCreate from app.services.storage import s3_service +ALEMBIC_INI = Path(__file__).resolve().parents[1] / "alembic.ini" + + +def _alembic_config() -> Config: + cfg = Config(str(ALEMBIC_INI)) + cfg.set_main_option("script_location", str(ALEMBIC_INI.parent / "migrations")) + return cfg + + +async def _run_alembic_upgrade() -> None: + """Run ``alembic upgrade head`` from inside a running event loop. + + ``env.py`` calls ``asyncio.run`` which cannot nest, so we offload the + command to a worker thread that owns its own event loop. + """ + cfg = _alembic_config() + await asyncio.to_thread(command.upgrade, cfg, "head") + dt_format = "%Y-%m-%dT%H:%M:%S.%f" now = datetime.now(UTC) @@ -142,11 +164,15 @@ async def get_test_session(): @pytest_asyncio.fixture(scope="function") async def async_session() -> AsyncSession: + # Reset schema using Alembic so tests exercise the same migration path + # as production startup. We drop everything first to guarantee a clean + # slate (including the alembic_version table) and then upgrade to head. async with engine.begin() as conn: - # Drop all tables first to ensure clean schema await conn.run_sync(SQLModel.metadata.drop_all) - # Then recreate with current schema including all fields - await conn.run_sync(SQLModel.metadata.create_all) + await conn.exec_driver_sql("DROP TABLE IF EXISTS alembic_version") + await engine.dispose() + + await _run_alembic_upgrade() async_session_maker = sessionmaker( engine, class_=AsyncSession, expire_on_commit=False From 4bc027712f84729221d821ae53ea9656b182f971 Mon Sep 17 00:00:00 2001 From: Mateo Date: Fri, 1 May 2026 07:21:23 +0200 Subject: [PATCH 5/5] fix: drop enum types in initial downgrade and enable compare_type online Address review feedback: PostgreSQL enum types are not removed by op.drop_table, so a downgrade-then-upgrade cycle would fail with duplicate enum errors. Also enable compare_type=True in the online migration context so autogenerate detects column type changes. --- annotation_api/src/migrations/env.py | 1 + .../versions/2026_05_01_0512-063dc76c9846_initial.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/annotation_api/src/migrations/env.py b/annotation_api/src/migrations/env.py index 1d559b6..32ddb8d 100644 --- a/annotation_api/src/migrations/env.py +++ b/annotation_api/src/migrations/env.py @@ -55,6 +55,7 @@ def do_run_migrations(connection: Connection) -> None: context.configure( connection=connection, target_metadata=target_metadata, + compare_type=True, render_item=render_item, ) diff --git a/annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py b/annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py index bc5ab70..10ab441 100644 --- a/annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py +++ b/annotation_api/src/migrations/versions/2026_05_01_0512-063dc76c9846_initial.py @@ -198,3 +198,13 @@ def downgrade() -> None: op.drop_index('ix_sequence_camera_id', table_name='sequences') op.drop_table('sequences') # ### end Alembic commands ### + + # Drop enum types created implicitly by sa.Enum columns above. + bind = op.get_bind() + for enum_name in ( + 'detectionannotationprocessingstage', + 'sequenceannotationprocessingstage', + 'annotationtype', + 'sourceapi', + ): + postgresql.ENUM(name=enum_name).drop(bind, checkfirst=True)