diff --git a/.gitignore b/.gitignore index d9fea9f..de57f6b 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,9 @@ celerybeat-schedule # Environments *.env .venv + +# Per-environment credential files (do not commit) +creds.json env/ venv/ ENV/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..58c08d0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,144 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Pyro-Annotator is a suite for annotating wildfire detection sequences. It combines a FastAPI backend, a React frontend, and data transfer scripts to collect, manage, and annotate fire detection data from multiple sources. + +## Repository Structure + +``` +pyro-annotator/ +├── annotation_api/ # FastAPI backend + data transfer scripts +├── frontend/ # React/TypeScript annotation UI +├── sam_based_bbox_propagation/ # SAM-based semi-automatic bbox tool (Dash, port 8050) +├── docker-compose.yml # Full stack orchestration +└── Makefile # Docker build/push targets +``` + +Each submodule has its own `CLAUDE.md` with detailed context — read those when working within a specific module. + +## Quick Start + +```bash +# Start all services (PostgreSQL, LocalStack S3, API, Frontend) +docker compose up -d + +# Services: +# Frontend: http://localhost:3000 +# API: http://localhost:5050 (docs at /docs) +# Database: localhost:5432 +# S3: http://localhost:4566 +``` + +## Backend (`annotation_api/`) + +**Stack**: FastAPI, Python 3.11+, uv, PostgreSQL (SQLModel/SQLAlchemy), S3-compatible storage, JWT auth + +```bash +cd annotation_api + +# Dev environment via Docker +make start # Start dev containers +make stop # Stop (preserves data) +make clean # Remove containers and volumes + +# Local dev (requires uv: curl -LsSf https://astral.sh/uv/install.sh | sh) +uv sync --group dev +uv run uvicorn app.main:app --reload --app-dir src + +# Quality +make lint # Format check + ruff + mypy +make fix # Auto-fix formatting/lint issues + +# Tests +make test # Full suite in Docker +make test-specific TEST=tests/test_foo.py::test_bar # Single test +``` + +## Frontend (`frontend/`) + +**Stack**: React 18, TypeScript, Vite, Tailwind CSS, Zustand, TanStack Query v5, Axios + +```bash +cd frontend +npm install +npm run dev # Dev server at http://localhost:5173 + +npm run build # TypeScript compile + Vite build +npm run lint # ESLint (strict) +npm run type-check # TypeScript check +npm run quality # All checks +npm run quality:fix # Fix all issues +npm run test # Vitest +``` + +## Data Transfer Scripts + +Scripts live in `annotation_api/scripts/data_transfer/ingestion/platform/`. Run from `annotation_api/` with `make` targets. + +### TP Pipeline (true positives — fire sequences) + +Pull annotated sequences, enrich bboxes with YOLO, visual check, push back. + +```bash +cd annotation_api + +# 1. Pull seq_annotation_done sequences (remote → local files, marks remote in_review) +make pull-seq-annotations MAX_SEQUENCES=20 + +# 2. Auto-fill missing bboxes with YOLO model +make auto-annotate + +# 3. Visual review in FiftyOne — tag "issue" on bad frames +make visual-check + +# 4. Push results: clean → annotated, issue → needs_manual +make apply-review +``` + +### FP Pipeline (false positives — no fire sequences) + +Pull sequences, visually confirm no fire was missed, push back with empty labels. + +```bash +cd annotation_api + +# 1. Pull seq_annotation_done sequences (separate output dir) +make pull-fp MAX_SEQUENCES=20 + +# 2. Visual check in FiftyOne — tag "issue" if fire was missed +make visual-check-fp + +# 3. Push results: clean → annotated (no labels), issue → needs_manual +make apply-review-fp +``` + +### Platform Import + +Credentials live in `annotation_api/.env` (copy from `.env.example` once). Each data-transfer script loads it at startup via `python-dotenv`; Make does not parse `.env`. + +```bash +# Import sequences from platform API (.env must define PLATFORM_LOGIN, PLATFORM_PASSWORD, +# PLATFORM_ADMIN_LOGIN, PLATFORM_ADMIN_PASSWORD, MAIN_ANNOTATION_LOGIN, MAIN_ANNOTATION_PASSWORD) +make import-platform DATE_FROM=2024-01-01 DATE_END=2024-01-02 +``` + +## Key Architecture Concepts + +**Backend data flow**: Platform API → ingestion scripts → annotation_api DB → frontend UI → human annotations + +**Processing stages** (sequence): `IMPORTED` → `READY_TO_ANNOTATE` → `UNDER_ANNOTATION` → `SEQ_ANNOTATION_DONE` → `IN_REVIEW` → `ANNOTATED` + +**Backend patterns**: CRUD modules per entity, Pydantic schemas separate from SQLModel, dependency injection, fastapi-pagination, IoU-based annotation generation service. + +**Frontend patterns**: Zustand for client state, TanStack Query for server state, canvas-based bbox drawing utilities, 13+ focused utility modules in `src/utils/`. + +## Pre-commit Hooks + +```bash +# Hooks run: ruff (format + lint), mypy, prevents commits to main +pre-commit install # Install hooks +pre-commit run --all-files # Run manually +``` diff --git a/README.md b/README.md index 1e3bc04..3c64457 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,12 @@ npm run dev # Vite dev server on port 5173 All workflows below assume the services are running locally (`docker compose up -d`) and that you have credentials to the remote annotation API. Run the `make` targets from the `annotation_api/` directory. -Before anything that talks to the remote API, export your credentials once per shell (or put them in `.envrc`): +Before anything that talks to the remote API, configure your credentials in `annotation_api/.env` (loaded by the data-transfer scripts at startup via python-dotenv): ```bash -export MAIN_ANNOTATION_LOGIN= -export MAIN_ANNOTATION_PASSWORD= +cd annotation_api +cp .env.example .env +# then edit .env and set MAIN_ANNOTATION_LOGIN / MAIN_ANNOTATION_PASSWORD ``` All make targets accept variable overrides inline, e.g. `make pull-sequences MAX_SEQUENCES=50 CLONE_STAGE=under_annotation`. Common variables: `REMOTE_API`, `LOCAL_API`, `MAX_SEQUENCES`, `CLONE_STAGE`, `DATA_ROOT`, `SMOKE_TYPE`, `DATASET_NAME`, `LOGLEVEL`. See `make help` for the full list. @@ -116,6 +117,30 @@ make apply-review - To preview changes without writing to the API, call the underlying script with `--dry-run`. - Override `DATASET_NAME` / `DATA_ROOT` if you used non-default values. +#### C. False-Positive (FP) Review + +For sequences with no fire (`smoke_types` is empty), confirm they are true false positives and push them as annotated with empty labels. + +**Step 1 — `pull-fp`**: pull `seq_annotation_done` FP sequences locally (moves remote stage to `in_review`): + +```bash +make pull-fp MAX_SEQUENCES=20 +``` + +**Step 2 — `visual-check-fp`**: review in FiftyOne — tag frames with `"issue"` if fire was actually missed: + +```bash +make visual-check-fp +``` + +**Step 3 — `apply-review-fp`**: push results back to the remote API: + +```bash +make apply-review-fp +``` +- Clean sequences (no `"issue"` tags) → moved to `annotated` with empty labels (confirmed FP). +- Issue sequences → moved to `needs_manual` for reannotation. + ##### Other commands **Reset stages on the remote API** (e.g., move `in_review` back to `seq_annotation_done` to retry a workflow): @@ -133,8 +158,9 @@ make update-stage-local FROM_STAGE=seq_annotation_done TO_STAGE=needs_manual MAX **Export images + YOLO labels from the remote API** (use smaller pages and a longer timeout for large datasets): ```bash -make export-dataset USERNAME= PASSWORD= OUTPUT_DIR=outputs/datasets LIMIT=1000 TIMEOUT=120 +make export-dataset OUTPUT_DIR=outputs/datasets LIMIT=1000 TIMEOUT=120 ``` +- Filter by category: `make export-dataset CATEGORY=fp` (also `wildfire`, `other_smoke`). Omit to export all. **Import a single sequence from an exported YOLO folder** (images + labels) into an API: @@ -156,14 +182,20 @@ make import-yolo-sequence \ If you manage the main dataset and have platform credentials, import directly from the platform into the target annotation API. This is the only entry point that brings new data into the system. -```bash -export PLATFORM_LOGIN= -export PLATFORM_PASSWORD= -export PLATFORM_ADMIN_LOGIN= -export PLATFORM_ADMIN_PASSWORD= -export MAIN_ANNOTATION_LOGIN= -export MAIN_ANNOTATION_PASSWORD= +Set the platform + target credentials in `annotation_api/.env` (see `.env.example`): + +``` +PLATFORM_LOGIN=... +PLATFORM_PASSWORD=... +PLATFORM_ADMIN_LOGIN=... +PLATFORM_ADMIN_PASSWORD=... +MAIN_ANNOTATION_LOGIN=... +MAIN_ANNOTATION_PASSWORD=... +``` + +Then run: +```bash cd annotation_api make import-platform DATE_FROM=2025-03-04 DATE_END=2025-03-04 MAX_SEQUENCES=10 ``` @@ -184,19 +216,24 @@ docker compose up -d curl http://localhost:5050/docs ``` -**Required Environment Variables:** -```bash +**Required Environment Variables (in `annotation_api/.env`):** + +Copy `annotation_api/.env.example` to `annotation_api/.env` and fill in the values you need: + +``` # Remote annotation API credentials (required for all workflows) -export MAIN_ANNOTATION_LOGIN="remote_user" -export MAIN_ANNOTATION_PASSWORD="remote_pass" +MAIN_ANNOTATION_LOGIN=remote_user +MAIN_ANNOTATION_PASSWORD=remote_pass # Platform API credentials (admin ingestion only) -export PLATFORM_LOGIN="your_platform_username" -export PLATFORM_PASSWORD="your_platform_password" -export PLATFORM_ADMIN_LOGIN="your_admin_username" -export PLATFORM_ADMIN_PASSWORD="your_admin_password" +PLATFORM_LOGIN=your_platform_username +PLATFORM_PASSWORD=your_platform_password +PLATFORM_ADMIN_LOGIN=your_admin_username +PLATFORM_ADMIN_PASSWORD=your_admin_password ``` +Each data-transfer script loads `annotation_api/.env` via `python-dotenv` at startup — no shell `export` or manual `source` needed. (Make does **not** parse `.env`, because Make's variable expansion would mangle values containing `$`, spaces, or quotes.) Shell-level env vars still take priority, so you can override per-invocation with `MAIN_ANNOTATION_LOGIN=foo make ...`. + ### Deployment Environments **Local Development (default):** diff --git a/annotation_api/.env.example b/annotation_api/.env.example new file mode 100644 index 0000000..10edb91 --- /dev/null +++ b/annotation_api/.env.example @@ -0,0 +1,35 @@ +# Copy to .env and fill in real values. +# +# Each data-transfer script loads this file via python-dotenv at startup, +# which respects dotenv quoting (single quotes for literal $, etc.). +# Make does NOT parse .env (it would mangle values with $ / quotes / spaces). +# To override a value per-invocation, pass it in the shell environment: +# MAIN_ANNOTATION_LOGIN=other_user make export-dataset + +# --- Database --- +POSTGRES_USER=dbadmin +POSTGRES_PASSWORD=changeme +POSTGRES_DB=pyroannotation + +# --- S3 / object storage --- +S3_ENDPOINT_URL=https://s3.gra.io.cloud.ovh.net/ +S3_ACCESS_KEY=your-access-key +S3_SECRET_KEY=your-secret-key +S3_REGION=gra + +# --- JWT signing secret --- +JWT_SECRET=change-me-to-a-secure-random-string + +# --- Annotation API auth (local target, used by scripts hitting localhost) --- +ANNOTATOR_LOGIN=admin +ANNOTATOR_PASSWORD=admin12345 + +# --- Remote annotation API auth (export-dataset, push-annotations, etc.) --- +MAIN_ANNOTATION_LOGIN=admin +MAIN_ANNOTATION_PASSWORD=changeme + +# --- Platform API credentials (only needed for `make import-platform`) --- +PLATFORM_LOGIN=your_platform_username +PLATFORM_PASSWORD=your_platform_password +PLATFORM_ADMIN_LOGIN=your_admin_username +PLATFORM_ADMIN_PASSWORD=your_admin_password diff --git a/annotation_api/.gitignore b/annotation_api/.gitignore index eda39f8..fbb737d 100644 --- a/annotation_api/.gitignore +++ b/annotation_api/.gitignore @@ -4,6 +4,7 @@ acme.json # Environment files (may contain secrets) .env .env.* +!.env.example # Python .venv/ diff --git a/annotation_api/CLAUDE.md b/annotation_api/CLAUDE.md index 79e8ad2..5f84959 100644 --- a/annotation_api/CLAUDE.md +++ b/annotation_api/CLAUDE.md @@ -37,7 +37,7 @@ annotation_api/ │ │ ├── models.py # SQLModel database models │ │ ├── main.py # FastAPI application entry point │ │ └── db.py # Database initialization -│ └── tests/ # Comprehensive test suite (192 tests) +│ └── tests/ # Test suite ├── docs/ # Complete user documentation │ ├── README.md # Documentation index │ ├── api-client-guide.md # User guide with examples @@ -45,10 +45,9 @@ annotation_api/ │ └── examples.md # Real-world implementation patterns ├── Dockerfile # Container configuration ├── scripts/ # Utility and data transfer scripts -│ ├── data_transfer/ # Data ingestion scripts -│ │ └── ingestion/ -│ │ └── platform/ # Platform API data fetching -│ └── setup.sh # Project setup automation +│ └── data_transfer/ # Data ingestion scripts +│ └── ingestion/ +│ └── platform/ # Platform API data fetching ├── pyproject.toml # uv dependencies and tool config ├── uv.lock # Locked dependencies for reproducible builds ├── docker-compose-dev.yml # Development environment @@ -98,35 +97,17 @@ The API provides enhanced endpoints with pagination, filtering, and ordering: ### Testing -- `make test` - Run comprehensive test suite with coverage in Docker containers (includes live code mounting) -- `make test-specific TEST=tests/endpoints/test_auth.py::test_login_valid_credentials` - Run specific test -- `uv run pytest src/tests/ -s --cov=app` - Run tests locally with coverage (requires local setup) - -**Note**: The `make test` command uses a specialized Docker setup with: -- Multi-stage Dockerfile with dedicated test target that includes dev dependencies -- Live code mounting via volumes (`./src/app:/app/app`, `./src/tests:/app/tests`) -- Proper test isolation with database cleanup between tests -- Fixed HTTPX AsyncClient integration using `ASGITransport` for FastAPI testing -- Comprehensive test suite with 192+ test cases covering all endpoints, services, and edge cases -- Authenticated test fixtures for API endpoint testing -- Clean test output with silenced debug messages and deprecation warnings - -**Test Coverage** includes: -- API endpoint testing with authentication -- Service layer testing for annotation generation -- Coordinate validation and error handling -- IoU clustering and temporal bbox grouping algorithms -- Edge cases like invalid predictions, malformed data, and clustering failures -- Database model validation and JSONB field validation +- `make test` - Run the test suite with coverage in Docker +- `make test-specific TEST=tests/endpoints/test_auth.py::test_login_valid_credentials` - Run a specific test +- `uv run pytest src/tests/ -s --cov=app` - Run tests locally (requires `uv sync --group dev`) + +`make test` uses `docker-compose-dev.yml` to spin up an ephemeral stack, mounts `./src/app` and `./src/tests` for live code, and tears down volumes when done. ### Development Server - `make start` - Start development environment with Docker Compose - `make stop` - Stop development environment (preserves data volumes) - `make clean` - Remove containers and volumes (fresh start, deletes all data) -- `make start-prod` - Start production environment -- `make stop-prod` - Stop production environment - `make docker-build` - Build Docker image -- `make setup` - Run setup tasks (creates acme.json, checks prerequisites) ### Database - Database migrations handled via Alembic @@ -167,30 +148,13 @@ Key environment variables (see `src/app/core/config.py`): ## Development Setup 1. Clone repository 2. Install uv: `curl -LsSf https://astral.sh/uv/install.sh | sh` -3. Copy `.env.template` to `.env` and configure (if needed) -4. **Automated setup**: Run `make start` - this automatically creates `acme.json` and checks prerequisites -5. Install dependencies (for local dev): `uv sync --group dev` -6. Run: `make start` (Docker - recommended) or `uv run uvicorn app.main:app --reload --app-dir src` (local) -7. Access API docs at: http://localhost:5050/docs +3. Copy `.env.example` to `.env` and configure +4. Install dependencies (for local dev): `uv sync --group dev` +5. Run: `make start` (Docker - recommended) or `uv run uvicorn app.main:app --reload --app-dir src` (local) +6. Access API docs at: http://localhost:5050/docs ### Authentication Setup -The API uses JWT authentication. Default credentials: -- **Username**: `admin` -- **Password**: `admin12345` - -Credentials are loaded from the `.envrc` file. To customize credentials, update your `.envrc` or set environment variables in `docker-compose.yml`: -```yaml -environment: - - AUTH_USERNAME=your_username - - AUTH_PASSWORD=your_password - - JWT_SECRET=your_secure_jwt_secret -``` - -### Setup Commands -- `make setup` - Run setup tasks only (creates acme.json, checks prerequisites) -- `make start` - Development environment with automatic setup -- `make start-prod` - Production environment with automatic setup -- `make stop` / `make stop-prod` - Stop respective environments +The API uses JWT authentication. Default dev credentials are `admin` / `admin12345`. Override via the `AUTH_USERNAME`, `AUTH_PASSWORD`, `JWT_SECRET` keys in `annotation_api/.env` (copied from `.env.example`). ## Code Standards - Python 3.11+ with type hints @@ -491,100 +455,6 @@ uv run python -m scripts.data_transfer.ingestion.platform.import \ - **Stage management** - Automatic transitions from platform data to READY_TO_ANNOTATE stage - **Parameter tuning** - Configurable confidence thresholds, IoU thresholds, and cluster sizes -## Migration Notes (Poetry → uv) - -### What Changed -- **Package Manager**: Migrated from Poetry to uv for 10-15x faster dependency installation -- **pyproject.toml**: Converted from Poetry format to standard PEP 621 format -- **Lock File**: `uv.lock` replaces `poetry.lock` for deterministic builds -- **Docker**: Multi-stage builds with better layer caching and performance optimizations -- **Dependencies**: Consolidated dev/quality/test groups into single `dev` group - -### Key Benefits -- **Performance**: Dramatically faster dependency resolution and installation -- **Standards Compliance**: Uses standard Python packaging specifications (PEP 621) -- **Docker Optimization**: Multi-stage builds with selective volume mounts for development -- **Simplified Workflow**: Single dependency group for all development tools - -### Docker Development Notes -- Development mode uses selective volume mounts to preserve virtual environment -- Production mode runs without volume mounts for optimal performance -- Both configurations use the same multi-stage Dockerfile with uv - -### Dependency Management -- Production dependencies: `uv sync` -- Development dependencies: `uv sync --group dev` -- Add dependencies: `uv add ` or `uv add --group dev ` -- Lock file automatically updated when dependencies change - -## Recent Enhancements (2024) - -### Authentication System Implementation -- **JWT Authentication**: Complete JWT-based authentication system protecting all API endpoints -- **Configurable Credentials**: Environment variable-based login credentials (AUTH_USERNAME/AUTH_PASSWORD) -- **Token Management**: Configurable token expiration and JWT secret management -- **Login Endpoint**: Dedicated `/api/v1/auth/login` endpoint for token acquisition -- **Middleware Integration**: FastAPI dependency injection for automatic endpoint protection -- **Test Authentication**: All 192 tests updated with authenticated client fixtures - -### Database & Schema Improvements -- **Added recorded_at field** to Sequence model as required field for temporal tracking -- **Enhanced database indexing** for optimal query performance on timestamp and filter fields -- **Improved unique constraints** to prevent data duplication and maintain integrity - -### API Enhancements -- **Pagination Support**: All list endpoints now return paginated responses with metadata -- **Advanced Filtering**: Filter sequences by source_api, camera_id, organisation_id, is_wildfire_alertapi -- **Flexible Ordering**: Order by created_at or recorded_at in ascending/descending order -- **Enhanced Error Handling**: Detailed validation messages with field-level error reporting - -### Test Suite Improvements -- **Comprehensive Coverage**: 192 test cases covering all endpoints and edge cases including authentication -- **Authentication Testing**: Authenticated client fixtures and JWT token validation tests -- **Clean Test Output**: Silenced debug messages and deprecation warnings for cleaner CI -- **Proper Isolation**: Database cleanup and sequence resets between tests -- **Pagination Testing**: Updated tests to handle new paginated response format -- **Modern AsyncClient**: Using ASGITransport for FastAPI testing best practices - -### Client Library & Documentation -- **Rich Exception Hierarchy**: Detailed error types with contextual information -- **Professional Documentation**: Complete user guides, API reference, and real-world examples -- **Validation Helpers**: Client-side validation to catch errors before API calls -- **Integration Patterns**: Examples for web apps, background tasks, and batch processing - -## Data Import Scripts - -### Platform Data Import -Single comprehensive script for fetching data from the Pyronear platform API and generating annotations: - -```bash -# Load environment variables from .envrc file (required for platform credentials) -source .envrc - -# End-to-end processing: fetch platform data → generate annotations -uv run python -m scripts.data_transfer.ingestion.platform.import \ - --date-from 2025-07-31 --date-end 2025-07-31 --loglevel info - -# Process with custom annotation parameters -uv run python -m scripts.data_transfer.ingestion.platform.import \ - --date-from 2025-07-31 --confidence-threshold 0.0 --iou-threshold 0.4 --loglevel info -``` - -#### Required Environment Variables (in .envrc) -- `PLATFORM_LOGIN` - Platform API login (e.g., sis-67) -- `PLATFORM_PASSWORD` - Platform API password -- `PLATFORM_ADMIN_LOGIN` - Admin login for organization access -- `PLATFORM_ADMIN_PASSWORD` - Admin password for organization access -- `ANNOTATOR_LOGIN` - Annotation API login for script authentication (default: `admin`) -- `ANNOTATOR_PASSWORD` - Annotation API password for script authentication (default: `admin12345`) - -#### Script Features -- **End-to-end workflow**: Complete pipeline from platform data to annotation-ready sequences -- **Automatic annotation generation**: Server-side clustering of AI predictions with confidence threshold 0.0 (includes all predictions) -- **Concurrent processing**: Multi-threading for faster data fetching -- **Progress tracking**: Rich progress bars for long-running operations -- **Stage management**: Automatic transitions to READY_TO_ANNOTATE stage - ## Troubleshooting ### Common Issues diff --git a/annotation_api/Makefile b/annotation_api/Makefile index 886ec14..69637bb 100644 --- a/annotation_api/Makefile +++ b/annotation_api/Makefile @@ -1,9 +1,16 @@ +# Note: .env is intentionally NOT included here. Make parses .env as Make +# syntax, which mangles values containing $, spaces, or quotes (e.g. +# pa$word becomes paord). Each Python script in scripts/data_transfer/ +# loads .env itself via python-dotenv, which handles dotenv quoting +# correctly and does not override existing shell variables. So: +# - Put credentials in annotation_api/.env (see .env.example). +# - Or override via your shell: MAIN_ANNOTATION_LOGIN=... make ... + # Show help with available commands help: @echo "Available commands:" @echo " lint - Run format check, linting, and type checking" @echo " fix - Auto-format code and fix linting issues" - @echo " setup - Setup project (creates acme.json, checks prerequisites)" @echo " docker-build - Build Docker image" @echo " start - Start development environment" @echo " stop - Stop development environment (preserves volumes)" @@ -24,11 +31,12 @@ help: @echo "False-positive workflow:" @echo " pull-fp - Pull seq_annotation_done sequences for FP review" @echo " visual-check-fp - Review FP sequences in FiftyOne (check no fire missed)" - @echo " apply-review-fp - Push clean FP sequences as 'managed' (no labels)" + @echo " apply-review-fp - Push clean FP sequences as 'annotated' (no labels)" @echo "" @echo "Other data commands:" @echo " export-dataset - Export images + YOLO labels from remote API" @echo " import-yolo-sequence - Import one YOLO folder into an API (needs SEQUENCE_DIR=...)" + @echo " import-local-yolo - Batch-import local YOLO sequences + sequences.csv (needs ROOT_DIR=...)" @echo " update-stage-remote - Move annotations between stages on the remote API" @echo " update-stage-local - Move annotations between stages on the local API" @echo " import-platform - (Admin) Import from platform API into annotation API" @@ -64,17 +72,12 @@ fix: uv run ruff format . uv run ruff check --fix . -# Setup the project (creates acme.json and checks prerequisites) -setup: - @bash scripts/setup.sh - # Build the docker docker-build: docker build -f Dockerfile . -t pyronear/annotation-api:latest -# Run the development environment (with automatic setup) +# Run the development environment start: - @bash scripts/setup.sh docker compose -f docker-compose.yml up --build # Stop the development environment (preserves volumes) @@ -104,8 +107,9 @@ test-specific: # ========================================================================= # Data workflow targets -# All targets requiring remote auth expect MAIN_ANNOTATION_LOGIN / MAIN_ANNOTATION_PASSWORD -# to be set in the environment (e.g. via .envrc or `export`). +# Targets that hit the remote API expect MAIN_ANNOTATION_LOGIN / +# MAIN_ANNOTATION_PASSWORD in annotation_api/.env. Each script loads .env +# itself via python-dotenv (Make does not parse .env, see note at top). # ========================================================================= # --- 1A. Sequence annotation workflow --- @@ -176,14 +180,15 @@ apply-review: FP_DATA_ROOT ?= outputs/fp_review FP_DATASET ?= fp_check -# Pull seq_annotation_done sequences for FP review. -# Usage: make pull-fp [MAX_SEQUENCES=20] [SMOKE_TYPE=wildfire] +# Pull seq_annotation_done sequences for FP review (empty smoke_types = false positives). +# Usage: make pull-fp [MAX_SEQUENCES=20] +# Note: smoke-type is hardcoded to "empty" for FP pulls. pull-fp: uv run python -m scripts.data_transfer.ingestion.platform.pull_sequence_annotations \ --remote-api $(REMOTE_API) \ --max-sequences $(MAX_SEQUENCES) \ --output-dir $(FP_DATA_ROOT) \ - --smoke-type $(SMOKE_TYPE) \ + --smoke-type empty \ --loglevel $(LOGLEVEL) # Visual check FP sequences in FiftyOne (confirm no fire was missed). @@ -207,20 +212,19 @@ apply-review-fp: # --- Other data commands --- # Export images + YOLO labels from a remote API. -# Usage: make export-dataset USERNAME= PASSWORD=

[OUTPUT_DIR=outputs/datasets] [LIMIT=1000] +# Usage: make export-dataset [OUTPUT_DIR=outputs/datasets] [LIMIT=1000] [CATEGORY=fp] +# Requires MAIN_ANNOTATION_LOGIN / MAIN_ANNOTATION_PASSWORD in +# annotation_api/.env (loaded by the script) or in your shell environment. +# Authentication will fail with a clear HTTP 401 if creds are missing/wrong. export-dataset: - @if [ -z "$(USERNAME)" ] || [ -z "$(PASSWORD)" ]; then \ - echo "ERROR: set USERNAME= PASSWORD= when invoking make export-dataset"; exit 1; \ - fi uv run python -m scripts.data_transfer.ingestion.platform.export_dataset \ --api-base $(REMOTE_API)/api/v1 \ - --username $(USERNAME) \ - --password $(PASSWORD) \ --verify-ssl \ --output-dir $(OUTPUT_DIR) \ --limit $(LIMIT) \ --timeout $(TIMEOUT) \ - --loglevel $(LOGLEVEL) + --loglevel $(LOGLEVEL) \ + $(if $(CATEGORY),--category $(CATEGORY),) # Import one YOLO sequence folder (images + labels) into an API. # Usage: make import-yolo-sequence SEQUENCE_DIR=path/to/seq ALERT_API_ID=123456 \ @@ -240,6 +244,30 @@ import-yolo-sequence: --sequence-stage $(SEQUENCE_STAGE) \ --loglevel $(LOGLEVEL) +# Batch-import a directory of local YOLO sequence folders paired with a sequences.csv. +# Usage: make import-local-yolo ROOT_DIR=/path/to/sdis-77-new_model \ +# [SEQUENCES_SUBDIR=sdis-77] [LABELS_DIR=labels_predictions] \ +# [API_BASE=http://localhost:5050] [MAX_WORKERS=4] [LIMIT_FOLDERS=0] [SKIP_FOLDERS=0] +SEQUENCES_SUBDIR ?= sdis-77 +LABELS_DIR ?= labels_predictions +LIMIT_FOLDERS ?= 0 +SKIP_FOLDERS ?= 0 +import-local-yolo: + @if [ -z "$(ROOT_DIR)" ]; then \ + echo "ERROR: set ROOT_DIR=path/to/dataset/root"; exit 1; \ + fi + uv run python -m scripts.data_transfer.ingestion.platform.batch_import_local_yolo \ + --root-dir $(ROOT_DIR) \ + --sequences-subdir $(SEQUENCES_SUBDIR) \ + --labels-dir-name $(LABELS_DIR) \ + --api-base $(API_BASE) \ + --source-api $(SOURCE_API) \ + --sequence-stage $(SEQUENCE_STAGE) \ + --max-workers $(MAX_WORKERS) \ + --limit $(LIMIT_FOLDERS) \ + --skip $(SKIP_FOLDERS) \ + --loglevel $(LOGLEVEL) + # Move annotations between stages on the remote API. # Usage: make update-stage-remote [FROM_STAGE=in_review] [TO_STAGE=seq_annotation_done] [MAX_SEQUENCES=0] update-stage-remote: @@ -281,9 +309,10 @@ import-platform: --max-workers $(MAX_WORKERS) \ --loglevel $(LOGLEVEL) -.PHONY: help lint fix setup docker-build start stop clean test test-specific \ +.PHONY: help lint fix docker-build start stop clean test test-specific \ pull-sequences push-annotations \ pull-seq-annotations auto-annotate visual-check apply-review \ pull-fp visual-check-fp apply-review-fp \ - export-dataset import-yolo-sequence update-stage-remote update-stage-local \ + export-dataset import-yolo-sequence import-local-yolo \ + update-stage-remote update-stage-local \ import-platform diff --git a/annotation_api/README.md b/annotation_api/README.md index 3434d0f..e2ce7f6 100644 --- a/annotation_api/README.md +++ b/annotation_api/README.md @@ -25,29 +25,15 @@ cd annotation_api The setup is now automated! Simply run: ```bash -# For development (recommended) make start - -# For production -make start-prod - -# Or run setup separately -make setup ``` -The setup automatically: -- Creates the required `acme.json` file for Let's Encrypt certificates -- Sets proper file permissions (600) -- Checks prerequisites (Docker, Docker Compose) +This builds the images (if needed) and starts the development stack via `docker-compose.yml`. #### 3 - Stop the services ```bash -# Stop development environment make stop - -# Stop production environment -make stop-prod ``` #### 4 - Check what you've deployed diff --git a/annotation_api/docs/data-ingestion-guide.md b/annotation_api/docs/data-ingestion-guide.md index 3caf072..ff83e6a 100644 --- a/annotation_api/docs/data-ingestion-guide.md +++ b/annotation_api/docs/data-ingestion-guide.md @@ -27,17 +27,8 @@ You need **both regular and admin credentials** for the platform API: ### Environment Variables -Create a `.env` file in your project root or export these environment variables: +All credentials live in `annotation_api/.env`. Copy `annotation_api/.env.example` to `annotation_api/.env` and fill in the values: -```bash -# Platform API Credentials -export PLATFORM_LOGIN="your_platform_username" -export PLATFORM_PASSWORD="your_platform_password" -export PLATFORM_ADMIN_LOGIN="your_admin_username" -export PLATFORM_ADMIN_PASSWORD="your_admin_password" -``` - -Or in a `.env` file: ```env PLATFORM_LOGIN=your_platform_username PLATFORM_PASSWORD=your_platform_password @@ -45,19 +36,7 @@ PLATFORM_ADMIN_LOGIN=your_admin_username PLATFORM_ADMIN_PASSWORD=your_admin_password ``` -### Load Environment Variables - -If using a `.env` file, load it before running scripts: -```bash -# Load environment variables from .env file -source .env - -# Or export individual variables -export PLATFORM_LOGIN="myusername" -export PLATFORM_PASSWORD="mypassword" -export PLATFORM_ADMIN_LOGIN="myadmin" -export PLATFORM_ADMIN_PASSWORD="myadminpassword" -``` +Each script in `scripts/data_transfer/ingestion/platform/` loads `.env` at startup via `python-dotenv` (which handles dotenv quoting correctly, including values with `$`). Make does **not** parse `.env`. Shell-level env vars take priority, so `MAIN_ANNOTATION_LOGIN=foo make ...` still overrides the file. ## Script Usage @@ -251,13 +230,14 @@ Final Statistics: #### 1. Missing Environment Variables **Error**: `Missing platform credentials...` -**Solution**: Ensure all four environment variables are set: -```bash -export PLATFORM_LOGIN="your_username" -export PLATFORM_PASSWORD="your_password" -export PLATFORM_ADMIN_LOGIN="your_admin" -export PLATFORM_ADMIN_PASSWORD="your_admin_password" +**Solution**: Ensure all four variables are set in `annotation_api/.env`: +```env +PLATFORM_LOGIN=your_username +PLATFORM_PASSWORD=your_password +PLATFORM_ADMIN_LOGIN=your_admin +PLATFORM_ADMIN_PASSWORD=your_admin_password ``` +(copied from `annotation_api/.env.example`). #### 2. Authentication Failures **Error**: `Failed to fetch access token` or `401 Unauthorized` diff --git a/annotation_api/import_all_platforms.sh b/annotation_api/import_all_platforms.sh new file mode 100755 index 0000000..88e2875 --- /dev/null +++ b/annotation_api/import_all_platforms.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CREDS_FILE="${SCRIPT_DIR}/../creds.json" + +if [ ! -f "$CREDS_FILE" ]; then + echo "ERROR: creds.json not found at $CREDS_FILE" + exit 1 +fi + +for login in $(jq -r 'keys[]' "$CREDS_FILE"); do + password=$(jq -r --arg k "$login" '.[$k]' "$CREDS_FILE") + echo "=========================================" + echo "Importing for: $login" + echo "=========================================" + export PLATFORM_LOGIN="$login" + export PLATFORM_PASSWORD="$password" + make -C "$SCRIPT_DIR" import-platform DATE_FROM=2025-01-01 DATE_END=2026-04-01 MAX_SEQUENCES=0 MAX_WORKERS=6 || { + echo "WARNING: import failed for $login, continuing..." + } +done + +echo "=========================================" +echo "All done." diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/apply_fiftyone_review.py b/annotation_api/scripts/data_transfer/ingestion/platform/apply_fiftyone_review.py index 603786a..0fb622a 100644 --- a/annotation_api/scripts/data_transfer/ingestion/platform/apply_fiftyone_review.py +++ b/annotation_api/scripts/data_transfer/ingestion/platform/apply_fiftyone_review.py @@ -77,7 +77,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--fp-mode", action="store_true", - help="False-positive mode: clean sequences get 'managed' stage with empty annotations instead of 'annotated'", + help="False-positive mode: clean sequences get 'annotated' stage with empty annotations (no detection labels)", ) parser.add_argument( "--dry-run", diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/batch_import_local_yolo.py b/annotation_api/scripts/data_transfer/ingestion/platform/batch_import_local_yolo.py new file mode 100644 index 0000000..a2113b8 --- /dev/null +++ b/annotation_api/scripts/data_transfer/ingestion/platform/batch_import_local_yolo.py @@ -0,0 +1,300 @@ +""" +Batch-import local YOLO sequence folders (paired with a sequences.csv metadata +file) into the annotation API. + +Layout expected:: + + / + sequences.csv + / + / + images/ + labels_predictions/ + / + ... + +Each sequence folder name must end with ``_sequence-`` so we can join it to +the ``sequence_id`` column of ``sequences.csv`` and pull organisation / camera / +lat / lon / azimuth metadata from there. + +For every folder this script invokes +``scripts.data_transfer.ingestion.platform.import_yolo_sequence`` as a +subprocess, so each import is isolated. +""" + +from __future__ import annotations + +import argparse +import csv +import logging +import re +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Dict, Optional + +from dotenv import load_dotenv + +load_dotenv() + +SEQUENCE_ID_RE = re.compile(r"_sequence-(\d+)$") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root-dir", + required=True, + help="Directory containing sequences.csv and the sequences subfolder.", + ) + parser.add_argument( + "--sequences-subdir", + default="sdis-77", + help="Subdirectory under --root-dir holding per-sequence folders.", + ) + parser.add_argument( + "--csv", + default=None, + help="sequences.csv path (default: /sequences.csv).", + ) + parser.add_argument( + "--labels-dir-name", + default="labels_predictions", + help="Labels subfolder name inside each sequence folder.", + ) + parser.add_argument( + "--api-base", + default="http://localhost:5050", + help="Annotation API base URL.", + ) + parser.add_argument( + "--source-api", + default="pyronear_french", + help="source_api value stored on each sequence.", + ) + parser.add_argument( + "--sequence-stage", + default="ready_to_annotate", + help="Processing stage to assign to imported sequence annotations.", + ) + parser.add_argument( + "--max-workers", + type=int, + default=4, + help="Parallel import workers (subprocesses).", + ) + parser.add_argument( + "--limit", + type=int, + default=0, + help="Only import the first N folders (0 = all).", + ) + parser.add_argument( + "--skip", + type=int, + default=0, + help="Skip the first N folders (useful for resuming).", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print the commands that would run without executing them.", + ) + parser.add_argument( + "--loglevel", + default="info", + choices=["debug", "info", "warning", "error"], + ) + return parser.parse_args() + + +def setup_logging(level: str) -> None: + logging.basicConfig( + level=getattr(logging, level.upper()), + format="%(levelname)s - %(message)s", + ) + + +def load_metadata(csv_path: Path) -> Dict[int, Dict[str, str]]: + meta: Dict[int, Dict[str, str]] = {} + with csv_path.open(newline="") as fp: + for row in csv.DictReader(fp): + try: + seq_id = int(row["sequence_id"]) + except (KeyError, TypeError, ValueError): + continue + meta.setdefault(seq_id, row) + return meta + + +def extract_sequence_id(folder_name: str) -> Optional[int]: + match = SEQUENCE_ID_RE.search(folder_name) + return int(match.group(1)) if match else None + + +def build_command( + folder: Path, + seq_id: int, + row: Dict[str, str], + args: argparse.Namespace, +) -> list[str]: + cmd = [ + sys.executable, + "-m", + "scripts.data_transfer.ingestion.platform.import_yolo_sequence", + "--sequence-dir", + str(folder), + "--api-base", + args.api_base, + "--alert-api-id", + str(seq_id), + "--source-api", + args.source_api, + "--sequence-stage", + args.sequence_stage, + "--labels-dir-name", + args.labels_dir_name, + "--organisation-id", + row["organization_id"], + "--organisation-name", + row["organization_name"], + "--camera-id", + row["camera_id"], + "--camera-name", + row["camera_name"], + "--lat", + row["camera_lat"], + "--lon", + row["camera_lon"], + "--loglevel", + args.loglevel, + ] + azimuth_raw = row.get("sequence_azimuth") or row.get("detection_azimuth") or "" + try: + cmd.extend(["--azimuth", str(int(float(azimuth_raw)))]) + except ValueError: + pass + return cmd + + +def run_one(cmd: list[str], label: str) -> tuple[str, int, str]: + result = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + return label, result.returncode, result.stdout + + +def main() -> int: + args = parse_args() + setup_logging(args.loglevel) + + root = Path(args.root_dir).resolve() + csv_path = Path(args.csv) if args.csv else root / "sequences.csv" + seq_root = root / args.sequences_subdir + + if not csv_path.exists(): + logging.error("sequences.csv not found at %s", csv_path) + return 1 + if not seq_root.is_dir(): + logging.error("Sequences directory not found at %s", seq_root) + return 1 + + metadata = load_metadata(csv_path) + logging.info("Loaded metadata for %d sequences from %s", len(metadata), csv_path) + + folders = sorted(p for p in seq_root.iterdir() if p.is_dir()) + if args.skip: + folders = folders[args.skip:] + if args.limit: + folders = folders[: args.limit] + + jobs: list[tuple[Path, int, Dict[str, str]]] = [] + skipped = 0 + for folder in folders: + seq_id = extract_sequence_id(folder.name) + if seq_id is None: + logging.warning("%s: cannot parse sequence_id — skipping", folder.name) + skipped += 1 + continue + row = metadata.get(seq_id) + if row is None: + logging.warning( + "%s: sequence_id=%d not in CSV — skipping", folder.name, seq_id + ) + skipped += 1 + continue + jobs.append((folder, seq_id, row)) + + logging.info( + "Planning %d imports (%d skipped) with %d worker(s)", + len(jobs), + skipped, + args.max_workers, + ) + + if args.dry_run: + for folder, seq_id, row in jobs: + cmd = build_command(folder, seq_id, row, args) + logging.info("[DRY-RUN] %s", " ".join(cmd)) + return 0 + + successes = 0 + failures = 0 + total = len(jobs) + with ThreadPoolExecutor(max_workers=max(1, args.max_workers)) as pool: + futures = { + pool.submit( + run_one, + build_command(folder, seq_id, row, args), + folder.name, + ): (folder, seq_id) + for folder, seq_id, row in jobs + } + for idx, future in enumerate(as_completed(futures), start=1): + folder, seq_id = futures[future] + try: + label, rc, output = future.result() + except Exception as exc: + failures += 1 + logging.error( + "[%d/%d] %s (seq_id=%d) crashed: %s", + idx, + total, + folder.name, + seq_id, + exc, + ) + continue + if rc == 0: + successes += 1 + logging.info( + "[%d/%d] OK %s (seq_id=%d)", idx, total, label, seq_id + ) + else: + failures += 1 + logging.error( + "[%d/%d] FAIL %s (seq_id=%d, rc=%d)\n%s", + idx, + total, + label, + seq_id, + rc, + output.strip(), + ) + + logging.info( + "Done. success=%d failure=%d skipped=%d total_folders=%d", + successes, + failures, + skipped, + len(folders), + ) + return 0 if failures == 0 else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/cleanup_duplicate_images.py b/annotation_api/scripts/data_transfer/ingestion/platform/cleanup_duplicate_images.py new file mode 100644 index 0000000..e67c756 --- /dev/null +++ b/annotation_api/scripts/data_transfer/ingestion/platform/cleanup_duplicate_images.py @@ -0,0 +1,256 @@ +""" +Delete sequences in READY_TO_ANNOTATE stage where at least 2 of the first 3 images are identical. + +This catches sequences with duplicate/frozen frames that are not useful for annotation. + +Usage: + # Dry run (list duplicates without deleting) + uv run python -m scripts.data_transfer.ingestion.platform.cleanup_duplicate_images \ + --url-api-annotation https://your-api-url --dry-run + + # Save images locally to double check before deleting + uv run python -m scripts.data_transfer.ingestion.platform.cleanup_duplicate_images \ + --url-api-annotation https://your-api-url --dry-run --save-images outputs/duplicate_check + + # Actually delete + uv run python -m scripts.data_transfer.ingestion.platform.cleanup_duplicate_images \ + --url-api-annotation https://your-api-url +""" + +import argparse +import hashlib +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from itertools import combinations +from pathlib import Path +from typing import Optional + +import requests +from dotenv import load_dotenv + +from app.clients import annotation_api + +from . import shared + +load_dotenv() + +logger = logging.getLogger(__name__) + + +def fetch_ready_sequences(base_url: str, auth_token: str) -> list[dict]: + """Fetch all sequence annotations in READY_TO_ANNOTATE stage, return their sequence info.""" + page = 1 + size = 100 + results: list[dict] = [] + while True: + resp = annotation_api.list_sequence_annotations( + base_url, + auth_token, + processing_stage="ready_to_annotate", + page=page, + size=size, + ) + items = resp.get("items", []) + results.extend(items) + if page >= resp.get("pages", 1): + break + page += 1 + return results + + +def get_first_n_detections( + base_url: str, auth_token: str, sequence_id: int, n: int = 3 +) -> list[dict]: + """Get the first N detections for a sequence, ordered by recorded_at.""" + resp = annotation_api.list_detections( + base_url, + auth_token, + sequence_id=sequence_id, + order_by="recorded_at", + order_direction="asc", + page=1, + size=n, + ) + return resp.get("items", []) + + +def download_image_bytes(url: str, timeout: int = 30) -> bytes: + """Download image from a URL and return raw bytes.""" + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + return resp.content + + +def image_hash(data: bytes) -> str: + """Compute MD5 hash of image bytes.""" + return hashlib.md5(data).hexdigest() + + +def check_duplicate_images( + base_url: str, + auth_token: str, + sequence_id: int, + save_dir: Optional[Path] = None, +) -> bool: + """Check if at least 2 of the first 3 images in a sequence are identical. + + If save_dir is provided, saves the downloaded images for manual review. + """ + detections = get_first_n_detections(base_url, auth_token, sequence_id) + + if len(detections) < 2: + return False + + hashes: list[Optional[str]] = [] + images: list[Optional[bytes]] = [] + for det in detections: + det_id = det["id"] + try: + url = annotation_api.get_detection_url(base_url, auth_token, det_id) + img_bytes = download_image_bytes(url) + hashes.append(image_hash(img_bytes)) + images.append(img_bytes) + except Exception: + logger.warning( + f"Failed to download detection {det_id} for sequence {sequence_id}" + ) + hashes.append(None) + images.append(None) + + # Check if any pair of hashes match + is_duplicate = False + for i, j in combinations(range(len(hashes)), 2): + if hashes[i] is not None and hashes[i] == hashes[j]: + is_duplicate = True + break + + # Save images if requested + if save_dir is not None: + label = "DUP" if is_duplicate else "OK" + seq_dir = save_dir / f"{label}_seq_{sequence_id}" + seq_dir.mkdir(parents=True, exist_ok=True) + for idx, (det, img_bytes) in enumerate(zip(detections, images)): + if img_bytes is not None: + h = hashes[idx] or "unknown" + filepath = seq_dir / f"det_{det['id']}_{h[:8]}.jpg" + filepath.write_bytes(img_bytes) + + return is_duplicate + + +def format_seq_info(seq_info: dict) -> str: + """Format sequence info for logging.""" + parts = [ + f"org={seq_info.get('organisation_name', '?')}", + f"camera={seq_info.get('camera_name', '?')}", + f"recorded={seq_info.get('recorded_at', '?')}", + f"alert_id={seq_info.get('alert_api_id', '?')}", + ] + return " | ".join(parts) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Delete READY_TO_ANNOTATE sequences with duplicate images" + ) + parser.add_argument( + "--url-api-annotation", + type=str, + default="http://localhost:5050", + help="Annotation API base URL", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="List duplicates without deleting", + ) + parser.add_argument( + "--save-images", + type=str, + default=None, + help="Save duplicate images to this directory for manual review", + ) + parser.add_argument( + "--workers", + type=int, + default=10, + help="Number of parallel workers (default: 10)", + ) + + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) + + save_dir = Path(args.save_images) if args.save_images else None + if save_dir: + save_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"Will save duplicate images to {save_dir}") + + login, password = shared.get_annotation_credentials(args.url_api_annotation) + token = annotation_api.get_auth_token(args.url_api_annotation, login, password) + + logger.info("Fetching READY_TO_ANNOTATE sequences...") + annotations = fetch_ready_sequences(args.url_api_annotation, token) + logger.info(f"Found {len(annotations)} sequences in READY_TO_ANNOTATE stage") + + def process_one(i: int, ann: dict) -> Optional[dict]: + seq_id = ann["sequence_id"] + try: + seq_info = annotation_api.get_sequence( + args.url_api_annotation, token, seq_id + ) + except Exception: + seq_info = {"id": seq_id} + + info_str = format_seq_info(seq_info) + logger.info(f"[{i + 1}/{len(annotations)}] seq {seq_id} | {info_str}") + + try: + if check_duplicate_images(args.url_api_annotation, token, seq_id, save_dir): + logger.info(f" -> DUPLICATE detected in seq {seq_id}") + return {"annotation": ann, "sequence": seq_info} + except Exception: + logger.exception(f" -> Error checking sequence {seq_id}") + return None + + duplicates: list[dict] = [] + with ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = { + pool.submit(process_one, i, ann): ann for i, ann in enumerate(annotations) + } + for future in as_completed(futures): + result = future.result() + if result is not None: + duplicates.append(result) + + logger.info( + f"\nFound {len(duplicates)} sequences with duplicate images out of {len(annotations)}" + ) + + if args.dry_run or not duplicates: + for dup in duplicates: + seq = dup["sequence"] + logger.info(f" Would delete seq {seq.get('id')} | {format_seq_info(seq)}") + if save_dir: + logger.info(f"Images saved to {save_dir} for review") + logger.info("Dry run or no duplicates; no deletions performed.") + return + + for dup in duplicates: + seq_id = dup["sequence"].get("id", dup["annotation"]["sequence_id"]) + try: + annotation_api.delete_sequence(args.url_api_annotation, token, seq_id) + logger.info( + f"Deleted sequence {seq_id} | {format_seq_info(dup['sequence'])}" + ) + except Exception: + logger.exception(f"Failed to delete sequence {seq_id}") + + logger.info(f"Done. Deleted {len(duplicates)} sequences.") + + +if __name__ == "__main__": + main() diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/export_annotations.py b/annotation_api/scripts/data_transfer/ingestion/platform/export_annotations.py index 4d4fa30..b85c8d1 100644 --- a/annotation_api/scripts/data_transfer/ingestion/platform/export_annotations.py +++ b/annotation_api/scripts/data_transfer/ingestion/platform/export_annotations.py @@ -14,28 +14,58 @@ import argparse import json import logging +import os from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List import requests +from dotenv import load_dotenv + +load_dotenv() def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Export sequences and annotations to JSON") - parser.add_argument("--api-base", default="http://localhost:5050/api/v1", help="Base URL of the API") - parser.add_argument("--username", default="admin", help="API username") - parser.add_argument("--password", default="admin12345", help="API password") - parser.add_argument("--page-size", type=int, default=50, help="Page size for pagination") - parser.add_argument("--timeout", type=int, default=30, help="HTTP request timeout in seconds") - parser.add_argument("--output", default="", help="Output JSON path, defaults to outputs/sequences_and_annotations_YYYYMMDD.json") - parser.add_argument("--loglevel", default="info", choices=["debug", "info", "warning", "error"]) - parser.add_argument("--verify-ssl", action="store_true", help="Verify TLS certificates") + parser = argparse.ArgumentParser( + description="Export sequences and annotations to JSON" + ) + parser.add_argument( + "--api-base", default="http://localhost:5050/api/v1", help="Base URL of the API" + ) + parser.add_argument( + "--username", + default=os.getenv("ANNOTATOR_LOGIN", "admin"), + help="API username, defaults to ANNOTATOR_LOGIN or admin", + ) + parser.add_argument( + "--password", + default=os.getenv("ANNOTATOR_PASSWORD", "admin12345"), + help="API password, defaults to ANNOTATOR_PASSWORD or admin12345", + ) + parser.add_argument( + "--page-size", type=int, default=50, help="Page size for pagination" + ) + parser.add_argument( + "--timeout", type=int, default=30, help="HTTP request timeout in seconds" + ) + parser.add_argument( + "--output", + default="", + help="Output JSON path, defaults to outputs/sequences_and_annotations_YYYYMMDD.json", + ) + parser.add_argument( + "--loglevel", default="info", choices=["debug", "info", "warning", "error"] + ) + parser.add_argument( + "--verify-ssl", action="store_true", help="Verify TLS certificates" + ) return parser.parse_args() def setup_logging(level: str) -> None: - logging.basicConfig(level=getattr(logging, level.upper()), format="[%(levelname)s] %(message)s") + logging.basicConfig( + level=getattr(logging, level.upper()), format="[%(levelname)s] %(message)s" + ) def iso_utc_now() -> str: @@ -49,7 +79,9 @@ def default_output_path() -> Path: return p -def get_token(api_base: str, username: str, password: str, timeout: int, verify_ssl: bool) -> str: +def get_token( + api_base: str, username: str, password: str, timeout: int, verify_ssl: bool +) -> str: login_url = f"{api_base}/auth/login" payload = {"username": username, "password": password} resp = requests.post(login_url, json=payload, timeout=timeout, verify=verify_ssl) @@ -62,12 +94,21 @@ def get_token(api_base: str, username: str, password: str, timeout: int, verify_ return token -def fetch_all_pages(url: str, headers: Dict[str, str], params_common: Dict[str, Any], page_size: int, timeout: int, verify_ssl: bool) -> List[Dict[str, Any]]: +def fetch_all_pages( + url: str, + headers: Dict[str, str], + params_common: Dict[str, Any], + page_size: int, + timeout: int, + verify_ssl: bool, +) -> List[Dict[str, Any]]: items: List[Dict[str, Any]] = [] page = 1 while True: params = {**params_common, "page": page, "size": page_size} - resp = requests.get(url, headers=headers, params=params, timeout=timeout, verify=verify_ssl) + resp = requests.get( + url, headers=headers, params=params, timeout=timeout, verify=verify_ssl + ) resp.raise_for_status() data = resp.json() page_items = data.get("items", []) @@ -75,14 +116,23 @@ def fetch_all_pages(url: str, headers: Dict[str, str], params_common: Dict[str, break items.extend(page_items) pages = data.get("pages") - logging.debug("Fetched page %s, items %s, total pages %s", page, len(page_items), pages) + logging.debug( + "Fetched page %s, items %s, total pages %s", page, len(page_items), pages + ) if pages is not None and page >= pages: break page += 1 return items -def export_all(api_base: str, username: str, password: str, page_size: int, timeout: int, verify_ssl: bool) -> Dict[str, Any]: +def export_all( + api_base: str, + username: str, + password: str, + page_size: int, + timeout: int, + verify_ssl: bool, +) -> Dict[str, Any]: token = get_token(api_base, username, password, timeout, verify_ssl) headers = {"accept": "application/json", "Authorization": f"Bearer {token}"} @@ -99,16 +149,22 @@ def export_all(api_base: str, username: str, password: str, page_size: int, time } logging.info("Fetching annotations") - annotations = fetch_all_pages(base_annot, headers, annot_params, page_size, timeout, verify_ssl) + annotations = fetch_all_pages( + base_annot, headers, annot_params, page_size, timeout, verify_ssl + ) logging.info("Fetching sequences") - sequences = fetch_all_pages(base_seq, headers, seq_params, page_size, timeout, verify_ssl) + sequences = fetch_all_pages( + base_seq, headers, seq_params, page_size, timeout, verify_ssl + ) payload = { "generated_at": iso_utc_now(), "annotations": {"count": len(annotations), "items": annotations}, "sequences": {"count": len(sequences), "items": sequences}, } - logging.info("Collected %s annotations and %s sequences", len(annotations), len(sequences)) + logging.info( + "Collected %s annotations and %s sequences", len(annotations), len(sequences) + ) return payload @@ -128,7 +184,9 @@ def main() -> None: verify_ssl=args.verify_ssl, ) - out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + out_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" + ) logging.info("Saved export to %s", out_path) diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/export_dataset.py b/annotation_api/scripts/data_transfer/ingestion/platform/export_dataset.py index 0eb20e4..b753050 100644 --- a/annotation_api/scripts/data_transfer/ingestion/platform/export_dataset.py +++ b/annotation_api/scripts/data_transfer/ingestion/platform/export_dataset.py @@ -14,10 +14,10 @@ labels/ {prefix}-{org}_{camera}_{azimuth}_{recorded_at}.txt # empty if no bbox -Categories: - wildfire - smoke_type == wildfire - other_smoke - smoke_type == industrial or other - fp - all false positive types +Categories (from sequence_smoke_types): + wildfire - "wildfire" in sequence_smoke_types + other_smoke - any other smoke type in sequence_smoke_types + fp - no smoke types (false positive sequence) YOLO format per line: class_id x_center y_center width height @@ -41,7 +41,6 @@ import os import re import unicodedata -from collections import defaultdict from datetime import datetime from multiprocessing import Pool, cpu_count from pathlib import Path @@ -71,13 +70,15 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--username", - default=os.getenv("ANNOTATOR_LOGIN", "admin"), - help="API username, defaults to ANNOTATOR_LOGIN env var or 'admin'", + default=os.getenv("MAIN_ANNOTATION_LOGIN") + or os.getenv("ANNOTATOR_LOGIN", "admin"), + help="API username, defaults to MAIN_ANNOTATION_LOGIN / ANNOTATOR_LOGIN env vars", ) parser.add_argument( "--password", - default=os.getenv("ANNOTATOR_PASSWORD", "admin12345"), - help="API password, defaults to ANNOTATOR_PASSWORD env var or 'admin12345'", + default=os.getenv("MAIN_ANNOTATION_PASSWORD") + or os.getenv("ANNOTATOR_PASSWORD", "admin12345"), + help="API password, defaults to MAIN_ANNOTATION_PASSWORD / ANNOTATOR_PASSWORD env vars", ) parser.add_argument( "--timeout", @@ -145,6 +146,13 @@ def parse_args() -> argparse.Namespace: default=0, help="Number of worker processes for downloads, zero uses CPU count", ) + parser.add_argument( + "--category", + type=str, + choices=["wildfire", "other_smoke", "fp"], + default=None, + help="Only export sequences of this category (wildfire, other_smoke, fp). Default: all.", + ) return parser.parse_args() @@ -317,62 +325,74 @@ def recorded_at_sort_key(row: Dict[str, Any]) -> Tuple[int, str]: return int(seq_id), rec_str -def compute_sequence_categories(rows: List[Dict[str, Any]]) -> Dict[int, List[str]]: +def sequence_category_from_row(row: Dict[str, Any]) -> str: + """Derive a single category from the sequence-level annotation fields. + + The export endpoint already filters to processing_stage=annotated, so + rows reaching this function come from annotated sequences. Sequences + with empty smoke_types and empty false_positive_types therefore + represent confirmed false positives from the FP review workflow + (a deliberate "no smoke, no FP type" annotation), not unannotated + sequences. + + Priority: + 1. "wildfire" in sequence_smoke_types → wildfire + 2. any other smoke type present → other_smoke + 3. otherwise (includes empty + empty) → fp """ - For each sequence, compute the set of top-level categories - (wildfire, other_smoke, fp) present in sequences_bbox. + smoke_types = row.get("sequence_smoke_types") or [] + if "wildfire" in smoke_types: + return CATEGORY_WILDFIRE + if smoke_types: + return CATEGORY_OTHER_SMOKE + return CATEGORY_FP + + +def compute_sequence_categories(rows: List[Dict[str, Any]]) -> Dict[int, str]: + """Map each sequence_id to a single category (wildfire, other_smoke, fp). + + The export endpoint filters to annotated sequences, so every row gets a + category (empty smoke_types + empty false_positive_types → fp). """ - seq_cats: Dict[int, set] = defaultdict(set) + seq_cats: Dict[int, str] = {} for row in rows: seq_id = row.get("sequence_id") if seq_id is None: continue - seq_ann = row.get("sequence_annotation") or {} - groups = seq_ann.get("sequences_bbox") or [] - for group in groups: - is_smoke = group.get("is_smoke", False) - smoke_type = group.get("smoke_type") - fp_types = group.get("false_positive_types") or [] - - if is_smoke and smoke_type: - seq_type = smoke_type - elif fp_types: - seq_type = fp_types[0] - else: - continue - - if seq_type not in CLASS_ID: - seq_type = "other" - seq_cats[int(seq_id)].add(seq_type_to_category(seq_type)) - - return {seq_id: sorted(cats) for seq_id, cats in seq_cats.items()} + seq_id_int = int(seq_id) + if seq_id_int not in seq_cats: + seq_cats[seq_id_int] = sequence_category_from_row(row) + return seq_cats -def extract_labels_for_detection(row: Dict[str, Any]) -> Dict[str, List[str]]: +def extract_labels_for_detection(row: Dict[str, Any]) -> List[str]: """ - From one export row, build: - { category: [ 'class_id x_center y_center width height', ... ] } + From one export row, build a list of YOLO label lines for this detection. - category is one of wildfire, other_smoke, fp. class_id uses the detailed type index from ALL_CLASSES. - Only boxes whose detection_id matches row["detection_id"] are used. - Boxes use normalized xyxyn coordinates [x1, y1, x2, y2]. + + When a group has is_smoke=True but no smoke_type, falls back to + the sequence-level smoke_types field. """ detection_id = row.get("detection_id") seq_ann = row.get("sequence_annotation") or {} sequences_bbox = seq_ann.get("sequences_bbox") or [] - labels_by_category: Dict[str, List[str]] = {} + # Fallback smoke type from the sequence-level derived field + seq_smoke_types = row.get("sequence_smoke_types") or [] + default_smoke_type = seq_smoke_types[0] if seq_smoke_types else "wildfire" + + labels: List[str] = [] for group in sequences_bbox: is_smoke = group.get("is_smoke", False) smoke_type = group.get("smoke_type") fp_types = group.get("false_positive_types") or [] - if is_smoke and smoke_type: - seq_type = smoke_type + if is_smoke: + seq_type = smoke_type if smoke_type else default_smoke_type elif fp_types: seq_type = fp_types[0] else: @@ -382,7 +402,6 @@ def extract_labels_for_detection(row: Dict[str, Any]) -> Dict[str, List[str]]: seq_type = "other" class_id = CLASS_ID[seq_type] - category = seq_type_to_category(seq_type) for bbox in group.get("bboxes", []): if bbox.get("detection_id") != detection_id: @@ -398,13 +417,11 @@ def extract_labels_for_detection(row: Dict[str, Any]) -> Dict[str, List[str]]: width = x2 - x1 height = y2 - y1 - line = ( - f"{class_id} " - f"{x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}" + labels.append( + f"{class_id} " f"{x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}" ) - labels_by_category.setdefault(category, []).append(line) - return labels_by_category + return labels def fetch_detections( @@ -538,8 +555,8 @@ def _process_task(task: Dict[str, Any]) -> Tuple[int, int]: seq_id_int: int = task["seq_id_int"] file_base: str = task["file_base"] image_url: str = task["image_url"] - types_for_seq: List[str] = task["types_for_seq"] - labels_by_type: Dict[str, List[str]] = task["labels_by_type"] + category: str = task["category"] + labels: List[str] = task["labels"] try: session = _get_session() @@ -550,37 +567,30 @@ def _process_task(task: Dict[str, Any]) -> Tuple[int, int]: logging.warning("Failed to download %s: %s", image_url, exc) return 0, 0 - img_filename = f"{file_base}.jpg" - label_filename = f"{file_base}.txt" - - images_written = 0 - labels_nonempty = 0 - - for seq_type in types_for_seq: - lines = labels_by_type.get(seq_type, []) + seq_folder_name = FOLDER_NAME_MAP.get((category, seq_id_int), file_base) - key = (seq_type, seq_id_int) - seq_folder_name = FOLDER_NAME_MAP.get(key, file_base) + base = BASE_DIR / category / seq_folder_name # type: ignore[operator] + img_dir = base / "images" + label_dir = base / "labels" + img_dir.mkdir(parents=True, exist_ok=True) + label_dir.mkdir(parents=True, exist_ok=True) - base = BASE_DIR / seq_type / seq_folder_name # type: ignore[operator] - img_dir = base / "images" - label_dir = base / "labels" - img_dir.mkdir(parents=True, exist_ok=True) - label_dir.mkdir(parents=True, exist_ok=True) + img_path = img_dir / f"{file_base}.jpg" + label_path = label_dir / f"{file_base}.txt" - img_path = img_dir / img_filename - label_path = label_dir / label_filename + images_written = 0 + labels_nonempty = 0 - if not img_path.exists(): - with open(img_path, "wb") as f: - f.write(img_bytes) - images_written += 1 + if not img_path.exists(): + with open(img_path, "wb") as f: + f.write(img_bytes) + images_written = 1 - with open(label_path, "w", encoding="utf-8") as f: - if lines: - f.write("\n".join(lines) + "\n") - if lines: - labels_nonempty += 1 + with open(label_path, "w", encoding="utf-8") as f: + if labels: + f.write("\n".join(labels) + "\n") + if labels: + labels_nonempty = 1 return images_written, labels_nonempty @@ -592,6 +602,7 @@ def build_dataset( verify_ssl: bool, headers: Dict[str, str], num_workers: int, + category_filter: Optional[str] = None, ) -> None: """ Build the dataset folder structure from the exported rows. @@ -611,8 +622,18 @@ def build_dataset( # sort rows by (sequence_id, recorded_at) rows_sorted = sorted(rows, key=recorded_at_sort_key) - # compute sequence -> list of categories present in annotation - seq_types_map = compute_sequence_categories(rows_sorted) + # compute sequence -> single category + seq_cat_map = compute_sequence_categories(rows_sorted) + + if category_filter: + before = len(seq_cat_map) + seq_cat_map = {k: v for k, v in seq_cat_map.items() if v == category_filter} + logging.info( + "Category filter '%s': kept %s/%s sequences", + category_filter, + len(seq_cat_map), + before, + ) # Prepare folder name map and tasks folder_name_map: Dict[Tuple[str, int], str] = {} @@ -629,9 +650,9 @@ def build_dataset( continue seq_id_int = int(seq_id) - types_for_seq = seq_types_map.get(seq_id_int) - if not types_for_seq: - types_for_seq = ["no_label"] + category = seq_cat_map.get(seq_id_int) + if category is None: + continue try: file_base = build_file_basename(row) @@ -643,21 +664,20 @@ def build_dataset( ) continue - labels_by_type = extract_labels_for_detection(row) + labels = extract_labels_for_detection(row) - # record first file_base per (seq_type, seq_id) for folder naming - for seq_type in types_for_seq: - key = (seq_type, seq_id_int) - if key not in folder_name_map: - folder_name_map[key] = file_base + # record first file_base per (category, seq_id) for folder naming + key = (category, seq_id_int) + if key not in folder_name_map: + folder_name_map[key] = file_base tasks.append( { "seq_id_int": seq_id_int, "file_base": file_base, "image_url": image_url, - "types_for_seq": types_for_seq, - "labels_by_type": labels_by_type, + "category": category, + "labels": labels, } ) @@ -741,6 +761,7 @@ def main() -> None: verify_ssl=args.verify_ssl, headers=headers, num_workers=args.num_workers, + category_filter=args.category, ) diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/import_annotations.py b/annotation_api/scripts/data_transfer/ingestion/platform/import_annotations.py index a7c4f7d..896eb57 100644 --- a/annotation_api/scripts/data_transfer/ingestion/platform/import_annotations.py +++ b/annotation_api/scripts/data_transfer/ingestion/platform/import_annotations.py @@ -27,6 +27,9 @@ from typing import Any, Dict, List, Optional import requests +from dotenv import load_dotenv + +load_dotenv() # Globals TOKEN: Optional[str] = None @@ -218,15 +221,11 @@ def build_payload_keep_bboxes_update_labels( ann_curr = deepcopy(curr_ann.get("annotation", {})) curr_groups = ( - ann_curr.get("sequences_bbox", []) - if isinstance(ann_curr, dict) - else [] + ann_curr.get("sequences_bbox", []) if isinstance(ann_curr, dict) else [] ) old_ann_raw = old_ann.get("annotation", {}) old_groups = ( - old_ann_raw.get("sequences_bbox", []) - if isinstance(old_ann_raw, dict) - else [] + old_ann_raw.get("sequences_bbox", []) if isinstance(old_ann_raw, dict) else [] ) for idx in range(min(len(curr_groups), len(old_groups))): @@ -376,9 +375,7 @@ def run_import( logging.info("Fetched new annotations: %s", len(new_annotations)) new_seq_by_id = { - seq.get("id"): seq - for seq in new_sequences - if seq.get("id") is not None + seq.get("id"): seq for seq in new_sequences if seq.get("id") is not None } # Loop and patch diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/import_yolo_sequence.py b/annotation_api/scripts/data_transfer/ingestion/platform/import_yolo_sequence.py index 7bdbd6d..e773047 100644 --- a/annotation_api/scripts/data_transfer/ingestion/platform/import_yolo_sequence.py +++ b/annotation_api/scripts/data_transfer/ingestion/platform/import_yolo_sequence.py @@ -66,7 +66,7 @@ ] ANNOTATION_TYPE_CHOICES = ["wildfire_smoke", "other_smoke", "other", "none"] -RECORDED_AT_RE = re.compile(r"-(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})$") +RECORDED_AT_RE = re.compile(r"[-_](\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})$") @dataclass @@ -183,6 +183,11 @@ def parse_args() -> argparse.Namespace: default=1, help="Starting alert_api_id for detections (incremented per image).", ) + parser.add_argument( + "--labels-dir-name", + default="labels", + help="Labels subdirectory name inside the sequence folder (e.g. 'labels' or 'labels_predictions').", + ) parser.add_argument( "--dry-run", action="store_true", @@ -422,7 +427,7 @@ def main() -> None: sequence_dir = Path(args.sequence_dir) images_dir = sequence_dir / "images" - labels_dir = sequence_dir / "labels" + labels_dir = sequence_dir / args.labels_dir_name if not images_dir.exists(): raise FileNotFoundError(f"Missing images folder: {images_dir}") if not labels_dir.exists(): diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/pull_sequence_annotations.py b/annotation_api/scripts/data_transfer/ingestion/platform/pull_sequence_annotations.py index f428eb9..aa5d680 100644 --- a/annotation_api/scripts/data_transfer/ingestion/platform/pull_sequence_annotations.py +++ b/annotation_api/scripts/data_transfer/ingestion/platform/pull_sequence_annotations.py @@ -34,13 +34,17 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--username", type=str, - default=os.getenv("MAIN_ANNOTATION_LOGIN", os.getenv("ANNOTATOR_LOGIN", "admin")), + default=os.getenv( + "MAIN_ANNOTATION_LOGIN", os.getenv("ANNOTATOR_LOGIN", "admin") + ), help="Remote API username", ) parser.add_argument( "--password", type=str, - default=os.getenv("MAIN_ANNOTATION_PASSWORD", os.getenv("ANNOTATOR_PASSWORD", "admin12345")), + default=os.getenv( + "MAIN_ANNOTATION_PASSWORD", os.getenv("ANNOTATOR_PASSWORD", "admin12345") + ), help="Remote API password", ) parser.add_argument( @@ -69,9 +73,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--smoke-type", type=str, - choices=["wildfire", "industrial", "other", "any"], + choices=["wildfire", "industrial", "other", "any", "empty"], default=None, - help="Only pull sequences whose annotation smoke_types includes this value (use 'any' to include all smoke types)", + help="Only pull sequences whose annotation smoke_types includes this value (use 'any' to include all smoke types, 'empty' to only include sequences with no smoke_types)", ) parser.add_argument( "--loglevel", @@ -101,10 +105,39 @@ def write_label(label_path: Path, bbox: List[float], class_name: str) -> None: label_path.write_text(f"{cid} " + " ".join(f"{v:.6f}" for v in yolo) + "\n") -def fetch_sequences(remote_api: str, token: str, max_sequences: int) -> List[Dict]: +def _passes_smoke_filter(seq: Dict, smoke_type: Optional[str]) -> bool: + """Apply the --smoke-type filter against the sequence's embedded annotation. + + Returns True if the sequence should be kept. Sequences without an embedded + annotation are always kept (the worker will fetch one and decide). + """ + if not smoke_type: + return True + ann = seq.get("annotation") + if not ann: + return True + smoke_types = ann.get("smoke_types") or [] + if smoke_type == "empty": + return not smoke_types + if smoke_type == "any": + return bool(smoke_types) + return smoke_type in smoke_types + + +def fetch_sequences( + remote_api: str, + token: str, + max_sequences: int, + smoke_type: Optional[str] = None, +) -> List[Dict]: + """Page through seq_annotation_done sequences, keeping only those that pass + the smoke-type filter, until we have ``max_sequences`` accepted (or we + exhaust all pages). This matters for FP pulls (smoke_type=empty) where the + first page can be entirely smoke sequences. + """ page = 1 size = 100 - results: List[Dict] = [] + accepted: List[Dict] = [] while True: resp = annotation_api.list_sequences( remote_api, @@ -115,13 +148,16 @@ def fetch_sequences(remote_api: str, token: str, max_sequences: int) -> List[Dic include_annotation=True, ) items = resp.get("items", []) - results.extend(items) - if max_sequences and len(results) >= max_sequences: - return results[:max_sequences] + for item in items: + if not _passes_smoke_filter(item, smoke_type): + continue + accepted.append(item) + if max_sequences and len(accepted) >= max_sequences: + return accepted if page >= resp.get("pages", 1): break page += 1 - return results + return accepted def get_sequence_annotation(remote_api: str, token: str, seq_id: int) -> Optional[Dict]: @@ -134,10 +170,17 @@ def get_sequence_annotation(remote_api: str, token: str, seq_id: int) -> Optiona def main() -> None: args = parse_args() - logging.basicConfig(level=args.loglevel.upper(), format="%(levelname)s - %(message)s") + logging.basicConfig( + level=args.loglevel.upper(), format="%(levelname)s - %(message)s" + ) token = annotation_api.get_auth_token(args.remote_api, args.username, args.password) - sequences = fetch_sequences(args.remote_api, token, args.max_sequences) + sequences = fetch_sequences( + args.remote_api, + token, + args.max_sequences, + smoke_type=args.smoke_type, + ) logging.info(f"Found {len(sequences)} sequence(s) with stage seq_annotation_done") logging.info("Processing with %s worker(s)", args.max_workers) @@ -151,10 +194,24 @@ def process_sequence(seq: Dict) -> Tuple[str, int]: logging.warning("No annotation for sequence %s, skipping", seq_id) return ("no_annotation", seq_id) + # Re-check against the freshly fetched annotation; the page payload is only a prefilter. smoke_types = ann.get("smoke_types", []) - if args.smoke_type and args.smoke_type != "any": + if args.smoke_type == "empty": + if smoke_types: + logging.info( + "Skipping sequence %s (smoke_types=%s, expected empty)", + seq_id, + smoke_types, + ) + return ("filter_skip", seq_id) + elif args.smoke_type and args.smoke_type != "any": if args.smoke_type not in smoke_types: - logging.info("Skipping sequence %s (smoke_types=%s not matching %s)", seq_id, smoke_types, args.smoke_type) + logging.info( + "Skipping sequence %s (smoke_types=%s not matching %s)", + seq_id, + smoke_types, + args.smoke_type, + ) return ("filter_skip", seq_id) elif args.smoke_type == "any": if not smoke_types: @@ -190,17 +247,25 @@ def process_sequence(seq: Dict) -> Tuple[str, int]: label_path = lbl_dir / (img_name.replace(".jpg", ".txt")) try: - resp = requests.get(image_url, timeout=30, verify=not args.skip_ssl_verify) + resp = requests.get( + image_url, timeout=30, verify=not args.skip_ssl_verify + ) resp.raise_for_status() img_path.write_bytes(resp.content) except Exception as exc: - logging.error("Failed to download image for detection %s: %s", det_id, exc) + logging.error( + "Failed to download image for detection %s: %s", det_id, exc + ) continue # Match by annotation detection_id (primary) then fall back to alert_api_id if present. bbox = ann_bboxes.get(det_id) or ann_bboxes.get(det.get("alert_api_id")) if bbox: - write_label(label_path, bbox.get("xyxyn", []), bbox.get("class_name", "wildfire")) + write_label( + label_path, + bbox.get("xyxyn", []), + bbox.get("class_name", "wildfire"), + ) else: label_path.write_text("") @@ -219,7 +284,13 @@ def process_sequence(seq: Dict) -> Tuple[str, int]: return ("ok", seq_id) - results: Dict[str, int] = {"ok": 0, "filter_skip": 0, "no_annotation": 0, "stage_update_failed": 0, "errors": 0} + results: Dict[str, int] = { + "ok": 0, + "filter_skip": 0, + "no_annotation": 0, + "stage_update_failed": 0, + "errors": 0, + } with ThreadPoolExecutor(max_workers=args.max_workers) as executor: future_map = {executor.submit(process_sequence, seq): seq for seq in sequences} diff --git a/annotation_api/scripts/data_transfer/ingestion/platform/shared.py b/annotation_api/scripts/data_transfer/ingestion/platform/shared.py index 847a411..72aa54b 100644 --- a/annotation_api/scripts/data_transfer/ingestion/platform/shared.py +++ b/annotation_api/scripts/data_transfer/ingestion/platform/shared.py @@ -14,6 +14,7 @@ from urllib.parse import urlparse import requests +from dotenv import load_dotenv from rich.progress import ( Progress, SpinnerColumn, @@ -32,6 +33,12 @@ ValidationError, ) +# Load .env at import time so any script that imports from this module +# (e.g. via get_annotation_credentials) picks up secrets from +# annotation_api/.env, even if the script forgets to call load_dotenv() +# itself. +load_dotenv() + # Import LogSuppressor from import module @@ -163,7 +170,9 @@ def transform_sequence_data(record: dict, source_api: str = "pyronear_french") - "camera_id": record["camera_id"], "organisation_name": record["organization_name"], "organisation_id": record["organization_id"], - "is_wildfire_alertapi": record["sequence_is_wildfire"], # Platform enum: 'wildfire_smoke', 'other_smoke', 'other' + "is_wildfire_alertapi": record[ + "sequence_is_wildfire" + ], # Platform enum: 'wildfire_smoke', 'other_smoke', 'other' "lat": record["camera_lat"], "lon": record["camera_lon"], "azimuth": record["sequence_azimuth"], @@ -341,17 +350,21 @@ def _process_single_detection( return result except ValidationError as e: - error_msg = f"Detection {record['detection_id']} validation failed: {e.message}" + error_msg = ( + f"Detection {record['detection_id']} validation failed: {e.message}" + ) logging.error(error_msg) if e.field_errors: for field_error in e.field_errors: - logging.error(f" - {field_error['field']}: {field_error['message']}") + logging.error( + f" - {field_error['field']}: {field_error['message']}" + ) result["error"] = error_msg return result except AnnotationAPIError as e: if e.status_code in (502, 503, 504) and attempt < max_retries: - delay = base_delay * (2 ** attempt) + delay = base_delay * (2**attempt) logging.warning( f"⚠️ Detection {record['detection_id']} got HTTP {e.status_code} " f"— retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries})" diff --git a/annotation_api/scripts/setup.sh b/annotation_api/scripts/setup.sh deleted file mode 100755 index 514f973..0000000 --- a/annotation_api/scripts/setup.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Pyronear Annotation API - Setup Script -# This script handles initial setup tasks for the annotation API - -set -e # Exit on any error - -echo "🔧 Setting up Pyronear Annotation API..." - - -# Function to check if required tools are installed -check_prerequisites() { - echo "🔍 Checking prerequisites..." - - if ! command -v docker &> /dev/null; then - echo "❌ Docker is not installed. Please install Docker first." - echo " See: https://docs.docker.com/engine/install/" - exit 1 - fi - - if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then - echo "❌ Docker Compose is not installed. Please install Docker Compose first." - echo " See: https://docs.docker.com/compose/install/" - exit 1 - fi - - echo "✅ Prerequisites check passed" -} - -# Main setup function -main() { - echo "🚀 Starting setup process..." - - check_prerequisites - - echo "" - echo "🎉 Setup completed successfully!" - echo "" - echo "Next steps:" - echo " - For development: make start" - echo " - Run tests: make test" - echo "" - echo "API will be available at: http://localhost:5050/docs" -} - -# Run main function -main "$@" diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 9589984..ccfd2ea 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -1,263 +1,147 @@ # PyroAnnotator Frontend - Claude Context -## Project Overview -The PyroAnnotator Frontend is a React/TypeScript application for wildfire detection annotation. It provides a modern interface for annotating detection sequences with smoke type classifications and false positive identification using data from the PyroAnnotator annotation API backend. - -## Technology Stack -- **Framework**: React 18 with TypeScript -- **Build Tool**: Vite 5.x -- **Package Manager**: npm -- **Styling**: Tailwind CSS 3.x -- **State Management**: Zustand 4.x -- **API Client**: TanStack Query v5 (React Query) + Axios -- **Routing**: React Router DOM v6 -- **Forms**: React Hook Form + Zod validation -- **Icons**: Lucide React -- **UI Components**: Headless UI + custom components -- **Container**: Docker with Nginx -- **Development**: Hot reload with Vite dev server +React/TypeScript SPA for annotating wildfire detection sequences against the PyroAnnotator API. + +## Stack +- React 18 + TypeScript, Vite 5 +- Tailwind CSS 3, Headless UI, Lucide icons +- React Router v6 +- TanStack Query v5 (server state) + Zustand 4 (client state) +- React Hook Form + Zod +- Axios +- Vitest + Testing Library +- ESLint (strict, max-warnings 0) + Prettier ## Project Structure -``` -frontend/ -├── src/ -│ ├── components/ -│ │ ├── annotation/ # Annotation-specific components -│ │ │ ├── AnnotationInterface.tsx -│ │ │ └── SequenceBboxCard.tsx -│ │ ├── layout/ # Layout components -│ │ │ └── AppLayout.tsx -│ │ ├── media/ # Media display components -│ │ └── ui/ # Reusable UI components -│ │ └── ProgressIndicator.tsx -│ ├── hooks/ # Custom React hooks -│ │ ├── useAnnotationStats.ts -│ │ └── useDetectionImage.ts -│ ├── pages/ # Route components -│ │ ├── AnnotationInterface.tsx -│ │ ├── AnnotationPage.tsx # Main annotation workflow -│ │ ├── AnnotationsPage.tsx # List view -│ │ ├── DashboardPage.tsx -│ │ ├── HomePage.tsx -│ │ ├── SequenceDetailPage.tsx -│ │ └── SequencesPage.tsx -│ ├── services/ -│ │ └── api.ts # API client with axios -│ ├── store/ # Zustand state management -│ │ ├── useAnnotationStore.ts -│ │ └── useSequenceStore.ts -│ ├── types/ -│ │ └── api.ts # TypeScript type definitions -│ ├── utils/ -│ │ └── constants.ts # App constants and enums -│ ├── App.tsx -│ └── main.tsx -├── public/ # Static assets -├── docker-compose.yml # Container orchestration -├── Dockerfile # Multi-stage build -├── nginx.conf # Nginx configuration -├── package.json -├── tsconfig.json -├── tailwind.config.js -└── vite.config.ts -``` - -## Development Commands -```bash -# Development server -npm run dev # Start Vite dev server on port 5173 - -# Build & Quality -npm run build # TypeScript compile + Vite build -npm run preview # Preview production build -npm run lint # ESLint check (strict: fails on warnings) -npm run lint:ci # ESLint check (CI-friendly: allows warnings) -npm run lint:fix # ESLint auto-fix -npm run format # Prettier formatting -npm run format:check # Check formatting -npm run type-check # TypeScript type checking -npm run quality # Run all quality checks (strict) -npm run quality:ci # Run all quality checks (CI-friendly) -npm run quality:fix # Fix all quality issues -# Docker -docker compose up # Start production container on port 3000 -docker compose up -d # Detached mode -docker compose down # Stop and remove containers -docker compose build # Rebuild image -docker compose build --no-cache # Force rebuild +``` +frontend/src/ +├── App.tsx # Router shell, auth gate, react-query provider +├── main.tsx # Entry point +├── components/ +│ ├── annotation/ # Sequence annotation pieces (CroppedImageSequence, FullImageSequence, ImageOverlays, SmokeTypeSelector) +│ ├── detection-annotation/ # Detection-level bbox annotation (canvas, toolbar, shortcuts modal, image card, progress header, submission) +│ ├── detection-sequence/ # DetectionGrid, DetectionHeader, ImageModal +│ ├── filters/ # FalsePositiveFilter, ModelAccuracyFilter, SmokeTypeFilter, TabbedFilters, shared/ +│ ├── layout/ # AppLayout +│ ├── sequence/ # SequencePlayer, SequenceReviewer, MediaControls, PlayerControls, MissedSmokePanel, MissedSmokeInstructionsModal +│ ├── sequence-annotation/ # AnnotationHeader, MissedSmokePanel, ProcessingStageMessages, SequenceAnnotationGrid +│ ├── sequences/ # Table headers/rows + pagination for annotate / review queues, plus SequencesLegend +│ └── ui/ # ContributorList, NotificationBadge, NotificationSystem, PasswordField, ProgressIndicator +├── hooks/ +│ ├── annotation/ # useDrawingCanvas, useKeyboardShortcuts +│ └── *.ts # useAnnotationCounts/Stats, useCameras, useOrganizations, useSourceApis, useSequenceDetections, useDetectionImage, useImagePreloader, usePersistedFilters, usePersistedTabState +├── pages/ +│ ├── LoginPage.tsx +│ ├── HomePage.tsx +│ ├── DashboardPage.tsx +│ ├── SequencesPage.tsx # Annotate queue +│ ├── SequencesPageWrapper.tsx # Stage-parameterized list (annotated, etc.) +│ ├── AnnotationInterface.tsx # Annotate one sequence +│ ├── DetectionAnnotatePage.tsx +│ ├── DetectionReviewPage.tsx +│ ├── DetectionSequenceAnnotatePage.tsx +│ └── UserManagementPage.tsx +├── services/api.ts # Axios client (interceptors, JWT) +├── store/ +│ ├── useAuthStore.ts # Token + user (persisted) +│ └── useSequenceStore.ts # Selection / in-progress sequence state +├── types/ +│ ├── api.ts # Mirrors backend schemas (Sequence, Detection, *Annotation, enums) +│ └── branded.ts # Branded ID types +└── utils/ + ├── annotation/ # annotationHandlers, canvasUtils, coordinateUtils, drawingUtils, effectUtils, imageUtils, keyboardUtils, navigationUtils, progressUtils, sequenceUtils, validationUtils, workflowUtils, index + ├── notification/toastUtils.ts + ├── api-functional.ts + ├── constants.ts + ├── filter-state.ts / filterHelpers.ts + ├── modelAccuracy.ts + ├── passwordUtils.ts + ├── playback-calculations.ts + └── processingStage.ts ``` -## API Integration -The frontend integrates with the PyroAnnotator annotation API backend: - -### Type Definitions (`src/types/api.ts`) -Based on backend SQLModel schemas with comprehensive typing: -- `Sequence` - Detection sequence metadata -- `SequenceAnnotation` - Human annotations for sequences -- `Detection` - Individual detection data -- `DetectionAnnotation` - Human annotations for detections -- `SmokeType` - 'wildfire' | 'industrial' | 'other' (from backend enum) -- `FalsePositiveType` - Extensive enum matching backend (antenna, building, cliff, etc.) -- `PaginatedResponse` - Paginated API responses - -### API Client (`src/services/api.ts`) -Axios-based client with: -- **Base Configuration**: Configurable base URL via `VITE_API_BASE_URL` -- **Request/Response Interceptors**: Logging and error handling -- **Comprehensive Methods**: - - Sequences: CRUD operations with filtering/pagination - - Detections: CRUD with image upload support - - Sequence Annotations: CRUD with complex filtering - - Detection Annotations: CRUD operations -- **Error Handling**: Typed `ApiError` responses - -### State Management -**Zustand Stores**: -- `useAnnotationStore`: Current annotation work, progress tracking -- `useSequenceStore`: Sequence data, filtering, pagination - -**TanStack Query Integration**: -- Caching with query keys from constants -- Optimistic updates for mutations -- Background refetching and error retry - -## Key Features & Components - -### Annotation Workflow -- **SequenceBboxCard**: Individual bbox annotation -- **AnnotationInterface**: Complete annotation workflow UI -- **Progress Tracking**: Visual progress indicators and statistics +## Routes (declared in `App.tsx`) -### Data Management -- **Pagination**: All list views support server-side pagination -- **Filtering**: Advanced filtering by smoke type, false positive type, processing stage -- **Search**: Real-time search across sequences and annotations -- **Caching**: Intelligent caching with TanStack Query +| Path | Component | +| ----------------------------------------------------- | --------------------------------- | +| `/login` | `LoginPage` | +| `/sequences/annotate` | `SequencesPage` | +| `/sequences/review` | `SequencesPageWrapper` | +| `/sequences/:id/annotate` | `AnnotationInterface` | +| `/detections/annotate` | `DetectionAnnotatePage` | +| `/detections/review` | `DetectionReviewPage` | +| `/detections/:sequenceId/annotate/:detectionId?` | `DetectionSequenceAnnotatePage` | +| `/users` | `UserManagementPage` | -### UI/UX -- **Responsive Design**: Mobile-friendly with Tailwind CSS -- **Dark Mode Ready**: CSS custom properties for theming -- **Accessible**: Semantic HTML and ARIA labels -- **Performance**: Code splitting and lazy loading +## Development Commands -## Environment Configuration ```bash -# Development (.env.local) -VITE_API_BASE_URL=http://localhost:5050 # Backend API URL -VITE_ENVIRONMENT=development - -# Production (docker-compose.yml) -VITE_API_BASE_URL=http://localhost:5050 # Backend API URL -VITE_ENVIRONMENT=production +npm run dev # Vite dev server, port 3000 +npm run build # tsc + vite build +npm run preview # serve dist/ + +npm run lint # ESLint, --max-warnings 0 +npm run lint:ci # ESLint, --max-warnings 100 (CI lenient) +npm run lint:fix +npm run format # Prettier write +npm run format:check +npm run type-check # tsc --noEmit +npm run quality # type-check + lint + format:check +npm run quality:fix # type-check + lint:fix + format + +npm test # Vitest run +npm run test:watch +npm run test:coverage ``` -## Docker Configuration - -### Multi-stage Dockerfile -- **Builder Stage**: Node 18 Alpine for building -- **Production Stage**: Nginx Alpine for serving -- **Build Process**: npm ci → npm run build → copy to nginx -- **Security**: Non-root user, minimal attack surface - -### Nginx Configuration (`nginx.conf`) -- **SPA Support**: Client-side routing with try_files -- **Compression**: Gzip for static assets -- **Caching**: Appropriate cache headers for different file types -- **Security Headers**: XSS protection, content type sniffing prevention -- **Health Check**: `/health` endpoint for container orchestration - -### Docker Compose -- **Port Mapping**: Host 3000 → Container 80 -- **Health Checks**: Built-in curl-based health monitoring -- **Restart Policy**: `unless-stopped` for reliability -- **Network**: Uses default bridge network (external network removed) +Docker: -## Common Issues & Solutions - -### TypeScript Configuration -- **Strict Mode**: Full TypeScript strict mode enabled -- **Path Mapping**: `@/*` aliases to `./src/*` -- **Unused Variable Detection**: `noUnusedLocals` and `noUnusedParameters` enabled - -### React Query v5 Migration -- **Breaking Change**: `cacheTime` → `gcTime` -- **Query Keys**: Use array format consistently -- **Error Handling**: Proper error type definitions - -### Build Issues ```bash -# Clear caches and rebuild -rm -rf node_modules dist -npm ci -npm run build - -# Docker cache issues -docker compose down +docker compose up # builds and serves at http://localhost:3000 docker compose build --no-cache -docker compose up ``` -### Nginx Configuration -- **gzip_proxied**: Valid values only (removed `must-revalidate`) -- **try_files**: Essential for SPA routing support -- **Cache Headers**: Different strategies for HTML vs assets +## API Integration + +- Client: `src/services/api.ts` (Axios). Base URL: `VITE_API_BASE_URL` (default `http://localhost:5050`). Request interceptor injects the JWT from `useAuthStore`. +- Types: `src/types/api.ts` mirrors backend schemas. Backend enums (`SmokeType`, `FalsePositiveType`) are the source of truth — no hardcoded label strings in components. +- Error type: `ApiError` for typed catch. -## Data Flow Architecture +Endpoints used: `/api/v1/sequences`, `/api/v1/detections`, `/api/v1/annotations/sequences`, `/api/v1/annotations/detections`, `/api/v1/auth/login`. -### API Data Sources -Backend enums are the source of truth: -- `SmokeType` enum from backend models -- `FalsePositiveType` enum from backend models -- No hardcoded label arrays in frontend +## State Management -### Annotation Workflow -1. **Sequence Selection**: Browse paginated sequence list -2. **Annotation Creation**: Auto-create annotation record if none exists -3. **Bbox Processing**: Iterate through detection bboxes -4. **Classification**: Select smoke type or false positive types -5. **Completion**: Mark annotation as complete +- **`useAuthStore`** — JWT token, current user. Persisted to localStorage. +- **`useSequenceStore`** — selection / in-progress sequence state during an annotation session. +- **TanStack Query** — all server reads. Query keys live alongside the hooks that own them. Use `invalidateQueries` after mutations rather than manual cache writes. +- **Local component state** — for transient UI only. -### State Synchronization -- **Optimistic Updates**: Immediate UI updates with server sync -- **Cache Invalidation**: Strategic cache updates after mutations -- **Error Recovery**: Rollback on failed mutations +## Conventions -## Migration Notes +- Path alias `@/*` → `./src/*`. +- TypeScript strict mode; `noUnusedLocals` / `noUnusedParameters` enabled — don't leave unused imports. +- Annotation logic lives in `src/utils/annotation/` and `src/hooks/annotation/`. Prefer extending those modules over inlining canvas/keyboard/coordinate logic into components. +- Notifications: use `src/utils/notification/toastUtils.ts` and the `NotificationSystem` UI — don't roll your own. +- Filter state: persisted via `usePersistedFilters` / `usePersistedTabState`. -### From Old Sequence Labeler -- **Removed Components**: Old `LabelSelector` component removed -- **Updated Types**: Use backend enum types instead of hardcoded labels -- **API Integration**: Full integration with new annotation API -- **State Management**: Migrated to Zustand from previous state solution +## Common Issues -### Recent Fixes (2024) -- **React Query v5**: Updated for latest API changes -- **Type Safety**: Comprehensive TypeScript error resolution -- **Docker Optimization**: Removed external network dependencies -- **Build Pipeline**: Fixed all compilation errors for production builds +- **TanStack Query v5**: `cacheTime` is `gcTime`; query keys are arrays. +- **Build fails after API change**: regenerate / update `src/types/api.ts`, then `npm run type-check`. +- **Auth bounces to `/login`**: check token persistence in `useAuthStore`; clear localStorage to reset. +- **CORS errors** in dev: backend must allow `http://localhost:3000`. +- **Cache rebuild**: `rm -rf node_modules dist && npm ci && npm run build`. For Docker: `docker compose build --no-cache`. -## Performance Considerations -- **Code Splitting**: Route-based code splitting with React Router -- **Image Optimization**: Lazy loading for image previews -- **API Efficiency**: Pagination and filtering to reduce data transfer -- **Caching Strategy**: Aggressive caching for static data, fresh data for annotations +## Docker -## Security -- **Environment Variables**: No secrets in frontend code -- **Content Security Policy**: Basic CSP headers in nginx -- **XSS Protection**: Browser security headers enabled -- **HTTPS Ready**: Production deployment assumes HTTPS termination upstream +- Multi-stage Dockerfile: Node 18 builder → nginx Alpine runtime. +- `nginx.conf` handles SPA routing (`try_files`), gzip, cache headers, and a `/health` endpoint. +- Container port 80, mapped to host 3000 in compose. -## Future Enhancements -- **Real-time Updates**: WebSocket integration for live annotation status -- **Batch Operations**: Bulk annotation processing -- **Export Features**: Annotation data export in various formats -- **Advanced Filtering**: More sophisticated search and filter options -- **Mobile App**: React Native version for field annotations +## Environment Variables -## Troubleshooting -- **Build Failures**: Check TypeScript configuration and dependency versions -- **Docker Issues**: Verify nginx config syntax and port mappings -- **API Connection**: Confirm backend is running and CORS is configured -- **State Issues**: Clear browser storage and check Zustand devtools \ No newline at end of file +| Variable | Default | Description | +| -------------------- | ------------------------ | --------------------- | +| `VITE_API_BASE_URL` | `http://localhost:5050` | Backend API base URL | +| `VITE_ENVIRONMENT` | `development` | Label only | diff --git a/frontend/README.md b/frontend/README.md index c475b3b..14d79a0 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,383 +1,122 @@ # PyroAnnotator Frontend -A modern React application for wildfire detection annotation, replacing the legacy Streamlit interface with a professional, scalable solution. +React/TypeScript UI for annotating wildfire detection sequences against the PyroAnnotator API. -## Features +## Stack -- 🔥 **Modern UI**: Built with React 18, TypeScript, and Tailwind CSS -- 📊 **Real-time Data**: Integration with PyroAnnotator API backend -- 🎯 **Advanced Filtering**: Filter sequences by source, camera, organization, and wildfire alerts -- 📱 **Responsive Design**: Works on desktop, tablet, and mobile devices -- 🚀 **Performance**: Optimized with React Query for caching and Zustand for state management -- 🎨 **Interactive Image Viewer**: Side-by-side image comparison with controls -- 📋 **Smart Annotation**: Intuitive interface matching API data models -- 🐳 **Docker Ready**: Containerized for easy deployment - -## Technology Stack - -### Core Framework -- **React 18** with TypeScript for type safety and modern concurrent features -- **Vite 5.x** for fast development server and optimized production builds -- **React Router DOM v6** for declarative client-side routing - -### Styling & UI -- **Tailwind CSS 3.x** for utility-first styling and responsive design -- **Lucide React** for consistent iconography and modern icon set -- **Headless UI** for accessible components and focus management - -### State Management & Data -- **Zustand 4.x** for lightweight, performant state management -- **TanStack Query v5 (React Query)** for server state, caching, and background refetching -- **React Hook Form** with Zod validation for type-safe form handling -- **Axios** for HTTP client with request/response interceptors - -### Architecture & Code Quality -- **13 Specialized Annotation Utilities**: Modular utility layer for maintainability -- **Comprehensive JSDoc Documentation**: Full coverage across all utilities and functions -- **TypeScript Strict Mode**: Enhanced type safety with strict compiler options -- **ESLint + Prettier**: Code formatting and quality enforcement -- **Functional Programming Patterns**: Pure functions and immutable state updates - -### API Integration & Types -- **Type-safe API Client**: Custom Axios client with comprehensive error handling -- **Backend Enum Synchronization**: Frontend constants matching backend SQLModel enums -- **Query Key Management**: Structured cache invalidation with centralized query keys -- **Optimistic Updates**: UI-first interactions with server synchronization +- React 18 + TypeScript, built with Vite 5 +- Tailwind CSS, Headless UI, Lucide icons +- Routing: React Router v6 +- Server state: TanStack Query v5 +- Client state: Zustand +- Forms: React Hook Form + Zod +- HTTP: Axios +- Tests: Vitest + Testing Library ## Prerequisites -- **Node.js** 18+ and npm -- **PyroAnnotator API** running on `http://localhost:5050` -- Modern web browser with ES2020+ support +- Node.js 18+ and npm +- A running annotation API (default `http://localhost:5050`) — see `../annotation_api/` ## Quick Start -### 1. Install Dependencies - ```bash cd frontend npm install +npm run dev # http://localhost:3000 ``` -### 2. Environment Configuration - -Create a `.env.local` file (optional): +Optional `frontend/.env.local`: -```bash -# API Base URL (default: http://localhost:5050) +``` VITE_API_BASE_URL=http://localhost:5050 - -# Environment (development/production) VITE_ENVIRONMENT=development ``` -### 3. Start Development Server - -```bash -npm run dev -``` - -The application will be available at `http://localhost:3000` - -### 4. Verify API Connection - -1. Ensure the PyroAnnotator API is running at `http://localhost:5050` -2. Check API health: `curl http://localhost:5050/status` -3. View API docs: `http://localhost:5050/docs` - -## Available Scripts - -### Development -- `npm run dev` - Start development server with hot reload -- `npm run build` - Build for production -- `npm run preview` - Preview production build locally - -### Code Quality -- `npm run lint` - Run ESLint for code linting -- `npm run type-check` - Run TypeScript type checking +## Scripts + +| Command | What it does | +| -------------------- | -------------------------------------------------------- | +| `npm run dev` | Vite dev server on port 3000 | +| `npm run build` | TypeScript compile + production build | +| `npm run preview` | Serve the production build locally | +| `npm run lint` | ESLint (strict — fails on warnings) | +| `npm run lint:fix` | ESLint auto-fix | +| `npm run format` | Prettier write | +| `npm run type-check` | `tsc --noEmit` | +| `npm run quality` | `type-check` + `lint` + `format:check` | +| `npm run quality:fix`| `type-check` + `lint:fix` + `format` | +| `npm test` | Vitest run | +| `npm run test:watch` | Vitest watch mode | ## Project Structure ``` -frontend/ -├── public/ # Static assets -├── src/ -│ ├── components/ # Reusable UI components -│ │ ├── annotation/ # Annotation-specific components -│ │ │ ├── AnnotationInterface.tsx # Main annotation interface (547 lines) -│ │ │ ├── DetectionAnnotationCanvas.tsx # Canvas for bbox annotation -│ │ │ ├── SubmissionControls.tsx # Annotation submission controls -│ │ │ ├── AnnotationToolbar.tsx # Annotation action toolbar -│ │ │ ├── KeyboardShortcutModal.tsx # Keyboard shortcuts help -│ │ │ └── SequenceBboxCard.tsx # Individual bbox annotation -│ │ ├── layout/ # Layout components -│ │ │ └── AppLayout.tsx -│ │ ├── media/ # Media display components -│ │ └── ui/ # Reusable UI components -│ │ └── ProgressIndicator.tsx -│ ├── pages/ # Route components -│ │ ├── HomePage.tsx # Landing page -│ │ ├── SequencesPage.tsx # Sequence browser -│ │ ├── AnnotationPage.tsx # Main annotation workflow -│ │ ├── AnnotationsPage.tsx # List view -│ │ ├── DashboardPage.tsx -│ │ ├── SequenceDetailPage.tsx -│ │ └── AnnotationInterface.tsx -│ ├── hooks/ # Custom React hooks -│ │ ├── useAnnotationStats.ts -│ │ └── useDetectionImage.ts -│ ├── services/ # API client services -│ │ └── api.ts # Main API client with axios -│ ├── store/ # Zustand state management -│ │ ├── useSequenceStore.ts -│ │ └── useAnnotationStore.ts -│ ├── types/ # TypeScript type definitions -│ │ └── api.ts # API response types -│ ├── utils/ # Utility functions and constants -│ │ ├── annotation/ # Annotation workflow utilities (13 files) -│ │ │ ├── sequenceUtils.ts # Sequence data utilities -│ │ │ ├── progressUtils.ts # Progress tracking utilities -│ │ │ ├── effectUtils.ts # useEffect hook utilities -│ │ │ ├── keyboardUtils.ts # Keyboard shortcut handling -│ │ │ ├── navigationUtils.ts # Navigation state utilities -│ │ │ ├── annotationHandlers.ts # Event handler utilities -│ │ │ ├── coordinateUtils.ts # Canvas coordinate utilities -│ │ │ ├── drawingUtils.ts # Canvas drawing utilities -│ │ │ ├── validationUtils.ts # Validation utilities -│ │ │ ├── canvasUtils.ts # Canvas manipulation utilities -│ │ │ ├── imageUtils.ts # Image processing utilities -│ │ │ ├── workflowUtils.ts # Annotation workflow utilities -│ │ │ └── index.ts # Utility exports -│ │ ├── notification/ # Notification utilities -│ │ │ └── toastUtils.ts # Toast notification management -│ │ ├── constants.ts # App constants and enums -│ │ ├── modelAccuracy.ts # Model accuracy analysis utilities -│ │ ├── processingStage.ts # Processing stage utilities -│ │ ├── passwordUtils.ts # Password validation utilities -│ │ └── filter-state.ts # Filter state management utilities -│ ├── App.tsx # Main app component -│ ├── main.tsx # App entry point -│ └── index.css # Global styles -├── package.json # Dependencies and scripts -├── tsconfig.json # TypeScript configuration -├── vite.config.ts # Vite configuration -├── tailwind.config.js # Tailwind CSS configuration -└── README.md # This file +frontend/src/ +├── components/ +│ ├── annotation/ # Sequence annotation UI (Cropped/Full image, overlays, smoke type) +│ ├── detection-annotation/ # Detection-level bbox annotation (canvas, toolbar, shortcuts modal) +│ ├── detection-sequence/ # Detection grid + image modal for a sequence +│ ├── filters/ # Filter UI (smoke type, FP, model accuracy, date range) +│ ├── layout/ # AppLayout +│ ├── sequence/ # Sequence player and missed-smoke panels +│ ├── sequence-annotation/ # Sequence-level annotation grid + headers +│ ├── sequences/ # Tables, pagination, legend for sequence lists +│ └── ui/ # Notifications, password field, progress, contributors +├── hooks/ # Custom hooks (annotation/, plus useCameras, useDetectionImage, etc.) +├── pages/ # Route components (see below) +├── services/api.ts # Axios client + request/response interceptors +├── store/ # Zustand stores: useAuthStore, useSequenceStore +├── types/ # api.ts (mirror of backend schemas), branded.ts +└── utils/ # annotation/ utilities, notification/, filter helpers, etc. ``` -## Key Components - -### AnnotationInterface (Refactored Architecture) -The main annotation component has been significantly refactored from 1,561 lines to 547 lines (65% reduction) through systematic extraction of utilities and modular design: - -- **Modular Components**: Extracted specialized components (DetectionAnnotationCanvas, SubmissionControls, AnnotationToolbar) -- **Utility Layer**: 13 annotation utilities handling specific concerns (sequence management, progress tracking, keyboard shortcuts, canvas operations) -- **Toast Notifications**: Centralized notification system with useToastNotifications hook -- **Event Handlers**: Extracted annotation event handlers into dedicated utilities -- **Comprehensive JSDoc**: Full documentation coverage for maintainability - -### Annotation Workflow Utilities -A comprehensive set of 13 specialized utilities powering the annotation interface: +## Routes -- **sequenceUtils.ts**: Sequence data management and processing -- **progressUtils.ts**: Progress tracking and statistics calculation -- **effectUtils.ts**: useEffect hook management and lifecycle utilities -- **keyboardUtils.ts**: Keyboard shortcut handling and navigation -- **navigationUtils.ts**: Navigation state management between sequences -- **annotationHandlers.ts**: Event handling for annotation interactions -- **coordinateUtils.ts**: Canvas coordinate transformations and calculations -- **drawingUtils.ts**: Canvas drawing operations and bbox rendering -- **validationUtils.ts**: Input validation and data integrity checking -- **canvasUtils.ts**: Canvas manipulation and image processing -- **imageUtils.ts**: Image loading, scaling, and optimization -- **workflowUtils.ts**: Annotation workflow state management -- **index.ts**: Centralized utility exports and public API - -### SequencesPage -- Browse and filter wildfire detection sequences -- Advanced filtering with model accuracy analysis -- Pagination and search functionality with URL state persistence -- Real-time data from API with intelligent caching - -### Annotation Pages -- **AnnotationPage**: Main annotation workflow with enhanced UI -- **AnnotationsPage**: List view with processing stage visualization -- **SequenceDetailPage**: Detailed sequence information and metadata -- Interactive image viewer with side-by-side comparison -- Progress tracking and batch operations -- Smart annotation interface matching backend API enums - -### API Client & State Management -- **Type-safe API client** with comprehensive error handling -- **Automatic request/response interceptors** with logging -- **TanStack Query v5** for advanced caching and background refetching -- **Zustand stores** for annotation and sequence state management -- **Query key management** for cache invalidation strategies +| Path | Page | Purpose | +| ----------------------------------------------------- | --------------------------------- | ---------------------------------------- | +| `/login` | `LoginPage` | Auth | +| `/sequences/annotate` | `SequencesPage` | Sequences awaiting annotation | +| `/sequences/review` | `SequencesPageWrapper` | Already-annotated sequence list | +| `/sequences/:id/annotate` | `AnnotationInterface` | Annotate a single sequence | +| `/detections/annotate` | `DetectionAnnotatePage` | Detection-level annotation queue | +| `/detections/review` | `DetectionReviewPage` | Review detection annotations | +| `/detections/:sequenceId/annotate/:detectionId?` | `DetectionSequenceAnnotatePage` | Detection annotation for one sequence | +| `/users` | `UserManagementPage` | User management | ## API Integration -The frontend integrates with the PyroAnnotator API using: +`src/services/api.ts` is an Axios client configured via `VITE_API_BASE_URL`. Backend enums (`SmokeType`, `FalsePositiveType`) live in `src/types/api.ts` and are the source of truth — don't hardcode label strings in components. -### Endpoints Used -- `GET /api/v1/sequences` - List sequences with filtering -- `GET /api/v1/sequences/{id}` - Get sequence details -- `GET /api/v1/annotations/sequences` - List sequence annotations -- `POST /api/v1/annotations/sequences` - Create annotations -- `PATCH /api/v1/annotations/sequences/{id}` - Update annotations +Endpoints used: `/api/v1/sequences`, `/api/v1/detections`, `/api/v1/annotations/sequences`, `/api/v1/annotations/detections`. JWT bearer auth; the token is held in `useAuthStore`. -### Data Models -The frontend uses TypeScript types that match the API's data models: +## State Management -- **Sequences**: Camera sequences with metadata -- **Annotations**: Human annotations with smoke/false positive classification -- **Enums**: SmokeType and FalsePositiveType matching backend definitions +- **TanStack Query** for server data (sequences, detections, annotations) — caching + invalidation. +- **Zustand** for cross-page client state: `useAuthStore` (token, user) and `useSequenceStore` (selection, in-progress work). +- **Local React state** for component-scoped UI. -## Environment Variables +## Docker -| Variable | Default | Description | -|----------|---------|-------------| -| `VITE_API_BASE_URL` | `http://localhost:5050` | Backend API base URL | -| `VITE_ENVIRONMENT` | `development` | Environment mode | - -## Docker Deployment - -### Build Docker Image +Multi-stage build (Node 18 → nginx Alpine). Expose container port 80, mapped to host 3000. ```bash docker build -t pyro-annotator-frontend . +docker run -p 3000:80 -e VITE_API_BASE_URL=http://your-api-host:5050 pyro-annotator-frontend ``` -### Run Container - -```bash -docker run -p 3000:80 \ - -e VITE_API_BASE_URL=http://your-api-host:5050 \ - pyro-annotator-frontend -``` - -### Docker Compose - -```yaml -version: '3.8' -services: - frontend: - build: . - ports: - - "3000:80" - environment: - - VITE_API_BASE_URL=http://backend:5050 - depends_on: - - backend -``` - -## Development Workflow - -### Refactored Architecture (2024) -The frontend has undergone significant architectural improvements: - -- **Component Reduction**: Main AnnotationInterface reduced from 1,561 lines to 547 lines (65% reduction) -- **Modular Utilities**: Created 13 specialized annotation utilities for maintainability -- **Comprehensive Documentation**: Added complete JSDoc coverage across all utilities -- **Enhanced Type Safety**: Full TypeScript strict mode with comprehensive error resolution -- **Performance Optimization**: Improved state management and caching strategies +The repo's top-level `docker-compose.yml` runs the frontend together with the backend stack. -### Adding New Features -1. Create feature branch: `git checkout -b feature/new-feature` -2. Identify appropriate utility modules in `src/utils/annotation/` -3. Add new components following modular patterns -4. Leverage existing utilities (sequenceUtils, progressUtils, etc.) -5. Update TypeScript types if needed -6. Add comprehensive JSDoc documentation -7. Add to routing in `App.tsx` -8. Test with API integration and run quality checks -9. Submit pull request - -### Working with Annotation Utilities -When modifying annotation functionality: -1. **Identify the right utility**: Use the appropriate utility from `src/utils/annotation/` -2. **Follow established patterns**: Each utility has a specific concern and API -3. **Maintain documentation**: Update JSDoc when modifying function signatures -4. **Test interactions**: Ensure utilities work together correctly -5. **Use the index.ts**: Import utilities through the centralized export - -### API Updates -When the backend API changes: -1. Update types in `src/types/api.ts` -2. Update API client in `src/services/api.ts` -3. Update constants in `src/utils/constants.ts` -4. Update model accuracy utilities in `src/utils/modelAccuracy.ts` -5. Test all affected components and utilities - -### State Management Strategy -- **Zustand** for app-level state (sequences, annotations, UI state) -- **TanStack Query v5** for server state (API data, caching, background refetching) -- **Local state** for component-specific data and temporary UI state -- **Toast notifications** managed through centralized toastUtils -- **Navigation state** handled by dedicated navigationUtils +## Environment Variables -### Code Quality Standards -- **TypeScript**: Full strict mode enabled with comprehensive type checking -- **ESLint**: Strict linting with warnings treated as errors in CI -- **JSDoc**: Comprehensive documentation for all public functions and interfaces -- **Testing**: Integration with API and component testing -- **Performance**: Monitoring bundle size and render performance +| Variable | Default | Description | +| -------------------- | ------------------------ | -------------------------- | +| `VITE_API_BASE_URL` | `http://localhost:5050` | Backend API base URL | +| `VITE_ENVIRONMENT` | `development` | Environment label | ## Troubleshooting -### Common Issues - -**API Connection Failed** -``` -Failed to load sequences -``` -- Check if backend API is running on `http://localhost:5050` -- Verify CORS settings in backend -- Check network connectivity - -**Build Errors** -``` -Module not found -``` -- Run `npm install` to ensure dependencies are installed -- Check import paths use `@/` alias for src directory -- Verify TypeScript configuration - -**Type Errors** -``` -Property does not exist on type -``` -- Update API types in `src/types/api.ts` -- Check backend API schema changes -- Run `npm run type-check` - -### Performance Issues - -**Slow Loading** -- Check React Query cache configuration -- Verify API response times -- Monitor network requests in DevTools - -**Memory Usage** -- Check for memory leaks in useEffect cleanup -- Verify image resources are properly disposed -- Monitor component re-renders - -## Contributing - -1. Follow TypeScript best practices -2. Use existing component patterns -3. Add proper error handling -4. Update documentation for new features -5. Test with real API data -6. Follow conventional commit messages - -## License - -This project is licensed under the Apache License 2.0 - see the [LICENSE](../LICENSE) file for details. - -## Support - -For issues and questions: -- Check the [API documentation](http://localhost:5050/docs) -- Review the [implementation plan](./IMPLEMENTATION_PLAN.md) -- Open issues on the project repository \ No newline at end of file +- **Sequences don't load** — check the backend is up (`curl http://localhost:5050/status`) and CORS allows your origin. +- **Type errors after API change** — update `src/types/api.ts` and `src/services/api.ts`, then `npm run type-check`. +- **Build fails** — `rm -rf node_modules dist && npm ci && npm run build`. +- **Stale auth state** — clear localStorage; `useAuthStore` persists the JWT. diff --git a/frontend/src/pages/DetectionAnnotatePage.tsx.backup b/frontend/src/pages/DetectionAnnotatePage.tsx.backup deleted file mode 100644 index fbb9a0b..0000000 --- a/frontend/src/pages/DetectionAnnotatePage.tsx.backup +++ /dev/null @@ -1,355 +0,0 @@ -import { useMemo } from 'react'; -import { useQuery } from '@tanstack/react-query'; -import { useNavigate } from 'react-router-dom'; -import { apiClient } from '@/services/api'; -import { ExtendedSequenceFilters, SequenceWithDetectionProgress } from '@/types/api'; -import { QUERY_KEYS } from '@/utils/constants'; -import { analyzeSequenceAccuracy } from '@/utils/modelAccuracy'; -import TabbedFilters from '@/components/filters/TabbedFilters'; -import { - DetectionReviewTableHeader, - SequencesLegend, - DetectionReviewTableRow, - DetectionReviewPagination, -} from '@/components/sequences'; -import { useCameras } from '@/hooks/useCameras'; -import { useOrganizations } from '@/hooks/useOrganizations'; -import { useSourceApis } from '@/hooks/useSourceApis'; -import { usePersistedFilters, createDefaultFilterState } from '@/hooks/usePersistedFilters'; -import { calculatePresetDateRange } from '@/components/filters/shared/DateRangeFilter'; -import { hasActiveUserFilters } from '@/utils/filterHelpers'; - -export default function DetectionReviewPage() { - const navigate = useNavigate(); - - // Create default state specific to detection review page - const defaultState = { - ...createDefaultFilterState('annotated'), - filters: { - ...createDefaultFilterState('annotated').filters, - detection_annotation_completion: 'complete' as const, - include_detection_stats: true, - processing_stage: 'annotated' as const, // Only show sequences that have completed sequence-level annotation - is_unsure: false, // Exclude unsure sequences from detection annotation workflow - }, - }; - - // Use persisted filters hook - const { - filters, - dateFrom, - dateTo, - selectedFalsePositiveTypes, - selectedSmokeTypes, - selectedModelAccuracy, - setFilters, - setDateFrom, - setDateTo, - setSelectedFalsePositiveTypes, - setSelectedSmokeTypes, - setSelectedModelAccuracy, - resetFilters, - } = usePersistedFilters('filters-detections-review', defaultState); - - // Fetch cameras, organizations, and source APIs for dropdown options - const { data: cameras = [], isLoading: camerasLoading } = useCameras(); - const { data: organizations = [], isLoading: organizationsLoading } = useOrganizations(); - const { data: sourceApis = [], isLoading: sourceApisLoading } = useSourceApis(); - - // Date range helper functions - const setDateRange = (preset: string) => { - const { dateFrom: startDateStr, dateTo: endDateStr } = calculatePresetDateRange(preset); - - setDateFrom(startDateStr); - setDateTo(endDateStr); - - // Convert to API datetime format if dates are valid - const startDateTime = startDateStr ? startDateStr + 'T00:00:00' : undefined; - const endDateTime = endDateStr ? endDateStr + 'T23:59:59' : undefined; - - handleFilterChange({ - recorded_at_gte: startDateTime, - recorded_at_lte: endDateTime, - }); - }; - - const clearDateRange = () => { - setDateFrom(''); - setDateTo(''); - handleFilterChange({ recorded_at_gte: undefined, recorded_at_lte: undefined }); - }; - - // Update filters when date range changes - const handleDateFromChange = (value: string) => { - setDateFrom(value); - const dateTimeValue = value ? value + 'T00:00:00' : undefined; - handleFilterChange({ recorded_at_gte: dateTimeValue }); - }; - - const handleDateToChange = (value: string) => { - setDateTo(value); - const dateTimeValue = value ? value + 'T23:59:59' : undefined; - handleFilterChange({ recorded_at_lte: dateTimeValue }); - }; - - // Fetch sequences with complete detection annotations - const { - data: sequences, - isLoading, - error, - } = useQuery({ - queryKey: [...QUERY_KEYS.SEQUENCES, 'detection-review', filters], - queryFn: () => apiClient.getSequences(filters), - }); - - // Fetch sequence annotations for model accuracy analysis - const { data: sequenceAnnotations } = useQuery({ - queryKey: [ - ...QUERY_KEYS.SEQUENCE_ANNOTATIONS, - 'detection-review', - sequences?.items?.map(s => s.id), - ], - queryFn: async () => { - if (!sequences?.items?.length) return []; - - const annotationPromises = sequences.items.map(sequence => - apiClient - .getSequenceAnnotations({ sequence_id: sequence.id, size: 1 }) - .then(response => ({ sequenceId: sequence.id, annotation: response.items[0] || null })) - .catch(() => ({ sequenceId: sequence.id, annotation: null })) - ); - - return Promise.all(annotationPromises); - }, - enabled: !!sequences?.items?.length, - }); - - // Create a map for quick annotation lookup - const annotationMap = - sequenceAnnotations?.reduce( - (acc, { sequenceId, annotation }) => { - acc[sequenceId] = annotation; - return acc; - }, - {} as Record - ) || {}; - - // Filter sequences by model accuracy - const filteredSequences = useMemo(() => { - if (!sequences || selectedModelAccuracy === 'all') { - return sequences; - } - - const filtered = sequences.items.filter(sequence => { - const annotation = annotationMap[sequence.id]; - if (!annotation) { - return selectedModelAccuracy === 'unknown'; - } - - const accuracy = analyzeSequenceAccuracy({ - ...sequence, - annotation: annotation, - }); - - return accuracy.type === selectedModelAccuracy; - }); - - return { - ...sequences, - items: filtered, - total: filtered.length, - pages: Math.ceil(filtered.length / sequences.size), - }; - }, [sequences, annotationMap, selectedModelAccuracy]); - - const handleFilterChange = (newFilters: Partial) => { - setFilters({ ...filters, ...newFilters, page: 1 }); - }; - - const handleFalsePositiveFilterChange = (selectedTypes: string[]) => { - // Only call setSelectedFalsePositiveTypes (which now does atomic update) - setSelectedFalsePositiveTypes(selectedTypes); - }; - - const handlePageChange = (page: number) => { - setFilters({ ...filters, page }); - }; - - const handleSequenceClick = (clickedSequence: SequenceWithDetectionProgress) => { - // Navigate to detection annotation interface for review purposes - navigate(`/detections/${clickedSequence.id}/annotate?from=detections-review`); - }; - - if (isLoading) { - return ( -

-
-
- ); - } - - if (error) { - return ( -
-
-

Failed to load sequences

-

{String(error)}

-
-
- ); - } - - // Empty state when no sequences are available for review - if (filteredSequences && filteredSequences.items.length === 0) { - // Check if user has applied filters - const hasFilters = hasActiveUserFilters( - filters, - dateFrom, - dateTo, - selectedFalsePositiveTypes, - selectedSmokeTypes, - selectedModelAccuracy, - 'all', // selectedUnsure - true, // showModelAccuracy - true, // showFalsePositiveTypes - true, // showSmokeTypes - false // showUnsureFilter - ); - - return ( -
- {/* Header */} -
-
-

Detections

-

Review and verify annotated wildfire detections

-
-
- - {/* Filters */} - - - {/* Empty state message */} -
-
- {hasFilters ? ( - // Filtered results - no matches - <> -
🔍
-

- No matching sequences found -

-

- No sequences with completed detection annotations match your current filters. -

-

Try adjusting your search criteria above.

- - ) : ( - // No filters - no sequences available -

- No sequences with completed detection annotations to review at the moment. -

- )} -
-
-
- ); - } - - return ( -
- {/* Header */} -
-
-

Detection Review

-

Review and verify annotated wildfire detections

-
-
- - {/* Filters */} - - - {/* Results */} - {filteredSequences && ( -
- - - - - {/* Sequence List */} -
- {filteredSequences.items.map(sequence => ( - - ))} -
- - -
- )} -
- ); -}