From adef325ab244cf6ba5803f3567c5737c2a40aa69 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Sat, 25 Jul 2026 21:53:05 +0300 Subject: [PATCH] feat(program_v2): management command and runbook to restore deleted program offers Deleting program offers via DeleteProgramOffers cascades to ResponseDimensionValue and PROGRAM_OFFER involvements, and SET_NULLs Program.program_offer_id for any offer already accepted into a Program. This adds an idempotent management command that diffs a PITR side instance (wired up via an optional "pitr" DB alias) against production to restore exactly what was lost, including relinking program_offer_id, plus a runbook documenting the end-to-end recovery process. Co-Authored-By: Claude Sonnet 5 --- .../docs/restore-deleted-program-offers.md | 122 ++++++++++++ ...ogram_v2_restore_deleted_program_offers.py | 182 ++++++++++++++++++ kompassi/settings.py | 16 ++ 3 files changed, 320 insertions(+) create mode 100644 kompassi/docs/restore-deleted-program-offers.md create mode 100644 kompassi/program_v2/management/commands/program_v2_restore_deleted_program_offers.py diff --git a/kompassi/docs/restore-deleted-program-offers.md b/kompassi/docs/restore-deleted-program-offers.md new file mode 100644 index 000000000..6307ff2f0 --- /dev/null +++ b/kompassi/docs/restore-deleted-program-offers.md @@ -0,0 +1,122 @@ +# Restoring deleted program offers from a PITR restore + +This runbook covers recovering program offers (and everything that cascaded from their deletion) +after they were deleted via the `DeleteProgramOffers` GraphQL mutation +(`kompassi/program_v2/graphql/mutations/delete_program_offers.py`) — whether by mistake or +maliciously. It uses the `program_v2_restore_deleted_program_offers` management command together +with a Point-In-Time-Recovery (PITR) restore of the database taken from before the deletion. + +The event log (`event_log_v2`) will contain an entry like this when this has happened: + +``` +entry type = program_v2.program_offer.deleted +other fields = {"event": "", "context": "...", "organization": "...", "count_deleted": } +``` + +Note that the log entry does **not** record which rows were deleted, only the count. The restore +process below determines the exact set of affected rows by diffing IDs between the PITR restore +and the current database, rather than trusting the log. + +## What actually gets deleted + +Program offers are `forms.Response` rows (survey responses to a survey with +`app_name=program_v2`, `purpose_slug=default`). Deleting them is a real SQL `DELETE`, which +cascades through foreign keys: + +| Table | Relation | `on_delete` | Effect | +|---|---|---|---| +| `forms_response` | — | — | the program offers themselves, plus any superseded old revisions | +| `forms_responsedimensionvalue` | `subject → Response` | `CASCADE` | deleted with the response | +| `involvement_involvement` | `response → Response` | `CASCADE` | deleted, but only `PROGRAM_OFFER`-type involvements — the mutation detaches (nulls `response_id` on) `PROGRAM_HOST` involvements first, so those are *not* deleted | +| `involvement_involvementdimensionvalue` | `subject → Involvement` | `CASCADE` | deleted transitively with the involvements above | +| `program_v2_program` | `program_offer → Response` | **`SET_NULL`** | **not deleted** — but any offer that had already been accepted into a `Program` now has `program_offer_id = NULL`. This is easy to miss and must be relinked, not just re-inserted. | + +`forms.Response` and `forms.ResponseDimensionValue`/`involvement.InvolvementDimensionValue` use +UUID or database-assigned primary keys that are never reused after a delete, so restoring rows +with their original IDs cannot collide with anything created after the incident. This is what +makes an in-place restore (rather than a full database rollback) safe. + +## 1. Take a PITR restore of the database into a side instance + +Restore to a target time a few seconds *before* the deletion. Given a log timestamp of +`2026-07-25 09:28:43.704 UTC`, target e.g. `2026-07-25 09:28:41 UTC`. + +Afterwards, confirm on the side instance that the `program_v2.program_offer.deleted` entry is +**absent** from `event_log_v2` — that proves the restore stopped short of the delete transaction. +Do not restore into or over the production database — this must be a separate, standalone +instance. + +## 2. Take a fresh backup of production + +Before running anything against production, take a fresh backup/snapshot distinct from the PITR +side instance, so that the restore procedure itself can be undone if something turns out to be +wrong with it. + +## 3. Point the backend at the side instance via the `pitr` database alias + +`kompassi/settings.py` wires up an optional second database alias named `pitr` when +`PITR_POSTGRES_HOSTNAME` is set (following the same `PITR_POSTGRES_HOSTNAME` / +`PITR_POSTGRES_DATABASE` / `PITR_POSTGRES_USERNAME` / `PITR_POSTGRES_PASSWORD` / +`PITR_POSTGRES_PORT` / `PITR_POSTGRES_SSLMODE` naming as the main `POSTGRES_*` settings). Set +these to point at the side instance from wherever you'll run `manage.py` against production, e.g.: + +```bash +export PITR_POSTGRES_HOSTNAME= +export PITR_POSTGRES_DATABASE= +export PITR_POSTGRES_USERNAME= +export PITR_POSTGRES_PASSWORD=<...> +``` + +A read-only user on the side instance is sufficient and preferred — the command never writes to +the `pitr` alias, only to `default`. + +## 4. Dry run + +```bash +python manage.py program_v2_restore_deleted_program_offers --expected-count +``` + +Without `--commit`, the command performs the entire restore inside a transaction and then rolls it +back, so you can review what it *would* do first. It reports: + +- how many responses exist on `pitr` but are missing on `default` (what will be restored) +- how many responses exist on `default` but not on `pitr` (created/edited after the PITR snapshot — + left untouched) +- counts for each table it restores or relinks + +Pass `--expected-count` set to `count_deleted` from the event log entry as a cross-check: the +command refuses to proceed if the number of rows it would restore doesn't match exactly. If it +doesn't match, stop and investigate before proceeding — don't just drop `--expected-count` to make +it pass. + +## 5. Commit + +Once the dry run's counts look right: + +```bash +python manage.py program_v2_restore_deleted_program_offers --expected-count --commit +``` + +This is safe to run against a live production database — it only inserts rows that don't already +exist and relinks `program_offer_id` where it is currently `NULL`; it never deletes or overwrites +existing rows. It re-derives `cached_dimensions`/`cached_key_fields` on the restored responses and +involvements from the (now restored) dimension values rather than trusting whatever was cached in +the PITR snapshot. + +The command is idempotent — running it again afterwards (with or without `--commit`) reports +nothing left to restore. + +## 6. Verify and clean up + +- Spot-check a few restored offers via the admin UI or GraphQL. +- Re-check `event_log_v2` if you want to record that the incident was remediated. +- Once you're confident, tear down the PITR side instance and unset the `PITR_POSTGRES_*` + variables — don't leave `pitr` wired up to a stale side instance. + +## Rehearsing this before touching production + +Before running any of this against production, rehearse it against a disposable copy: clone the +current database into a second one on the same Postgres server (e.g. `pg_dump | psql` into a new +database), point `pitr` at *that*, delete a handful of program offers for a test event the same +way `DeleteProgramOffers` does, and confirm the command restores them correctly and idempotently. +This is exactly how the command was validated during development. diff --git a/kompassi/program_v2/management/commands/program_v2_restore_deleted_program_offers.py b/kompassi/program_v2/management/commands/program_v2_restore_deleted_program_offers.py new file mode 100644 index 000000000..3825e9269 --- /dev/null +++ b/kompassi/program_v2/management/commands/program_v2_restore_deleted_program_offers.py @@ -0,0 +1,182 @@ +import logging + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from kompassi.core.models.event import Event +from kompassi.dimensions.models.enums import DimensionApp +from kompassi.forms.models.enums import SurveyPurpose +from kompassi.forms.models.form import Form +from kompassi.forms.models.response import Response +from kompassi.forms.models.response_dimension_value import ResponseDimensionValue +from kompassi.involvement.models.enums import InvolvementApp, InvolvementType +from kompassi.involvement.models.involvement import Involvement +from kompassi.involvement.models.involvement_dimension_value import InvolvementDimensionValue +from kompassi.program_v2.models.program import Program + +logger = logging.getLogger(__name__) + +PITR_ALIAS = "pitr" + + +class Command(BaseCommand): + # See kompassi/docs/restore-deleted-program-offers.md for the full runbook. + help = ( + "Restore program offers (and everything that cascaded from their deletion) for one event " + f"from a PITR side instance wired up as the {PITR_ALIAS!r} database alias. " + "Runs as a dry run (rolled back) unless --commit is passed." + ) + + def add_arguments(self, parser): + parser.add_argument("event_slug", metavar="EVENT_SLUG") + parser.add_argument( + "--expected-count", + type=int, + default=None, + help=( + "Abort unless exactly this many forms.Response rows would be restored. " + "Use this to cross-check against count_deleted in the program_v2.program_offer.deleted " + "event log entry before committing." + ), + ) + parser.add_argument( + "--commit", + default=False, + action="store_true", + help="Actually commit the restore. Without this flag, the restore is rolled back at the end (dry run).", + ) + + def handle(self, *args, **opts): + event_slug: str = opts["event_slug"] + expected_count: int | None = opts["expected_count"] + commit: bool = opts["commit"] + + if PITR_ALIAS not in settings.DATABASES: + raise CommandError( + f"No {PITR_ALIAS!r} database alias configured. Set the PITR_POSTGRES_* environment " + "variables to point it at the PITR side instance before running this command." + ) + + event = Event.objects.get(slug=event_slug) + + offer_forms = Form.objects.filter( + survey__event=event, + survey__app_name=DimensionApp.PROGRAM_V2.value, + survey__purpose_slug=SurveyPurpose.DEFAULT.value, + ) + form_ids = set(offer_forms.values_list("id", flat=True)) + pitr_form_ids = set( + Form.objects.using(PITR_ALIAS) + .filter( + survey__event__slug=event_slug, + survey__app_name=DimensionApp.PROGRAM_V2.value, + survey__purpose_slug=SurveyPurpose.DEFAULT.value, + ) + .values_list("id", flat=True) + ) + if form_ids != pitr_form_ids: + raise CommandError( + f"Program offer forms differ between default ({len(form_ids)} forms) and {PITR_ALIAS} " + f"({len(pitr_form_ids)} forms) for {event_slug}. Forms are never touched by " + "DeleteProgramOffers, so this is unexpected - investigate before proceeding." + ) + + prod_response_ids = set(Response.objects.filter(form_id__in=form_ids).values_list("id", flat=True)) + pitr_response_ids = set( + Response.objects.using(PITR_ALIAS).filter(form_id__in=form_ids).values_list("id", flat=True) + ) + missing_response_ids = pitr_response_ids - prod_response_ids + extra_response_ids = prod_response_ids - pitr_response_ids + + self.stdout.write( + f"Responses present on {PITR_ALIAS} but missing on default (to be restored): {len(missing_response_ids)}" + ) + self.stdout.write( + f"Responses present on default but not on {PITR_ALIAS} " + f"(created/edited after the PITR snapshot, left untouched): {len(extra_response_ids)}" + ) + + if not missing_response_ids: + self.stdout.write("Nothing to restore.") + return + + if expected_count is not None and len(missing_response_ids) != expected_count: + raise CommandError( + f"Expected to restore exactly {expected_count} responses, found {len(missing_response_ids)}. " + "Refusing to proceed." + ) + + with transaction.atomic(): + self._restore(missing_response_ids) + + if commit: + self.stdout.write( + self.style.SUCCESS(f"Committed restore of {len(missing_response_ids)} program offers.") + ) + else: + self.stdout.write( + self.style.WARNING("Dry run complete, rolling back (pass --commit to apply for real).") + ) + transaction.set_rollback(True) + + def _restore(self, response_ids: set): + # 1. forms.Response. superseded_by is a self-FK, so insert with it nulled out first and + # patch it in a second pass once every row in this batch exists. + pitr_responses = list(Response.objects.using(PITR_ALIAS).filter(id__in=response_ids)) + superseded_by = { + response.id: response.superseded_by_id for response in pitr_responses if response.superseded_by_id + } + for response in pitr_responses: + response.superseded_by_id = None + Response.objects.bulk_create(pitr_responses) + self.stdout.write(f" Restored {len(pitr_responses)} forms.Response rows.") + + for response_id, superseded_by_id in superseded_by.items(): + Response.objects.filter(id=response_id).update(superseded_by_id=superseded_by_id) + self.stdout.write(f" Relinked superseded_by on {len(superseded_by)} responses.") + + # 2. forms.ResponseDimensionValue. Its own PK is not referenced anywhere else, so let the + # database assign fresh ids; only (subject, value) need to match the original state. + dimension_values = list(ResponseDimensionValue.objects.using(PITR_ALIAS).filter(subject_id__in=response_ids)) + for dimension_value in dimension_values: + dimension_value.id = None + ResponseDimensionValue.objects.bulk_create(dimension_values) + self.stdout.write(f" Restored {len(dimension_values)} forms.ResponseDimensionValue rows.") + + # 3. involvement.Involvement, PROGRAM_OFFER only. PROGRAM_HOST involvements were detached + # (response set to NULL) rather than deleted by DeleteProgramOffers, so they are untouched. + pitr_involvements = list( + Involvement.objects.using(PITR_ALIAS).filter( + response_id__in=response_ids, + app=InvolvementApp.PROGRAM, + type=InvolvementType.PROGRAM_OFFER, + ) + ) + Involvement.objects.bulk_create(pitr_involvements) + involvement_ids = [involvement.id for involvement in pitr_involvements] + self.stdout.write(f" Restored {len(pitr_involvements)} involvement.Involvement rows.") + + # 4. involvement.InvolvementDimensionValue, same reasoning as (2). + involvement_dimension_values = list( + InvolvementDimensionValue.objects.using(PITR_ALIAS).filter(subject_id__in=involvement_ids) + ) + for dimension_value in involvement_dimension_values: + dimension_value.id = None + InvolvementDimensionValue.objects.bulk_create(involvement_dimension_values) + self.stdout.write(f" Restored {len(involvement_dimension_values)} involvement.InvolvementDimensionValue rows.") + + # 5. program_v2.Program.program_offer_id was SET NULL, not deleted. Relink it, but only + # where it is still NULL in default, in case it was legitimately changed since. + relinked = 0 + for program in Program.objects.using(PITR_ALIAS).filter(program_offer_id__in=response_ids): + relinked += Program.objects.filter(id=program.id, program_offer_id__isnull=True).update( + program_offer_id=program.program_offer_id + ) + self.stdout.write(f" Relinked program_offer_id on {relinked} program_v2.Program rows.") + + # 6. Recompute denormalized caches from the now-restored dimension values rather than + # trusting the (possibly stale) cached_dimensions/cached_key_fields copied from pitr. + Response.refresh_cached_fields_qs(Response.objects.filter(id__in=response_ids)) + Involvement.refresh_cached_dimensions_qs(Involvement.objects.filter(id__in=involvement_ids)) + self.stdout.write(" Refreshed cached dimensions/key fields on restored responses and involvements.") diff --git a/kompassi/settings.py b/kompassi/settings.py index 66ad7276b..6c898ceea 100644 --- a/kompassi/settings.py +++ b/kompassi/settings.py @@ -46,6 +46,22 @@ }, } +# Optional read-only connection to a side instance (eg. a PITR restore) used by one-off data +# recovery management commands such as program_v2_restore_deleted_program_offers. Only wired up +# when PITR_POSTGRES_HOSTNAME is set, so it has no effect on normal deployments. +if PITR_POSTGRES_HOSTNAME := env("PITR_POSTGRES_HOSTNAME", default=""): + DATABASES["pitr"] = { + "HOST": PITR_POSTGRES_HOSTNAME, + "NAME": env("PITR_POSTGRES_DATABASE", default="kompassi"), + "USER": env("PITR_POSTGRES_USERNAME", default="kompassi"), + "PORT": env("PITR_POSTGRES_PORT", default="5432"), + "PASSWORD": env("PITR_POSTGRES_PASSWORD", default="secret"), + "OPTIONS": { + "sslmode": env("PITR_POSTGRES_SSLMODE", default="allow"), + }, + "ENGINE": "django.db.backends.postgresql", + } + SESSION_ENGINE = "django.contrib.sessions.backends.cache" DEFAULT_AUTO_FIELD = "django.db.models.AutoField"