Support PostgreSQL 18+ properly, guard existing data against version bumps

.env.defaults

@@ -31,11 +31,23 @@ RESTART_POLICY=always
 # the list of URLs under which this application is available
 ALLOWED_HOSTS=localhost, ip6-localhost, 127.0.0.1, [::1], rdmo, rdc-rdmo, rdmo.ils-geomonitoring.de
 
-# NOTE: avoid "latest" here. Postgres 18 refuses to start against a bind
-# mount at .../data directly (it expects the new pg_upgrade-friendly layout,
-# a single mount at /var/lib/postgresql instead), which is what this stack
-# uses. Pin an actual major version and upgrade deliberately.
+# NOTE: avoid "latest" here, pin an actual major version and upgrade
+# deliberately. Postgres major-version upgrades need pg_dump/pg_upgrade
+# with both binary versions available - they are NOT automatic here, and
+# fixperms will back up vol/postgres and refuse to start rather than let a
+# version bump silently touch data it can't actually read.
+#
+# Postgres 18+ also changed where it expects to be mounted: <=17 wants a
+# bind mount directly at .../data (POSTGRES_DATA_MOUNT's default below), 18+
+# wants a single mount one level up, at /var/lib/postgresql, and manages its
+# own version-named subdirectory under that. If you bump POSTGRES_VERSION to
+# 18 or newer, also set:
+#   POSTGRES_DATA_MOUNT=/var/lib/postgresql
+# Only do this for a fresh deployment, or after manually migrating existing
+# data (pg_dump from the old version, pg_restore into the new one) - it does
+# not on its own convert already-existing <=17 data to the 18+ layout.
 POSTGRES_VERSION=16
+POSTGRES_DATA_MOUNT=/var/lib/postgresql/data
 POSTGRES_USER=rdmo
 POSTGRES_PASSWORD=rdmopgpass19
 POSTGRES_DB=rdmo

.gitignore

@@ -7,3 +7,5 @@ vol/
 
 *.toml.zip
 /p2r.yaml
+
+rdmo_src

docker-compose.yaml

@@ -15,13 +15,20 @@ services:
   # exist yet, docker creates it as root, which that unprivileged user can't
   # write to. This fixes ownership once, as root, before the real services
   # start, without changing their (deliberately) unprivileged runtime user.
+  # It also refuses to start (after taking a backup) if the postgres major
+  # version on disk doesn't match POSTGRES_VERSION - see fixperms.sh.
   fixperms:
     image: busybox:latest
     container_name: ${GLOBAL_PREFIX:-rdc}-fixperms
     restart: "no"
-    command: sh -c "mkdir -p /vol/log /vol/postgres && chown -R ${LOCAL_UID:-1000}:${LOCAL_GID:-1000} /vol"
+    entrypoint: ["sh", "/fixperms.sh"]
+    environment:
+      LOCAL_UID: ${LOCAL_UID:-1000}
+      LOCAL_GID: ${LOCAL_GID:-1000}
+      POSTGRES_VERSION: ${POSTGRES_VERSION:-16}
     volumes:
       - ${VOLDIR:-./vol}:/vol
+      - ./docker/fixperms/fixperms.sh:/fixperms.sh:ro
 
   caddy:
     build:
@@ -64,7 +71,7 @@ services:
       fixperms:
         condition: service_completed_successfully
     volumes:
-      - ${VOLDIR:-./vol}/postgres:/var/lib/postgresql/data
+      - ${VOLDIR:-./vol}/postgres:${POSTGRES_DATA_MOUNT:-/var/lib/postgresql/data}
     env_file:
       - .env.defaults
       - path: .env

docker/fixperms/fixperms.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+set -eu
+
+log() { echo "[fixperms] $*"; }
+
+TARGET_UID="${LOCAL_UID:-1000}"
+TARGET_GID="${LOCAL_GID:-1000}"
+REQUESTED_VERSION="${POSTGRES_VERSION:-16}"
+PGDIR="/vol/postgres"
+
+mkdir -p /vol/log "${PGDIR}"
+
+# Find any existing PG_VERSION marker: either flat directly under $PGDIR
+# (the layout postgres <=17 images use), or nested under a major-version
+# subdir (the layout postgres >=18 images use, e.g. $PGDIR/18/docker).
+existing_version_file="$(find "${PGDIR}" -maxdepth 3 -name PG_VERSION 2>/dev/null | head -n1)"
+
+if [ -n "${existing_version_file}" ]; then
+  existing_major="$(cat "${existing_version_file}")"
+  case "${REQUESTED_VERSION}" in
+    latest | "")
+      log "POSTGRES_VERSION is '${REQUESTED_VERSION}' - can't determine its major version ahead of time, skipping the version-mismatch check."
+      ;;
+    *)
+      requested_major="${REQUESTED_VERSION%%.*}"
+      if [ "${existing_major}" != "${requested_major}" ]; then
+        backup="/vol/postgres-backup-${existing_major}-$(date +%Y%m%d%H%M%S).tar.gz"
+        log "MISMATCH: existing data on disk is PostgreSQL major ${existing_major}, but POSTGRES_VERSION=${REQUESTED_VERSION} was requested."
+        log "Backing up ${PGDIR} to ${backup} before refusing to start ..."
+        tar -czf "${backup}" -C /vol postgres
+        log "Backup done: ${backup}"
+        log "Refusing to start: postgres major-version upgrades are not automatic here (they need pg_dump/pg_upgrade with"
+        log "both binary versions available, not just a file copy). Either set POSTGRES_VERSION back to ${existing_major}.x,"
+        log "or perform a deliberate upgrade (see readme.md) - the backup above is your safety net either way."
+        exit 1
+      fi
+      log "Existing data is PostgreSQL major ${existing_major}, matches requested POSTGRES_VERSION=${REQUESTED_VERSION}."
+      ;;
+  esac
+else
+  log "No existing PostgreSQL data directory found under ${PGDIR} - nothing to check, starting fresh."
+fi
+
+chown -R "${TARGET_UID}:${TARGET_GID}" /vol
+log "vol ownership set to ${TARGET_UID}:${TARGET_GID}."

readme.md

@@ -8,6 +8,7 @@
   - [Dockers](#dockers)
   - [Volumes](#volumes)
 - [Configuration & Usage](#configuration--usage)
+- [Upgrading PostgreSQL](#upgrading-postgresql)
 - [Multiple RDMO Instances on a Single Docker Host](#multiple-rdmo-instances-on-a-single-docker-host)
 
 <!-- /toc -->
@@ -20,6 +21,8 @@ This repository contains RDMO docker images that are held together by [docker co
 
 Four containers are going to be created: `Caddy`, `PostgreSQL`, `RDMO`, and a short-lived `fixperms` helper. `caddy`, `postgres` and `rdmo` each run as an unprivileged, UID/GID-mapped user rather than root. `fixperms` runs once, as root, before the other three start: it creates `vol/log` and `vol/postgres` if they don't exist yet and makes sure they (and the rest of `vol/`) are owned by that same UID/GID, then exits. This is only needed because docker would otherwise create missing bind-mount folders as root, which the unprivileged containers couldn't write to; the three long-running services never run as root themselves.
 
+`fixperms` also compares the PostgreSQL major version already on disk (if any) against `POSTGRES_VERSION`. If they don't match, it backs up `vol/postgres` to a timestamped `.tar.gz` next to it and refuses to start the rest of the stack, rather than let a newer postgres either fail confusingly or silently start a fresh, empty database next to your real one. See [Upgrading PostgreSQL](#upgrading-postgresql) if you actually want to move to a new major version.
+
 ### Volumes
 
 `VOLDIR` on the docker host (`vol/` next to `docker-compose.yaml` by default, see `VOLDIR` in [Configuration & Usage](#configuration--usage) to relocate it, e.g. to a different disk) holds everything that needs to persist or be shared between containers:
@@ -63,6 +66,29 @@ Note that `VOLDIR` is bind-mounted directly (not a named docker volume), so any
 
    A fresh RDMO installation does not contain any data. You may want to import `conditions`, `domains`, `options`, `questions`, `tasks` and `views`. In the `RDMO container` there is a shell script that automatically clones the [rdmo-catalog repo](https://github.com/rdmorganiser/rdmo-catalog) and imports everything in it. If you consider it being helpful you could do `import-github-catalogues.sh`.
 
+## Upgrading PostgreSQL
+
+PostgreSQL 18 changed where it expects to be mounted: versions up to 17 want a bind mount directly at `/var/lib/postgresql/data`, which is what `POSTGRES_DATA_MOUNT` defaults to; 18 and newer want a single mount one level up, at `/var/lib/postgresql`, and manage their own version-named subdirectory (e.g. `vol/postgres/18/docker`) underneath it themselves.
+
+Bumping `POSTGRES_VERSION` alone does not convert existing data to a new major version's format - PostgreSQL major versions can't read each other's on-disk files directly, that needs `pg_dump`/`pg_restore` (or `pg_upgrade` with both binary versions available, which this setup doesn't wire up). `fixperms` guards against doing this by accident: it compares the major version already on disk against `POSTGRES_VERSION`, and if they differ, backs up `vol/postgres` and refuses to start rather than risk it (see [Structure](#structure)).
+
+To actually move to PostgreSQL 18+ on an existing instance:
+
+1. With the stack still on the old version, back up your data properly, e.g.:
+   ```shell
+   docker exec -t rdc-postgres pg_dump -U rdmo -d rdmo -Fc -f /tmp/rdmo.dump
+   docker cp rdc-postgres:/tmp/rdmo.dump ./rdmo.dump
+   ```
+2. Set `POSTGRES_VERSION=18` (or newer) and `POSTGRES_DATA_MOUNT=/var/lib/postgresql` in your `.env`.
+3. Start the stack. `fixperms` will find no existing data under the new layout and let postgres initialize a fresh, empty database.
+4. Restore your dump into it:
+   ```shell
+   docker cp ./rdmo.dump rdc-postgres:/tmp/rdmo.dump
+   docker exec -t rdc-postgres pg_restore -U rdmo -d rdmo /tmp/rdmo.dump
+   ```
+
+Starting a brand-new deployment directly on PostgreSQL 18+ is simpler: just set both variables above before the first `up` - there's no existing data for `fixperms` to compare against, so it proceeds normally.
+
 ## Multiple RDMO Instances on a Single Docker Host
 
 You can have multiple running RDMO instances on a single docker host as long as you pay attention to three things.