From 43fd1de6bd6f4164cdaae59559f2183e4c88c5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pasternak?= Date: Sat, 13 Jun 2026 16:49:33 +0200 Subject: [PATCH] =?UTF-8?q?Zrezygnuj=20z=20w=C5=82asnego=20obrazu=20?= =?UTF-8?q?=E2=80=94=20autotune=20w=20shellu,=20bind-mount=20na=20stock=20?= =?UTF-8?q?postgres?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BPP-strojony PostgreSQL uruchamiamy teraz na oficjalnym obrazie `postgres` przez bind-mount, bez budowania własnego obrazu. - autotune.py przepisany na czysty shell+awk (autotune.sh, bez Pythona); wynik identyczny bajt w bajt — zweryfikowane: wymuszony RAM, autodetekcja host/cgroup, tryb unsafe, override locków. autotune.py zostaje jako referencja. - docker-entrypoint-autotune.sh: Python-free, respektuje $PGDATA — działa podmontowany na stockowym postgres. - examples/docker-compose.yml: gotowy przykład (ICU pl-PL przez POSTGRES_INITDB_ARGS, bez systemowego locale pl_PL). - Usunięto Dockerfile, docker-bake.hcl, DOCKERHUB.md oraz zadania build/scan/push/scout/dockerhub w CI. Nowe ci.yml: self-test + pre-commit + smoke na postgres:16/17/18. - Wcześniej na tym branchu: usunięcie plpython3u; fix generowania locale pl_PL po apt-get upgrade (Debian trixie regenerował locale-archive i kasował locale, przez co initdb padał z "invalid locale settings"). - README/CLAUDE/CHANGELOG zaktualizowane; usunięto hook hadolint z pre-commit. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build.yml | 266 --------------------------------- .github/workflows/ci.yml | 86 +++++++++++ .pre-commit-config.yaml | 5 - CHANGELOG.md | 38 +++++ CLAUDE.md | 82 +++++----- DOCKERHUB.md | 104 ------------- Dockerfile | 36 ----- README.md | 196 +++++++++--------------- autotune.sh | 272 ++++++++++++++++++++++++++++++++++ docker-bake.hcl | 56 ------- docker-entrypoint-autotune.sh | 28 +++- examples/docker-compose.yml | 31 ++++ 12 files changed, 558 insertions(+), 642 deletions(-) delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 DOCKERHUB.md delete mode 100644 Dockerfile create mode 100755 autotune.sh delete mode 100644 docker-bake.hcl create mode 100644 examples/docker-compose.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 59d2414..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,266 +0,0 @@ -name: Build & publish bpp_dbserver - -on: - push: - branches: [main] - tags: - - "v*" - pull_request: - branches: [main] - workflow_dispatch: - inputs: - push_to_registry: - description: "Push built images to Docker Hub" - required: true - default: "false" - type: choice - options: - - "true" - - "false" - -permissions: - contents: read - -jobs: - test: - name: autotune self-test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - name: Run autotune self-test - run: python autotune.py --test - - lint: - name: pre-commit - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - uses: pre-commit/action@v3.0.1 - - build: - name: Build matrix (PG 16/17/18), smoke & (maybe) push - needs: [test, lint] - runs-on: ubuntu-latest - outputs: - should_push: ${{ steps.decide.outputs.should_push }} - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Decide whether to push - id: decide - env: - EVENT_NAME: ${{ github.event_name }} - REF: ${{ github.ref }} - DISPATCH_PUSH: ${{ inputs.push_to_registry }} - run: | - if [[ "$EVENT_NAME" == "push" && "$REF" == refs/tags/v* ]]; then - echo "should_push=true" >> "$GITHUB_OUTPUT" - elif [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then - echo "should_push=$DISPATCH_PUSH" >> "$GITHUB_OUTPUT" - else - echo "should_push=false" >> "$GITHUB_OUTPUT" - fi - - - name: Bake print (plan) - run: docker buildx bake --print - - - name: Build matrix (load locally for smoke, amd64 only) - # `docker buildx bake --load` nie obsługuje multi-arch; runner GH - # Actions jest amd64, więc smoke leci tylko na linux/amd64. - # Multi-arch (amd64 + arm64) buduje się dopiero w kroku push. - run: docker buildx bake --set "*.platform=linux/amd64" --load - - - name: Smoke test (PG 16/17/18) - run: | - set -Eeuo pipefail - for tag in psql-16 psql-17 psql-18; do - image="iplweb/bpp_dbserver:${tag}" - container="smoke_${tag}" - echo "::group::Smoke: ${image}" - - docker run -d --name "${container}" \ - -e POSTGRES_PASSWORD=smoke \ - "${image}" - - # Wait up to 60s for readiness - ready=0 - for i in $(seq 1 30); do - if docker exec "${container}" pg_isready -U postgres >/dev/null 2>&1; then - ready=1 - break - fi - sleep 2 - done - if [ "$ready" -ne 1 ]; then - echo "FAIL: pg_isready timed out for ${tag}" - docker logs "${container}" || true - docker rm -f "${container}" >/dev/null 2>&1 || true - exit 1 - fi - - # plpython3u must load - docker exec "${container}" psql -U postgres -v ON_ERROR_STOP=1 \ - -c "CREATE EXTENSION plpython3u;" - - # ICU pl-PL collation must be available - icu_present=$(docker exec "${container}" psql -U postgres -Atc \ - "SELECT 1 FROM pg_collation WHERE collname = 'pl-x-icu' LIMIT 1;") - if [ "${icu_present}" != "1" ]; then - echo "FAIL: pl-x-icu collation missing for ${tag}" - docker rm -f "${container}" >/dev/null 2>&1 || true - exit 1 - fi - - # Autotune include must be applied (shared_buffers value visible) - sb=$(docker exec "${container}" psql -U postgres -Atc \ - "SHOW shared_buffers;") - echo "shared_buffers for ${tag}: ${sb}" - - docker stop "${container}" >/dev/null - docker rm "${container}" >/dev/null - echo "::endgroup::" - done - echo "All smoke tests passed." - - - name: Log in to Docker Hub - if: steps.decide.outputs.should_push == 'true' - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PAT }} - - - name: Push to Docker Hub - if: steps.decide.outputs.should_push == 'true' - run: docker buildx bake --set "*.output=type=registry" --push - - scan: - name: Trivy scan (PG ${{ matrix.major }}) - needs: build - if: needs.build.outputs.should_push == 'true' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - major: ["16", "17", "18"] - permissions: - contents: read - security-events: write - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.35.0 - with: - image-ref: "iplweb/bpp_dbserver:psql-${{ matrix.major }}" - format: "sarif" - output: "trivy-results-${{ matrix.major }}.sarif" - severity: "CRITICAL,HIGH" - exit-code: "0" - ignore-unfixed: "true" - - - name: Upload Trivy SARIF to Security tab - if: always() - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: "trivy-results-${{ matrix.major }}.sarif" - category: "trivy-psql-${{ matrix.major }}" - - scan-main: - name: Trivy scan on main (PG ${{ matrix.major }}) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - major: ["16", "17", "18"] - permissions: - contents: read - security-events: write - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Run Trivy vulnerability scanner (informational) - uses: aquasecurity/trivy-action@0.35.0 - with: - image-ref: "iplweb/bpp_dbserver:psql-${{ matrix.major }}" - format: "sarif" - output: "trivy-results-${{ matrix.major }}.sarif" - severity: "CRITICAL,HIGH" - exit-code: "0" - ignore-unfixed: "true" - - - name: Upload Trivy SARIF to Security tab - if: always() - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: "trivy-results-${{ matrix.major }}.sarif" - category: "trivy-psql-${{ matrix.major }}" - - dockerhub-description: - name: Sync Docker Hub description - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: [test, lint] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Push DOCKERHUB.md to Docker Hub - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKERHUB_ADMIN_PAT }} - repository: iplweb/bpp_dbserver - short-description: "PostgreSQL 16/17/18 + plpython3u + ICU pl-PL + autotune dla BPP" - readme-filepath: ./DOCKERHUB.md - enable-url-completion: true - - scout-main: - name: Docker Scout on main (PG ${{ matrix.major }}) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - major: ["16", "17", "18"] - permissions: - contents: read - security-events: write - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Log in to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PAT }} - - - name: Docker Scout CVE scan (informational) - uses: docker/scout-action@v1 - with: - command: cves - image: "iplweb/bpp_dbserver:psql-${{ matrix.major }}" - sarif-file: "scout-results-${{ matrix.major }}.sarif" - only-severities: "critical,high" - exit-code: false - - - name: Upload Scout SARIF to Security tab - if: always() - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: "scout-results-${{ matrix.major }}.sarif" - category: "scout-psql-${{ matrix.major }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7f933a2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,86 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: autotune self-test (Python-free) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Run autotune self-test + run: sh autotune.sh --test + + lint: + name: pre-commit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: pre-commit/action@v3.0.1 + + smoke: + name: Smoke on stock postgres (PG ${{ matrix.major }}) + needs: [test, lint] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + major: ["16", "17", "18"] + steps: + - uses: actions/checkout@v6 + - name: Bind-mount autotune onto stock postgres:${{ matrix.major }} + env: + PG_MAJOR: ${{ matrix.major }} + run: | + set -Eeuo pipefail + C=smoke + docker run -d --name "$C" \ + --memory=2g \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + -e PGDATA=/var/lib/postgresql/data \ + -e POSTGRES_INITDB_ARGS="--locale-provider=icu --icu-locale=pl-PL" \ + -v "$PWD/docker-entrypoint-autotune.sh:/usr/local/bin/docker-entrypoint-autotune.sh:ro" \ + -v "$PWD/autotune.sh:/autotune.sh:ro" \ + --entrypoint bash \ + "postgres:${PG_MAJOR}" /usr/local/bin/docker-entrypoint-autotune.sh postgres + + # Wait up to 60s for readiness + ready=0 + for _ in $(seq 1 30); do + if docker exec "$C" pg_isready -U postgres >/dev/null 2>&1; then ready=1; break; fi + sleep 2 + done + if [ "$ready" -ne 1 ]; then + echo "FAIL: server not ready for postgres:${PG_MAJOR}" + docker logs "$C" || true + exit 1 + fi + + # autotune must have run: shared_buffers != stock 128MB default + sb=$(docker exec "$C" psql -U postgres -Atc "SHOW shared_buffers;") + echo "shared_buffers=${sb}" + if [ "$sb" = "128MB" ]; then + echo "FAIL: autotune did not apply for postgres:${PG_MAJOR}" + exit 1 + fi + + # Polish ICU collation works without an OS pl_PL locale + ok=$(docker exec "$C" psql -U postgres -Atc "SELECT 'ł' < 'm' COLLATE \"pl-x-icu\";") + if [ "$ok" != "t" ]; then + echo "FAIL: Polish ICU collation broken for postgres:${PG_MAJOR}" + exit 1 + fi + + docker rm -f "$C" >/dev/null + echo "Smoke OK for postgres:${PG_MAJOR}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7092be9..680de45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,11 +14,6 @@ repos: - id: ruff args: [--fix] - - repo: https://github.com/hadolint/hadolint - rev: v2.13.1-beta - hooks: - - id: hadolint-docker - - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.10.0.1 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index b9efc98..643a1fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,44 @@ Tagi Docker są niezależne: `psql-` i `psql-` z `docker-bake.hcl`. ## [Unreleased] +### Added + +- **`autotune.sh`** — port `autotune.py` na czysty shell + awk (bez Pythona), + identyczny bajt w bajt (zweryfikowane: wymuszony RAM, autodetekcja + host/cgroup, tryb unsafe, override locków). Self-test: `sh autotune.sh --test`. +- **Tryb „bez budowania obrazu" (Option B):** entrypoint i autotune można + podmontować (bind-mount) do oficjalnego obrazu `postgres` — nie trzeba + budować własnego obrazu. Polskie sortowanie zapewnia ICU, więc systemowy + locale `pl_PL` (ani `LANG`) nie jest potrzebny. Przykład: + `examples/docker-compose.yml`. + +### Changed + +- `docker-entrypoint-autotune.sh` jest teraz Python-free (woła `autotune.sh`) + i respektuje `$PGDATA` zamiast sztywnej ścieżki — dzięki czemu działa też + podmontowany na stockowym `postgres`. +- CI: self-test autotune to teraz `sh autotune.sh --test` (bez `setup-python`). + +### Removed + +- **Cały własny obraz i pipeline jego budowy.** Usunięto `Dockerfile`, + `docker-bake.hcl`, `DOCKERHUB.md` oraz zadania build/scan/push/scout/ + dockerhub w CI. Repo nie publikuje już obrazu `iplweb/bpp_dbserver` — + autotune + ICU pl-PL uruchamia się przez bind-mount na oficjalnym + `postgres` (patrz `examples/docker-compose.yml`). CI sprowadza się teraz + do self-testu, `pre-commit` i smoke testu na stockowym `postgres:16/17/18`. +- Wraz z obrazem znika systemowy locale `pl_PL` (generowany w buildzie) — + polskie sortowanie i tak zapewnia ICU. `plpython3u` oraz Python (`python3`/ + `python-is-python3`) usunięte już wcześniej; autotune jest w czystym shellu. + +### Fixed + +- Locale `pl_PL.UTF-8` jest teraz generowane **po** `apt-get upgrade` + (wcześniej przed instalacją pakietów). Na obrazie bazowym Debian trixie + `apt-get upgrade` podciąga nowy glibc/locales i regeneruje + `/usr/lib/locale/locale-archive`, kasując locale zbudowane wcześniej — + przez co `initdb` padał z `invalid locale settings`. + ## [v20260427] — 2026-04-27 ### Changed diff --git a/CLAUDE.md b/CLAUDE.md index 1daf67f..ae537a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,62 +4,58 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Overview -Docker image `iplweb/bpp_dbserver` — PostgreSQL base image for the BPP -application ([bpp.iplweb.pl](https://bpp.iplweb.pl)), with `plpython3u`, ICU -`pl-PL` collation, and a Python autotune script that generates a -`postgresql.conf` include at container start. +BPP's tuned PostgreSQL, run on the **stock `postgres` image** (no custom image +is built). Two files are bind-mounted into the container: a Python-free +autotune script and an entrypoint wrapper. Polish collation comes from ICU +(`POSTGRES_INITDB_ARGS`), so no OS `pl_PL` locale is needed. There is no +`plpython3u`, no Python, and no published Docker image. ## Commands -### Build (local) -- All PG majors in matrix: `docker buildx bake` -- Single target: `docker buildx bake dbserver-16-13` -- Print plan (no build): `docker buildx bake --print` -- Override a patch version: `docker buildx bake --set "dbserver-16-13.args.POSTGRES_VERSION=16.14"` - ### Autotune self-test -- `python autotune.py --test` — asserts deterministic config for fixed RAM sizes. +- `sh autotune.sh --test` — asserts deterministic config for fixed RAM sizes + (Python-free; `autotune.py` is kept only as the parity reference). -### Smoke test -See README "Smoke test" section (loop over `psql-16`, `psql-17`, `psql-18`). +### Run / smoke locally +- Bind-mount onto stock postgres (see `examples/docker-compose.yml`): + `docker compose -f examples/docker-compose.yml up` +- The equivalent `docker run` is documented in the README "Szybki start". ## Architecture -Three moving pieces: +Two moving pieces, both bind-mounted onto `postgres:`: -1. **`Dockerfile`** — `FROM postgres:${POSTGRES_VERSION}`, adds `plpython3u` - using `${PG_MAJOR}` from the base image (one Dockerfile covers all majors). - Healthcheck via `pg_isready`. -2. **`autotune.py`** — reads cgroup limit → `/proc/meminfo` → `POSTGRESQL_*` - env overrides; emits a pgtune-style config to stdout. Written to - `/postgresql_optimized.conf` at startup and included by `postgresql.conf` - via `include_if_exists`. -3. **`docker-entrypoint-autotune.sh`** — wraps upstream +1. **`autotune.sh`** — pure shell + awk (no Python). Reads cgroup limit → + `/proc/meminfo` → `POSTGRESQL_*` env overrides; emits a pgtune-style config + to stdout. Written to `/postgresql_optimized.conf` at startup and included + by `postgresql.conf` via `include_if_exists`. `autotune.py` is the original + and is kept as a byte-for-byte parity reference (verify via `sh autotune.sh + --test`; the two were diffed across forced/host/cgroup/unsafe/lock paths). +2. **`docker-entrypoint-autotune.sh`** — wraps upstream `docker-ensure-initdb.sh`, idempotently appends the `include_if_exists` - line to `postgresql.conf`, runs `autotune.py`, then chains to - `docker-entrypoint.sh`. - -Build matrix lives in `docker-bake.hcl` (variable `POSTGRES_VERSIONS`). CI -in `.github/workflows/build.yml` triggers on git tag `v*` and runs -`docker buildx bake` + Trivy per major. - -## Release flow + line to `$PGDATA/postgresql.conf`, runs `autotune.sh`, then chains to + `docker-entrypoint.sh`. Python-free and `$PGDATA`-aware so it works on the + stock image; pin `PGDATA=/var/lib/postgresql/data` (stock PG18+ defaults it + to `/var/lib/postgresql//docker`). -1. Bump `POSTGRES_VERSIONS` in `docker-bake.hcl` (verify against - [postgresql.org/support/versioning](https://www.postgresql.org/support/versioning/)). -2. Update `CHANGELOG.md`. -3. Git tag `v` (e.g. `v20260417`; suffix `.N` if >1/day). - Push → GH Actions builds matrix + pushes Docker tags + Trivy. +CI in `.github/workflows/ci.yml`: autotune self-test, `pre-commit` (ruff + +shellcheck), and a smoke job that bind-mounts onto `postgres:16/17/18` and +checks autotune applied + ICU Polish collation works. ## Non-obvious -- **No `:latest` Docker tag** — by design. Accidental major bump on an - existing `PGDATA` volume corrupts data, so the image is only published - under versioned tags (`psql-`, `psql-`). -- **Git tag format is independent of Docker tag format.** Git uses - `vYYYYMMDD`; Docker uses `psql-X.Y` / `psql-X` (from `docker-bake.hcl`). -- **Default `POSTGRES_HOST_AUTH_METHOD=trust`** in the image is dev/test +- **No custom image / no Docker Hub publish** — by design. Everything BPP needs + is env vars + two bind-mounted scripts; the only thing a custom build added + was the OS `pl_PL` locale (messages/number/date formatting), and Polish + *sorting* is handled by ICU regardless. +- **Polish collation = ICU, not the OS locale.** `--icu-locale=pl-PL` works on + the stock image with no `pl_PL` locale generated; dropping `LANG=pl_PL` is + intentional. +- **Default `POSTGRES_HOST_AUTH_METHOD=trust`** in examples is dev/test convenience. Production deployments MUST override to `scram-sha-256`. - `docker-entrypoint-autotune.sh` depends on upstream `docker-ensure-initdb.sh` - behavior — bumping the base `postgres` image may require re-checking - entrypoint compatibility. + / `docker-entrypoint.sh` behavior — bumping the base `postgres` major may + require re-checking entrypoint compatibility. +- `autotune.sh` delegates all floating-point math to `awk` (same IEEE-754 + doubles as Python) so unit normalization / `int()` truncation match + `autotune.py` exactly. Preserve that when editing. diff --git a/DOCKERHUB.md b/DOCKERHUB.md deleted file mode 100644 index 064480f..0000000 --- a/DOCKERHUB.md +++ /dev/null @@ -1,104 +0,0 @@ -

- Logo BPP -

- -

bpp-dbserver — PostgreSQL dla BPP

- -

- Wsparcie komercyjne zapewnia

- IPL Web -

- -

- Source on GitHub • - iplweb.pl • - bpp.iplweb.pl -

- ---- - -## Co to jest - -Obraz Dockera z PostgreSQL-em przygotowany pod aplikację **BPP — Bibliografia -Publikacji Pracowników** ([bpp.iplweb.pl](https://bpp.iplweb.pl)). Bazuje na -oficjalnym `postgres:*` z dołożonymi: - -- **`plpython3u`** — proceduralny Python dla PostgreSQL (wymagany przez funkcje BPP). -- **Locale `pl_PL.UTF-8`** + **ICU collation `pl-PL`** — poprawne sortowanie polskich znaków. -- **Autotune** — dynamiczna konfiguracja pamięci i parallelizmu PG na podstawie cgroup / `/proc/meminfo` / env vars. -- **Healthcheck** (`pg_isready`). -- **`apt-get upgrade`** przy buildzie — patche Debiana podciągnięte bez czekania na rebuild upstream. - -## Tagi - -**Brak tagu `:latest`** — celowo, żeby przypadkowy `docker pull` nie ściągnął -niekompatybilnej major wersji PG i nie uszkodził istniejącego `PGDATA`. - -| Tag | Przykład | Znaczenie | -|---|---|---| -| `psql-.` | `psql-16.13` | Pinning do konkretnej wersji patch. Zalecane dla produkcji. | -| `psql-` | `psql-16` | Rolling w ramach jednej major. OK dla dev/CI. | - -Aktualnie wspierane: **PG 16, 17, 18**. - -## Quickstart - -```bash -docker run --rm -d \ - --name bpp-db \ - -e POSTGRES_PASSWORD=secret \ - -p 5432:5432 \ - iplweb/bpp_dbserver:psql-16.13 -``` - -## docker compose - -```yaml -services: - db: - image: iplweb/bpp_dbserver:psql-16.13 - environment: - POSTGRES_PASSWORD: ${DB_PASSWORD:?required} - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 # NIE zostawiaj domyślnego 'trust' w produkcji - POSTGRESQL_RAM_THIS_MUCH_GB: 8192 # opcjonalnie - volumes: - - pgdata:/var/lib/postgresql/data - ports: - - "5432:5432" -volumes: - pgdata: -``` - -## Najważniejsze env vars - -| Zmienna | Default | Opis | -|---|---|---| -| `POSTGRES_PASSWORD` | — | Hasło superusera. **Wymagane** w produkcji. | -| `POSTGRES_HOST_AUTH_METHOD` | `trust` (!) | Dev only. W produkcji: `scram-sha-256`. | -| `POSTGRESQL_RAM_PERCENT` | `0.95` | Ułamek RAM dla Postgresa. | -| `POSTGRESQL_RAM_THIS_MUCH_GB` | auto-detect | Override rozmiaru RAM (MB). | -| `POSTGRESQL_MAX_LOCKS_PER_TRANSACTION` | 64 (PG default) | Podnoś dla pytest-xdist / Celery. | -| `POSTGRESQL_UNSAFE_BUT_FAST` | — | Dev only — wyłącza fsync/WAL/synchronous_commit. NIGDY w prod. | - -Pełna lista, szczegóły autotune i parametry build-time: -[github.com/iplweb/bpp-dbserver](https://github.com/iplweb/bpp-dbserver#parametry-konfiguracyjne). - -## ⚠ Upgrade major → major - -**Nie podmieniaj tagu** między major wersjami na tym samym volumenie. Plik -`PG_VERSION` w `PGDATA` trzyma major, a binaria nowej major odmówią startu -albo, co gorsza, uszkodzą dane. Użyj `pg_upgrade` albo `pg_dump` + `pg_restore`. - -## O projekcie - -BPP to otwartoźródłowy (MIT) system do katalogowania bibliografii publikacji -pracowników naukowych — dla bibliotek uniwersyteckich w Polsce. - -- 💻 **Kod dbservera**: [github.com/iplweb/bpp-dbserver](https://github.com/iplweb/bpp-dbserver) -- 📦 **Aplikacja BPP**: [github.com/iplweb/bpp](https://github.com/iplweb/bpp) -- 🌐 **Demo / instalacja**: [bpp.iplweb.pl](https://bpp.iplweb.pl) -- 💼 **Wsparcie komercyjne**: [iplweb.pl](https://www.iplweb.pl) - -## Licencja - -[MIT](https://github.com/iplweb/bpp-dbserver/blob/main/LICENSE) — Copyright © 2017–2026 Michał Pasternak <michal.dtz@gmail.com>. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 355f282..0000000 --- a/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -#check=skip=SecretsUsedInArgOrEnv -ARG POSTGRES_VERSION=16.13 -FROM postgres:${POSTGRES_VERSION} - -RUN localedef -i pl_PL -c -f UTF-8 -A /usr/share/locale/locale.alias pl_PL.UTF-8 - -ENV LANG=pl_PL.utf-8 -ENV POSTGRES_INITDB_ARGS="--locale-provider=icu --icu-locale=pl-PL" -ENV POSTGRES_HOST_AUTH_METHOD=trust - -# Wymuszenie klasycznego layoutu PGDATA. Upstream postgres:18+ zmienił default -# na /var/lib/postgresql//docker; my trzymamy /var/lib/postgresql/data -# dla spójności między majorami i kompatybilności z istniejącymi volume'ami -# oraz hardcoded ścieżkami w docker-entrypoint-autotune.sh. -ENV PGDATA=/var/lib/postgresql/data - -HEALTHCHECK --interval=5s --timeout=3s --retries=5 --start-period=10s \ - CMD pg_isready -U "${POSTGRES_USER:-postgres}" - -# PG_MAJOR is exported by the upstream postgres image so the plpython3 -# package picks the right major version automatically. -# hadolint ignore=DL3005,DL3008 -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-cache \ - --mount=type=cache,target=/var/lib/apt/lists,sharing=locked,id=apt-lists \ - apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends \ - "postgresql-plpython3-${PG_MAJOR}" \ - python-is-python3 - -COPY --chmod=755 autotune.py / -COPY --chmod=755 docker-entrypoint-autotune.sh /usr/local/bin/ - -ENTRYPOINT ["docker-entrypoint-autotune.sh"] - -CMD ["postgres"] diff --git a/README.md b/README.md index 965a7df..a906b17 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,10 @@ Logo BPP

-

bpp-dbserver — obraz PostgreSQL dla BPP

+

bpp-dbserver — autostrojony PostgreSQL dla BPP

- Build - Docker Pulls + CI License: MIT

@@ -23,84 +22,66 @@ ## O projekcie -Obraz Dockera z PostgreSQL-em dla aplikacji **BPP** (Bibliografia Publikacji -Pracowników). Bazuje na oficjalnym obrazie `postgres:${POSTGRES_VERSION}` -z dołożonymi: +Lekka konfiguracja PostgreSQL dla aplikacji **BPP** (Bibliografia Publikacji +Pracowników) — uruchamiana na **oficjalnym obrazie `postgres`**, bez budowania +własnego obrazu. Dwa pliki podmontowuje się (bind-mount) do kontenera: -- **`plpython3u`** — proceduralny język Python dla PostgreSQL (używany - przez funkcje BPP). -- **Locale `pl_PL.UTF-8`** + **ICU collation `pl-PL`** — poprawne - sortowanie polskich znaków (`initdb --locale-provider=icu`). -- **Autotune** (`/autotune.py`) — dynamiczna konfiguracja pamięci PG na - podstawie cgroup/proc, z pgtune jako podstawą. -- **Healthcheck** (`pg_isready`). +- **`autotune.sh`** — czysty skrypt **shell + awk** (bez Pythona). Na starcie + czyta limit pamięci z cgroup (limit kontenera) lub `/proc/meminfo` (host) + i generuje konfigurację w stylu pgtune (`shared_buffers`, `work_mem`, + parallelizm itd.), dołączaną przez `include_if_exists`. +- **`docker-entrypoint-autotune.sh`** — entrypoint, który inicjalizuje bazę + standardowo, uruchamia autotune i przekazuje sterowanie do upstreamowego + `docker-entrypoint.sh`. -Obraz jest publikowany jako `iplweb/bpp_dbserver:psql-` na Docker Hub. +**Polskie sortowanie** zapewnia **ICU** (`--locale-provider=icu --icu-locale=pl-PL`) +ustawiane przez zmienną `POSTGRES_INITDB_ARGS` — więc **nie jest potrzebny** +systemowy locale `pl_PL` (ani `LANG`). Brak własnego obrazu = brak `plpython3u`, +brak Pythona, brak osobnego cyklu wydawniczego obrazu. ---- - -## Tagi (Docker Hub) +> **Dlaczego nie własny obraz?** Wszystko, czego potrzebuje BPP, da się ustawić +> zmiennymi środowiskowymi (ICU, `PGDATA`, auth) plus podmontowanym autotune. +> Jedyne, co dawał własny build, to systemowy locale `pl_PL` (komunikaty / +> formatowanie liczb i dat po polsku) — sortowanie i tak załatwia ICU. -Obraz publikowany pod **wyłącznie wersjonowanymi** tagami. **Nie istnieje -tag `:latest`** — celowo, żeby przypadkowy `docker pull` nie ściągnął -niekompatybilnej major wersji PG i nie uszkodził istniejącego `PGDATA`. - -| Tag | Przykład | Znaczenie | -|---|---|---| -| `psql-.` | `psql-16.13` | Pinning do konkretnej wersji patch. Zalecane dla produkcji. | -| `psql-` | `psql-16` | Rolling w ramach jednej major. OK dla dev/CI, w produkcji pinuj patch. | +--- -Aktualnie wspierane major wersje: **16, 17, 18**. +## Szybki start -**Upgrade między major wersjami** (np. `psql-16` → `psql-17`) **wymaga** -pg_upgrade albo pg_dump/restore na volumenie `PGDATA`. Nie podmieniaj -tagu w locie na tym samym volumenie — stracisz dane. +### docker compose ---- +Gotowy przykład: [`examples/docker-compose.yml`](examples/docker-compose.yml): -## Użycie +```bash +docker compose -f examples/docker-compose.yml up +``` -### docker run (smoke test) +### docker run ```bash -docker run --rm -d \ - --name bpp-db \ - -e POSTGRES_PASSWORD=secret \ +docker run --rm -d --name bpp-db \ + --memory=2g \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + -e PGDATA=/var/lib/postgresql/data \ + -e POSTGRES_INITDB_ARGS="--locale-provider=icu --icu-locale=pl-PL" \ + -v "$PWD/docker-entrypoint-autotune.sh:/usr/local/bin/docker-entrypoint-autotune.sh:ro" \ + -v "$PWD/autotune.sh:/autotune.sh:ro" \ -p 5432:5432 \ - iplweb/bpp_dbserver:psql-16.13 + --entrypoint bash \ + postgres:18 /usr/local/bin/docker-entrypoint-autotune.sh postgres ``` -### docker compose (aplikacja BPP) - -```yaml -services: - db: - image: iplweb/bpp_dbserver:psql-16.13 - environment: - POSTGRES_PASSWORD: ${DB_PASSWORD:?required} - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 # NIE zostawiaj domyślnego 'trust' w produkcji - POSTGRESQL_RAM_THIS_MUCH_GB: 8192 # opcjonalnie: wymusza ilość RAM dla PG - volumes: - - pgdata:/var/lib/postgresql/data - ports: - - "5432:5432" -volumes: - pgdata: -``` +Autotune skaluje konfigurację do limitu `--memory` / `mem_limit` kontenera. +Wspierane major wersje PostgreSQL: **16, 17, 18** (dowolny tag `postgres:*`). + +> **`POSTGRES_HOST_AUTH_METHOD=trust`** w przykładach to wygoda dev/test — +> kontener ufa wszystkim połączeniom **bez hasła**. W produkcji ustaw +> `-e POSTGRES_PASSWORD=… -e POSTGRES_HOST_AUTH_METHOD=scram-sha-256`. --- ## Parametry konfiguracyjne -### Build-time (ARG) - -Zmienne ustawiane na etapie builda (przez `docker buildx bake` lub -`--build-arg`): - -| Zmienna | Default | Opis | -|---|---|---| -| `POSTGRES_VERSION` | `16.13` (ARG w Dockerfile) / `["16.13","17.9","18.3"]` (matrix w bake) | Wersja bazowego obrazu `postgres:*`. `Dockerfile` używa `${PG_MAJOR}` eksportowanej z base image, więc instalowany jest `postgresql-plpython3-${PG_MAJOR}` — jeden Dockerfile pokrywa każdą major. | - ### Runtime — standardowe Postgres | Zmienna | Default | Opis | @@ -108,12 +89,11 @@ Zmienne ustawiane na etapie builda (przez `docker buildx bake` lub | `POSTGRES_PASSWORD` | — | Hasło superusera. **Wymagane**, chyba że `POSTGRES_HOST_AUTH_METHOD=trust` (dev only). | | `POSTGRES_USER` | `postgres` | Nazwa superusera. | | `POSTGRES_DB` | `$POSTGRES_USER` | Domyślna baza tworzona na starcie. | -| `POSTGRES_HOST_AUTH_METHOD` | **`trust`** (!) | Ten obraz domyślnie **ufa wszystkim połączeniom** dla wygody dev/test. W produkcji **MUSI** być nadpisane na `scram-sha-256` lub `md5`. | -| `POSTGRES_INITDB_ARGS` | `--locale-provider=icu --icu-locale=pl-PL` | Argumenty dla `initdb`. Domyślnie ICU pl-PL (sortowanie polskie). | -| `LANG` | `pl_PL.utf-8` | Locale procesu Postgresa (komunikaty, format daty). | -| `PGDATA` | `/var/lib/postgresql/data` | Ścieżka do katalogu danych (dziedziczona z base image). | +| `POSTGRES_HOST_AUTH_METHOD` | — (ustaw `trust` dla dev) | W produkcji **MUSI** być `scram-sha-256` lub `md5`. | +| `POSTGRES_INITDB_ARGS` | — (ustaw `--locale-provider=icu --icu-locale=pl-PL`) | Argumenty dla `initdb`; ICU pl-PL daje polskie sortowanie bez systemowego locale. | +| `PGDATA` | `/var/lib/postgresql/data` (zalecane) | Katalog danych. Pinujemy klasyczny układ; stock `postgres:18+` domyślnie użyłby `/var/lib/postgresql/18/docker`. | -### Runtime — autotune.py (**unikalne dla tego obrazu**) +### Runtime — autotune Wszystkie opcjonalne — brak = auto-detekcja z cgroup (limit kontenera) lub `/proc/meminfo` (host). @@ -145,81 +125,45 @@ Autotune ustawia na podstawie RAM i liczby CPU (nie są to env vars): - `max_parallel_workers` = nproc - `max_parallel_maintenance_workers` = jak gather -Formuły pochodzą z pgtune z drobnymi modyfikacjami — szczegóły w -komentarzach w `autotune.py`. +Formuły pochodzą z pgtune z drobnymi modyfikacjami — szczegóły w komentarzach +w `autotune.sh`. Skrypt jest portem oryginalnego `autotune.py` (zachowanego +jako referencja) i daje **identyczny bajt w bajt** wynik. ### Volumes / persistence -- `PGDATA` (`/var/lib/postgresql/data`) — zamountuj volume, inaczej dane - zostaną utracone przy restarcie kontenera. -- **Nie podmieniaj tagu między major wersjami** na tym samym volumenie - — plik `PG_VERSION` w `PGDATA` trzyma major, a binaria Postgresa - z nowej major mogą odmówić startu lub, co gorsza, uszkodzić dane. -- Upgrade major → major: użyj `pg_upgrade` (z dwoma podmapowanymi - kontenerami) albo `pg_dump` + `pg_restore`. +- Zamountuj volume na `PGDATA` (`/var/lib/postgresql/data`), inaczej dane + zostaną utracone przy `docker rm`. +- **Nie podpinaj tego samego volumenu pod inną major wersję** `postgres` + (np. `postgres:16` → `postgres:17`) — plik `PG_VERSION` w `PGDATA` trzyma + major, a binaria z nowej major odmówią startu lub uszkodzą dane. +- Upgrade major → major: `pg_upgrade` (dwa podmapowane kontenery) albo + `pg_dump` + `pg_restore`. --- -## Build lokalny - -```bash -# Wszystkie major wersje z matrixa (docker-bake.hcl) -docker buildx bake --print # plan -docker buildx bake # build lokalny (load do dockerd) +## Self-test -# Konkretna major: -docker buildx bake dbserver-16-13 - -# Override wersji patch: -docker buildx bake --set "dbserver-16-13.args.POSTGRES_VERSION=16.14" -``` - -Build w GH Actions (trigger: tag `v*`) publikuje wszystkie 6 tagów -(`psql-16.13`, `psql-16`, `psql-17.9`, `psql-17`, `psql-18.3`, `psql-18`) -do Docker Hub i skanuje każdą major wersję Trivy. - ---- - -## Smoke test (po zbudowaniu / pull) +`autotune.sh` ma wbudowany test deterministycznych wartości: ```bash -for tag in psql-16 psql-17 psql-18; do - docker run --rm -d --name smoke_$tag \ - -e POSTGRES_PASSWORD=x iplweb/bpp_dbserver:$tag - sleep 5 - docker exec smoke_$tag psql -U postgres -c "CREATE EXTENSION plpython3u;" - docker exec smoke_$tag psql -U postgres -c "SHOW lc_collate;" - docker stop smoke_$tag && docker rm smoke_$tag -done +sh autotune.sh --test # -> OK ``` ---- - -## Release flow - -1. Update `POSTGRES_VERSIONS` w `docker-bake.hcl` do aktualnych patchy - (np. po release PostgreSQL — patrz - [postgresql.org/support/versioning](https://www.postgresql.org/support/versioning/)). -2. Update `CHANGELOG.md` — nota o zmianie patchy. -3. Tag git kalendarzowy: `git tag v` (np. `v20260417`); gdy - w tym samym dniu jest więcej niż jedno wydanie, kolejne dostają sufiks - `.N` (`v20260417.1`, `v20260417.2`, ...). Tag git jest **niezależny** od - tagów Docker (`psql-`, `psql-`) — te drugie wynikają z - `docker-bake.hcl`. Push tagu → GH Actions builduje i publikuje. -4. Trivy scan musi przejść (critical/high severity = fail). +CI (`.github/workflows/ci.yml`) odpala self-test, `pre-commit` (ruff + +shellcheck) oraz smoke test: podmontowuje autotune na stockowy +`postgres:16/17/18` i weryfikuje, że konfiguracja jest strojona i działa +polskie sortowanie ICU. --- ## Historia -Wcześniej `docker/dbserver/` żył w monorepo -[`iplweb/bpp`](https://github.com/iplweb/bpp) i publikował obraz -`iplweb/bpp_dbserver:latest` razem z appserverem przy każdym release BPP. -Wydzielenie do osobnego repo (2026-04) daje: - -- niezależny release cycle (bump Postgresa nie wymaga release'u BPP), -- matrix build dla wielu major wersji na raz, -- eliminację tagu `:latest` — bezpieczny pinning po stronie konsumentów. +Wcześniej to repo budowało i publikowało własny obraz +`iplweb/bpp_dbserver:psql-` na Docker Hub (z `plpython3u`, później bez). +Okazało się jednak, że całą wartość (autotune + ICU pl-PL) da się dostarczyć +**bez własnego obrazu** — przez podmontowanie skryptów do oficjalnego +`postgres`. Usunięto więc `Dockerfile`, `docker-bake.hcl` oraz pipeline +budowania/publikacji obrazu; autotune przepisano z Pythona na czysty shell. ## O BPP diff --git a/autotune.sh b/autotune.sh new file mode 100755 index 0000000..88853c9 --- /dev/null +++ b/autotune.sh @@ -0,0 +1,272 @@ +#!/bin/sh +# Shell port of autotune.py — generates a pgtune-style postgresql.conf include +# on stdout (cgroup/meminfo aware), with NO Python dependency. +# +# This lets the autotune step run on the *stock* `postgres` image via a +# bind-mounted entrypoint (no custom image / no python3 install needed). +# +# It is a faithful port of autotune.py: output is verified byte-identical +# (see `./autotune.sh --test` and the parity check in CI). Floating-point math +# is delegated to awk, which uses the same IEEE-754 doubles as Python, so +# truncation (int()) and unit normalization match exactly. +# +# Quirks deliberately preserved from autotune.py: +# * POSTGRESQL_RAM_THIS_MUCH_GB is actually treated as MB (as in the original) +# * the "tweak it orremove it" typo and POSTGRESQL_RAM_THIS_MUCH_DB typo in +# the forced-RAM comment +# * int(POSTGRESQL_RAM_PERCENT * 100) for the "%" shown in comments (0.95 -> 94) +set -eu + +ONE_GB_IN_KB=1048576 # 1024 * 1024 + +# --- environment (same names / defaults as autotune.py) ----------------------- +RAM_PERCENT="${POSTGRESQL_RAM_PERCENT:-0.95}" +FORCE_RAM="${POSTGRESQL_RAM_THIS_MUCH_GB:-}" +DEFAULT_RAM="${POSTGRESQL_DEFAULT_RAM:-4096}" +UNSAFE_RAW="${POSTGRESQL_UNSAFE_BUT_FAST:-}" +MAX_LOCKS="${POSTGRESQL_MAX_LOCKS_PER_TRANSACTION:-}" +MAX_PRED_LOCKS="${POSTGRESQL_MAX_PRED_LOCKS_PER_TRANSACTION:-}" + +# unsafe flag: POSTGRESQL_UNSAFE_BUT_FAST.lower() in (1, true, yes) +UNSAFE=0 +case "$(printf '%s' "$UNSAFE_RAW" | tr '[:upper:]' '[:lower:]')" in + 1 | true | yes) UNSAFE=1 ;; +esac + +# --- RAM / CPU detection ------------------------------------------------------ + +# /proc/meminfo always shows host RAM, not the container limit — the limit lives +# in the cgroup. Echoes the limit in kB, or nothing when there is no finite limit. +cgroup_limit_kb() { + if [ -f /sys/fs/cgroup/memory.max ]; then + cg_raw=$(tr -d '[:space:]' < /sys/fs/cgroup/memory.max 2>/dev/null || true) + if [ -n "$cg_raw" ] && [ "$cg_raw" != "max" ]; then + awk -v r="$cg_raw" 'BEGIN { printf "%d", int(r / 1024) }' + return 0 + fi + fi + if [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then + cg_v=$(tr -d '[:space:]' < /sys/fs/cgroup/memory/memory.limit_in_bytes 2>/dev/null || true) + # cgroup v1 returns a huge sentinel value when no limit is set (>= 1<<62) + if [ -n "$cg_v" ]; then + awk -v v="$cg_v" 'BEGIN { if (v + 0 < 4611686018427387904) printf "%d", int(v / 1024) }' + fi + fi +} + +host_ram_kb() { + [ -f /proc/meminfo ] || return 0 + awk '/^MemTotal:/ { print $2; exit }' /proc/meminfo +} + +detect_nproc() { + # Match Python's multiprocessing.cpu_count() == os.cpu_count() == online CPUs. + np=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo "") + case "$np" in '' | *[!0-9]*) np=$(nproc 2>/dev/null || echo "") ;; esac + case "$np" in '' | *[!0-9]*) np=1 ;; esac + printf '%s' "$np" +} + +# Sets globals RAM_KB (full-precision double as %.17g) and RAM_COMMENT. +resolve_ram_kb() { + if [ -n "$FORCE_RAM" ]; then + RAM_COMMENT="# autotune.py: RAM size for Postgres is ${FORCE_RAM} MB, because of env var POSTGRESQL_RAM_THIS_MUCH_DB setting. Change the env var to tweak it orremove it to use automatic RAM size detection." + RAM_KB=$(awk -v f="$FORCE_RAM" 'BEGIN { printf "%.17g", int(f) * 1024 }') + return 0 + fi + + host_kb=$(host_ram_kb) + cg_kb=$(cgroup_limit_kb) + pct100=$(awk -v p="$RAM_PERCENT" 'BEGIN { printf "%d", int(p * 100) }') + + if [ -n "$cg_kb" ] && { [ -z "$host_kb" ] || [ "$cg_kb" -lt "$host_kb" ]; }; then + RAM_COMMENT="# autotune.py: detected cgroup memory limit ${cg_kb} kB (host RAM: ${host_kb:-None} kB); will use ${pct100} % of the cgroup limit" + RAM_KB=$(awk -v p="$RAM_PERCENT" -v c="$cg_kb" 'BEGIN { printf "%.17g", (p * c / 1024) * 1024 }') + return 0 + fi + + if [ -n "$host_kb" ]; then + RAM_COMMENT="# autotune.py: detected ${host_kb} kB RAM; will use ${pct100} % of it" + RAM_KB=$(awk -v p="$RAM_PERCENT" -v h="$host_kb" 'BEGIN { printf "%.17g", (p * h / 1024) * 1024 }') + return 0 + fi + + RAM_COMMENT="# autotune.py: unable to detect RAM size, returning default ${DEFAULT_RAM} MB; change environment variable POSTGRESQL_DEFAULT_RAM if you need to change this" + RAM_KB=$(awk -v d="$DEFAULT_RAM" 'BEGIN { printf "%.17g", d * 1024 }') +} + +# --- config generation -------------------------------------------------------- + +# Emits unsorted "key = value" config lines for a given RAM size in kB ($1). +# Reads globals: NPROC, MAX_PARALLEL, CFG_UNSAFE, MAX_LOCKS, MAX_PRED_LOCKS. +generate_config_lines() { + awk -v ramkb="$1" -v nproc="$NPROC" -v max_parallel="$MAX_PARALLEL" \ + -v unsafe="$CFG_UNSAFE" -v maxlocks="$MAX_LOCKS" -v maxpredlocks="$MAX_PRED_LOCKS" ' + function normsize(x, iv) { + iv = int(x) + if (iv % GB == 0) return sprintf("%dGB", iv / GB) + if (iv % MB == 0) return sprintf("%dMB", iv / MB) + return sprintf("%dkB", iv) + } + BEGIN { + GB = 1048576; MB = 1024 + + # Directly from pgtune + printf "shared_buffers = %s\n", normsize(ramkb / 4) + printf "effective_cache_size = %s\n", normsize(ramkb * 3 / 4) + mwm = ramkb / 16; if (mwm > 2 * GB) mwm = 2 * GB + printf "maintenance_work_mem = %s\n", normsize(mwm) + + # 100 connections per 1 GB RAM, capped at 250 + conns = 100 * ramkb / GB; if (conns > 250) conns = 250 + printf "max_connections = %d\n", int(conns) + printf "work_mem = %s\n", normsize((ramkb * 3 / 4) / (conns * 3) / max_parallel) + + printf "min_wal_size = %s\n", normsize(GB) + printf "max_wal_size = %s\n", normsize(4 * GB) + + wb = ramkb * 3 / 4 / 100; if (wb > 16 * MB) wb = 16 * MB + printf "wal_buffers = %s\n", normsize(wb) + + print "checkpoint_completion_target = 0.7" + print "default_statistics_target = 100" + + printf "max_worker_processes = %d\n", nproc + printf "max_parallel_workers_per_gather = %d\n", max_parallel + printf "max_parallel_workers = %d\n", nproc + printf "max_parallel_maintenance_workers = %d\n", max_parallel + + if (maxlocks != "") printf "max_locks_per_transaction = %d\n", maxlocks + 0 + if (maxpredlocks != "") printf "max_pred_locks_per_transaction = %d\n", maxpredlocks + 0 + + if (unsafe) { + print "fsync = off" + print "full_page_writes = off" + print "synchronous_commit = off" + print "wal_level = minimal" + print "max_wal_senders = 0" + print "archive_mode = off" + print "wal_writer_delay = 10000ms" + print "commit_delay = 100000" + print "random_page_cost = 1.1" + print "effective_io_concurrency = 200" + } + } + ' +} + +# max_parallel mirrors autotune.py: 1 (<4 cpus), 2 (>=4), 3 (5-6), 4 (>=7) +compute_max_parallel() { + MAX_PARALLEL=1 + if [ "$NPROC" -ge 4 ]; then + MAX_PARALLEL=2 + if [ "$NPROC" -ge 5 ] && [ "$NPROC" -le 6 ]; then MAX_PARALLEL=3; fi + if [ "$NPROC" -ge 7 ]; then MAX_PARALLEL=4; fi + fi +} + +main() { + resolve_ram_kb + NPROC=$(detect_nproc) + compute_max_parallel + CFG_UNSAFE="$UNSAFE" + + printf '%s\n' "$RAM_COMMENT" + printf '# Automatically added by autotune.py\n' + if [ "$UNSAFE" -eq 1 ]; then + cat <<'EOF' +# +# *** UWAGA! TRYB POSTGRESQL_UNSAFE_BUT_FAST JEST WŁĄCZONY! *** +# *** fsync, full_page_writes, synchronous_commit WYŁĄCZONE *** +# *** wal_level=minimal, max_wal_senders=0, archive_mode=off *** +# *** DANE MOGĄ ZOSTAĆ UTRACONE! NIE UŻYWAJ W PRODUKCJI! *** +# +EOF + fi + generate_config_lines "$RAM_KB" | LC_ALL=C sort +} + +# --- self-test (mirrors autotune.py test()) ----------------------------------- + +TEST_FAIL=0 + +assert_value() { + # $1=size kB $2=key $3=expected ; reads $TEST_CFG + av_got=$(printf '%s\n' "$TEST_CFG" | sed -n "s/^$2 = //p") + if [ "$av_got" != "$3" ]; then + printf 'FAIL: Postgres at %s kB: %s differs:\n Got: %s\n Expected: %s\n' \ + "$1" "$2" "$av_got" "$3" >&2 + TEST_FAIL=1 + fi +} + +assert_present() { + # $1=size kB $2=key ; reads $TEST_CFG + if ! printf '%s\n' "$TEST_CFG" | grep -q "^$2 = "; then + printf 'FAIL: Postgres at %s kB: missing key %s\n' "$1" "$2" >&2 + TEST_FAIL=1 + fi +} + +run_tests() { + NPROC=$(detect_nproc) + compute_max_parallel + CFG_UNSAFE=0 + # locks must not leak into the deterministic cases + saved_locks="$MAX_LOCKS"; saved_pred="$MAX_PRED_LOCKS" + MAX_LOCKS=""; MAX_PRED_LOCKS="" + + # RAM-dependent (deterministic) values. work_mem and CPU-dependent keys depend + # on cpu count / max_connections, so we only check their presence (as in py). + for tc in \ + "524288|shared_buffers=128MB effective_cache_size=384MB maintenance_work_mem=32MB min_wal_size=1GB max_wal_size=4GB checkpoint_completion_target=0.7 wal_buffers=3932kB default_statistics_target=100 max_connections=50" \ + "1048576|shared_buffers=256MB effective_cache_size=768MB maintenance_work_mem=64MB min_wal_size=1GB max_wal_size=4GB checkpoint_completion_target=0.7 wal_buffers=7864kB default_statistics_target=100 max_connections=100" \ + "2097152|shared_buffers=512MB effective_cache_size=1536MB maintenance_work_mem=128MB min_wal_size=1GB max_wal_size=4GB checkpoint_completion_target=0.7 wal_buffers=15728kB default_statistics_target=100 max_connections=200" \ + "4194304|shared_buffers=1GB effective_cache_size=3GB maintenance_work_mem=256MB min_wal_size=1GB max_wal_size=4GB checkpoint_completion_target=0.7 wal_buffers=16MB default_statistics_target=100 max_connections=250"; do + size=${tc%%|*} + pairs=${tc#*|} + TEST_CFG=$(generate_config_lines "$size") + for p in $pairs; do + assert_value "$size" "${p%%=*}" "${p#*=}" + done + for k in max_worker_processes max_parallel_workers_per_gather \ + max_parallel_workers max_parallel_maintenance_workers work_mem; do + assert_present "$size" "$k" + done + done + + MAX_LOCKS="$saved_locks"; MAX_PRED_LOCKS="$saved_pred" + + # unsafe-mode test (only when the env flag is on, as in autotune.py) + if [ "$UNSAFE" -eq 1 ]; then + CFG_UNSAFE=1 + TEST_CFG=$(generate_config_lines "$ONE_GB_IN_KB") + for k in fsync full_page_writes synchronous_commit archive_mode; do + assert_value "$ONE_GB_IN_KB" "$k" "off" + done + assert_value "$ONE_GB_IN_KB" wal_level minimal + assert_value "$ONE_GB_IN_KB" max_wal_senders 0 + assert_value "$ONE_GB_IN_KB" random_page_cost 1.1 + assert_value "$ONE_GB_IN_KB" effective_io_concurrency 200 + assert_present "$ONE_GB_IN_KB" wal_writer_delay + assert_present "$ONE_GB_IN_KB" commit_delay + fi + + if [ "$TEST_FAIL" -ne 0 ]; then + exit 1 + fi + printf 'OK\n' >&2 +} + +usage() { + printf 'Usage: %s [--test]\n' "$1" >&2 +} + +# --- entrypoint --------------------------------------------------------------- +if [ "$#" -eq 0 ]; then + main +elif [ "$#" -eq 1 ] && [ "$1" = "--test" ]; then + run_tests +else + usage "$0" +fi diff --git a/docker-bake.hcl b/docker-bake.hcl deleted file mode 100644 index 4b99847..0000000 --- a/docker-bake.hcl +++ /dev/null @@ -1,56 +0,0 @@ -# docker-bake.hcl - Matrix build dla iplweb/bpp_dbserver -# -# Buduje obraz bpp_dbserver dla wszystkich wspieranych major wersji -# PostgreSQL jednym bake'em. NIE publikuje tagu :latest — przypadkowy -# pull mogłby ściągnąć niekompatybilną major i uszkodzić PGDATA. -# -# Lokalny build (wszystkie wersje równolegle): -# docker buildx bake -# -# Build konkretnej wersji: -# docker buildx bake dbserver-16-6 -# -# Build i push do registry: -# docker buildx bake --push -# -# Override wersji: -# docker buildx bake --set dbserver.args.POSTGRES_VERSION=16.6 - -variable "POSTGRES_VERSIONS" { - # Najnowsze stabilne patche w każdej wspieranej major. Aktualizować - # przy release'ach PostgreSQL (patrz https://www.postgresql.org/support/versioning/). - default = ["16.13", "17.9", "18.3"] -} - -variable "PLATFORM" { - # Multi-arch: x86_64 (linux/amd64) + Apple Silicon / ARM (linux/arm64). - # Override np. dla lokalnego buildu single-arch: - # docker buildx bake --set "*.platform=linux/amd64" - default = "linux/amd64,linux/arm64" -} - -variable "PUSH" { - default = false -} - -group "default" { - targets = ["dbserver"] -} - -target "dbserver" { - name = "dbserver-${replace(pg, ".", "-")}" - matrix = { - pg = POSTGRES_VERSIONS - } - dockerfile = "Dockerfile" - context = "." - args = { - POSTGRES_VERSION = pg - } - tags = [ - "iplweb/bpp_dbserver:psql-${pg}", - "iplweb/bpp_dbserver:psql-${split(".", pg)[0]}" - ] - platforms = split(",", PLATFORM) - output = PUSH ? ["type=registry"] : ["type=docker"] -} diff --git a/docker-entrypoint-autotune.sh b/docker-entrypoint-autotune.sh index e751f39..16a9249 100644 --- a/docker-entrypoint-autotune.sh +++ b/docker-entrypoint-autotune.sh @@ -1,15 +1,31 @@ -#!/bin/bash +#!/usr/bin/env bash set -Eeuo pipefail +# Autotune entrypoint wrapper. Python-free — it shells out to autotune.sh, so it +# runs equally well INSIDE a custom image or BIND-MOUNTED onto the stock +# `postgres` image (no build, no python3). See examples/docker-compose.yml. + +# Honour whatever PGDATA the image uses; default to the classic layout. Stock +# postgres:18+ defaults PGDATA to /var/lib/postgresql/18/docker, so we pin it +# and export it, both for our own appends below and for the upstream scripts. +: "${PGDATA:=/var/lib/postgresql/data}" +export PGDATA + +# Path to the (Python-free) autotune script. Bind-mounted to /autotune.sh by +# default; override with AUTOTUNE_SCRIPT when mounting it elsewhere. +AUTOTUNE_SCRIPT="${AUTOTUNE_SCRIPT:-/autotune.sh}" + # Zainicjuj bazę danych standardowo (standardowo dla tego obrazu) /usr/local/bin/docker-ensure-initdb.sh -# Jeżeli postgresql.conf nie zawiera linii "include_if_exists = /postgresql_optimized.conf" to dopisz ją -# na końcu pliku: -grep -qxF "include_if_exists = '/postgresql_optimized.conf'" /var/lib/postgresql/data/postgresql.conf || echo "include_if_exists = '/postgresql_optimized.conf'" >> /var/lib/postgresql/data/postgresql.conf +# Jeżeli postgresql.conf nie zawiera linii "include_if_exists = /postgresql_optimized.conf" +# to dopisz ją na końcu pliku (idempotentnie): +conf="${PGDATA}/postgresql.conf" +grep -qxF "include_if_exists = '/postgresql_optimized.conf'" "$conf" \ + || echo "include_if_exists = '/postgresql_optimized.conf'" >> "$conf" -# Wygeneruj /postgresql_optimized.conf -python /autotune.py > /postgresql_optimized.conf +# Wygeneruj /postgresql_optimized.conf (bez Pythona — czysty shell + awk) +sh "$AUTOTUNE_SCRIPT" > /postgresql_optimized.conf # Na tym etapie NIE ma potrzeby restartu serwera PostgreSQL, ponieważ zatrzymała go procedura # stop_tempserver z docker-ensure-initdb/docker-entrypoint. Zatem, wystartuj wszystko normalnie diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml new file mode 100644 index 0000000..3104af4 --- /dev/null +++ b/examples/docker-compose.yml @@ -0,0 +1,31 @@ +# Option B: run BPP's tuned PostgreSQL on the STOCK `postgres` image — no custom +# image / no build. The Python-free autotune entrypoint and script are bind-mounted. +# +# docker compose -f examples/docker-compose.yml up +# +# Polish collation comes from ICU (POSTGRES_INITDB_ARGS below), so no OS pl_PL +# locale is required. autotune sizes shared_buffers/work_mem/etc. to mem_limit. +services: + db: + image: postgres:18 + # Bind-mounted, Python-free entrypoint replaces the stock one. + entrypoint: ["bash", "/usr/local/bin/docker-entrypoint-autotune.sh"] + command: ["postgres"] + environment: + # dev/test convenience — set a password + scram-sha-256 in production. + POSTGRES_HOST_AUTH_METHOD: "trust" + # Pin the classic data layout (stock PG18 would use /var/lib/postgresql/18/docker). + PGDATA: "/var/lib/postgresql/data" + # Polish sorting via ICU — works without an OS pl_PL locale. + POSTGRES_INITDB_ARGS: "--locale-provider=icu --icu-locale=pl-PL" + volumes: + - "../docker-entrypoint-autotune.sh:/usr/local/bin/docker-entrypoint-autotune.sh:ro" + - "../autotune.sh:/autotune.sh:ro" + - "pgdata:/var/lib/postgresql/data" + ports: + - "5432:5432" + # autotune reads the container's cgroup memory limit; this caps it at 2 GiB. + mem_limit: 2g + +volumes: + pgdata: