diff --git a/README.md b/README.md index 966444a..25bc016 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,10 @@ the whole round-trip end to end. - **Kubernetes** — runs as a CronJob / Deployment; verified end-to-end against CloudNativePG. - Content-addressed **deduplication** (FastCDC, page-aligned splits) - — no incremental chains to break. + — every base backup is a self-contained full, deduplicated against + what's already in the repo, so storage stays incremental-sized + without a mandatory chain. PG 17+ incremental backups + (`backup --incremental-from`) are supported too when you want them. - **AES-256-GCM** envelope encryption; a FIPS / BoringCrypto build variant is available. - **4 Tier-1 KMS providers** — AWS KMS · GCP KMS · Azure Key Vault · @@ -305,8 +308,9 @@ build. Or via the canonical Makefile: ```sh -make # build bin/pg_hardstorage + bin/pg_hardstorage_testkit -make build-simple # the interactive quick-start helper +make build # build bin/pg_hardstorage (bare `make` prints the help menu) +make all-binaries # build + testkit + interactive helper +make build-simple # just the interactive quick-start helper make build-compat # the pgBackRest / Barman / WAL-G shims make build-fips # FIPS / BoringCrypto variant make build-firecracker # microVM verifier-sandbox variant diff --git a/docs/CONTRIBUTING-DOCS.md b/docs/CONTRIBUTING-DOCS.md index 9f941df..e2941d3 100644 --- a/docs/CONTRIBUTING-DOCS.md +++ b/docs/CONTRIBUTING-DOCS.md @@ -141,7 +141,7 @@ can't bit-rot: ````markdown ```bash # RUNNABLE -pg_hardstorage init --pg-connection postgres://… --repo fs:///tmp/hsr --yes +pg_hardstorage init --pg-connection postgres://… --repo file:///tmp/hsr --yes ``` ```` diff --git a/docs/compliance/audit-evidence-bundles.md b/docs/compliance/audit-evidence-bundles.md index 416287f..fdfa8ad 100644 --- a/docs/compliance/audit-evidence-bundles.md +++ b/docs/compliance/audit-evidence-bundles.md @@ -35,7 +35,7 @@ pg_hardstorage audit export-bundle \ --until 2026-05-01T00:00:00Z \ --include-anchors \ --operator "ops@acme" \ - -o ./acme-april-2026.tar.gz + --out ./acme-april-2026.tar.gz ``` ### Verify diff --git a/docs/compliance/data-residency-pinning.md b/docs/compliance/data-residency-pinning.md index 9152f10..50adf4f 100644 --- a/docs/compliance/data-residency-pinning.md +++ b/docs/compliance/data-residency-pinning.md @@ -126,12 +126,17 @@ For v0.1, the safety is layered: - An operator who configures a violating repo is responsible for re-checking after configuration changes. -For v1.0+ the planned enforcement is: +What ships today is the read-only `residency check` +command: `pg_hardstorage residency check ` +reports `verify.residency_violation` (exit 9) when the +configured repo's region is outside the policy. It does not +mutate anything. + +Planned enforcement (roadmap, not yet shipped): - `residency check` becomes part of the pre-flight refusal gate; backup commit refuses with `verify.residency_violation` - (exit 4) when the configured repo's region is outside the - policy. + when the configured repo's region is outside the policy. - The audit chain records every residency check + violation attempt. @@ -141,16 +146,15 @@ For v1.0+ the planned enforcement is: A deployment with residency `["eu"]` whose repo is also async-replicated to a US region for DR is a policy -violation at the replica side. The replication subsystem -respects per-deployment residency: - -- `repo replicate --from --to ` for a - deployment with `residency: ["eu"]` refuses with - `verify.residency_violation` unless explicitly - overridden via `--allow-cross-region`. -- The `--allow-cross-region` flag is recorded in the audit - chain — an explicit operator decision, not an - invisible default. +violation at the replica side. + +Residency enforcement on the replication path is on the +roadmap and not yet shipped: `repo replicate` does not +consult per-deployment residency, does not refuse +cross-region copies, and there is no `--allow-cross-region` +flag. Until it lands, run `pg_hardstorage residency check` +against the replica repo out-of-band to detect a +cross-boundary replica. --- diff --git a/docs/compliance/fedramp.md b/docs/compliance/fedramp.md index a8728a4..8728b59 100644 --- a/docs/compliance/fedramp.md +++ b/docs/compliance/fedramp.md @@ -28,10 +28,10 @@ JSON is `fedramp`. | Control | Title | Product feature | Command | Audit event | | --- | --- | --- | --- | --- | -| AC-2 | Account management | RBAC scopes; per-tenant KEK; JIT tokens | `pg_hardstorage rbac ...`, `jit issue ...` | `rbac.*`, `jit.*` | +| AC-2 | Account management | RBAC scopes (enforced server-side / SCIM-provisioned; no operator CLI verb); per-tenant KEK; JIT tokens | `pg_hardstorage jit issue ...` | `jit.*` | | AC-3 | Access enforcement | n-of-m approval gates on destructive ops | `pg_hardstorage approval request ...` | `approval.request`, `approval.approve` | | AC-6 | Least privilege | RBAC scope per principal; JIT for break-glass elevation | `pg_hardstorage jit issue --scope ...` | `jit.issue` | -| AC-6(7) | Review of user privileges | `audit search --action-prefix rbac.` | `pg_hardstorage audit search ...` | `rbac.*` | +| AC-6(7) | Review of user privileges | JIT grant/revoke history via audit search; RBAC changes reviewed server-side | `pg_hardstorage audit search --action-prefix jit.` | `jit.*` | | AC-7 | Unsuccessful logon attempts | RBAC denials recorded | (automatic) | `auth.denied` | | AC-12 | Session termination | JIT tokens auto-expire (max 24h) | (automatic) | `jit.issue` | @@ -125,8 +125,10 @@ make build-fips The variant uses `GOEXPERIMENT=boringcrypto` and links against the BoringCrypto FIPS-validated module. Refuses to -start if `crypto/tls` reports non-FIPS. `--fips-strict` -panics on any non-FIPS plugin. +start if `crypto/tls` reports non-FIPS. In that FIPS build, +`--fips-strict` panics on any non-FIPS plugin. (The FIPS build is +not yet shipped; `--fips-strict` is not present in the default +binary.) ### FedRAMP High vs Moderate @@ -146,9 +148,13 @@ Use `residency` to pin to GovCloud regions: pg_hardstorage residency set fedramp-prod us-gov-east-1 us-gov-west-1 ``` -The replication subsystem refuses cross-boundary -replication unless explicitly overridden via -`--allow-cross-region` (which is itself audit-logged). +Cross-boundary residency enforcement on replication is on +the roadmap and not yet shipped — `repo replicate` does not +refuse cross-region copies, and there is no +`--allow-cross-region` flag. What ships today is the +read-only `pg_hardstorage residency check `, +which reports `verify.residency_violation` (exit 9) when the +configured repo's region is outside the policy. --- diff --git a/docs/compliance/hipaa.md b/docs/compliance/hipaa.md index d40adf6..746a7f1 100644 --- a/docs/compliance/hipaa.md +++ b/docs/compliance/hipaa.md @@ -37,7 +37,7 @@ JSON is `hipaa`. | Section | Description | Product feature | Command | Audit event | | --- | --- | --- | --- | --- | -| §164.312(a)(1) | Access control | Per-tenant KEK + RBAC scopes | `pg_hardstorage rbac ...` | `rbac.*` | +| §164.312(a)(1) | Access control | Per-tenant KEK + RBAC scopes (RBAC enforced server-side / SCIM-provisioned; no operator CLI verb) | (server-side) | (server-side) | | §164.312(a)(2)(i) | Unique user identification | RBAC actor identity recorded on every event | (automatic) | (every event records `actor`) | | §164.312(a)(2)(ii) | Emergency access procedure | JIT tokens + n-of-m approval | `pg_hardstorage jit issue ...`, `approval request ...` | `jit.issue`, `approval.request` | | §164.312(a)(2)(iii) | Automatic logoff | JIT tokens auto-expire (max 24h) | (automatic) | `jit.issue` | @@ -72,7 +72,9 @@ make build-fips ``` The FIPS variant refuses to start if `crypto/tls` reports -non-FIPS. `--fips-strict` panics on any non-FIPS plugin. +non-FIPS. In that FIPS build, `--fips-strict` panics on any +non-FIPS plugin. (The FIPS build is not yet shipped; `--fips-strict` +is not present in the default binary.) --- diff --git a/docs/compliance/iso-27001-control-mapping.md b/docs/compliance/iso-27001-control-mapping.md index 7303f72..f91c2ba 100644 --- a/docs/compliance/iso-27001-control-mapping.md +++ b/docs/compliance/iso-27001-control-mapping.md @@ -22,7 +22,7 @@ JSON is `iso27001`. | Control | Title | Product feature | Command | Audit event | | --- | --- | --- | --- | --- | | A.5.10 | Acceptable use of information | RBAC scopes + JIT tokens for break-glass | `pg_hardstorage jit issue ...` | `jit.issue`, `jit.revoke` | -| A.5.15 | Access control | Per-tenant KEK + RBAC scope | `pg_hardstorage rbac ...` | `rbac.*` | +| A.5.15 | Access control | Per-tenant KEK + RBAC scope (RBAC enforced server-side / SCIM-provisioned; no operator CLI verb) | (server-side) | (server-side) | | A.5.23 | Information security for use of cloud services | Storage plugin per-region scoping; air-gap mode | (config) | (config) | | A.5.30 | ICT readiness for business continuity | Disaster runbooks R1–R7; recovery drills with measured RTO | `pg_hardstorage recovery drill ...` | `recovery.drill_failed` (on fail) | | A.5.34 | Privacy and protection of PII | Data residency + classification | `pg_hardstorage residency set ...`, `classify set ...` | (config), (manifest tag) | diff --git a/docs/compliance/soc2-control-mapping.md b/docs/compliance/soc2-control-mapping.md index 0cfee34..e99a15f 100644 --- a/docs/compliance/soc2-control-mapping.md +++ b/docs/compliance/soc2-control-mapping.md @@ -27,7 +27,7 @@ The framework string in the report's JSON is `soc2`. | --- | --- | --- | --- | --- | | CC6.1 | Logical access security | Three-layer envelope encryption (RKEK → BDEK → per-chunk Kc) | `pg_hardstorage backup ...` | `backup.create` | | CC6.6 | Logical access — boundary protection | Storage plugin per-region scoping; air-gap mode rejects public endpoints | `pg_hardstorage residency set ...` | (config) | -| CC6.7 | Restricted access to information assets | Per-tenant KEK + RBAC scopes | `pg_hardstorage rbac ...` | `rbac.*` | +| CC6.7 | Restricted access to information assets | Per-tenant KEK + RBAC scopes (RBAC enforced server-side / SCIM-provisioned; no operator CLI verb) | (server-side) | (server-side) | | CC6.8 | Detection and prevention of unauthorised software | SLSA L3 build provenance + cosign signatures on every release | (build-time) | (cosign attest) | | CC7.1 | Detection of system anomalies | Anomaly score (Z-score over 30-day baseline) per backup | `pg_hardstorage backup ...` | `anomaly.detected` | | CC7.2 | System events logged in tamper-evident chain | Hash-chained Merkle audit log | `pg_hardstorage audit verify-chain` | (every event) | diff --git a/docs/explanation/architecture-tour.md b/docs/explanation/architecture-tour.md index 6eec65a..68a8e66 100644 --- a/docs/explanation/architecture-tour.md +++ b/docs/explanation/architecture-tour.md @@ -147,7 +147,7 @@ manifest is independently verifiable without external state. "encryption": { "scheme": "aes-256-gcm", "wrapped_dek": "base64...", - "kek_ref": "local-keyring://default", + "kek_ref": "local:default", "envelope_version": 2 }, "tablespaces": [{"oid":1663,"location":"pg_default"}], @@ -363,13 +363,15 @@ cooperating mechanisms: - **Strategy C (fallback):** recreate the slot on detection. The agent runs `IDENTIFY_SYSTEM` after reconnect; if the slot doesn't exist, recreate it and report any gap loudly. -3. **Dual-slot mode** (≥ 50 TB or `availability=high`). Two slots - on two nodes feed the same CAS; either stream can fail without - RPO impact. Shipped in v1.0. -4. **Synchronous-target mode** (`wal_mode: synchronous`). Agent - advertises as a `synchronous_standby_names` candidate, PG waits - for our flush ACK before commit. RPO = 0 at the cost of write - latency. Shipped in v1.0. +3. **Dual-slot mode** (≥ 50 TB, opt-in via a `patroni.slots:` list + of `{name, role}` entries). Two slots on two nodes feed the + same CAS; either stream can fail without RPO impact. +4. **Synchronous-target mode** (`wal_mode: synchronous`) — *not + implemented*. The aspiration is for the agent to advertise as a + `synchronous_standby_names` candidate so PG waits for our flush + ACK before commit (RPO = 0 at the cost of write latency). Today + there is no `wal_mode: synchronous` config key; only preflight + *detection* of where `sync_standby` slots are placed ships. Strategy C is honest about loss — it never silently glosses over a gap. Manifests record the gap; PITR inside the gap window is diff --git a/docs/explanation/audit-chain.md b/docs/explanation/audit-chain.md index 30b54aa..51e69b1 100644 --- a/docs/explanation/audit-chain.md +++ b/docs/explanation/audit-chain.md @@ -212,14 +212,13 @@ The default 5-minute cadence is a tradeoff between two costs: - Sparse anchoring widens the rewrite window — events between two anchors are still rewritable until the next anchor. -`pg_hardstorage repo set audit.anchor_interval` accepts any -duration from 30 s to 1 h. Production tuning typically picks -between 1 min (security-sensitive, accept higher log volume) and -15 min (capacity-constrained). +The cadence is set per deployment via the `schedule.audit_anchor` +spec in `pg_hardstorage.yaml` (e.g. `every: 5m`). Production +tuning typically picks between 1 min (security-sensitive, accept +higher log volume) and 15 min (capacity-constrained). -The `pg_hardstorage_audit_anchor_lag_seconds` metric exposes how -long it's been since the last successful anchor; the default -alerting recipe fires if it exceeds 3× the configured interval. +Track anchor freshness from the `audit verify-chain` output, which +reports how long it's been since the last successful anchor. --- @@ -284,7 +283,7 @@ reporting. --- -## What's deferred to v1.0 +## What's deferred past v1.0 The chain is real and verifiable; the audit subsystem ships hash-chained from v0.1 onward and the durable monotonic diff --git a/docs/explanation/comparison-pgbackrest-walg-barman.md b/docs/explanation/comparison-pgbackrest-walg-barman.md index 572ee63..2bfa26c 100644 --- a/docs/explanation/comparison-pgbackrest-walg-barman.md +++ b/docs/explanation/comparison-pgbackrest-walg-barman.md @@ -190,7 +190,7 @@ differentiation: - **K8s with the operator-shim model**. The WAL-G shim and pgBackRest shim let you swap into existing operator-managed clusters without rewriting the operator. (A native CNPG-I - provider is on the roadmap for v0.5.) + provider is on the roadmap.) - **Transparent Data Encryption (TDE) at the source.** PG forks that encrypt heap / index / WAL at rest — CYBERTEC PGEE, @@ -211,9 +211,6 @@ Where we are *honest* about being behind: - **Ecosystem of expert operators.** More humans in the world know pgBackRest than know `pg_hardstorage` today. Onboarding cost is real. -- **Storage backends.** S3 and FS are first-class in v0.1. - Azure Blob and GCS are second-class until v0.5; SFTP and tar - are scoped for v0.5 / v1.0. --- diff --git a/docs/explanation/envelope-encryption.md b/docs/explanation/envelope-encryption.md index 3442926..63d0f9b 100644 --- a/docs/explanation/envelope-encryption.md +++ b/docs/explanation/envelope-encryption.md @@ -138,9 +138,11 @@ Single-org installs get a default tenant they never see — but the tenant boundary exists in the data layout regardless. This is a deliberate architectural choice, baked into the data layout: -- **GDPR crypto-shred** is `pg_hardstorage kms shred --confirm-keyring - --reason "GDPR Art. 17 request #4421"`. One key-destruction op. The - audit event is the compliance artifact. +- **GDPR crypto-shred** is `pg_hardstorage kms shred --repo + --require-approval --confirm-keyring + --reason "GDPR Art. 17 request #4421"`. One key-destruction op, + gated behind an approved n-of-m request. The audit event is the + compliance artifact. - **Multi-tenant SaaS** can isolate customer A from customer B cryptographically: even if every chunk leaks, only the owner's KEK can decrypt their tenant's BDEKs. @@ -157,8 +159,10 @@ this effectively free for steady-state workloads. ## KEK rotation flow -`pg_hardstorage kms rotate --repo --old-kek-ref --new-kek-ref ` walks -every manifest in the repo wrapped under the old KEK ref: +`pg_hardstorage kms rotate --repo --old-kek-ref +--new-kek-ref --old-kek-file --new-kek-file + --apply` walks every manifest in the repo wrapped +under the old KEK ref (omit `--apply` for a dry-run preview): ```mermaid sequenceDiagram @@ -200,7 +204,7 @@ A few properties worth highlighting: After retirement the old KEK can be scheduled for deletion via the KMS's own retention policy. -`pg_hardstorage repair attestation ` exists for the +`pg_hardstorage repair attestation ` exists for the adjacent case — re-signing a manifest whose Ed25519 signature no longer validates after a rotation that also rotated the attestation key. See the architecture tour for the full repair diff --git a/docs/explanation/index.md b/docs/explanation/index.md index ca0d5ea..da66019 100644 --- a/docs/explanation/index.md +++ b/docs/explanation/index.md @@ -63,7 +63,7 @@ SPEC is the source of truth and we want to know — file an issue. - [Coordination without etcd](coordination-without-etcd.md) — the progressive ladder from JSON state files up to opt-in etcd. - [Tier-1 vs Tier-2 plugins](tier1-vs-tier2-plugins.md) — - compile-time vs `go-plugin`, trust posture, registry roadmap. + compile-time vs stdio-JSON-RPC subprocess, trust posture, registry roadmap. - [Verify-sandbox tradeoffs](verify-sandbox-tradeoffs.md) — Docker default vs Firecracker microVM; isolation vs setup cost. diff --git a/docs/explanation/llm-safety-stack.md b/docs/explanation/llm-safety-stack.md index 1e1e8b5..a537baf 100644 --- a/docs/explanation/llm-safety-stack.md +++ b/docs/explanation/llm-safety-stack.md @@ -270,29 +270,28 @@ independent layers of evidence. --- -## Skills are signed YAML files, not Go code +## Skills are YAML files, not Go code -The skills (`restore`, `incident`, `ask`, `explain`, `runbook`, -`postmortem`) are versioned, declarative YAML files — not Go -functions baked into the binary. Implications: +The builtin skills (`ask`, `explain`, `incident`, `restore`) are +versioned, declarative YAML files — not Go functions baked into +the binary. Implications: - **Hot-fix loop in minutes, not weeks.** A bad skill response in - production gets a same-day patch — drop a new YAML file in - `/etc/pg_hardstorage/skills/`, increment the version, restart - the agent (or `pg_hardstorage llm reload-skills`). No binary - rebuild, no Debian package release. -- **Skill isolation.** A bug in the postmortem skill cannot + production gets a same-day patch — author a new YAML file, + increment the version, and `pg_hardstorage llm skill install + ` into the operator overlay (any existing version is + snapshotted for rollback). No binary rebuild, no Debian package + release. +- **Skill isolation.** A bug in the incident skill cannot touch the restore skill. Each skill loads independently, has its own tool allowlist, its own guardrails, its own RBAC scope. - **Inspectable, with signing on the roadmap.** Skills load as plain, reviewable YAML today. Cryptographic skill signing and signature verification (a project key for shipped skills, the operator's key for local ones) are planned, not yet shipped. -- **Linted + golden-tested.** `pg_hardstorage llm skill lint - ` validates the schema and static-checks the tool list - (no banned tools, no missing required ones). - `pg_hardstorage llm skill test ` runs golden test cases - against a pinned model checkpoint per release. +- **Linted.** `pg_hardstorage llm skill lint ` validates + the schema and static-checks the tool list (no banned tools, no + missing required ones). A skill file declares which tools are available to the LLM in that session. The `restore` skill explicitly excludes @@ -334,8 +333,10 @@ specific set of failure modes: - **"You released a bad model update."** The bundle records the exact model id and version. Provider-side model versioning is captured. -- **"The skill was malicious."** Skills are signed; unsigned - skills require a flag whose use is audited. +- **"The skill was malicious."** Skills load as reviewable YAML + and are schema-linted before install; the exact skill name and + version are recorded in the audit bundle. (Cryptographic skill + signing is roadmap, not yet shipped.) - **"You hid prompts from me."** `/show-context` plus the export bundle prove otherwise. - **"A prompt injection in a manifest description executed diff --git a/docs/explanation/patroni-failover-deep-dive.md b/docs/explanation/patroni-failover-deep-dive.md index dcca9b3..8233047 100644 --- a/docs/explanation/patroni-failover-deep-dive.md +++ b/docs/explanation/patroni-failover-deep-dive.md @@ -179,11 +179,10 @@ the dual-slot mode eliminates this entire class of problem: ```yaml deployments: db1: - physical_wal: - mode: dual + patroni: slots: - - { node: primary, name: pg_hardstorage_db1_primary } - - { node: replica1, name: pg_hardstorage_db1_replica } + - { name: pg_hardstorage_db1_primary, role: leader } + - { name: pg_hardstorage_db1_replica, role: replica } ``` Both slots stream concurrently into the same content-addressed @@ -200,24 +199,22 @@ the gap is typically zero. --- -## Mechanism 4 — synchronous-target mode +## Mechanism 4 — synchronous-target mode (not implemented) -The opt-in `wal_mode: synchronous` makes the agent advertise -itself as a candidate in `synchronous_standby_names`. PG **will -not commit a transaction until our agent has fsynced the WAL into -the repo**. On failover, no committed transaction can be missing, -because none committed without our ACK. RTO unaffected; RPO = 0 -across the failover boundary. +Synchronous-target mode is a design aspiration, not a shipped +feature. The intended posture: an opt-in `wal_mode: synchronous` +would make the agent advertise itself as a candidate in +`synchronous_standby_names`, so PG **would not commit a +transaction until our agent had fsynced the WAL into the repo**. +On failover, no committed transaction could be missing, because +none committed without our ACK — RPO = 0 across the failover +boundary. -The cost is a real one: write latency increases by the -backup-side flush latency, which on cloud object storage is on the -order of tens to low hundreds of milliseconds. Recommended only -for tier-0 deployments where the customer has explicitly accepted -the tradeoff. - -This is the strongest available posture and the only one that -makes "no transaction loss across failover" a structural -guarantee rather than a probabilistic one. +What ships today is *detection only*: preflight checks report +where `sync_standby` slots are placed and flag configurations that +would strand WAL, but there is no `wal_mode: synchronous` config +key and the agent does not itself join +`synchronous_standby_names`. The always-on ACK path is roadmap. --- @@ -320,8 +317,13 @@ A few combinations the design refuses by construction: - Strategy A — Patroni `permanent_slots` integration. - Strategy B — PG 17+ synced slots. -- Dual-slot mode (≥ 50 TB or `availability=high`). -- Sync-target mode (`wal_mode: synchronous`). +- Dual-slot mode (≥ 50 TB, via a `patroni.slots:` list). + +**Roadmap:** + +- Sync-target mode (`wal_mode: synchronous`) — detection of + `sync_standby` placement ships today; the always-on ACK path is + not yet implemented. --- diff --git a/docs/explanation/three-execution-modes.md b/docs/explanation/three-execution-modes.md index 0ed773d..4795240 100644 --- a/docs/explanation/three-execution-modes.md +++ b/docs/explanation/three-execution-modes.md @@ -141,14 +141,16 @@ election — no extra coordination service needed. The K8s integrations specifically: -- **CNPG-I provider** for CloudNativePG (v0.5 roadmap). +- **CloudNativePG** is supported today via the barman-cloud + drop-in shim (`compat/barmancloud`); a native CNPG-I gRPC + provider is still roadmap. - **WAL-G drop-in shim** (`pg-hardstorage-walg` binary) for - Zalando — shipped in v1.1. + Zalando — shipped (v1.0.7). - **pgBackRest drop-in shim** (`pg-hardstorage-pgbackrest` - binary) for Crunchy PGO — shipped in v1.1. + binary) for Crunchy PGO — shipped (v1.0.7). - **Barman drop-in shim** (`pg-hardstorage-barman` + `pg-hardstorage-barman-wal-archive` binaries) for in-pod - Barman wrappers — shipped in v1.1. + Barman wrappers — shipped (v1.0.7). - **Helm charts**: `charts/pg-hardstorage-server` (control plane) and `charts/pg-hardstorage-sidecar` (per-Pod sidecar) — shipped. diff --git a/docs/explanation/tier1-vs-tier2-plugins.md b/docs/explanation/tier1-vs-tier2-plugins.md index 32fabd6..c455e96 100644 --- a/docs/explanation/tier1-vs-tier2-plugins.md +++ b/docs/explanation/tier1-vs-tier2-plugins.md @@ -22,8 +22,8 @@ question is *where the trust boundary sits*. path. - **Tier 2 plugins** (third-party storage, third-party sinks, domain-specific extensions) ship as separate binaries discovered - on `$HSPLUGIN_PATH`, talking to the agent over - `hashicorp/go-plugin` (gRPC over Unix-domain stdio). + on `$HSPLUGIN_PATH`, talking to the agent over a one-shot + stdio JSON-RPC protocol (`pg_hardstorage.plugin.v1`). Crash-isolated, language-agnostic. This page explains the trust posture each tier implies, and why @@ -75,7 +75,7 @@ that it doesn't bottleneck users. --- -## Why Tier 2 is `hashicorp/go-plugin` +## Why Tier 2 is a stdio JSON-RPC subprocess Tier 2 exists for plugins we don't (or shouldn't) maintain in-tree: @@ -93,9 +93,9 @@ in-tree: structured error, and falls through to whatever fallback the caller chose (typically retry-with-degraded-mode). -2. **Language-agnostic.** The wire protocol is gRPC over - Unix-domain stdio. Plugin authors can write in Go, Rust, - Python — anything with a gRPC binding. +2. **Language-agnostic.** The wire protocol is one-shot + stdio JSON-RPC. Plugin authors can write in Go, Rust, + Python — anything that can read and write JSON on stdio. 3. **Independent release cadence.** A vendor can ship a new plugin without waiting for our release cycle. @@ -140,9 +140,10 @@ Default policy: emits a critical audit event, and is recorded in every subsequent operation's audit trail. -This mirrors the [LLM safety stack](llm-safety-stack.md)'s -posture for skill files: signed by default, unsigned-with-flag -for development, the flag's use is audited. +Skill files follow a different posture today: the +[LLM safety stack](llm-safety-stack.md)'s skills are unsigned +YAML, gated by schema linting rather than signatures. Skill +signing is on the roadmap, not shipped. --- diff --git a/docs/explanation/verify-sandbox-tradeoffs.md b/docs/explanation/verify-sandbox-tradeoffs.md index d990802..61c2032 100644 --- a/docs/explanation/verify-sandbox-tradeoffs.md +++ b/docs/explanation/verify-sandbox-tradeoffs.md @@ -132,10 +132,12 @@ Firecracker pays for itself. --- -## The k8s `Job` middle ground +## The k8s `Job` middle ground (roadmap) -In K8s installs, the natural sandbox is a `Job` running in a -dedicated namespace with strict `PodSecurity`: +A k8s `Job` backend is on the roadmap, not yet shipped — only the +`docker` and `firecracker` backends are registered in the binary +today. When it lands, the natural K8s sandbox is a `Job` running +in a dedicated namespace with strict `PodSecurity`: - `runAsNonRoot: true` - `readOnlyRootFilesystem: true` @@ -143,13 +145,13 @@ dedicated namespace with strict `PodSecurity`: - `capabilities: drop: [ALL]` - NetworkPolicy restricting egress to the repo and KMS only -This sits between Docker and Firecracker on the isolation +This would sit between Docker and Firecracker on the isolation spectrum. The kernel boundary is the host's kernel, but the combination of namespaces + cgroups + seccomp + minimal capabilities is sufficient for most threat models. -The verifier sub-system uses k8s `Job` automatically when -running in K8s; the operator doesn't have to choose. +Until it ships, K8s installs select `docker` or `firecracker` +explicitly via `Options.Backend`. --- @@ -194,7 +196,7 @@ backup, so an auditor can reconstruct coverage. ## Further reading - [Architecture tour: verifier] - (architecture-tour.md#9-resilience-design) — where verification + (architecture-tour.md#8-resilience-design) — where verification sits in the larger architecture. - [Verify how-to](../how-to/verify/index.md) — the operator recipes. diff --git a/docs/faq.md b/docs/faq.md index 6e0fe0c..efecf28 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -74,9 +74,8 @@ for the full reasoning. #### What PostgreSQL versions are supported? -PG 15, 16, and 17 are first-class — required to pass CI. -PG 18 runs in the test matrix under allow-failure guardrails -while readiness is confirmed. PG 14 and earlier are +PG 15 through 18 are supported and required to pass CI; +PG 18 is first-class. PG 14 and earlier are out of scope — `BASE_BACKUP` non-exclusive mode and `pg_backup_start` / `pg_backup_stop` are PG 15-only APIs we rely on. @@ -476,8 +475,10 @@ binary. Index: #### Does it produce SOC 2 evidence? Yes — the audit log is hash-chained Merkle, periodically -anchored to a transparency log (Rekor by default, -customer-managed log on request). `compliance report` +anchored to a self-hosted, storage-backed transparency log. +External-witness anchoring (a real Rekor / Sigstore backend) +is on the roadmap and drops in behind the same interface. +`compliance report` auto-generates a monthly PDF that maps observed events to SOC 2 control IDs. Mapping page: [SOC 2 control mapping](compliance/soc2-control-mapping.md). @@ -563,7 +564,8 @@ Tier-1 plugins are first-party, compiled into the binary, self-register via `init()`, and ride the same signed release. Tier-2 plugins are third-party, ship as separate binaries, and the agent discovers them on `$HSPLUGIN_PATH` -via `hashicorp/go-plugin` (gRPC over stdio). Side-by-side +via a one-shot stdio JSON-RPC handshake +(`pg_hardstorage.plugin.v1`). Side-by-side table: [Tier-1 vs Tier-2](explanation/tier1-vs-tier2-plugins.md); contract reference: @@ -579,8 +581,9 @@ walks through writing one end-to-end. #### Can I write a plugin in a language other than Go? -Yes — Tier-2 plugins use `hashicorp/go-plugin`'s gRPC -transport, so anything that speaks gRPC works. The +Yes — Tier-2 plugins speak stdio JSON-RPC (each request is +a line of JSON the plugin reads from stdin; each response a +line it writes to stdout), so any language works. The [Tier-2 protocol reference](reference/plugins/tier2-go-plugin-protocol.md) documents the wire format. diff --git a/docs/glossary.md b/docs/glossary.md index f1993ef..40ead83 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -301,11 +301,13 @@ isolation when verifying untrusted backups. Linux + KVM only. See [firecracker sandbox](how-to/verify/firecracker-sandbox.md). -#### `go-plugin` +#### Tier-2 plugin transport -`hashicorp/go-plugin` — the gRPC-over-stdio framework -pg_hardstorage uses to host Tier-2 plugins. Crash- -isolated, language-agnostic. See +The wire contract pg_hardstorage uses to host Tier-2 +plugins: a one-shot stdio JSON-RPC protocol +(`pg_hardstorage.plugin.v1`) — the host launches the plugin +executable per operation and exchanges line-delimited JSON +over stdin/stdout. Crash-isolated, language-agnostic. See [Tier-2 protocol](reference/plugins/tier2-go-plugin-protocol.md). #### `HSREPO` @@ -334,7 +336,7 @@ crypto-shred primitive. See The URL-shaped reference to a KEK (`aws-kms://arn:...`, `gcp-kms://...`, `vault-transit://...`, -`local-keyring://...`, etc.). Stored in +`local:default`, etc.). Stored in `manifest.json.encryption.kek_ref`. Schemes documented in the auto-generated KEKRef reference page. @@ -366,7 +368,7 @@ confirmation. Read-only by default. See The plugin tier for chat-completion backends (OpenAI, Bedrock, Vertex, Ollama, llama.cpp, Hugging Face, …). -Tier-1 in-tree, Tier-2 external via `go-plugin`. See the +Tier-1 in-tree, Tier-2 external via stdio JSON-RPC. See the [LLM provider contract](reference/plugins/llm-provider-contract.md). #### Manifest @@ -460,7 +462,7 @@ encryption, compression, renderer, sink, and LLM provider. Tier-1 plugins are first-party and compiled into the binary; Tier-2 plugins are third-party, ship as separate binaries, and are discovered on `$HSPLUGIN_PATH` via -`go-plugin`. See +stdio JSON-RPC. See [Tier-1 vs Tier-2](explanation/tier1-vs-tier2-plugins.md). #### Recovery slot @@ -639,8 +641,8 @@ pg_hardstorage we use [Deployment](#deployment). #### Storage URL scheme The URL prefix that selects a storage backend — -`s3://`, `gs://`, `azblob://`, `sftp://`, `scp://`, -`fs://`, etc. Auto-generated reference page lists all +`s3://`, `gcs://`, `azblob://`, `sftp://`, `scp://`, +`file://`, etc. Auto-generated reference page lists all built-in schemes. #### Subject @@ -682,8 +684,8 @@ audit, FIPS-build, and ship. See #### Tier-2 plugin A third-party plugin shipped as a separate binary, -discovered on `$HSPLUGIN_PATH`, hosted via `go-plugin` -(gRPC over stdio). Crash-isolated, language-agnostic. +discovered on `$HSPLUGIN_PATH`, hosted via one-shot stdio +JSON-RPC. Crash-isolated, language-agnostic. #### Timeline diff --git a/docs/how-to/adding/deployment.md b/docs/how-to/adding/deployment.md index bdc628d..59f16b8 100644 --- a/docs/how-to/adding/deployment.md +++ b/docs/how-to/adding/deployment.md @@ -37,11 +37,7 @@ pg_hardstorage deployment add db1 \ ``` ```console -deployment "db1" added - pg_connection: postgres://pgbackup@db1.example.com/postgres - repo: file:///srv/pg_hardstorage/repo - schedule: backup=every 6h, rotate=daily_at 04:00 - probe: ok (PG 17.2) +✓ Deployment "db1" added ``` The CLI runs a connection probe by default; pass `--skip-probe` diff --git a/docs/how-to/adding/kms-hsm-pkcs11.md b/docs/how-to/adding/kms-hsm-pkcs11.md index a4a25a7..7dba0d4 100644 --- a/docs/how-to/adding/kms-hsm-pkcs11.md +++ b/docs/how-to/adding/kms-hsm-pkcs11.md @@ -59,8 +59,7 @@ Or pull the official `pg-hardstorage-fips` artifact — built with `-tags pkcs11` by default since FIPS-validated HSM-backed envelopes are the canonical FIPS posture. See [the FIPS variant build how-to](../packaging/fips-variant.md) -when that page lands; until then, ship `-tags pkcs11` -yourselves. +for the full build recipe. ## What you need @@ -86,11 +85,13 @@ pkcs11-tool \ ```bash pg_hardstorage kms verify \ + --repo file:///srv/pg_hardstorage/repo \ --kek-ref 'pkcs11://test-token/db-kek?module=/usr/lib/softhsm/libsofthsm2.so&pin=1234' ``` -`inspect` reads the key's metadata only — confirms the binary -has the PKCS#11 backend and the URL resolves. No mutation. +`kms verify --repo … --kek-ref` reads the key's metadata only — +confirms the binary has the PKCS#11 backend and the URL resolves. +No mutation. ### 2. Production: file-based PIN diff --git a/docs/how-to/adding/repository-azblob.md b/docs/how-to/adding/repository-azblob.md index 8e54da3..5ca80d7 100644 --- a/docs/how-to/adding/repository-azblob.md +++ b/docs/how-to/adding/repository-azblob.md @@ -61,8 +61,11 @@ pg_hardstorage repo init 'azblob://acmebackups/prod' ``` ```console -repo: azblob://acmebackups/prod -mode: ok region: +✓ Repository initialised + URL: azblob://acmebackups/prod + ID: b7d2f1a0c3e94856a1f0d9b8c7e6a5f4 + Schema: pg_hardstorage.repo.v1 + Created: 2026-07-06T13:56:12Z ``` The agent's pod identity / VM-managed identity is read by diff --git a/docs/how-to/adding/repository-gcs.md b/docs/how-to/adding/repository-gcs.md index 0cf5bb2..da6c019 100644 --- a/docs/how-to/adding/repository-gcs.md +++ b/docs/how-to/adding/repository-gcs.md @@ -47,8 +47,11 @@ pg_hardstorage repo init 'gcs://acme-pg-backups' ``` ```console -repo: gcs://acme-pg-backups -mode: ok +✓ Repository initialised + URL: gcs://acme-pg-backups + ID: 54663a3581cb6b76c87e8c962daa186b + Schema: pg_hardstorage.repo.v1 + Created: 2026-07-06T13:55:51Z ``` Workload Identity on GKE: bind the Kubernetes service account to diff --git a/docs/how-to/adding/repository-s3.md b/docs/how-to/adding/repository-s3.md index 4fe121b..0bbf716 100644 --- a/docs/how-to/adding/repository-s3.md +++ b/docs/how-to/adding/repository-s3.md @@ -51,8 +51,11 @@ pg_hardstorage repo init 's3://acme-pg-backups/?region=eu-central-1' ``` ```console -repo: s3://acme-pg-backups/?region=eu-central-1 -mode: ok region: eu-central-1 +✓ Repository initialised + URL: s3://acme-pg-backups/?region=eu-central-1 + ID: a96bbb87cdf922544033b4fe285a8025 + Schema: pg_hardstorage.repo.v1 + Created: 2026-07-06T13:54:25Z ``` ### 2. AWS S3 with a prefix @@ -102,8 +105,9 @@ unset. pg_hardstorage repo check s3://acme-pg-backups/ ``` -Roundtrips a manifest signature, sample-checks a chunk reference, -and reports the total live byte count. Mismatches return exit 9. +Reports the live-manifest count, verifies every manifest +signature, counts distinct chunk references, and flags any +missing chunks. Mismatches return exit 9. ## IAM policy (minimum) diff --git a/docs/how-to/adding/repository-scp.md b/docs/how-to/adding/repository-scp.md index 24e51b2..9c80697 100644 --- a/docs/how-to/adding/repository-scp.md +++ b/docs/how-to/adding/repository-scp.md @@ -121,8 +121,11 @@ pg_hardstorage repo init scp://backup@nas.example.com/srv/pg-hardstorage ``` ```console -repo: scp://backup@nas.example.com/srv/pg-hardstorage -mode: ok +✓ Repository initialised + URL: scp://backup@nas.example.com/srv/pg-hardstorage + ID: 3f8c1e5b9a2d4076c8e1f3a5b7d9c0e2 + Schema: pg_hardstorage.repo.v1 + Created: 2026-07-06T13:56:34Z ``` The plugin reads `identity_file`, `identity_passphrase`, and diff --git a/docs/how-to/adding/repository-sftp.md b/docs/how-to/adding/repository-sftp.md index dc85c5d..253fc40 100644 --- a/docs/how-to/adding/repository-sftp.md +++ b/docs/how-to/adding/repository-sftp.md @@ -99,8 +99,11 @@ pg_hardstorage repo init sftp://backup@nas.example.com/srv/pg-hardstorage ``` ```console -repo: sftp://backup@nas.example.com/srv/pg-hardstorage -mode: ok +✓ Repository initialised + URL: sftp://backup@nas.example.com/srv/pg-hardstorage + ID: 6a0d2f4b8c1e3057a9b2d4f6c8e0a1b3 + Schema: pg_hardstorage.repo.v1 + Created: 2026-07-06T13:56:58Z ``` The plugin reads `identity_file`, `identity_passphrase`, and diff --git a/docs/how-to/adding/sink-jira.md b/docs/how-to/adding/sink-jira.md index 81fd711..a782948 100644 --- a/docs/how-to/adding/sink-jira.md +++ b/docs/how-to/adding/sink-jira.md @@ -43,7 +43,7 @@ pg_hardstorage notify add jira \ ``` ```console -sink "ops-jira" added (plugin=jira) +✓ Sink added — ops-jira (plugin jira) ``` The builder validates `base_url` parses, `project` is set, and diff --git a/docs/how-to/adding/sink-other.md b/docs/how-to/adding/sink-other.md index 496f6f6..8e88c52 100644 --- a/docs/how-to/adding/sink-other.md +++ b/docs/how-to/adding/sink-other.md @@ -55,7 +55,7 @@ sinks: CEF severity is rendered as 0-10 from the RFC 5424 ladder. -## Datadog (`datadog`) +## Datadog (`datadog-events`) Posts events to Datadog's `/api/v1/events` endpoint with the Datadog API key. @@ -63,7 +63,7 @@ Datadog API key. ```yaml sinks: - name: dd - plugin: datadog + plugin: datadog-events config: api_key: site: datadoghq.com # or datadoghq.eu, ddog-gov.com @@ -85,10 +85,10 @@ sinks: config: smtp_host: smtp.example.com smtp_port: 587 - tls_mode: starttls - auth_mode: plain + tls: starttls + auth: plain username: pg-hardstorage - password_secret: kms-secret://ops/smtp-password + password: kms-secret://ops/smtp-password from: backups@example.com to: ["dba@example.com"] cc: ["ops@example.com"] @@ -140,8 +140,8 @@ sinks: plugin: opsgenie config: api_key: - region: us # us | eu - team: dba-on-call # routes to the team's escalation + api_url: https://api.opsgenie.com # EU shard: https://api.eu.opsgenie.com + teams: [dba-on-call] # routes to each team's escalation ``` ## ServiceNow (`servicenow`) @@ -154,14 +154,14 @@ sinks: - name: sn plugin: servicenow config: - base_url: https://acme.service-now.com + instance_url: https://acme.service-now.com username: svc-pg-hardstorage password: - table: incident + category: software assignment_group: dba-team ``` -## Splunk HEC (`splunkhec`) +## Splunk HEC (`splunk-hec`) HTTP Event Collector for Splunk. POSTs to the HEC endpoint with the per-token auth header. @@ -169,7 +169,7 @@ with the per-token auth header. ```yaml sinks: - name: splunk - plugin: splunkhec + plugin: splunk-hec config: url: https://splunk.example.com:8088/services/collector token: @@ -177,7 +177,7 @@ sinks: index: pg_hardstorage ``` -## OpenTelemetry events (`otelevents`) +## OpenTelemetry events (`otel-events`) Emits events as OTLP/HTTP log records to an OpenTelemetry collector. Pair with the agent's `--otel-endpoint` for the @@ -186,7 +186,7 @@ trace path; this sink covers the structured-log path. ```yaml sinks: - name: otel - plugin: otelevents + plugin: otel-events config: endpoint: http://otel-collector:4318 service_name: pg_hardstorage diff --git a/docs/how-to/adding/sink-pagerduty.md b/docs/how-to/adding/sink-pagerduty.md index d43b6b8..5d76cf0 100644 --- a/docs/how-to/adding/sink-pagerduty.md +++ b/docs/how-to/adding/sink-pagerduty.md @@ -35,7 +35,7 @@ pg_hardstorage notify add pagerduty \ ``` ```console -sink "ops-pd" added (plugin=pagerduty) +✓ Sink added — ops-pd (plugin pagerduty) ``` ### 2. Verify @@ -68,7 +68,7 @@ sinks: | --- | --- | --- | | `routing_key` | required | 32-hex Events API v2 integration key. | | `source` | `pg_hardstorage` | Free-form; appears as the alert's source. Pair with the deployment name. | -| `client` | empty | Optional client name on the incident. | +| `client` | `pg_hardstorage` | Client name on the incident. | | `client_url` | empty | Optional URL on the incident — runbook link. | | `min_severity` | `error` | PD is for waking people; the floor is deliberately strict. | diff --git a/docs/how-to/adding/sink-slack.md b/docs/how-to/adding/sink-slack.md index 25a0df6..68b745f 100644 --- a/docs/how-to/adding/sink-slack.md +++ b/docs/how-to/adding/sink-slack.md @@ -36,7 +36,7 @@ pg_hardstorage notify add slack \ ``` ```console -sink "ops-slack" added (plugin=slack) +✓ Sink added — ops-slack (plugin slack) ``` The CLI runs the same builder the agent uses at start-up, so a @@ -50,8 +50,11 @@ pg_hardstorage notify list ``` ```console -NAME PLUGIN MIN_SEVERITY COMPONENTS -ops-slack slack warning * +1 sink(s) configured + ops-slack + plugin: slack + endpoint: https://hooks.slack.com/**** + min_severity: warning ``` ### 3. Smoke-test (optional) diff --git a/docs/how-to/adding/sink-syslog.md b/docs/how-to/adding/sink-syslog.md index ac0a366..1562c7a 100644 --- a/docs/how-to/adding/sink-syslog.md +++ b/docs/how-to/adding/sink-syslog.md @@ -38,7 +38,7 @@ pg_hardstorage notify add syslog \ ``` ```console -sink "prod-syslog" added (plugin=syslog) +✓ Sink added — prod-syslog (plugin syslog) ``` `--set` covers top-level keys only. Add the nested `tls:` diff --git a/docs/how-to/adding/sink-webhook.md b/docs/how-to/adding/sink-webhook.md index bb83904..1487b18 100644 --- a/docs/how-to/adding/sink-webhook.md +++ b/docs/how-to/adding/sink-webhook.md @@ -34,7 +34,7 @@ pg_hardstorage notify add webhook \ ``` ```console -sink "ops-webhook" added (plugin=webhook) +✓ Sink added — ops-webhook (plugin webhook) ``` ### 2. Verify diff --git a/docs/how-to/air-gapped/transport-bundle-import.md b/docs/how-to/air-gapped/transport-bundle-import.md index d687473..23b4dd2 100644 --- a/docs/how-to/air-gapped/transport-bundle-import.md +++ b/docs/how-to/air-gapped/transport-bundle-import.md @@ -20,8 +20,8 @@ tags: ## What you need - The destination repository URL (must be a valid storage - scheme: `s3://`, `file://`, `azure://`, `gs://`, `sftp://`, - …). + scheme: `s3://`, `file://`, `azblob://`, `gcs://`, `sftp://`, + `scp://`). - The path to the bundle tar received over the air gap. - Read access to the bundle, write access to the destination repo. diff --git a/docs/how-to/kubernetes/barman-shim.md b/docs/how-to/kubernetes/barman-shim.md index 62e9f00..c993b58 100644 --- a/docs/how-to/kubernetes/barman-shim.md +++ b/docs/how-to/kubernetes/barman-shim.md @@ -43,30 +43,44 @@ Both patterns benefit from the shim: the wrapper script, job spec, and on-disk config stay identical, but the bytes that land in the repository are pg_hardstorage-flavour. -If you're not running such a pod, the -[host-package](../packaging/debian-rpm.md) install with -the Debian or RPM `pg-hardstorage-compat-barman` subpackage -plus a `/usr/local/bin/barman` symlink is the simpler path. +If you're not running such a pod, a +[host-package](../packaging/debian-rpm.md) install of +pg_hardstorage plus a `/usr/local/bin/barman` symlink is the +simpler path. Note that the `.deb`/`.rpm` packages ship only +the `pg_hardstorage` binary — the compat shims are not +packaged, so you build `pg-hardstorage-barman` from source +(see below) and drop it in yourself. ## What you need - A pod or container image that today exec's `barman` / `barman-wal-archive`. -- The pg_hardstorage v1.1 image carrying both shim - binaries. +- The two shim binaries. These are **not** shipped in any + published image or package — the `ghcr.io/cybertec-postgresql/pg_hardstorage` + image is distroless and carries only the `pg_hardstorage` + binary. Build the shims from source (`cmd/pg-hardstorage-barman` + and `cmd/pg-hardstorage-barman-wal-archive`) as shown below. - A repository URL accessible from the pod. ## Steps ### 1. Build the image with the shim +Compile the shim binaries from the pg_hardstorage source +tree in a builder stage, then copy them into your image: + ```dockerfile +FROM golang:1.26 AS shim-builder +WORKDIR /src +# Clone or COPY the pg_hardstorage source here. +RUN git clone https://github.com/cybertec-postgresql/pg_hardstorage . \ + && CGO_ENABLED=0 go build -o /out/pg-hardstorage-barman ./cmd/pg-hardstorage-barman \ + && CGO_ENABLED=0 go build -o /out/pg-hardstorage-barman-wal-archive ./cmd/pg-hardstorage-barman-wal-archive + FROM your-existing-barman-image:tag -COPY --from=ghcr.io/cybertec-postgresql/pg_hardstorage:v1.1 \ - /usr/bin/pg-hardstorage-barman /usr/bin/pg-hardstorage-barman -COPY --from=ghcr.io/cybertec-postgresql/pg_hardstorage:v1.1 \ - /usr/bin/pg-hardstorage-barman-wal-archive /usr/bin/pg-hardstorage-barman-wal-archive +COPY --from=shim-builder /out/pg-hardstorage-barman /usr/bin/pg-hardstorage-barman +COPY --from=shim-builder /out/pg-hardstorage-barman-wal-archive /usr/bin/pg-hardstorage-barman-wal-archive # Drop-in: barman invocations land on our binary. RUN ln -sf /usr/bin/pg-hardstorage-barman /usr/local/bin/barman \ diff --git a/docs/how-to/kubernetes/cnpg-i-provider.md b/docs/how-to/kubernetes/cnpg-i-provider.md index 30e7e1e..64917b9 100644 --- a/docs/how-to/kubernetes/cnpg-i-provider.md +++ b/docs/how-to/kubernetes/cnpg-i-provider.md @@ -39,6 +39,11 @@ tags: ### 1. Install the control plane +> **Not yet published.** This is a v0.5 roadmap surface. The +> `oci://ghcr.io/cybertec-postgresql/pg-hardstorage-server` +> chart below does not exist yet — no OCI Helm chart is +> published today. The command shows the intended shape only. + ```bash helm install pg-hardstorage \ oci://ghcr.io/cybertec-postgresql/pg-hardstorage-server \ diff --git a/docs/how-to/kubernetes/helm-sidecar-chart.md b/docs/how-to/kubernetes/helm-sidecar-chart.md index 2c6711b..fcc691f 100644 --- a/docs/how-to/kubernetes/helm-sidecar-chart.md +++ b/docs/how-to/kubernetes/helm-sidecar-chart.md @@ -42,9 +42,12 @@ helm install pg-hardstorage \ --values my-values.yaml ``` -(Once the OCI registry is published, the install is a -single-line `helm install pg-hardstorage -oci://ghcr.io/cybertec-postgresql/pg-hardstorage-sidecar`.) +The in-tree chart is the only supported install path today. +No OCI Helm chart is published yet; +`oci://ghcr.io/cybertec-postgresql/pg-hardstorage-sidecar` +does not exist. Once it is published, the install will +collapse to a single-line `helm install pg-hardstorage +oci://ghcr.io/cybertec-postgresql/pg-hardstorage-sidecar`. ### 2. Provide a values file diff --git a/docs/how-to/kubernetes/pgbackrest-shim.md b/docs/how-to/kubernetes/pgbackrest-shim.md index 68d2429..9534d4b 100644 --- a/docs/how-to/kubernetes/pgbackrest-shim.md +++ b/docs/how-to/kubernetes/pgbackrest-shim.md @@ -57,18 +57,26 @@ signed, sandbox-verifiable). ### 1. Build a PGO image with the shim ```dockerfile +FROM golang:1.26 AS shim-builder +WORKDIR /src +# Clone or COPY the pg_hardstorage source here. +RUN git clone https://github.com/cybertec-postgresql/pg_hardstorage . \ + && CGO_ENABLED=0 go build -o /out/pg-hardstorage-pgbackrest ./cmd/pg-hardstorage-pgbackrest + FROM registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-15 -COPY --from=ghcr.io/cybertec-postgresql/pg_hardstorage:v1.1 \ - /usr/bin/pg-hardstorage-pgbackrest /usr/bin/pg-hardstorage-pgbackrest +COPY --from=shim-builder /out/pg-hardstorage-pgbackrest /usr/bin/pg-hardstorage-pgbackrest # Drop-in: pgbackrest invocations land at our binary. RUN ln -sf /usr/bin/pg-hardstorage-pgbackrest /usr/bin/pgbackrest ``` -The official ghcr.io tag carries every `compat/*` shim -binary; pulling them all costs <30 MiB (single static Go -binaries). +The compat shims are **not** published in any image: the +`ghcr.io/cybertec-postgresql/pg_hardstorage` image is +distroless and carries only the `pg_hardstorage` binary. You +build each shim from its `cmd/pg-hardstorage-*` package (a +single static Go binary, <30 MiB) as shown in the builder +stage above. ### 2. Configure via standard PGO repo spec diff --git a/docs/how-to/kubernetes/walg-shim.md b/docs/how-to/kubernetes/walg-shim.md index 410a60a..cee34ca 100644 --- a/docs/how-to/kubernetes/walg-shim.md +++ b/docs/how-to/kubernetes/walg-shim.md @@ -55,10 +55,15 @@ chunks, signed manifests, sandbox-verifiable). ### 1. Build a Spilo image with the shim ```dockerfile +FROM golang:1.26 AS shim-builder +WORKDIR /src +# Clone or COPY the pg_hardstorage source here. +RUN git clone https://github.com/cybertec-postgresql/pg_hardstorage . \ + && CGO_ENABLED=0 go build -o /out/pg-hardstorage-walg ./cmd/pg-hardstorage-walg + FROM ghcr.io/zalando/spilo-15:latest -COPY --from=ghcr.io/cybertec-postgresql/pg_hardstorage:v1.1 \ - /usr/bin/pg-hardstorage-walg /usr/bin/pg-hardstorage-walg +COPY --from=shim-builder /out/pg-hardstorage-walg /usr/bin/pg-hardstorage-walg # Replace wal-g with the shim wrapper. RUN ln -sf /usr/bin/pg-hardstorage-walg /usr/bin/wal-g @@ -68,9 +73,12 @@ RUN ln -sf /usr/bin/pg-hardstorage-walg /usr/bin/wal-g # the shim if you'd rather not symlink. ``` -The official ghcr.io tag carries every `compat/*` shim -binary; pulling them all costs <30 MiB (single static Go -binaries). +The compat shims are **not** published in any image: the +`ghcr.io/cybertec-postgresql/pg_hardstorage` image is +distroless and carries only the `pg_hardstorage` binary. You +build `pg-hardstorage-walg` from its `cmd/pg-hardstorage-walg` +package (a single static Go binary, <30 MiB) as shown in the +builder stage above. ### 2. Configure via standard Zalando env vars diff --git a/docs/how-to/migration/from-barman.md b/docs/how-to/migration/from-barman.md index bcc01f4..efca51a 100644 --- a/docs/how-to/migration/from-barman.md +++ b/docs/how-to/migration/from-barman.md @@ -74,7 +74,12 @@ $EDITOR /etc/pg_hardstorage/pg_hardstorage.yaml # 4. Initialise the new repo. pg_hardstorage repo init -# 5. Drop both shim binaries onto PATH. +# 5. Build the shim binaries from source (they are not +# installed by any release artifact) and drop them on PATH. +go build -o /usr/local/bin/pg-hardstorage-barman \ + ./cmd/pg-hardstorage-barman +go build -o /usr/local/bin/pg-hardstorage-barman-wal-archive \ + ./cmd/pg-hardstorage-barman-wal-archive sudo ln -sf /usr/local/bin/pg-hardstorage-barman \ /usr/local/bin/barman sudo ln -sf /usr/local/bin/pg-hardstorage-barman-wal-archive \ diff --git a/docs/how-to/migration/from-pgbackrest.md b/docs/how-to/migration/from-pgbackrest.md index 015fa93..16dddc8 100644 --- a/docs/how-to/migration/from-pgbackrest.md +++ b/docs/how-to/migration/from-pgbackrest.md @@ -86,7 +86,11 @@ $EDITOR /etc/pg_hardstorage/pg_hardstorage.yaml # pgbackrest repo). pg_hardstorage repo init -# 5. Drop the shim onto PATH, ahead of /usr/bin/pgbackrest. +# 5. Build the shim from source (it is not installed by any +# release artifact) and drop it onto PATH, ahead of +# /usr/bin/pgbackrest. +go build -o /usr/local/bin/pg-hardstorage-pgbackrest \ + ./cmd/pg-hardstorage-pgbackrest sudo ln -sf /usr/local/bin/pg-hardstorage-pgbackrest \ /usr/local/bin/pgbackrest diff --git a/docs/how-to/migration/from-walg.md b/docs/how-to/migration/from-walg.md index 1de5e6a..067bff0 100644 --- a/docs/how-to/migration/from-walg.md +++ b/docs/how-to/migration/from-walg.md @@ -86,7 +86,11 @@ $EDITOR /etc/pg_hardstorage/pg_hardstorage.yaml # WAL-G prefix). pg_hardstorage repo init -# 5. Drop the shim onto PATH, ahead of /usr/bin/wal-g. +# 5. Build the shim from source (it is not installed by any +# release artifact) and drop it onto PATH, ahead of +# /usr/bin/wal-g. +go build -o /usr/local/bin/pg-hardstorage-walg \ + ./cmd/pg-hardstorage-walg sudo ln -sf /usr/local/bin/pg-hardstorage-walg \ /usr/local/bin/wal-g diff --git a/docs/how-to/migration/index.md b/docs/how-to/migration/index.md index ad3aeaf..c071f53 100644 --- a/docs/how-to/migration/index.md +++ b/docs/how-to/migration/index.md @@ -22,12 +22,16 @@ old repo when its window expires. Old backups stay restorable by the legacy tool itself for as long as you keep the binary around. -For pgBackRest and Barman, **v1.1 ships drop-in shim +For pgBackRest and Barman, the one-shot config translator +(`pg_hardstorage compat translate --from ...`) ships in the +main `pg_hardstorage` binary today. The **drop-in shim binaries** (`pg-hardstorage-pgbackrest`, -`pg-hardstorage-barman`) plus a one-shot config -translator (`pg_hardstorage compat translate --from ...`) -so existing cron jobs and `archive_command` settings keep -working but produce native pg_hardstorage backups. See +`pg-hardstorage-barman`) are **built from source** — their +`cmd/pg-hardstorage-*` packages live in the repo, but no +release artifact (tarball, `.deb`/`.rpm`, or container +image) ships them. Compile them yourself, then drop them on +PATH so existing cron jobs and `archive_command` settings +keep working but produce native pg_hardstorage backups. See the per-tool pages. ## Pages diff --git a/docs/how-to/operating/configuration-file.md b/docs/how-to/operating/configuration-file.md index b2b75d9..9a8cd95 100644 --- a/docs/how-to/operating/configuration-file.md +++ b/docs/how-to/operating/configuration-file.md @@ -45,15 +45,16 @@ A minimum viable config for one deployment: # /etc/pg_hardstorage/pg_hardstorage.yaml schema: pg_hardstorage.config.v1 -# Repository shared across deployments unless overridden per-deployment. -repo: s3://my-backups/prod - +# `deployments` is a MAP keyed by deployment name — not a list. +# There is no top-level `repo:` field; the repository is configured +# per deployment. deployments: - - name: prod - connection: postgresql://backup_user@db.host:5432/postgres + prod: + pg_connection: postgresql://backup_user@db.host:5432/postgres + repo: s3://my-backups/prod schedule: - backup: "daily_at 02:00" - rotate: "daily_at 04:00" + backup: { daily_at: "02:00" } + rotate: { daily_at: "04:00" } retention: policy: simple keep_for: 30d diff --git a/docs/how-to/operating/crypto-shred.md b/docs/how-to/operating/crypto-shred.md index c4fed45..c40304d 100644 --- a/docs/how-to/operating/crypto-shred.md +++ b/docs/how-to/operating/crypto-shred.md @@ -38,17 +38,14 @@ tags: ### 1. Dry-run: enumerate the affected backups ```bash -pg_hardstorage kms shred --dry-run +pg_hardstorage kms shred --dry-run --repo file:///srv/pg_hardstorage/repo ``` ```console -shred dry-run - keyring: /etc/pg_hardstorage/keys - affected manifests: 1184 - affected deployments: db1, db2 - affected total bytes: 487.3 GiB - oldest manifest: 2024-08-12T03:11:04Z - newest manifest: 2026-04-28T14:21:08Z +✓ kms shred --dry-run — preview only, KEK NOT destroyed + Keyring: /etc/pg_hardstorage/keyring + Affected: 0 backup(s) would become unrecoverable (none — no encrypted backups in this repo were wrapped with this KEK) + Note: re-run without --dry-run plus --require-approval / --confirm-keyring / --yes to actually destroy the KEK ``` Dry-run never writes to the audit chain (no state change worth @@ -61,7 +58,7 @@ size the consequences before requesting approval. pg_hardstorage approval request \ --repo file:///srv/pg_hardstorage/repo \ --op kms.shred \ - --target /etc/pg_hardstorage/keys \ + --target /etc/pg_hardstorage/keyring \ --reason "GDPR Art 17 #4421 — subject deletion request" \ --threshold 2 \ --approver-key /etc/pg_hardstorage/approvers/alice.pub \ @@ -70,10 +67,13 @@ pg_hardstorage approval request \ ``` ```console -request_id: apr-2026-04-28-7f3a1b2c -threshold: 2 of 3 -ttl: 24h0m0s -status: pending +✓ approval request created + ID: appr-6a4bb4064d13c6f0 + Op: kms.shred + Target: /etc/pg_hardstorage/keyring + Threshold: 2 of 3 allowlisted approvers + Expires: 2026-07-07T13:56:22Z + Approve: pg_hardstorage approval approve appr-6a4bb4064d13c6f0 --repo ``` The request is signed against the operator's keypair; tampering @@ -99,7 +99,9 @@ See [n-of-m approvals](n-of-m-approvals.md) for the full flow. ```bash pg_hardstorage kms shred \ - --confirm-keyring /etc/pg_hardstorage/keys \ + --repo file:///srv/pg_hardstorage/repo \ + --require-approval appr-6a4bb4064d13c6f0 \ + --confirm-keyring /etc/pg_hardstorage/keyring \ --reason "GDPR Art 17 #4421" \ --yes ``` @@ -115,13 +117,12 @@ The command: affected scope. ```console -shred complete - keyring: /etc/pg_hardstorage/keys - destroyed_at: 2026-04-28T14:24:11Z - approval_request_id: apr-2026-04-28-7f3a1b2c - approvers: alice@acme.example.com, bob@acme.example.com - audit_event: shred-2026-04-28-c91e84f0 - affected_backups_count: 1184 +✓ kms shred — KEK irreversibly destroyed + Keyring: /etc/pg_hardstorage/keyring + Reason: GDPR Art 17 #4421 + Affected: 0 backup(s) now unrecoverable (none — no encrypted backups in this repo were wrapped with this KEK) + Approval: appr-6a4bb4064d13c6f0 + Note: every backup wrapped with this KEK is now permanently unrecoverable ``` The audit event is your compliance artefact: it records @@ -196,15 +197,18 @@ contract. ## Troubleshooting -**`shred.no_approval`** — no approved n-of-m request matches -`op=kms.shred, target=`. Open one, get it approved. +**`usage.missing_flag`** — `--require-approval` was omitted; +`kms shred` refuses to run without an approved n-of-m gate. +Open an approval request for `op=kms.shred, target=`, +get it approved, then pass its ID. -**`shred.confirm_keyring_mismatch`** — `--confirm-keyring` +**`usage.confirmation_mismatch`** — `--confirm-keyring` doesn't match the resolved keyring path. Compare against `pg_hardstorage kms inspect`. -**`shred.yes_required`** — running interactively without -`--yes`. Add it. +**`usage.confirmation_required`** — `--confirm-keyring` or +`--yes` was omitted. Both are required for a live shred; add +whichever is missing. **Provider-side destroy refused (HSM)** — the HSM has policy gates (operator card, ACS quorum) beyond pg_hardstorage's diff --git a/docs/how-to/operating/data-residency.md b/docs/how-to/operating/data-residency.md index 15e5192..5cb228d 100644 --- a/docs/how-to/operating/data-residency.md +++ b/docs/how-to/operating/data-residency.md @@ -48,7 +48,7 @@ pg_hardstorage residency set db1 eu-west-1 ``` ```console -residency: db1 → ["eu-west-1"] +✓ db1: residency = [eu-west-1] (was []) ``` ### 2. Pin to a continent @@ -77,10 +77,11 @@ pg_hardstorage residency list ``` ```console -DEPLOYMENT POLICY -db1 eu-west-1 -db2 eu -db3 (none) +3 deployment(s) + DEPLOYMENT RESIDENCY + db1 eu-west-1 + db2 eu + db3 — ``` ### 5. Verify the configured repo's region matches @@ -90,13 +91,20 @@ pg_hardstorage residency check db1 ``` ```console -db1 policy=["eu-west-1"] repo_region=eu-west-1 ok +residency check — db1 + Repo: s3://acme-pg-backups/?region=eu-west-1 + Region: eu-west-1 + Allowed: eu-west-1 + ✓ region "eu-west-1" matches policy entry "eu-west-1" exactly ``` -A mismatch returns exit code 9: +A mismatch returns exit code 9 and a `verify.residency_violation` +error: ```console -db1 policy=["eu-west-1"] repo_region=us-east-1 FAIL +ERROR verify.residency_violation: residency check: region "us-east-1" does not match any allowed entry [eu-west-1] + hint: either update the deployment's repo to a region that matches the policy, or relax the policy with `pg_hardstorage residency set` / `clear`. + run: pg_hardstorage residency list ``` `doctor` runs the same check and surfaces the result among its @@ -151,12 +159,13 @@ See [Add AWS KMS](../adding/kms-aws.md), [GCP KMS](../adding/kms-gcp.md), ## Troubleshooting -**`residency.fs_unknown`** — the deployment's repo is `file://…`, -which doesn't report a region. Move to an object store, or -clear the residency policy. +**`verify.residency_violation`** (repo region unknown) — the +deployment's repo is `file://…`, which doesn't report a region. +Move to an object store, or clear the residency policy. -**`residency.mismatch`** — `repo_region` doesn't match the -allowlist. Either move the repo or relax the policy. +**`verify.residency_violation`** (region mismatch) — the repo's +region doesn't match the allowlist. Either move the repo or relax +the policy. **`region` empty on S3-compatible endpoints** — MinIO, R2, Wasabi often don't report a meaningful region. Pin diff --git a/docs/how-to/operating/legal-hold.md b/docs/how-to/operating/legal-hold.md index 0fe1c93..55be3a6 100644 --- a/docs/how-to/operating/legal-hold.md +++ b/docs/how-to/operating/legal-hold.md @@ -36,10 +36,11 @@ pg_hardstorage list db1 ``` ```console -ID TYPE WHEN SIZE STATUS -db1.full.20260427T020000Z full 2026-04-27 02:00 UTC 12.3 GiB ok -db1.full.20260420T020000Z full 2026-04-20 02:00 UTC 12.1 GiB ok -db1.full.20260413T020000Z full 2026-04-13 02:00 UTC 11.9 GiB ok +Backups for db1 (3): + BACKUP ID TYPE WHEN FILES SIZE DEDUP DURATION + db1.full.20260427T020000Z full 2026-04-27 02:00 1284 12.3 GiB 1.42x 38210ms + db1.full.20260420T020000Z full 2026-04-20 02:00 1281 12.1 GiB 1.41x 37544ms + db1.full.20260413T020000Z full 2026-04-13 02:00 1279 11.9 GiB 1.40x 36988ms ``` ### 2. Place an indefinite hold @@ -111,10 +112,15 @@ pg_hardstorage hold list db1 --repo file:///srv/pg_hardstorage/repo ### 5. Purge expired holds ```bash -pg_hardstorage hold purge-expired --repo file:///srv/pg_hardstorage/repo +# Preview first (no mutations, no audit emits): +pg_hardstorage hold purge-expired --repo file:///srv/pg_hardstorage/repo --dry-run + +# Then actually remove: +pg_hardstorage hold purge-expired --repo file:///srv/pg_hardstorage/repo --yes ``` -Removes every marker whose `ExpiresAt` is in the past. The +`--yes` is required for the live removal (`--dry-run` is the only +way to run it without `--yes`). Removes every marker whose `ExpiresAt` is in the past. The markers themselves are gone, but the underlying manifest is now eligible for rotation (it was never deleted; just protected). @@ -137,12 +143,14 @@ side of the marker, and it should be deliberate. the soft-delete sweep: ```console -rotate db1 policy=gfs - ... - held: 2 backup(s) pinned by legal hold - db1.full.20260420T020000Z (ops@acme.example.com) - db1.full.20260413T020000Z (counsel@acme.example.com) - tombstoned: 0 backup(s) +✓ Rotation applied + Policy: gfs + + db1 + keep: 3 + delete: 0 + held: 2 (excluded from delete: db1.full.20260420T020000Z, db1.full.20260413T020000Z) + applied: 0 ``` The result body's `held` / `held_ids` fields capture what diff --git a/docs/how-to/operating/n-of-m-approvals.md b/docs/how-to/operating/n-of-m-approvals.md index fe28c93..0c23ef7 100644 --- a/docs/how-to/operating/n-of-m-approvals.md +++ b/docs/how-to/operating/n-of-m-approvals.md @@ -50,7 +50,7 @@ the approval is not an operator surface. pg_hardstorage approval request \ --repo file:///srv/pg_hardstorage/repo \ --op kms.shred \ - --target /etc/pg_hardstorage/keys \ + --target /etc/pg_hardstorage/keyring \ --reason "GDPR Art 17 #4421 — subject deletion request" \ --threshold 2 \ --ttl 24h \ @@ -60,13 +60,13 @@ pg_hardstorage approval request \ ``` ```console -request_id: apr-2026-04-28-7f3a1b2c -op: kms.shred -target: /etc/pg_hardstorage/keys -threshold: 2 of 3 -ttl: 24h0m0s -status: pending -created_at: 2026-04-28T14:21:08Z +✓ approval request created + ID: appr-6a4bb4064d13c6f0 + Op: kms.shred + Target: /etc/pg_hardstorage/keyring + Threshold: 2 of 3 allowlisted approvers + Expires: 2026-07-07T13:56:22Z + Approve: pg_hardstorage approval approve appr-6a4bb4064d13c6f0 --repo ``` The request is signed against the operator's keypair and @@ -77,27 +77,26 @@ invalidates every existing approval. ```bash # On Alice's machine -pg_hardstorage approval status apr-2026-04-28-7f3a1b2c \ +pg_hardstorage approval status appr-6a4bb4064d13c6f0 \ --repo file:///srv/pg_hardstorage/repo ``` ```console -op: kms.shred -target: /etc/pg_hardstorage/keys -reason: GDPR Art 17 #4421 — subject deletion request -threshold: 2 of 3 (1 approval so far) -status: pending -ttl_remaining: 23h47m -approvers: - - alice.pub (not signed) - - bob.pub (not signed) - - carol.pub (signed at 2026-04-28T14:24:11Z) +approval appr-6a4bb4064d13c6f0 + Op: kms.shred + Initiator: hardstorage + Target: /etc/pg_hardstorage/keyring + Reason: GDPR Art 17 #4421 — subject deletion request + Status: pending (1/2 approvals) + Expires: 2026-07-07T13:56:22Z + Approvals: + 2026-07-06T13:56:34Z carol@acme.example.com confirm subject 4421 deletion ``` If Alice agrees: ```bash -pg_hardstorage approval approve apr-2026-04-28-7f3a1b2c \ +pg_hardstorage approval approve appr-6a4bb4064d13c6f0 \ --repo file:///srv/pg_hardstorage/repo \ --approver alice@acme.example.com \ --key /home/alice/.ssh/pg_hardstorage_alice.pem \ @@ -111,28 +110,30 @@ ed25519 signature. ### 3. Watch for "approved" ```bash -pg_hardstorage approval status apr-2026-04-28-7f3a1b2c \ +pg_hardstorage approval status appr-6a4bb4064d13c6f0 \ --repo file:///srv/pg_hardstorage/repo ``` ```console -op: kms.shred -target: /etc/pg_hardstorage/keys -threshold: 2 of 3 (2 approvals — APPROVED) -status: approved -approved_at: 2026-04-28T14:31:42Z -ttl_remaining: 23h36m -approvers: - - alice.pub (signed at 2026-04-28T14:31:42Z by alice@acme.example.com) - - bob.pub (not signed) - - carol.pub (signed at 2026-04-28T14:24:11Z by carol@acme.example.com) +approval appr-6a4bb4064d13c6f0 + Op: kms.shred + Initiator: hardstorage + Target: /etc/pg_hardstorage/keyring + Reason: GDPR Art 17 #4421 — subject deletion request + Status: approved (2/2 approvals) + Expires: 2026-07-07T13:56:22Z + Approvals: + 2026-07-06T13:56:42Z alice@acme.example.com I confirm subject 4421's deletion is in flight + 2026-07-06T13:56:34Z carol@acme.example.com confirm subject 4421 deletion ``` ### 4. Consume the approval ```bash pg_hardstorage kms shred \ - --confirm-keyring /etc/pg_hardstorage/keys \ + --repo file:///srv/pg_hardstorage/repo \ + --require-approval appr-6a4bb4064d13c6f0 \ + --confirm-keyring /etc/pg_hardstorage/keyring \ --reason "GDPR Art 17 #4421" \ --yes ``` @@ -145,7 +146,7 @@ to prevent replay. ### 5. (Optional) Revoke a request before approval ```bash -pg_hardstorage approval revoke apr-2026-04-28-7f3a1b2c \ +pg_hardstorage approval revoke appr-6a4bb4064d13c6f0 \ --repo file:///srv/pg_hardstorage/repo \ --reason "Wrong target — opened against staging keyring" ``` @@ -161,9 +162,9 @@ pg_hardstorage approval list \ ``` ```console -ID OP TARGET THRESHOLD STATUS TTL_REMAINING -apr-2026-04-28-7f3a1b2c kms.shred /etc/pg_hardstorage/keys 2 of 3 approved 23h36m -apr-2026-04-27-3e1f9a04 repo.wipe s3://acme-pg-backups/ 3 of 5 pending 8h12m +ID OP STATUS APPROVALS EXPIRES TARGET +appr-6a4bb4064d13c6f0 kms.shred approved 2/2 2026-07-07T13:56:22Z /etc/pg_hardstorage/keyring +appr-3e1f9a04c2b18d5a repo.wipe pending 1/3 2026-07-06T22:12:00Z s3://acme-pg-backups/ ``` ## Threshold sizing diff --git a/docs/how-to/operating/schedule-backups.md b/docs/how-to/operating/schedule-backups.md index df25c9e..2e61f2e 100644 --- a/docs/how-to/operating/schedule-backups.md +++ b/docs/how-to/operating/schedule-backups.md @@ -48,10 +48,14 @@ pg_hardstorage schedule ``` ```console -DEPLOYMENT BACKUP ROTATE -db1 every 6h daily_at 04:00 -db2 daily_at 02:00 daily_at 03:30 -db3 (unscheduled) (unscheduled) +Schedules for 3 deployment(s): + DEPLOYMENT TASK WHEN + db1 backup every 6h0m0s + db1 rotate daily at 04:00 (Local) + db2 backup daily at 02:00 (Local) + db2 rotate daily at 03:30 (Local) + db3 backup off + db3 rotate off ``` The fleet listing surfaces both tasks — useful for spotting @@ -65,7 +69,9 @@ pg_hardstorage schedule db1 ``` ```console -db1 backup: every 6h +Schedule for db1.backup: + every 6h0m0s + every: 6h ``` `--task=rotate` shows the rotate schedule instead: diff --git a/docs/how-to/operating/scrub-and-heal.md b/docs/how-to/operating/scrub-and-heal.md index 27dd123..f6e681a 100644 --- a/docs/how-to/operating/scrub-and-heal.md +++ b/docs/how-to/operating/scrub-and-heal.md @@ -37,8 +37,14 @@ pg_hardstorage repo scrub file:///srv/pg_hardstorage/repo --sample-percent 1 ``` ```console -scrub repo=file:///srv/pg_hardstorage/repo sample=1% - chunks_checked=412 mismatches=0 ok +repo scrub — 1% sample + Referenced chunks: 41200 + Sampled: 412 + OK: 412 + Mismatches: 0 + Bytes scanned: 3.2 GiB + Duration: 8140 ms + ✓ no integrity findings ``` Wire this into a cron job: @@ -84,14 +90,23 @@ pg_hardstorage repair scrub \ ``` ```console -chunks_checked=12453 mismatches=2 healed=2 not_at_replica=0 - healed: sha256=abc123... refetched=4 KiB - healed: sha256=def456... refetched=64 KiB +repair scrub + sampled 12453 / 12453 referenced chunks (68.4 GiB verified) + ✗ 2 chunk(s) failed integrity check: + sha256:abc123… + sha256:def456… + heal — replica file:///srv/pg_hardstorage/repo-replica + Healed: 2 + Already OK: 0 + Not at replica: 0 + Failed: 0 + Bytes copied: 68 KiB + ✓ all mismatches healed ``` Heal is **best-effort**: a chunk missing at the replica is -reported as `NotAtReplica` and the run continues with the next -mismatch. To preview without writing, run scrub **without** +counted in the result body's `Not at replica` line and the run +continues with the next mismatch. To preview without writing, run scrub **without** `--heal` — it reports every mismatch (exactly what a heal would repair) and touches nothing: @@ -158,13 +173,16 @@ attach that to your incident postmortem. ## Troubleshooting -**`scrub.unreachable`** — the repo URL doesn't resolve. -Confirm credentials / network. +**`verify.scrub_mismatch`** — one or more sampled chunks failed +the hash check (exit code 9). The storage backend has corrupted +bytes for the listed chunks; heal from a replica with +`pg_hardstorage repair scrub --heal --replica `. -**`heal.NotAtReplica` for every mismatch** — the replica -isn't fully populated, or it was cut from a cold archive that -predates the corrupted chunks. Re-run `repo replicate` to -top up. +**`Not at replica` equals the mismatch count** — this is a +result-body field, not an error code: every corrupted chunk was +also missing at the replica. The replica isn't fully populated, +or it was cut from a cold archive that predates the corrupted +chunks. Re-run `repo replicate` to top up. **Heal runs but `repo check` still fails** — there's a deeper problem (missing manifest, broken signature, GC collision). diff --git a/docs/how-to/operating/set-retention.md b/docs/how-to/operating/set-retention.md index 4803854..c567e5f 100644 --- a/docs/how-to/operating/set-retention.md +++ b/docs/how-to/operating/set-retention.md @@ -58,7 +58,7 @@ calendar buckets. Use durations: `30d`, `2w`, `12h`, `45m`. ```yaml retention: policy: count - keep_full_count: 14 + keep_fulls: 14 ``` WAL is kept while needed for PITR onto any retained full. @@ -79,13 +79,16 @@ pg_hardstorage rotate db1 --repo file:///srv/pg_hardstorage/repo ``` ```console -deployment: db1 policy: gfs - kept id reason - ✓ db1.full.20260427T020000Z newest (always kept) - ✓ db1.full.20260426T020000Z daily slot 1 - ✓ db1.full.20260420T020000Z weekly slot 1 - ✗ db1.full.20260419T020000Z superseded by 20260420 - ... +Rotation plan (dry-run — no manifests soft-deleted) + Policy: gfs + + db1 + keep: 3 + delete: 1 + [keep] db1.full.20260427T020000Z @ 2026-04-27T02:00:00Z (daily-1,weekly-1) + [keep] db1.full.20260426T020000Z @ 2026-04-26T02:00:00Z (daily-2) + [keep] db1.full.20260420T020000Z @ 2026-04-20T02:00:00Z (weekly-2) + [del ] db1.full.20260419T020000Z @ 2026-04-19T02:00:00Z ``` `rotate` defaults to dry-run. Read the table; nothing's diff --git a/docs/how-to/operating/verify-fast-vs-full.md b/docs/how-to/operating/verify-fast-vs-full.md index 5fd84b2..7361573 100644 --- a/docs/how-to/operating/verify-fast-vs-full.md +++ b/docs/how-to/operating/verify-fast-vs-full.md @@ -22,7 +22,7 @@ tags: | Mode | Reads | Decrypts | Restores | pg_verifybackup | Typical use | | --- | --- | --- | --- | --- | --- | | Fast (default) | every chunk | yes | no | no | Hourly / daily cron | -| Sampled (`--sample N`) | N random chunks | yes | no | no | Spot-check a cold archive | +| Sampled (`--sample N`) | first N chunks in hash-sorted order | yes | no | no | Spot-check a cold archive | | Existence-only (`--existence-only`) | metadata only (Stat) | no | no | no | Pre-flight before `backup undelete` | | Full (`--full`) | every chunk | yes | yes (sandbox) | yes | Quarterly / pre-DR drill | @@ -39,7 +39,10 @@ pg_hardstorage verify db1 latest ``` ```console -db1.full.20260427T020000Z verified 12.3 GiB 3245 chunks 9.4s +verify db1/db1.full.20260427T020000Z (full) + manifest signature: valid + chunks: 3245 referenced, 3245 unique, 3245 sampled + ✓ 3245 chunk(s) verified — 12.3 GiB in 9412ms ``` `--repo ` overrides the deployment's configured repo for @@ -55,9 +58,11 @@ the chunk loop runs. pg_hardstorage verify db1 --sample 1000 ``` -Caps the chunk count to a random subset. Useful for cheap -periodic sanity checks on cold archives where running the full -verify weekly is overkill. +Caps the chunk count to the first N chunks in hash-sorted +order — deterministic, so repeated runs check the same subset +and results are reproducible. Useful for cheap periodic sanity +checks on cold archives where running the full verify weekly is +overkill. `--sample 0` and omitting the flag both mean "full chunk list." @@ -148,12 +153,12 @@ expected SHA-256. Either the storage backend corrupted them (scrub the rest of the repo with [scrub-and-heal](scrub-and-heal.md)) or the chunk was tampered with. -**`verify.missing_chunks`** — a referenced chunk is absent. +**`verify.chunks_missing`** — a referenced chunk is absent. Could be: a partial replicate, an over-aggressive `repo gc`, or storage-backend data loss. Pair with `pg_hardstorage repair manifest` and a replica. -**`verify.signature_mismatch`** — the manifest's Ed25519 +**`verify.manifest_signature`** — the manifest's Ed25519 signature is invalid. Treat as untrusted. Either the signing key changed and the manifest was never re-signed, or the manifest was modified after commit. Cross-check with the diff --git a/docs/how-to/packaging/build-from-source.md b/docs/how-to/packaging/build-from-source.md index 2d244b4..0cca56a 100644 --- a/docs/how-to/packaging/build-from-source.md +++ b/docs/how-to/packaging/build-from-source.md @@ -58,12 +58,16 @@ make build-testkit Writes `bin/pg_hardstorage_testkit`. Same flags; different entry point under `cmd/pg_hardstorage_testkit`. -### 3. Build both at once +### 3. Build all binaries at once ```bash make all-binaries ``` +This runs `build`, `build-testkit`, and `build-simple` — +three binaries: `bin/pg_hardstorage`, +`bin/pg_hardstorage_testkit`, and `bin/pg_hardstorage_simple`. + ### 4. Confirm the build is stamped ```bash @@ -71,13 +75,16 @@ make all-binaries ``` ```console -pg_hardstorage v1.0.x - commit: abcdef1 - date: 2026-05-04T08:13:42Z - fips: false - build: default +pg_hardstorage v1.0.x (abcdef1, built 2026-05-04T08:13:42Z) ``` +The output is a single line: `pg_hardstorage +[FIPS] (, built )`. The `[FIPS]` marker only +appears on the FIPS variant. For machine-readable output run +`version --output json`; the JSON carries `version`, +`commit`, `date`, `variant` (`"default"` or `"fips"`), and +`fips` fields. + A clean checkout that's not on a tagged commit produces `v1.0.x-N-gabcdef1` — which is what you want, because that's the unique identifier of the build that's running. diff --git a/docs/how-to/packaging/fips-variant.md b/docs/how-to/packaging/fips-variant.md index 7ed25dd..c9b6774 100644 --- a/docs/how-to/packaging/fips-variant.md +++ b/docs/how-to/packaging/fips-variant.md @@ -18,7 +18,8 @@ tags: ## What you need -- Go 1.19 or later. `GOEXPERIMENT=boringcrypto` is built +- Go 1.26+ (the version pinned in `go.mod`). + `GOEXPERIMENT=boringcrypto` is built into the upstream Go toolchain on `linux/amd64`; on every other GOOS/GOARCH the experiment fails with a clear error. @@ -54,10 +55,9 @@ GOEXPERIMENT=boringcrypto CGO_ENABLED=1 \ ``` The `fips` build tag is the runtime selector: -`internal/fips.Enabled()` returns `true` in this flavour; -`pg_hardstorage doctor` surfaces it under the system -section so operators see at a glance which variant is -running. +`internal/fips.Enabled()` returns `true` in this flavour. +The `pg_hardstorage version` subcommand surfaces the variant +so operators see at a glance which flavour is running. ### 2. Confirm BoringCrypto symbols are linked in @@ -78,25 +78,20 @@ verify `GOEXPERIMENT=boringcrypto` was set during the build. ### 3. Confirm the runtime flag -```bash -./bin/pg_hardstorage-fips doctor -``` - -The system section reports `fips: true`. The `version` -subcommand also surfaces the flag: +The `version` subcommand surfaces the variant: ```bash ./bin/pg_hardstorage-fips version ``` ```console -pg_hardstorage v1.0.x - commit: abcdef1 - ... - fips: true - build: fips +pg_hardstorage v1.0.x [FIPS] (abcdef1, built 2026-01-02T03:04:05Z) ``` +The `[FIPS]` marker in the one-line output is your runtime +confirmation. The JSON form (`version --output json`) carries +`"variant": "fips"` and `"fips": true`. + ## What just happened `GOEXPERIMENT=boringcrypto` swaps Go's standard crypto @@ -112,7 +107,7 @@ in older versions) refuse at runtime under The `fips` build tag activates the project's own FIPS posture: refuses non-FIPS encryption providers, refuses unsigned manifests in compliance mode, and surfaces the -flag in `doctor` output. +variant in `version` output. ## What FIPS mode actually changes for the operator @@ -121,7 +116,7 @@ flag in `doctor` output. | Storage envelope | AES-256-GCM-SIV via BoringCrypto. | | TLS to control plane / repos | Only FIPS-approved ciphersuites. | | KMS providers | Refuses providers that aren't FIPS-approved. | -| `doctor` | Reports `fips: true` and a "FIPS mode active" event. | +| `version` | Prints the `[FIPS]` marker (JSON `variant: "fips"`, `fips: true`). | | Unit / integration tests | Same suite, run with the FIPS build. | Operators who don't need FIPS should stay on the default @@ -138,8 +133,11 @@ The packaging story is roadmap from the SPEC: | v0.5 | `pg-hardstorage-fips` `.deb` / `.rpm` shipped with goreleaser; `Conflicts: pg-hardstorage` on Debian so the two binaries don't co-install. | -| v0.5+ | `ghcr.io/cybertec-postgresql/pg_hardstorage-fips:` - distroless container image, cosign-signed, SBOM via syft. | +| v0.5+ | A distroless FIPS container image, cosign-signed, + SBOM via syft. **Not yet published** — no `-fips` image + variant exists today; only the default + `ghcr.io/cybertec-postgresql/pg_hardstorage` image is + published, and it is not FIPS-built. | For v0.1 the path is "build it yourself with the instructions on this page." diff --git a/docs/how-to/packaging/firecracker-variant.md b/docs/how-to/packaging/firecracker-variant.md index 2e95891..2e9bdfd 100644 --- a/docs/how-to/packaging/firecracker-variant.md +++ b/docs/how-to/packaging/firecracker-variant.md @@ -65,34 +65,28 @@ go build -tags firecracker \ `CGO_ENABLED=0` works for this variant — the SDK is pure Go. The Makefile inherits the project default `CGO_ENABLED=0`. -### 3. Confirm the firecracker backend is registered +### 3. Confirm the firecracker backend is compiled in -```bash -./bin/pg_hardstorage-firecracker doctor -``` - -The system section lists registered sandbox backends. On -the firecracker build: - -```console -Sandbox backends: docker, firecracker -``` - -On the default build: - -```console -Sandbox backends: docker -``` +The `firecracker` build tag swaps the default stub backend +(`backend_firecracker_stub.go`, which refuses with a clear +"this build doesn't include the firecracker backend" +message) for the real implementation +(`backend_firecracker_real.go`). There is no `doctor` output +that enumerates sandbox backends; the confirmation is simply +that a firecracker-backed verify succeeds instead of +returning the stub's refusal. ### 4. Run a verify with the new backend +The sandbox backend and its Firecracker kernel / rootfs +paths are configured in `pg_hardstorage.yaml`, not via CLI +flags — there is no `--sandbox-backend`, `--firecracker-kernel`, +or `--firecracker-rootfs` flag. Run the verify as usual: + ```bash pg_hardstorage-firecracker verify db1 latest \ --repo s3://acme-pg-backups \ - --full \ - --sandbox-backend firecracker \ - --firecracker-kernel /var/lib/pg_hardstorage/firecracker/vmlinux \ - --firecracker-rootfs /var/lib/pg_hardstorage/firecracker/rootfs.ext4 + --full ``` See [Verify with the Firecracker sandbox](../verify/firecracker-sandbox.md) @@ -128,7 +122,9 @@ needing KVM in CI. | Portable across non-Linux operator workstations | Yes | No (Linux + KVM only) | If both Docker and Firecracker are available on a host, -operators select per-verify via `--sandbox-backend`. +operators select the backend through the sandbox +configuration in `pg_hardstorage.yaml` (there is no +`--sandbox-backend` CLI flag). ## Distribution diff --git a/docs/how-to/packaging/pkcs11-variant.md b/docs/how-to/packaging/pkcs11-variant.md index 1916c45..fe579cb 100644 --- a/docs/how-to/packaging/pkcs11-variant.md +++ b/docs/how-to/packaging/pkcs11-variant.md @@ -72,28 +72,34 @@ CGO_ENABLED=1 go build -tags pkcs11 \ ### 3. Configure a KEK against your HSM -```yaml -# /etc/pg_hardstorage/pg_hardstorage.yaml -kms: - default: - type: pkcs11 - pkcs11: - module: /usr/lib/softhsm/libsofthsm2.so - slot: 0 - pin_file: /etc/pg_hardstorage/keyring/hsm-pin - key_label: pg_hardstorage_master_kek +There is no `kms:` YAML block. A PKCS#11 KEK is addressed by +a `pkcs11://` KEKRef URL, whose form is: + +``` +pkcs11:///?module=&pin=&slot=&mech= ``` -The `module:` path is whatever your HSM vendor ships; -`slot:` and `key_label:` come from your HSM admin console. -`pin_file` is mode-0600, owner `pgbackup`, one-line file -containing the PIN — the same posture as the rest of the -keyring directory. +For example, against SoftHSM: + +``` +pkcs11://prod-token/pg_hardstorage_master_kek?module=/usr/lib/softhsm/libsofthsm2.so&slot=0&pin_source=/etc/pg_hardstorage/keyring/hsm-pin +``` + +The `module=` path is whatever your HSM vendor ships; +``, ``, and `slot=` come from your +HSM admin console. Rather than embedding the PIN in the URL, +point `pin_source=` at a mode-0600 file owned by `pgbackup` +(one line, the PIN) — the same posture as the rest of the +keyring directory. `module=` can also be supplied out of +band via `$PKCS11_MODULE_PATH`. ### 4. Run `kms verify` +Pass the full `pkcs11://` KEKRef to `--kek-ref`: + ```bash -pg_hardstorage-pkcs11 kms verify --kek-ref pkcs11://default +pg_hardstorage-pkcs11 kms verify --repo \ + --kek-ref 'pkcs11://prod-token/pg_hardstorage_master_kek?module=/usr/lib/softhsm/libsofthsm2.so&slot=0&pin_source=/etc/pg_hardstorage/keyring/hsm-pin' ``` This is the post-install smoke check: round-trip a tiny diff --git a/docs/how-to/windows-install.md b/docs/how-to/windows-install.md index aa6fd44..0a402a4 100644 --- a/docs/how-to/windows-install.md +++ b/docs/how-to/windows-install.md @@ -12,14 +12,15 @@ tags: # Install pg_hardstorage on Windows -> Status — **alpha**. Every command in the CLI is -> cross-compiled for `windows/amd64` and `windows/arm64` -> on every commit (CI gate `build (windows/amd64)`), but -> the agent's scheduled-job runner has no Windows Service -> integration yet, and the installer (`.msi`, signed -> bundle) is not yet packaged. This page covers the -> **manual binary install** that an operator can use -> today against a remote PostgreSQL. +> Status — **alpha**. CI compile-tests every command for +> both `windows/amd64` and `windows/arm64` on every commit +> (the `build (windows/…)` gate), but only `windows/amd64` +> is shipped as a tagged release asset — `windows/arm64` is +> CI-verified, not released. The agent's scheduled-job +> runner has no Windows Service integration yet, and the +> installer (`.msi`, signed bundle) is not yet packaged. +> This page covers the **manual binary install** that an +> operator can use today against a remote PostgreSQL. ## What works on Windows today @@ -64,20 +65,30 @@ tags: ### 1. Download the binaries -CI uploads `pg_hardstorage-windows--.zip` -artifacts on every commit. For a tagged release, -download the corresponding asset from the GitHub release -page. The bundle contains seven `.exe` files: - -``` -pg_hardstorage.exe -pg_hardstorage_testkit.exe -pg-hardstorage-pgbackrest.exe -pg-hardstorage-barman.exe -pg-hardstorage-barman-wal-archive.exe -pg-hardstorage-walg.exe -pg-hardstorage-compat.exe -``` +There are two distinct sources, and they ship different +sets of binaries: + +- **Per-commit CI artifact** — `pg_hardstorage-windows--` + is uploaded on every commit (7-day retention) for both + `amd64` and `arm64`. This bundle contains seven `.exe` + files: + + ``` + pg_hardstorage.exe + pg_hardstorage_testkit.exe + pg-hardstorage-pgbackrest.exe + pg-hardstorage-barman.exe + pg-hardstorage-barman-wal-archive.exe + pg-hardstorage-walg.exe + pg-hardstorage-compat.exe + ``` + +- **Tagged release** — the GitHub release ships only + `pg_hardstorage.exe`, and only for `windows/amd64` + (goreleaser publishes no `windows/arm64` release asset and + none of the shim/testkit `.exe` files). If you need the + shims or testkit on Windows, grab the per-commit CI + artifact or build them from source with `GOOS=windows`. ### 2. Place them on PATH @@ -119,7 +130,7 @@ keyring location, and any environment misconfiguration. Following Microsoft's Known Folders guidance: -| Domain | User mode (default) | System mode (`--mode system`) | +| Domain | User mode (default) | System mode (elevated / LocalSystem) | |---|---|---| | Config | `%APPDATA%\pg_hardstorage\` (roams) | `%PROGRAMDATA%\pg_hardstorage\config\` | | State | `%LOCALAPPDATA%\pg_hardstorage\state\` | `%PROGRAMDATA%\pg_hardstorage\state\` | @@ -128,6 +139,11 @@ Following Microsoft's Known Folders guidance: | Runtime | `%LOCALAPPDATA%\pg_hardstorage\run\` | `%PROGRAMDATA%\pg_hardstorage\run\` | | Shared data | `%PROGRAMDATA%\pg_hardstorage\` | `%PROGRAMDATA%\pg_hardstorage\share\` | +System mode is not selected by a flag — there is no +`--mode` switch. The resolver auto-detects it: an elevated +Administrator / LocalSystem context resolves to system mode, +the interactive user resolves to user mode. + Resolved paths show in `pg_hardstorage doctor` with `source: windows` so it's unambiguous why the resolver picked a particular directory. diff --git a/docs/index.md b/docs/index.md index e1eaf52..ae94b87 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ title: PostgreSQL backup, WAL streaming & PITR description: >- pg_hardstorage is an open-source PostgreSQL backup agent and CLI: continuous WAL streaming, point-in-time recovery, content-addressed - deduplication, envelope encryption, and signed manifests. PG 15+, + deduplication, envelope encryption, and signed manifests. PG 15–18, Apache 2.0. --- @@ -33,7 +33,7 @@ content-addressed chunks, AES-256-GCM-encrypts under a per-backup DEK wrapped by a configurable KEK, and signs every manifest with Ed25519. -PG 15+, Apache 2.0. Targets single-host deployments, +PG 15–18, Apache 2.0. Targets single-host deployments, multi-tenant SaaS, and Patroni clusters with the same binary. diff --git a/docs/llm-helper.md b/docs/llm-helper.md index 9e2a20f..a50d5d8 100644 --- a/docs/llm-helper.md +++ b/docs/llm-helper.md @@ -64,7 +64,7 @@ Every `llm ask` invocation returns a structured body: ```json { "skill": "ask", - "skill_version": "0.2.0", + "skill_version": "1.0.0", "provider": "openai", "question": "...", "answer": "...", diff --git a/docs/operations/cost-reporting.md b/docs/operations/cost-reporting.md index 96ea652..259a4da 100644 --- a/docs/operations/cost-reporting.md +++ b/docs/operations/cost-reporting.md @@ -83,45 +83,21 @@ production fleets. --- -## Per-tenant exports +## Per-deployment exports -A tenant boundary in `pg_hardstorage` is a logical grouping -of deployments under a shared KEK. The report splits per -tenant when the deployments are tagged: +The JSON report carries a `deployments[]` array — one row per +deployment. Each row has `name`, `backup_count`, +`logical_bytes` (pre-dedup), `manifest_bytes`, and `wal_bytes`: ```sh -pg_hardstorage cost report --repo s3://acme-backups/ \ - -o json +pg_hardstorage cost report --repo s3://acme-backups/ -o json ``` -Returns one row per tenant in the result body. The KEK -namespace alignment makes tenant-level chunk attribution -exact: chunks under tenant T's KEK never dedup against -tenant U's chunks (different per-chunk keys), so per-tenant -chunk bytes is the sum of chunks encrypted under T's -wrapped DEK. - -JSON shape (excerpt): - -```json -{ - "schema": "pg_hardstorage.cost.v1", - "by_tenant": [ - { - "tenant": "acme-prod", - "deployments": ["db1", "db2"], - "physical_bytes": 1389567800832, - "estimated_monthly_usd": 29.78 - }, - { - "tenant": "acme-staging", - "deployments": ["db1-staging"], - "physical_bytes": 64236000000, - "estimated_monthly_usd": 1.38 - } - ] -} -``` +Per-tenant splitting (a `by_tenant[]` breakdown grouping +deployments under a shared KEK) is **not yet shipped** — the +report has no `by_tenant` field. Aggregate deployments into +tenants downstream from the `deployments[]` rows if you need +tenant-level rollups today. --- @@ -149,9 +125,9 @@ chargeback pipeline: ```sh pg_hardstorage cost report --repo s3://acme-backups/ \ -o json \ - | jq -c '.result.body.by_tenant[] | - {tenant, physical_gb: (.physical_bytes / 1e9), - monthly_usd: .estimated_monthly_usd}' \ + | jq -c '.result.body.deployments[] | + {name, logical_gb: (.logical_bytes / 1e9), + manifest_bytes, wal_bytes}' \ | curl --data-binary @- https://billing.acme.example.com/api/v1/usage ``` @@ -173,7 +149,7 @@ pg_hardstorage repo usage --repo s3://acme-backups/ -o json \ > usage.json pg_hardstorage cost report --repo s3://acme-backups/ -o json \ > cost.json -jq -s '.[0].result.body.bytes_total - .[1].result.body.total_physical_bytes' \ +jq -s '.[0].result.body.total_bytes - .[1].result.body.total_physical_bytes' \ usage.json cost.json # Expect: 0 ``` diff --git a/docs/operations/operator-guide.md b/docs/operations/operator-guide.md index 0ef6098..1ff1cff 100644 --- a/docs/operations/operator-guide.md +++ b/docs/operations/operator-guide.md @@ -169,7 +169,7 @@ retention: ```yaml retention: policy: count - keep_full_count: 14 # keep last N fulls; WAL kept while needed for PITR + keep_fulls: 14 # keep last N fulls; WAL kept while needed for PITR ``` ### Soft-delete via tombstones @@ -231,10 +231,14 @@ on mismatch. ### Full verify (sandbox restore + pg_verifybackup) -The `verify` command's full mode is in v0.5; today, the -`pg_verifybackup` gate runs automatically after every `restore` — -that is the integration test. Skip it with `--verify=skip` only after -acknowledging that exit 9 is the contract. +`verify --full` performs a full restore into a Docker sandbox and runs +`pg_verifybackup` against it (requires Docker locally). To dispatch the +sandbox verify to an agent instead of running Docker on your host, pass +`--control-plane ` — it always implies `--full` semantics. + +In addition, the `pg_verifybackup` gate runs automatically after every +`restore` — that is the integration test. Skip it with `--verify=skip` +only after acknowledging that exit 9 is the contract. `--verify=auto` (default) runs `pg_verifybackup` if the binary is on `$PATH`. `--verify=require` returns `usage.no_pg_verifybackup` (exit 2) @@ -297,11 +301,22 @@ pg_hardstorage repo usage file:///srv/backups Bytes by category — chunks, primary manifests, replica manifests, trash, WAL, audit. Useful for explaining the bill. -### scrub stub +### scrub + +`repo scrub` is the periodic bit-rot check: it samples a percentage of +referenced chunks and re-hashes each against its key, surfacing any +mismatch as a "storage backend corrupted bytes" finding. + +```sh +pg_hardstorage repo scrub file:///srv/backups # 1% sample +pg_hardstorage repo scrub file:///srv/backups --full # every chunk +``` -`repo scrub` is the periodic auto-heal job (re-hash a percentage of -chunks per day, heal from replica region on mismatch). It is a stub -in v0.1; use `repair scrub` for the full pass today. +`--sample-percent` defaults to `1` (1% per run) — the operator-friendly +cadence for an hourly cron. Pass `--full` (shorthand for +`--sample-percent 100`) for exhaustive quarterly checks. Mismatches map +to exit 9 so a cron-wired scrub alarms when integrity slips; findings +are also captured in the hash-chained audit log. --- @@ -349,8 +364,23 @@ environments double-archive), the binary doubles as a shim: archive_command = '/usr/bin/pg_hardstorage wal push db1 %p --repo file:///srv/backups' ``` -`wal push` is a stub in v0.1 (lands in v0.5+); for now stick with -streaming as the only path. +`wal push` ships: PG invokes it from `archive_command`, and it reads +the `%p` segment, chunks it through the CAS, and commits a segment +manifest atomically. Re-pushes of an already-committed segment are +no-ops. + +The segment's `system_identifier` (stamped on every manifest so +cross-cluster contamination is detectable) is derived without a libpq +round-trip: it reads `xlp_sysid` from the segment's first-page header, +or you may pass `--system-identifier ` (the unsigned-decimal +form `pg_control_system()` reports) to skip the header parse. Pass +`--tde` when the source PG has Transparent Data Encryption enabled — +the on-disk segment is ciphertext, so header parsing is skipped and you +must supply `--system-identifier` or `--pg-connection`. + +Exit-code contract: `0` on success (including already-present), `>0` on +error, whereupon PG retries per `archive_timeout`. Run +`pg_hardstorage wal push --help` for the full flag set. ### restore_command shim @@ -387,24 +417,41 @@ works across compression posture, encryption setting, and re-runs. ### KEK rotation ```sh -pg_hardstorage kms rotate # v0.5 — walks all manifests, rewraps DEKs +pg_hardstorage kms rotate \ + --repo s3://acme-backups/ \ + --old-kek-ref --old-kek-file old-kek.bin \ + --new-kek-ref --new-kek-file new-kek.bin \ + --apply ``` -`kms rotate` is deferred to v0.5. For v0.1, the practical path is to -run a final backup under the old KEK, then move the keyring path to -the new KEK and start fresh — old backups are still readable as long -as the old keyring is preserved. +`kms rotate` ships: it walks every committed manifest wrapped with +`--old-kek-ref`, decrypts the wrapped DEK with `--old-kek-file`, +re-wraps it under `--new-kek-file`, records `--new-kek-ref`, re-signs, +and atomically rewrites the manifest. Chunks are **not** re-encrypted — +per-chunk keys derive from the unchanged BDEK, so rotation is +O(manifest count). Default mode is dry-run; pass `--apply` to actually +rewrite. Manifests under other KEK refs (other tenants) are skipped, +and an interrupted rotation is safely re-runnable. ### Crypto-shred ```sh -pg_hardstorage kms shred --confirm-keyring --reason "GDPR Art 17 #4421" --yes -``` - -`kms shred` is deferred to v0.5. The semantic is "destroy the KEK, -all backups become bit-for-bit unrecoverable, audit log entry is the -compliance artefact." Plan for it; do not rely on it shipping in -v0.1. +pg_hardstorage kms shred \ + --repo s3://acme-backups/ \ + --require-approval \ + --confirm-keyring \ + --reason "GDPR Art 17 #4421" --yes +``` + +`kms shred` ships: it destroys the local KEK, after which every backup +whose DEK was wrapped with it is permanently unrecoverable — the audit +log entry is the compliance artefact. The op is gated by three +independent safety mechanisms: a mandatory n-of-m approval workflow +(`--require-approval ` is **required**; shred is refused without an +approved gate), a typed-confirmation flag (`--confirm-keyring`, where +you repeat the literal keyring directory path), and an acknowledgement +flag (`--yes`). Use `--dry-run` to enumerate the affected backups +without destroying the KEK or requiring the gates. ### Inspect the keyring diff --git a/docs/operations/slo-as-code.md b/docs/operations/slo-as-code.md index 873ea97..97d3732 100644 --- a/docs/operations/slo-as-code.md +++ b/docs/operations/slo-as-code.md @@ -182,9 +182,9 @@ straightforward: ```sh pg_hardstorage slo report -o json \ | jq '.result.body.deployments[] | - {deployment, target: .rpo_target, - actual: .rpo_actual, - headroom: (.rpo_target - .rpo_actual)}' + {deployment, target: .rpo_target_seconds, + actual: .rpo_actual_seconds, + headroom: (.rpo_target_seconds - .rpo_actual_seconds)}' ``` Push to your monitoring backend; alert on `headroom < 0.2 * diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 221ab63..a591814 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -114,8 +114,8 @@ WAL inventory. PITR inside the gap window is then explicitly refused. ## WAL gap detected **Symptom.** A `wal_gap_detected` notice in `wal stream` logs, or a -restore returns `wal.gap_in_target` (exit 6) when the requested LSN -falls inside a known gap. +restore returns `restore.target_in_wal_gap` (exit 7) when the requested +LSN falls inside a known gap. **What it means.** Some range of WAL LSNs is missing from the repo. The gap auditor records the start and end of every gap it knows @@ -260,22 +260,22 @@ with the exact next step. Common refusals: --- -## Patroni split-brain refusal +## Split-brain / repo-contamination refusal -**Symptom.** `wal stream` or `backup` exits 4 with -`preflight.patroni_split_brain`. +**Symptom.** `wal push` refuses a segment with +`splitbrain.system_identifier_mismatch` or `splitbrain.content_mismatch`. -**What it means.** The agent's view of "who is the leader" disagrees -with PG's view. Either Patroni's REST endpoint reports a different -leader than the connected node thinks it is, or two candidates -report `role=primary`. Continuing would risk writing WAL or backup -chunks tagged with the wrong timeline. +**What it means.** The repo already holds a segment at that name whose +`system_identifier` — or whose bytes, at the same name — differ from the +one being pushed. That means two different clusters (or two divergent +timelines after a mishandled failover) are archiving into the same +deployment path. Continuing would let one cluster's WAL overwrite or +interleave with another's, corrupting the recovery chain. **What to do.** This is an operator-only resolution. See [runbooks/R7-patroni-split-brain](../reference/runbooks/R7-patroni-split-brain.md). -Do not `--force` past it. - -**Doctor check.** `Patroni topology` flags it as a critical finding. +Do not force past it — separate the clusters into distinct deployment +paths first. --- diff --git a/docs/reference/index.md b/docs/reference/index.md index bf45290..96e98ac 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -68,7 +68,9 @@ reflectors land (see [`DOC_PLAN.md`](../DOC_PLAN.md) - [Skill schema](skill-schema.md) — LLM skill YAML format and tool allowlist. - [Metric catalogue](metric-catalogue.md) — Prometheus - metric names; **implementation pending** (see SPEC drift). + metric names; the catalogue is largely **live** (~20 metric + families registered), only the Reserved families are still + outstanding (see SPEC drift). ## Runbooks diff --git a/docs/reference/kekref-schemes.md b/docs/reference/kekref-schemes.md index 41dd42d..200aabb 100644 --- a/docs/reference/kekref-schemes.md +++ b/docs/reference/kekref-schemes.md @@ -13,8 +13,13 @@ tags: A **KEKRef** is the manifest-stamped pointer to the Key-Encryption Key that wraps a backup's per-backup DEK. -The first segment of the KEKRef selects a `kms.Provider` -implementation in `internal/kms`; resolution is by +The first segment of the KEKRef selects how the DEK is +unwrapped. `local:` is resolved inline by the keystore +([`internal/backup/keystore/unwrap.go`](https://github.com/cybertec-postgresql/pg_hardstorage/blob/main/internal/backup/keystore/unwrap.go)), +*not* via `kms.DefaultRegistry`. Every other (cloud / HSM) +scheme — `aws-kms`, `gcp-kms`, `azure-kv`, `vault-transit`, +`pkcs11` — selects a `kms.Provider` implementation in +`internal/kms` and is dispatched by [`kms.DefaultRegistry`](https://github.com/cybertec-postgresql/pg_hardstorage/blob/main/internal/kms/kms.go). | Scheme | Provider | Built in | FIPS posture | Source | diff --git a/docs/reference/metric-catalogue.md b/docs/reference/metric-catalogue.md index 706a3bc..4259f98 100644 --- a/docs/reference/metric-catalogue.md +++ b/docs/reference/metric-catalogue.md @@ -26,8 +26,8 @@ and is exposed in Prometheus text exposition format !!! info "Live vs. reserved (SPEC drift #7)" The families marked **Live** below emit real samples today. The families marked **Reserved** have a committed name + - label layout but no producer yet — they render as - `# HELP`/`# TYPE` headers with no samples, and are tracked + label layout but no producer and no registration yet — they + don't appear in the exposition at all, and are tracked as drift items #7/#8 in [`docs/SPEC_DRIFT.md`](../SPEC_DRIFT.md). Until a reserved signal lands, operators read it from the diff --git a/docs/reference/plugins/encryption-contract.md b/docs/reference/plugins/encryption-contract.md index 99d1480..1d4e241 100644 --- a/docs/reference/plugins/encryption-contract.md +++ b/docs/reference/plugins/encryption-contract.md @@ -31,7 +31,7 @@ on-disk envelope they share. HashiCorp Vault Transit; on-prem-friendly, FIPS via Vault Enterprise. The local-keystore provider at - `internal/keystore/` is the dev / single-host default + `internal/backup/keystore/` is the dev / single-host default (`local:default`) and a useful minimal example of the interface shape. diff --git a/docs/reference/plugins/storage-contract.md b/docs/reference/plugins/storage-contract.md index be3dde6..1f348fa 100644 --- a/docs/reference/plugins/storage-contract.md +++ b/docs/reference/plugins/storage-contract.md @@ -46,6 +46,7 @@ type StoragePlugin interface { Delete(ctx context.Context, key string) error RenameIfNotExists(ctx context.Context, src, dst string) error SetRetention(ctx context.Context, key string, until time.Time, mode WORMMode) error + Barrier(ctx context.Context) error // makes deferred Puts crash-durable Capabilities() Capabilities Close() error } diff --git a/docs/reference/runbooks/R2-kms-key-destroyed.md b/docs/reference/runbooks/R2-kms-key-destroyed.md index 4e1a544..108a857 100644 --- a/docs/reference/runbooks/R2-kms-key-destroyed.md +++ b/docs/reference/runbooks/R2-kms-key-destroyed.md @@ -39,10 +39,18 @@ new backups from being written under a missing key. 2. **Inventory affected backups.** The `KEKRef` lives in each manifest under `encryption.kek_ref`: + `list` does not carry the `encryption` block; the per-backup + manifest does. Enumerate backups, then read each manifest with + `manifest show` (whose body embeds the full manifest, including + `encryption.kek_ref` and `backup_id`): + ```sh for d in $(pg_hardstorage deployment list -o json | jq -r '.result.body.deployments[].name'); do - pg_hardstorage list "$d" -o json | jq -r --arg key "" \ - '.result.body.backups[] | select(.encryption.kek_ref == $key) | "\($d) \(.id)"' + for b in $(pg_hardstorage list "$d" -o json | jq -r '.result.body.backups[].backup_id'); do + pg_hardstorage manifest show "$d" "$b" -o json | jq -r --arg key "" \ + 'select(.result.body.encryption.kek_ref == $key) | "\(.result.body.backup_id)"' \ + | sed "s|^|$d |" + done done ``` diff --git a/docs/reference/runbooks/R3-cold-start-from-backups.md b/docs/reference/runbooks/R3-cold-start-from-backups.md index 5a4a223..9452607 100644 --- a/docs/reference/runbooks/R3-cold-start-from-backups.md +++ b/docs/reference/runbooks/R3-cold-start-from-backups.md @@ -90,8 +90,9 @@ Before touching anything, verify the inputs you have: WAL gaps (the preview will refuse if not). Confirm the tablespace mapping matches what you expect on this host. -3. **Run the restore.** Same flags, drop `--preview`, add - `--confirm` to suppress the second prompt if you're scripting: +3. **Run the restore.** Same flags, drop `--preview`. Pass + `--force` if the target directory is non-empty (the default + refuses to overwrite): ```sh pg_hardstorage restore \ diff --git a/docs/reference/runbooks/R4-repo-corruption-at-rest.md b/docs/reference/runbooks/R4-repo-corruption-at-rest.md index 34d1d1a..c9af54d 100644 --- a/docs/reference/runbooks/R4-repo-corruption-at-rest.md +++ b/docs/reference/runbooks/R4-repo-corruption-at-rest.md @@ -39,8 +39,12 @@ what isn't. pg_hardstorage repair chunks --missing --repo -o json | tee /tmp/missing.json ``` -2. **If a replica exists, fetch known-good chunks.** Manual today - (auto-heal lands in v0.5+). For each corrupt chunk hash, copy +2. **If a replica exists, auto-heal from it.** The fastest path is + `pg_hardstorage repair scrub --heal --replica `, + which re-fetches every mismatched chunk from the replica and + rewrites it locally (use `--dry-run-heal` first to preview). + + To heal by hand instead, for each corrupt chunk hash copy the corresponding object from the replica region's `chunks/sha256/aa/bb/aabb.chk` to the same path under the primary region. Fail-safe because writes are CAS — the diff --git a/docs/reference/runbooks/R6-slot-dropped-gap.md b/docs/reference/runbooks/R6-slot-dropped-gap.md index 86cbeed..d67e021 100644 --- a/docs/reference/runbooks/R6-slot-dropped-gap.md +++ b/docs/reference/runbooks/R6-slot-dropped-gap.md @@ -12,7 +12,7 @@ gap window is now refused. - `wal stream` logs include a `wal.slot_recreated` notice with non-zero `gap_bytes` or `gap_seconds`. - An attempted PITR within the gap window fails with - `wal.gap_in_target` (exit 6). + `restore.target_in_wal_gap` (exit 7). - Patroni's REST endpoint shows a recent leader change. ## Pre-flight diff --git a/docs/reference/runbooks/R7-patroni-split-brain.md b/docs/reference/runbooks/R7-patroni-split-brain.md index 61d8c21..0f7b586 100644 --- a/docs/reference/runbooks/R7-patroni-split-brain.md +++ b/docs/reference/runbooks/R7-patroni-split-brain.md @@ -2,14 +2,19 @@ Two nodes both claim to be primary. Patroni's REST endpoint disagrees with the connected PG instance about the leader, or two -Patroni instances both report `role=primary`. The agent's pre-flight -refuses to write WAL or take backups in this state because it -cannot determine which timeline is canonical. +Patroni instances both report `role=primary`. There is no +Patroni-specific preflight that refuses in this state; instead, +`wal push` refuses a segment that a divergent writer already +archived (raising a `splitbrain.*` code), because it cannot +determine which timeline is canonical. ## Symptoms -- `wal stream` and `backup` exit 4 with - `preflight.patroni_split_brain`. +- A `wal push` of a segment already archived by a divergent + writer fails with a `splitbrain.*` code — + `splitbrain.system_identifier_mismatch` (different cluster) or + `splitbrain.content_mismatch` (same cluster, conflicting + content). There is no Patroni-specific preflight refusal. - `pg_hardstorage doctor` flags `Patroni topology` as critical. - `curl http://patroni-a:8008/cluster` and `curl http://patroni-b:8008/cluster` show conflicting `members[].role` diff --git a/docs/reference/runbooks/control-plane-setup.md b/docs/reference/runbooks/control-plane-setup.md index c7bdaca..8e7bdc3 100644 --- a/docs/reference/runbooks/control-plane-setup.md +++ b/docs/reference/runbooks/control-plane-setup.md @@ -15,15 +15,20 @@ control plane becomes useful when: - Your fleet sits behind **mTLS** and you want one auth-policy surface instead of per-host configuration. -What this runbook does NOT cover (yet — those land in v0.5+): +Current limitations (still on the roadmap): -- gRPC. v0.4 ships REST-only. The same handlers will be exposed - over gRPC in v0.5; the proto schema isn't stable yet. -- OIDC + per-verb RBAC. v0.4 is single-token. Use mTLS for fleets +- gRPC. The control plane ships REST-only. The same handlers are + planned over gRPC; the proto schema isn't stable yet. +- OIDC + per-verb RBAC. Auth is single-token. Use mTLS for fleets that need richer auth today. -- Persistent dispatch state. v0.4's JobRegistry is in-memory; a - control-plane restart loses queued + in-flight jobs. Persistent - PG-backed state lands in v0.5. + +Already shipped and covered below: + +- Persistent dispatch state is optional. The default JobRegistry + is in-memory (`--coord-backend memory`); a control-plane restart + then loses queued + in-flight jobs. Pass `--coord-backend pg` + with `--coord-dsn` for a persistent, multi-instance-HA + PostgreSQL-backed registry (used throughout this runbook). --- @@ -326,17 +331,19 @@ sudo ufw allow out to port 5432 proto tcp ### Multi-AZ availability -v0.4's control plane is single-instance. For HA today: +With the in-memory backend the control plane is effectively +single-instance. For HA: - Run two control-plane instances behind a load balancer; agents - heartbeat the LB hostname. **Caveat:** in-memory job state isn't - shared, so a job enqueued on instance A is invisible to instance - B until A processes it. Use sticky sessions on the LB to - mitigate, or queue outside the control plane (Kafka, Redis) and - have the control plane be a thin REST adapter. -- v0.5's PG-backed JobRegistry removes this caveat: any - control-plane instance can see + dispatch any job. Wait for v0.5 - if you need multi-instance dispatch correctness today. + heartbeat the LB hostname. **Caveat (in-memory backend only):** + with `--coord-backend memory`, job state isn't shared, so a job + enqueued on instance A is invisible to instance B until A + processes it. Use sticky sessions on the LB to mitigate, or + queue outside the control plane (Kafka, Redis) and have the + control plane be a thin REST adapter. +- The PG-backed JobRegistry (`--coord-backend pg`) removes this + caveat: any control-plane instance can see + dispatch any job. + Use it if you need multi-instance dispatch correctness. ### Observability @@ -350,8 +357,9 @@ visible signals: pg_hardstorage.server.v1`, `error.code`, `error.message`) For Prometheus scraping, point the scraper at `/v1/version` (cheap, -authenticated) for liveness; v0.5 adds dedicated `/metrics` and -control-plane-specific gauges. +authenticated) for liveness, or at the dedicated `/metrics` +endpoint (unauthenticated, like the health probes) for +control-plane and job gauges. --- @@ -404,10 +412,13 @@ wrong bucket. To fix: ### Restore-time job shows `pre-condition not met` -v0.4 doesn't dispatch restores yet (the agent-side executor returns -a v0.5-deferred error for `restore` and `verify` kinds). Run -restores via `pg_hardstorage restore` directly on the agent host -until v0.5 lands. +Restore and verify jobs are dispatched today: `POST +/v1/deployments//restores` and `.../verifies` enqueue +`JobRestore` / `JobVerify`, which the agent's RestoreExecutor / +VerifyExecutor run. A `pre-condition not met` result means the +job's guardrails (e.g. repo-URL match, non-empty target) failed — +check the job result body. You can still run restores via +`pg_hardstorage restore` directly on the agent host. ### Control plane refuses agent client cert @@ -425,13 +436,19 @@ CA to the bundle and restart the control plane. --- -## What ships in v0.5+ +## Already shipped + +- PostgreSQL-backed JobRegistry (`--coord-backend pg`) for + multi-instance HA dispatch +- Dispatch of restore / verify kinds (`JobRestore` / `JobVerify`) +- Dedicated `/metrics` endpoint + +## Still on the roadmap - gRPC alongside REST (same handlers, proto schema) -- PostgreSQL-backed JobRegistry (multi-instance HA dispatch) - pg_timetable integration as the recommended scheduler - OIDC + multi-token + per-verb RBAC -- Dispatch of restore / verify kinds with full progress streaming +- Full progress streaming for restore / verify dispatch - Job result persistence beyond the registry's in-memory window This runbook is updated alongside each milestone. diff --git a/docs/tutorials/build-a-storage-plugin.md b/docs/tutorials/build-a-storage-plugin.md index 9461109..dea9c4a 100644 --- a/docs/tutorials/build-a-storage-plugin.md +++ b/docs/tutorials/build-a-storage-plugin.md @@ -30,8 +30,8 @@ You author Tier-2. The host launches your binary on demand for each RPC; persistence and concurrency are not your problem. For the precise contract — every method signature, every error code, -the full handler set — see the [Tier-2 plugin reference](../reference/plugins/index.md) -(Agent F). This page is the guided "build one" walkthrough. +the full handler set — see the [Tier-2 plugin reference](../reference/plugins/index.md). +This page is the guided "build one" walkthrough. --- @@ -277,59 +277,57 @@ probe time is dropped from the registry with a logged warning. ```bash # RUNNABLE skip-in-ci="needs scaffolded Go plugin in /tmp/hs-plugin-tutorial" HSPLUGIN_PATH=/tmp/hs-plugin-tutorial \ - pg_hardstorage doctor + pg_hardstorage plugin list ``` ```console -... -Plugins - ✓ mem (storage, scheme=mem://) v0.1.0 · /tmp/hs-plugin-tutorial/pg-hardstorage-plugin-mem +NAME KIND VERSION PATH +mem storage 0.1.0 /tmp/hs-plugin-tutorial/pg-hardstorage-plugin-mem ``` -Your scheme (`mem://`) is now valid wherever `--repo` is accepted. +The host walked `$HSPLUGIN_PATH`, probed the binary, and got a valid +handshake back — so it lists here. ### 7. Smoke-test by reading repo audit (no real backup) A real backup against `mem://` requires the storage interface methods this tutorial does *not* implement (`Delete`, -`RenameIfNotExists`, `SetRetention`, `Capabilities`, -`Close`). The minimal set above is enough for `pg_hardstorage repo -init` and `pg_hardstorage repo audit` — the surface that exercises -discovery and the put/get/stat path: +`RenameIfNotExists`, `SetRetention`, `Barrier`, `Capabilities`, +`Close`) — the full interface is 12 methods. + +!!! warning "Not yet wired end-to-end" + + The steps below are **aspirational**. Tier-2 external plugins are + discovered and probed (step 6 above works today), but they are not + yet registered as storage factories, so a `--repo mem://…` URL does + not resolve through the external plugin. `pg_hardstorage repo init` + / `repo audit` against a plugin scheme will not run until that + wiring lands. Treat the following as a preview of the intended + surface, not a runnable step. ```bash +# ASPIRATIONAL — mem:// does not resolve through a Tier-2 plugin yet HSPLUGIN_PATH=/tmp/hs-plugin-tutorial \ pg_hardstorage repo init mem:///hs-tutorial ``` ```bash +# ASPIRATIONAL — see the warning above HSPLUGIN_PATH=/tmp/hs-plugin-tutorial \ pg_hardstorage repo audit mem:///hs-tutorial ``` -Inspect the backing JSON: - -```bash -cat /tmp/hs-plugin-tutorial/store.json | jq 'keys' -``` - -```console -[ - "HSREPO", - "config/repo.json" -] -``` - -Two objects — the magic file and the repo config. The plugin moved -real bytes through your handler. +Once wired, the backing JSON would hold two objects — the `HSREPO` +magic file and the repo config — showing that the plugin moved real +bytes through your handler. ### 8. Watch the host re-spawn the plugin -Tail the plugin's stderr to watch every RPC: +Tail the plugin's stderr to watch the host spawn and probe it: ```bash HSPLUGIN_PATH=/tmp/hs-plugin-tutorial \ - pg_hardstorage repo audit mem:///hs-tutorial 2>>/tmp/plugin-stderr.log + pg_hardstorage plugin list 2>>/tmp/plugin-stderr.log ``` Each call to your binary is one process: launch, read one request, @@ -371,14 +369,14 @@ away: - **Probe is the contract.** Get the probe response right and registration "just works"; get it wrong (missing `protocol` field, wrong version, slow exit) and your plugin silently doesn't show up - in `doctor`. + in `pg_hardstorage plugin list`. --- ## Next steps - [Tier-2 plugin reference](../reference/plugins/index.md) — the - full method set, param schemas, error codes (Agent F). + full method set, param schemas, error codes. - [Architecture tour](../explanation/architecture-tour.md) — where Tier-1 vs Tier-2 sits in the data plane. - [Operator guide](../operations/operator-guide.md) — running with diff --git a/docs/tutorials/encryption-walkthrough.md b/docs/tutorials/encryption-walkthrough.md index 2dfe870..94da3b4 100644 --- a/docs/tutorials/encryption-walkthrough.md +++ b/docs/tutorials/encryption-walkthrough.md @@ -85,10 +85,14 @@ pg_hardstorage kms inspect ``` ```console -Keyring: /home/you/.config/pg_hardstorage/keyring - signing-public.pem ed25519 fp=ab12...ef34 - signing-private.pem 0600 - kek.bin 0600 · 32 bytes +keyring at /home/you/.config/pg_hardstorage/keyring + manifest_signing.ed25519 + present: yes (mode -rw-------, 165 bytes, 2026-07-06T12:49:28Z) + manifest_signing.pub + present: yes (mode -rw-r--r--, 159 bytes, 2026-07-06T12:49:28Z) + fingerprint: sha256:4324b42cc21f7259 + kek.bin + present: yes (mode -rw-------, 32 bytes, 2026-07-06T12:49:28Z) ``` If you already have a keyring without a KEK, generate one with: @@ -123,8 +127,9 @@ pg_hardstorage show db1 latest \ ```console { + "scheme": "aes-256-gcm", "kek_ref": "local:default", - "wrapped_dek_b64": "...", + "wrapped_dek": "...", "envelope_version": 1 } ``` @@ -218,8 +223,9 @@ pg_hardstorage show db1 latest \ ```console { + "scheme": "aes-256-gcm", "kek_ref": "aws-kms://arn:aws:kms:eu-central-1:123456789012:key/abc-...", - "wrapped_dek_b64": "...", + "wrapped_dek": "...", "envelope_version": 1 } ``` @@ -238,8 +244,9 @@ pg_hardstorage restore db1 latest \ No extra flags. The restore reads the KEKRef, opens the matching provider, and asks AWS KMS to unwrap. If the calling principal lacks -`kms:Decrypt`, you get `kms.unauthorized` (exit 8) with the suggested -IAM statement to add. +`kms:Decrypt`, the unwrap fails as an authorization error (exit 3) with +the suggested IAM statement to add. (Exit 8 is reserved for a KMS +endpoint that is *unreachable*, not one that denies the call.) #### 5. Verify the envelope across every backup @@ -253,12 +260,17 @@ limited, so a large repo takes longer than the local-KEK case. --- -## Crypto-shred and rotation (preview) +## Crypto-shred and rotation -Two operations are wired in v0.2 but locked behind n-of-m approval: +Both operations ship today. `kms shred` destroys the KEK so every +backup wrapped by it becomes unrecoverable — it is gated behind an +n-of-m approval (`--require-approval`) plus a typed-keyring confirmation +(`--confirm-keyring`). `kms rotate` re-wraps each manifest's DEK under a +new KEK (no chunk data is re-read) and is not approval-gated. ```bash -pg_hardstorage kms shred --confirm-keyring --require-approval ... +pg_hardstorage kms shred --repo "$REPO" \ + --confirm-keyring --require-approval pg_hardstorage kms rotate --repo "$REPO" \ --old-kek-ref local:default --old-kek-file old-kek.bin \ --new-kek-ref aws-kms://alias/pg-hardstorage-demo-2026 --new-kek-file new-kek.bin --apply diff --git a/docs/tutorials/first-backup-restore.md b/docs/tutorials/first-backup-restore.md index eb142b9..a58de78 100644 --- a/docs/tutorials/first-backup-restore.md +++ b/docs/tutorials/first-backup-restore.md @@ -91,9 +91,19 @@ keypair is generated under your keyring directory (run Sample output: ```console -✓ Connected to PostgreSQL 17.x -✓ Backup db1.full.20260504T120000Z.a1b2 complete · 33 MB physical · 1 chunk -✓ Manifest committed (signed, ed25519) +✓ Backup committed + ID: db1.full.20260504T120000Z.a1b2 + Deployment: db1 + PostgreSQL: 17 + Cluster ID: 7659398055633653799 + Stop LSN / TLI: 0/2000100 / 1 + Files: 967 in 1 tablespace(s) + Logical bytes: 22.2 MiB + Unique chunks: 363 (8.7 MiB after dedup) + Dedup ratio: 2.56x + Duration: 1722 ms + Encryption: none + Manifest: manifests/db1/backups/db1.full.20260504T120000Z.a1b2/manifest.json ``` The backup ID has the shape `db1.full.YYYYMMDDThhmmssZ.` — @@ -108,8 +118,9 @@ pg_hardstorage list db1 --repo file:///tmp/hs-tutorial-repo ``` ```console -ID STARTED SIZE STATE -db1.full.20260504T120000Z.a1b2 2026-05-04 12:00:00 33 MB committed +Backups for db1 (1): + BACKUP ID TYPE WHEN FILES SIZE DEDUP DURATION + db1.full.20260504T120000Z.a1b2 full 2026-05-04 12:00 967 22.2 MiB 2.56x 1617ms ``` The backup ID has the shape `db1.full.YYYYMMDDThhmmssZ.` — @@ -131,8 +142,8 @@ pg_hardstorage show db1 "$BACKUP_ID" \ --repo file:///tmp/hs-tutorial-repo ``` -`show` prints LSN range, timeline, dedup ratio, encryption envelope -state, and a verification record once one is written. Pipe through +`show` prints the LSN range, timeline, compression, tablespaces, dedup +ratio, and the manifest's ed25519 signature fingerprint. Pipe through `-o json` if you want to parse it. ### 5. Verify the manifest without restoring @@ -164,14 +175,23 @@ pg_hardstorage restore db1 latest \ Sample output: ```console -✓ Pre-flight: repo reachable, KMS reachable, target empty -✓ Restored 1 chunk · 33 MB to /tmp/hs-tutorial-restored -✓ pg_verifybackup OK -``` - -The post-restore `pg_verifybackup` is the gate that decides exit -code: 0 on success, 9 if the verifier rejects anything. `--verify=skip` -turns it off (audited; do not skip in production). +✓ Restore complete + Backup: db1.full.20260504T120000Z.a1b2 + Deployment: db1 + Target: /tmp/hs-tutorial-restored + Files: 967 + Bytes written: 22.2 MiB + Chunks: 919 + backup_label: 230 bytes + Duration: 868 ms + Verification: passed +``` + +By default (`--verify=auto`) a `pg_verifybackup` manifest check runs +against the restored data dir when the matching `postgresql-client` is +on the runner's PATH, and its result shows in the `Verification:` line. +`--verify=require` makes a failed check fail the restore (exit 9); +`--verify=skip` turns it off (audited; do not skip in production). ### 7. Boot the restored data dir diff --git a/docs/tutorials/getting-started.md b/docs/tutorials/getting-started.md index be0077d..bd310d6 100644 --- a/docs/tutorials/getting-started.md +++ b/docs/tutorials/getting-started.md @@ -65,27 +65,29 @@ Same layout as the `.deb`. ### Container image +Pre-built images are not yet published to a registry. Build a +distroless image locally from the in-tree Dockerfile: + ```sh -VERSION=1.0.7 # latest release tag -docker pull "ghcr.io/cybertec-postgresql/pg_hardstorage:v${VERSION}" -docker run --rm "ghcr.io/cybertec-postgresql/pg_hardstorage:v${VERSION}" version +docker build -t pg_hardstorage:local -f deploy/docker/Dockerfile . +docker run --rm pg_hardstorage:local version ``` -The image is distroless. Mount a config dir at `/etc/pg_hardstorage` -and a state dir at `/var/lib/pg_hardstorage`; both must be writable by -UID 65532. +The image is distroless and runs as `nonroot`. Mount a config dir at +`/etc/pg_hardstorage` and a state dir at `/var/lib/pg_hardstorage`; +both must be writable by UID 65532. ### From source ```sh git clone https://github.com/cybertec-postgresql/pg_hardstorage cd pg_hardstorage -make # produces bin/pg_hardstorage +make build # produces bin/pg_hardstorage (bare `make` prints the help menu) sudo install -m 0755 bin/pg_hardstorage /usr/local/bin/ ``` Requires Go 1.26+. `make test` runs the full unit suite under the race -detector; `make test-integration` exercises a real PostgreSQL 17 +detector; `make test-integration` exercises a real PostgreSQL container via testcontainers-go (needs Docker). --- @@ -260,21 +262,36 @@ $ pg_hardstorage version pg_hardstorage v1.0.7 (abc1234, built 2026-04-29T12:00:00Z) ``` -`doctor` is the single-command "is anything wrong" check: +`doctor` is the single-command "is anything wrong" check. It prints a +sectioned report — resolved PATHS, CONFIG, KEYSTORE, AIRGAP posture, +REPOS reachability, and an ISSUES list: ```sh $ pg_hardstorage doctor -db1 — PG 17.2 — primary @ db1.example.com - ✓ PostgreSQL reachable - ✓ Replication slot 'pg_hardstorage_db1' active, lag 12s - ✓ Last backup 47m ago - ✓ Repository file:///srv/backups writable - ✓ KMS keyring present (~/.config/pg_hardstorage/keyring) - ✓ Schedule: next at 04:00 UTC -Summary: 1 healthy. +Mode: user + +CONFIG + Status: configured + Schema: pg_hardstorage.config.v1 + Deployments: 1 (db1) + db1 class: internal + Source files: + [loaded ] ~/.config/pg_hardstorage/pg_hardstorage.yaml + +KEYSTORE + Signing key: ✓ present + KEK: ✓ present (encryption ON by default for new backups) + +REPOS + file:///srv/backups — reachable + audit chain: 5 event(s) + +ISSUES + [WARNING] audit.anchor_missing: 5 audit event(s) but no transparency-log anchor; run `pg_hardstorage audit anchor` + hint: run `pg_hardstorage audit anchor --repo ` ``` -Any `✗` line carries a `Suggested fix:` block underneath. Run +Each issue line carries an indented `hint:` line underneath. Run `pg_hardstorage doctor -o json` for a machine-readable form. `doctor` exits 0 when healthy and exit 10 with `--exit-on-issues` when diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 46415a1..7db7abf 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -44,4 +44,4 @@ take the full getting-started instead. ## Extending pg_hardstorage - [Build a Tier-2 storage plugin](build-a-storage-plugin.md) — author - an out-of-tree storage backend over the `go-plugin` contract. + an out-of-tree storage backend over the Tier-2 JSON-RPC/stdio contract. diff --git a/docs/tutorials/llm-incident-walkthrough.md b/docs/tutorials/llm-incident-walkthrough.md index f9a141c..775213d 100644 --- a/docs/tutorials/llm-incident-walkthrough.md +++ b/docs/tutorials/llm-incident-walkthrough.md @@ -109,16 +109,23 @@ pg_hardstorage wal stream db1 \ --repo file:///tmp/hs-llm-repo ``` +`wal stream` does **not** abort. Its reconnect path runs +`EnsureSlot`, which finds the slot missing and recreates it with +`CREATE_REPLICATION_SLOT ... RESERVE_WAL`. Because the recreated slot +starts at the server's current position — past the agent's last +confirmed LSN — the run emits a structured `wal_gap_detected` event: + ```console -ERROR: WAL stream replication slot 'pg_hardstorage_db1' is not present on the server. -What to do: the slot was probably dropped by an admin. Recreate it -with: - pg_hardstorage wal repair db1 +[wal.follower] wal_gap_detected the new leader's slot was recreated and +has advanced past the agent's last confirmed LSN; PITR within the gap is +impossible from this repo. Backup taken after this point will note the +gap so PITR is refused. Investigate via `pg_hardstorage repair slot db1` +for diagnostics. ``` -The exit code is non-zero and the structured event ID is -`wal_gap_detected`. You could `pg_hardstorage wal repair` blindly — -or you can ask the helper to talk you through it. +The slot is back, but a WAL gap is now recorded in the repo. You could +inspect it with `pg_hardstorage repair slot db1` blindly — or you can +ask the helper to talk you through what the gap means. ### 3. Drop into the incident skill @@ -190,6 +197,7 @@ read_runbook search_docs suggest_command preview_command +read_command_help ``` ```text @@ -230,7 +238,7 @@ The recorded gap is the exact range during which the slot was absent. You can chase it with `wal audit` or accept it as a known recovery limitation depending on your RPO target. -### 7. Export the signed evidence bundle +### 7. Export the evidence bundle Note the session ID printed in the chat header (`session=01HXX...`) and pass it to `llm export-session`: @@ -242,18 +250,26 @@ pg_hardstorage llm export-session 01HXX... \ ``` ```console -✓ exported 14 audit events -✓ Merkle root anchored: sha256:c1aa... -✓ wrote /tmp/hs-llm-incident.tar.gz (signed, ed25519) +✓ exported LLM session 01HXX... + path: /tmp/hs-llm-incident.tar.gz + events: 14 + bundle: 8213 bytes + first hash: sha256:a13f... + last hash: sha256:c1aa... + verify: pg_hardstorage audit verify-chain --repo ``` +The bundle is not signed today; its integrity rests on the audit +chain it was drawn from — verify that chain with the `verify:` +command above. + Inside the tarball: ```text -session.json # session metadata (skill, provider, principal) -events.ndjson # every llm.* event in order, hash-chained -manifest.json # bundle index + audit chain head -manifest.sig # ed25519 signature over manifest.json +transcript.ndjson # every llm.* event in commit order +tool_results/*.json # full tool-result bodies for tool calls +manifest.json # schema, session_id, event count +audit_chain_proof.json # chain head + first/last event hashes ``` This is the artefact you hand to the auditor: **what the operator diff --git a/docs/tutorials/patroni-cluster.md b/docs/tutorials/patroni-cluster.md index 36d47b9..ac42bee 100644 --- a/docs/tutorials/patroni-cluster.md +++ b/docs/tutorials/patroni-cluster.md @@ -55,11 +55,13 @@ pg_hardstorage patroni status \ ``` ```console -Cluster: pg-tutorial · scope=patroni-tutorial - Member Role State TLI Lag Host:Port - patroni-1 Leader running 3 0 10.0.0.11:5432 - patroni-2 Replica running 3 24kB 10.0.0.12:5432 - patroni-3 Replica running 3 8kB 10.0.0.13:5432 +patroni cluster "patroni-tutorial" + Leader: patroni-1 (TLI 3) + + NAME ROLE STATE HOST PORT TLI LAG + patroni-1 Leader running 10.0.0.11 5432 3 0 + patroni-2 Replica streaming 10.0.0.12 5432 3 0 + patroni-3 Replica streaming 10.0.0.13 5432 3 0 ``` `patroni history` prints the timeline-history events that record @@ -157,9 +159,9 @@ pg_hardstorage backup pg-tutorial \ --repo file:///srv/hs-patroni-repo ``` -For clusters at ≥ 5 TB the agent prefers a Patroni *replica* for the -base backup to keep primary I/O free. Replica preference is -automatic — there is no flag to flip. +The base backup connects to the endpoint you give it. To take it from +a replica (to keep primary I/O free), point `--pg-connection` at a +replica's host — there is no automatic replica routing. ### 6. Force a failover and watch the slot survive @@ -176,18 +178,29 @@ pg_hardstorage patroni follow \ --url http://patroni-leader:8008 ``` +`patroni follow` streams a `patroni.leader_change` event each time the +leader changes (and `patroni.poll_error` on a transient REST error); +each event carries the old and new leader's name and host:port. In +text mode it renders through the generic event renderer: + ```console -2026-05-04T11:02:11Z initial-leader patroni-1 → 10.0.0.11:5432 TLI=3 -2026-05-04T11:03:42Z leader-change patroni-1 → patroni-2 TLI=3 → 4 +11:03:42 [NOTICE] patroni.leader_change ``` -In the `wal stream` terminal you will see the agent reconnect: +In the `wal stream` terminal you will see the follower coordinator +react. It emits events under the `wal.follower` namespace — the ones +that matter on a failover: + +- `wal.follower.leader_change` — the coordinator saw the new leader; +- `wal.follower.slot_reconciled` — the physical slot was found (or + recreated) on the new leader; +- `wal.follower.wal_gap_detected` — **critical**, emitted only if the + reconnect could not bridge the WAL (e.g. the slot was lost and + recreated past the last archived LSN). ```console -patroni.leader_change from=patroni-1 to=patroni-2 new_lsn=0/A0000000 -slot.repair slot=pg_hardstorage_pg_tutorial result=created -wal.gap_recorded range=[0/9F800000..0/A0000000] cause=patroni-failover -wal.stream receiving from patroni-2:5432 +11:03:42 [NOTICE] wal.follower.leader_change +11:03:43 [NOTICE] wal.follower.slot_reconciled ``` The agent recreated its replication slot on the new primary at the diff --git a/docs/tutorials/pitr-tutorial.md b/docs/tutorials/pitr-tutorial.md index efbf8b2..6866d25 100644 --- a/docs/tutorials/pitr-tutorial.md +++ b/docs/tutorials/pitr-tutorial.md @@ -129,19 +129,22 @@ backup, computes the WAL replay range, estimates RTO, and prints the checklist *without writing anything*: ```console -PITR plan for db1 - Source backup db1.full.20260504T120000Z (full · 33 MB) - Replay WAL to 2026-05-04 11:55:00 UTC (resolved from "5 minutes ago") - Target /tmp/hs-tutorial-pitr (empty ✓) - Verify gate auto (pg_verifybackup will run) - RTO estimate ~30s -Pre-flight checks - ✓ Repository reachable (file:///tmp/hs-tutorial-repo) - ✓ Keystore reachable - ✓ WAL coverage [0/1A000000 .. 0/22000000] available - ✓ Target directory empty -This is a preview — no changes were written. Re-run without --preview -to apply. +Restore plan (preview only — no files written) + Backup: db1.full.20260504T120000Z.a1b2 + Deployment: db1 + Target: /tmp/hs-tutorial-pitr + PostgreSQL: 17 + Cluster ID: 7659399478052106285 + Backup stop LSN: 0/2000100 (TLI 1) + Recovery target: time 2026-05-04T11:55:00Z (inclusive=true) + On target reached: pause + Recovery TLI: latest + Files: 967 + Total bytes: 22.2 MiB + Chunk refs: 919 (363 unique, 8.7 MiB after dedup) + backup_label: 230 bytes + Estimated RTO: 222 ms (assuming 100.0 MiB/s) + Pre-flight: ✓ ready ``` Natural-language parsing supports ` minutes/hours/days ago`, @@ -179,11 +182,21 @@ appends a managed `recovery_target_*` block to recovery proceeds. ```console -✓ Restored 1 chunk · 33 MB to /tmp/hs-tutorial-pitr -✓ recovery.signal armed -✓ recovery_target_time = '2026-05-04 11:55:00 UTC' -✓ pg_verifybackup OK -RTO actual: 28s +✓ Restore complete + Backup: db1.full.20260504T120000Z.a1b2 + Deployment: db1 + Target: /tmp/hs-tutorial-pitr + Files: 967 + Bytes written: 22.2 MiB + Chunks: 919 + backup_label: 230 bytes + Duration: 790 ms + Verification: passed + Recovery armed: + Stop at time: 2026-05-04T11:55:00Z + Action: pause + Timeline: latest + Inclusive: true ``` ### 6. Boot the restored cluster and confirm diff --git a/internal/restore/verify.go b/internal/restore/verify.go index 7137b9b..c3f4e24 100644 --- a/internal/restore/verify.go +++ b/internal/restore/verify.go @@ -128,7 +128,14 @@ func Verify(ctx context.Context, target string, mode VerifyMode) (*VerifyResult, res.ToolPath = path start := time.Now() - cmd := exec.CommandContext(ctx, path, target) + // -n / --no-parse-wal: verify the backup_manifest + file checksums + // only, WITHOUT trying to parse pg_wal. pg_hardstorage restores a + // base backup on its own; the WAL it needs to reach consistency is + // fetched at recovery time via the restore_command, so the restored + // data directory legitimately has no WAL segments yet. Without -n, + // pg_verifybackup fails every normal restore with "could not find + // any WAL file", defeating the gate. + cmd := exec.CommandContext(ctx, path, "-n", target) var stdout, stderr strings.Builder cmd.Stdout = &stdout cmd.Stderr = &stderr diff --git a/internal/restore/verify_noparsewal_test.go b/internal/restore/verify_noparsewal_test.go new file mode 100644 index 0000000..049973a --- /dev/null +++ b/internal/restore/verify_noparsewal_test.go @@ -0,0 +1,52 @@ +package restore + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Verify must invoke pg_verifybackup with -n (--no-parse-wal). A +// pg_hardstorage restore lays down the base backup only; the WAL needed +// to reach consistency is fetched at recovery time via restore_command, +// so the restored data dir has no pg_wal segments yet. Without -n, +// pg_verifybackup fails every normal restore with "could not find any +// WAL file", defeating the --verify gate. +func TestVerify_PassesNoParseWAL(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake-script seam is POSIX") + } + dir := t.TempDir() + argfile := filepath.Join(dir, "args") + fake := filepath.Join(dir, "pg_verifybackup") + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argfile + "\nexit 0\n" + if err := os.WriteFile(fake, []byte(script), 0o755); err != nil { + t.Fatalf("write fake pg_verifybackup: %v", err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + + res, err := Verify(context.Background(), t.TempDir(), VerifyRequire) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if res.Status != "passed" { + t.Errorf("status = %q, want passed", res.Status) + } + got, err := os.ReadFile(argfile) + if err != nil { + t.Fatalf("read recorded args: %v", err) + } + args := strings.Fields(string(got)) + found := false + for _, a := range args { + if a == "-n" || a == "--no-parse-wal" { + found = true + } + } + if !found { + t.Errorf("pg_verifybackup args = %v, want to include -n / --no-parse-wal", args) + } +} diff --git a/internal/simple/flow_setup.go b/internal/simple/flow_setup.go index dd12f88..5805faf 100644 --- a/internal/simple/flow_setup.go +++ b/internal/simple/flow_setup.go @@ -226,11 +226,11 @@ func validateDeploymentName(s string) error { return nil } -// validateRepoURL accepts any URL scheme the runner accepts. The -// short list (file://, s3://, gs://, azure://, sftp://, scp://) is -// validated by trying a probe write only for file:// — remote -// schemes can't be probed without credentials and we don't ask for -// those at setup time. +// validateRepoURL accepts the URL schemes the storage registry actually +// registers (file, s3, gcs, azblob, sftp, scp — see +// internal/plugin/storage/*). It is validated by trying a probe write +// only for file:// — remote schemes can't be probed without credentials +// and we don't ask for those at setup time. func validateRepoURL(s string) error { s = strings.TrimSpace(s) if s == "" { @@ -247,12 +247,13 @@ func validateRepoURL(s string) error { } return nil } - for _, p := range []string{"s3://", "gs://", "azure://", "sftp://", "scp://"} { + // Must match the schemes registered in internal/plugin/storage/*. + for _, p := range []string{"s3://", "gcs://", "azblob://", "sftp://", "scp://"} { if strings.HasPrefix(s, p) { return nil } } - return errors.New(`unknown URL scheme; expected file:// / s3:// / gs:// / azure:// / sftp:// / scp://`) + return errors.New(`unknown URL scheme; expected file:// / s3:// / gcs:// / azblob:// / sftp:// / scp://`) } // dbnameFromDSN pulls the dbname out of a libpq URI, used as the diff --git a/internal/simple/flow_setup_scheme_test.go b/internal/simple/flow_setup_scheme_test.go new file mode 100644 index 0000000..4df70ae --- /dev/null +++ b/internal/simple/flow_setup_scheme_test.go @@ -0,0 +1,35 @@ +package simple + +import "testing" + +// validateRepoURL must accept exactly the schemes the storage registry +// registers (internal/plugin/storage/*): file, s3, gcs, azblob, sftp, +// scp. It previously accepted gs:// and azure:// — schemes with no +// registered backend — while rejecting the real gcs:// and azblob://. +func TestValidateRepoURL_Schemes(t *testing.T) { + accept := []string{ + "file:///srv/backups", + "s3://bucket/prefix", + "gcs://bucket/prefix", + "azblob://acct.blob.core.windows.net/container/", + "sftp://host/srv/repo", + "scp://host/srv/repo", + } + reject := []string{ + "gs://bucket/prefix", // not a registered scheme + "azure://container/", // not a registered scheme + "http://example.com", // unsupported + "file://relative/path", // file:// must be absolute + "", // empty + } + for _, u := range accept { + if err := validateRepoURL(u); err != nil { + t.Errorf("validateRepoURL(%q) = %v, want accepted", u, err) + } + } + for _, u := range reject { + if err := validateRepoURL(u); err == nil { + t.Errorf("validateRepoURL(%q) = nil, want rejected", u) + } + } +}