From a1a430f36b57fecb33b71f00e200820165e48d5f Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:17:59 +0300 Subject: [PATCH 1/9] Add secure hosted production canary contract --- .github/workflows/deploy-bundle.yml | 3 + Makefile | 8 +- deploy/hosted-managed/Caddyfile | 15 + deploy/hosted-managed/production.env.example | 18 + ...pose.hosted-managed-production.example.yml | 220 +++++++++ docs/deployment.md | 22 + docs/hosted-managed-operations.md | 209 ++++++++ docs/ontology-specgraph-specspace-roadmap.md | 17 +- .../hosted_managed_production_preflight.py | 293 +++++++++++ scripts/hosted_managed_production_probe.py | 248 ++++++++++ scripts/hosted_managed_production_signoff.py | 331 +++++++++++++ scripts/hosted_managed_runtime_backup.py | 463 ++++++++++++++++++ ...idate_hosted_managed_production_compose.py | 288 +++++++++++ .../test_hosted_managed_operation_postgres.py | 64 +++ tests/test_hosted_managed_production.py | 373 ++++++++++++++ 15 files changed, 2565 insertions(+), 7 deletions(-) create mode 100644 deploy/hosted-managed/Caddyfile create mode 100644 deploy/hosted-managed/production.env.example create mode 100644 docker-compose.hosted-managed-production.example.yml create mode 100644 scripts/hosted_managed_production_preflight.py create mode 100644 scripts/hosted_managed_production_probe.py create mode 100644 scripts/hosted_managed_production_signoff.py create mode 100644 scripts/hosted_managed_runtime_backup.py create mode 100644 scripts/validate_hosted_managed_production_compose.py create mode 100644 tests/test_hosted_managed_production.py diff --git a/.github/workflows/deploy-bundle.yml b/.github/workflows/deploy-bundle.yml index cb2591d..910c890 100644 --- a/.github/workflows/deploy-bundle.yml +++ b/.github/workflows/deploy-bundle.yml @@ -31,6 +31,9 @@ jobs: - name: Validate standalone hosted runtime Compose contract run: make hosted-managed-runtime-compose-contract PYTHON=python3 + - name: Validate TLS-fronted production hosted runtime contract + run: make hosted-managed-production-contract PYTHON=python3 + - name: Build standalone hosted runtime image run: >- docker build --file Dockerfile.hosted-managed diff --git a/Makefile b/Makefile index be41c13..fda62ab 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) -.PHONY: python-quality test hosted-managed-contract hosted-managed-compose-contract hosted-managed-runtime-compose-contract hosted-managed-postgres-integration hosted-managed-compose-smoke +.PHONY: python-quality test hosted-managed-contract hosted-managed-compose-contract hosted-managed-runtime-compose-contract hosted-managed-production-compose-contract hosted-managed-production-contract hosted-managed-postgres-integration hosted-managed-compose-smoke python-quality: $(PYTHON) -m unittest discover -s tests @@ -20,6 +20,12 @@ hosted-managed-compose-contract: hosted-managed-runtime-compose-contract: $(PYTHON) scripts/validate_hosted_managed_runtime_compose.py +hosted-managed-production-compose-contract: + $(PYTHON) scripts/validate_hosted_managed_production_compose.py + +hosted-managed-production-contract: hosted-managed-production-compose-contract + $(PYTHON) -m unittest tests.test_hosted_managed_production + hosted-managed-postgres-integration: @test -n "$(PLATFORM_TEST_POSTGRES_URL)" || \ (echo "PLATFORM_TEST_POSTGRES_URL is required" >&2; exit 2) diff --git a/deploy/hosted-managed/Caddyfile b/deploy/hosted-managed/Caddyfile new file mode 100644 index 0000000..2e65500 --- /dev/null +++ b/deploy/hosted-managed/Caddyfile @@ -0,0 +1,15 @@ +{ + admin off + auto_https off +} + +:8443 { + tls /run/secrets/managed_operation_tls_certificate /run/secrets/managed_operation_tls_private_key + encode zstd gzip + header -Server + reverse_proxy managed-operation-service:8091 + log { + output stdout + format json + } +} diff --git a/deploy/hosted-managed/production.env.example b/deploy/hosted-managed/production.env.example new file mode 100644 index 0000000..19c9afb --- /dev/null +++ b/deploy/hosted-managed/production.env.example @@ -0,0 +1,18 @@ +# Non-secret deployment inputs only. Keep secret values in the referenced files. +PLATFORM_MANAGED_OPERATION_IMAGE=ghcr.io/0al-spec/platform@sha256: +PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE=postgres@sha256: +PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE=caddy@sha256: +PLATFORM_MANAGED_OPERATION_ALLOWLIST=review_status_execute + +PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT=/srv/0al/specgraph +PLATFORM_MANAGED_OPERATION_STATE_DIR=/srv/0al/specspace-state +PLATFORM_MANAGED_OPERATION_BACKUP_ROOT=/srv/0al/backups +PLATFORM_MANAGED_OPERATION_INGRESS_BIND_IP=0.0.0.0 +PLATFORM_MANAGED_OPERATION_INGRESS_PORT=443 + +PLATFORM_MANAGED_OPERATION_TOKEN_FILE=/srv/0al/secrets/service-token +PLATFORM_MANAGED_OPERATION_DB_PASSWORD_FILE=/srv/0al/secrets/database-password +PLATFORM_MANAGED_OPERATION_DATABASE_URL_FILE=/srv/0al/secrets/database-url +PLATFORM_MANAGED_OPERATION_GITHUB_TOKEN_FILE=/srv/0al/secrets/github-token +PLATFORM_MANAGED_OPERATION_TLS_CERTIFICATE_FILE=/srv/0al/secrets/tls-certificate.pem +PLATFORM_MANAGED_OPERATION_TLS_PRIVATE_KEY_FILE=/srv/0al/secrets/tls-private-key.pem diff --git a/docker-compose.hosted-managed-production.example.yml b/docker-compose.hosted-managed-production.example.yml new file mode 100644 index 0000000..6efe422 --- /dev/null +++ b/docker-compose.hosted-managed-production.example.yml @@ -0,0 +1,220 @@ +services: + managed-operation-postgres: + image: "${PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE:?set a digest-pinned PostgreSQL image}" + restart: unless-stopped + environment: + POSTGRES_DB: "${PLATFORM_MANAGED_OPERATION_DB:-managed_operations}" + POSTGRES_USER: "${PLATFORM_MANAGED_OPERATION_DB_USER:-managed_operations}" + POSTGRES_PASSWORD_FILE: /run/secrets/managed_operation_db_password + volumes: + - managed-operation-postgres:/var/lib/postgresql/data + secrets: + - managed_operation_db_password + networks: + - managed-backend + healthcheck: + test: + - CMD-SHELL + - >- + pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}" + interval: 5s + timeout: 5s + retries: 12 + + managed-operation-service: + image: "${PLATFORM_MANAGED_OPERATION_IMAGE:?set a digest-pinned Platform image}" + restart: unless-stopped + command: + - python3 + - scripts/platform.py + - managed-operation + - serve + - --queue-adapter + - postgresql + - --database-url-file + - /run/secrets/managed_operation_database_url + - --artifact-root + - /workspace/SpecGraph + - --state-dir + - /data/specspace-state + - --specgraph-dir + - /workspace/SpecGraph + - --host + - 0.0.0.0 + - --port + - "8091" + - --auth-token-file + - /run/secrets/managed_operation_token + expose: + - "8091" + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + volumes: + - "${PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT:?set PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT}:/workspace/SpecGraph:ro" + - "${PLATFORM_MANAGED_OPERATION_STATE_DIR:?set PLATFORM_MANAGED_OPERATION_STATE_DIR}:/data/specspace-state:ro" + secrets: + - managed_operation_token + - managed_operation_database_url + environment: + PLATFORM_MANAGED_OPERATION_ALLOWLIST: "${PLATFORM_MANAGED_OPERATION_ALLOWLIST:?set PLATFORM_MANAGED_OPERATION_ALLOWLIST}" + networks: + - managed-backend + depends_on: + managed-operation-postgres: + condition: service_healthy + healthcheck: + test: + - CMD-SHELL + - >- + python3 -c "import urllib.request; + urllib.request.urlopen('http://127.0.0.1:8091/v1/health', timeout=3).read()" + interval: 10s + timeout: 5s + retries: 6 + + managed-operation-worker: + image: "${PLATFORM_MANAGED_OPERATION_IMAGE:?set a digest-pinned Platform image}" + restart: unless-stopped + entrypoint: + - /bin/sh + - -ec + command: + - >- + export GH_TOKEN="$$(cat /run/secrets/managed_operation_github_token)"; + exec python3 scripts/platform.py managed-operation worker + --queue-adapter postgresql + --database-url-file /run/secrets/managed_operation_database_url + --artifact-root /workspace/SpecGraph + --state-dir /data/specspace-state + --specgraph-dir /workspace/SpecGraph + --worker-id "$${HOSTNAME}" + --health-file /tmp/managed-operation-worker-health.json + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + volumes: + - "${PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT:?set PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT}:/workspace/SpecGraph" + - "${PLATFORM_MANAGED_OPERATION_STATE_DIR:?set PLATFORM_MANAGED_OPERATION_STATE_DIR}:/data/specspace-state:ro" + secrets: + - managed_operation_database_url + - managed_operation_github_token + environment: + PLATFORM_MANAGED_OPERATION_ALLOWLIST: "${PLATFORM_MANAGED_OPERATION_ALLOWLIST:?set PLATFORM_MANAGED_OPERATION_ALLOWLIST}" + networks: + - managed-backend + - managed-egress + depends_on: + managed-operation-postgres: + condition: service_healthy + healthcheck: + test: + - CMD-SHELL + - >- + python3 -c "import json,pathlib,time; + p=pathlib.Path('/tmp/managed-operation-worker-health.json'); + d=json.loads(p.read_text()); + raise SystemExit(0 if d.get('ok') is True and time.time()-p.stat().st_mtime < 15 else 1)" + interval: 10s + timeout: 5s + retries: 6 + start_period: 30s + + managed-operation-ingress: + image: "${PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE:?set a digest-pinned Caddy image}" + restart: unless-stopped + user: "1000:1000" + read_only: true + tmpfs: + - /tmp + - /data + - /config + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + command: + - caddy + - run + - --config + - /etc/caddy/Caddyfile + - --adapter + - caddyfile + ports: + - "${PLATFORM_MANAGED_OPERATION_INGRESS_BIND_IP:-0.0.0.0}:${PLATFORM_MANAGED_OPERATION_INGRESS_PORT:-443}:8443" + volumes: + - ./deploy/hosted-managed/Caddyfile:/etc/caddy/Caddyfile:ro + secrets: + - managed_operation_tls_certificate + - managed_operation_tls_private_key + networks: + - managed-backend + depends_on: + managed-operation-service: + condition: service_healthy + healthcheck: + test: + - CMD-SHELL + - >- + wget --quiet --no-check-certificate --output-document=- + https://127.0.0.1:8443/v1/health >/dev/null + interval: 10s + timeout: 5s + retries: 6 + + managed-operation-maintenance: + image: "${PLATFORM_MANAGED_OPERATION_IMAGE:?set a digest-pinned Platform image}" + profiles: + - maintenance + user: "1000:1000" + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + command: + - python3 + - scripts/hosted_managed_runtime_backup.py + - --help + volumes: + - "${PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT:?set PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT}:/workspace/SpecGraph:ro" + - "${PLATFORM_MANAGED_OPERATION_BACKUP_ROOT:?set PLATFORM_MANAGED_OPERATION_BACKUP_ROOT}:/backups" + secrets: + - managed_operation_database_url + networks: + - managed-backend + depends_on: + managed-operation-postgres: + condition: service_healthy + +volumes: + managed-operation-postgres: + +networks: + managed-backend: + internal: true + managed-egress: {} + +secrets: + managed_operation_token: + file: "${PLATFORM_MANAGED_OPERATION_TOKEN_FILE:?set PLATFORM_MANAGED_OPERATION_TOKEN_FILE}" + managed_operation_db_password: + file: "${PLATFORM_MANAGED_OPERATION_DB_PASSWORD_FILE:?set PLATFORM_MANAGED_OPERATION_DB_PASSWORD_FILE}" + managed_operation_database_url: + file: "${PLATFORM_MANAGED_OPERATION_DATABASE_URL_FILE:?set PLATFORM_MANAGED_OPERATION_DATABASE_URL_FILE}" + managed_operation_github_token: + file: "${PLATFORM_MANAGED_OPERATION_GITHUB_TOKEN_FILE:?set PLATFORM_MANAGED_OPERATION_GITHUB_TOKEN_FILE}" + managed_operation_tls_certificate: + file: "${PLATFORM_MANAGED_OPERATION_TLS_CERTIFICATE_FILE:?set PLATFORM_MANAGED_OPERATION_TLS_CERTIFICATE_FILE}" + managed_operation_tls_private_key: + file: "${PLATFORM_MANAGED_OPERATION_TLS_PRIVATE_KEY_FILE:?set PLATFORM_MANAGED_OPERATION_TLS_PRIVATE_KEY_FILE}" diff --git a/docs/deployment.md b/docs/deployment.md index 66253c0..0c8b71f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -175,6 +175,28 @@ ambiguous consume-on-attempt or irreversible leases are quarantined. It fails if a receipt contradicts the operation registry, so a deployment cannot hide a replay-policy drift behind a green recovery command. +### TLS-fronted production profile + +The standalone VM profile proves the container boundary but publishes only on +host loopback. Use `docker-compose.hosted-managed-production.example.yml` for a +remote production SpecSpace connection. This separate profile adds a +digest-pinned Caddy ingress and keeps PostgreSQL and the Platform HTTP service +on an internal Docker network. Only the worker receives an egress network for +read-only GitHub review inspection. The Platform service has no direct host +port. + +Production rollout and sign-off are documented in +[`hosted-managed-operations.md`](hosted-managed-operations.md#production-rollout-and-sign-off). +The production contract is checked in CI with: + +```bash +make hosted-managed-production-contract +``` + +It fails when images are mutable, the allowlist is absent, the Platform service +publishes a direct port, TLS ingress is missing, maintenance tooling is enabled +by default, or service/worker network authority expands. + ## Local Compose Entry Point The working plan for this phase is maintained in diff --git a/docs/hosted-managed-operations.md b/docs/hosted-managed-operations.md index 66a626c..ec8ba00 100644 --- a/docs/hosted-managed-operations.md +++ b/docs/hosted-managed-operations.md @@ -283,6 +283,215 @@ terminal receipt with the same `request_id`, `idempotency_key`, and attempt number. It must not execute the operation a second time. PostgreSQL data and workspace reports must remain present after the reboot. +## Production Rollout And Sign-Off + +The clean-VM runtime is staging evidence. A production deployment uses +`docker-compose.hosted-managed-production.example.yml` and is not signed off +until its TLS, backup, reboot, replay, SpecSpace cutover, and rollback evidence +passes the final audit. + +### Provisioning boundary + +Provision DNS and a certificate for a dedicated managed-operation origin. Do +not reuse the public SpecSpace origin or expose port `8091`. The production +profile publishes only TLS ingress on port `443`; Caddy proxies to the internal +Platform service, which continues to require its bearer token. + +All runtime images must be immutable digest refs: + +```bash +export PLATFORM_MANAGED_OPERATION_IMAGE='ghcr.io/0al-spec/platform@sha256:' +export PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE='postgres@sha256:' +export PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE='caddy@sha256:' +export PLATFORM_MANAGED_OPERATION_ALLOWLIST=review_status_execute +export PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT=/srv/0al/specgraph +export PLATFORM_MANAGED_OPERATION_STATE_DIR=/srv/0al/specspace-state +export PLATFORM_MANAGED_OPERATION_BACKUP_ROOT=/srv/0al/backups +``` + +`deploy/hosted-managed/production.env.example` contains the complete non-secret +environment inventory. It may be copied to a root-readable deployment env file, +but the referenced secret values remain separate files and must never be added +to that env file. + +Create independent service, database, and GitHub credentials. The GitHub token +for the first canary needs read-only pull-request/repository metadata only. It +must not have repository write, workflow, administration, package-write, or +organization authority. Provide a certificate and private key through separate +files: + +```bash +export PLATFORM_MANAGED_OPERATION_TOKEN_FILE=/srv/0al/secrets/service-token +export PLATFORM_MANAGED_OPERATION_DB_PASSWORD_FILE=/srv/0al/secrets/database-password +export PLATFORM_MANAGED_OPERATION_DATABASE_URL_FILE=/srv/0al/secrets/database-url +export PLATFORM_MANAGED_OPERATION_GITHUB_TOKEN_FILE=/srv/0al/secrets/github-token +export PLATFORM_MANAGED_OPERATION_TLS_CERTIFICATE_FILE=/srv/0al/secrets/tls-certificate.pem +export PLATFORM_MANAGED_OPERATION_TLS_PRIVATE_KEY_FILE=/srv/0al/secrets/tls-private-key.pem + +sudo chown root:1000 /srv/0al/secrets/* +sudo chmod 0440 /srv/0al/secrets/* +sudo chown -R 1000:1000 \ + "$PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT" \ + "$PLATFORM_MANAGED_OPERATION_STATE_DIR" \ + "$PLATFORM_MANAGED_OPERATION_BACKUP_ROOT" +``` + +Secret values must not appear in `.env`, Compose YAML, shell history, queue +requests, canary reports, or container image layers. Rotate them by atomically +replacing the corresponding file and recreating only the consumers that read +it. Service-token rotation must update SpecSpace and Platform in one bounded +cutover. Database credential rotation must update PostgreSQL first, then both +database secret files, then recreate service, worker, and maintenance +containers. Never reuse the service bearer token as a GitHub or database token. + +Run the fail-closed host preflight as root so ownership checks are meaningful: + +```bash +sudo --preserve-env .venv/bin/python \ + scripts/hosted_managed_production_preflight.py \ + --service-url https://managed.example.org \ + --output /srv/0al/evidence/production-preflight.json +``` + +The preflight report contains no secret values, paths, or local filesystem +metadata. It requires exact read-only canary scope, `0440` root/runtime-group +secret files, digest-pinned images, a clean HTTPS URL, and runtime-owned data +directories. + +### Start and probe + +Validate and start the production profile: + +```bash +make hosted-managed-production-contract +docker compose --project-name platform-managed-production \ + --file docker-compose.hosted-managed-production.example.yml config >/dev/null +docker compose --project-name platform-managed-production \ + --file docker-compose.hosted-managed-production.example.yml up --detach + +.venv/bin/python scripts/hosted_managed_production_probe.py \ + --service-url https://managed.example.org \ + --compose-file "$PWD/docker-compose.hosted-managed-production.example.yml" \ + --project-name platform-managed-production \ + --output /srv/0al/evidence/probe-before-reboot.json +``` + +The probe requires all four runtime services to be healthy, PostgreSQL as the +queue adapter, a fresh worker heartbeat, and exactly +`review_status_execute` in service health. The report is public-safe and omits +Compose paths and credentials. + +Initial operational targets are diagnostic objectives, not an external SLA: + +- alert immediately on any quarantined request or expanded allowlist; +- alert after two failed probes or a worker heartbeat older than 30 seconds; +- alert when a read-only canary does not terminate within its bounded window; +- run a read-only canary at least daily during rollout and after each deploy; +- run strict recovery and queue-drain audit after unclean shutdowns; +- retain private backup reports with the encrypted off-host backup they pin. + +### Private backup and restore smoke + +Stop new enqueueing and stop the worker after its current lease before backup. +The backup tool rejects queued/leased/running jobs, active workspace locks, +symlinks, concurrent artifact changes, and an existing backup id. It stores a +transaction-consistent versioned export of the three queue tables plus a +digest-inventoried private archive of `runs/` artifacts. It never includes +secret files. + +Run the isolated maintenance profile explicitly; it is not started by normal +`up`: + +```bash +backup_id="production-$(date -u +%Y%m%dT%H%M%SZ)" +docker compose --project-name platform-managed-production \ + --file docker-compose.hosted-managed-production.example.yml \ + --profile maintenance run --rm managed-operation-maintenance \ + python3 scripts/hosted_managed_runtime_backup.py backup \ + --database-url-file /run/secrets/managed_operation_database_url \ + --artifact-root /workspace/SpecGraph \ + --backup-root /backups \ + --backup-id "$backup_id" + +docker compose --project-name platform-managed-production \ + --file docker-compose.hosted-managed-production.example.yml \ + --profile maintenance run --rm managed-operation-maintenance \ + python3 scripts/hosted_managed_runtime_backup.py restore-smoke \ + --database-url-file /run/secrets/managed_operation_database_url \ + --backup-root /backups \ + --backup-id "$backup_id" \ + --output "/backups/$backup_id/restore-smoke-report.json" +``` + +`restore-smoke` creates a temporary PostgreSQL database, restores the versioned +queue export, compares every table count, verifies every archived artifact +digest without extracting unsafe paths, then forcibly removes the temporary +database. It has no authority to restore over production. Copy the entire +private backup directory to encrypted off-host storage; copying only its public +summary is not a backup. + +### Canary, reboot, replay, and rollback + +Use an open review PR and a workspace-bound, queue-safe +`review_status_execute` request. Run the existing canary through the public TLS +origin with its bearer token file and with host-local artifact access so output +bytes are checked against receipt digests: + +```bash +.venv/bin/python scripts/platform.py managed-operation canary \ + --service-url https://managed.example.org \ + --auth-token-file "$PLATFORM_MANAGED_OPERATION_TOKEN_FILE" \ + --request /srv/0al/canary/review-status-request.json \ + --artifact-root "$PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT" \ + --output /srv/0al/evidence/canary.json \ + --format json +``` + +Verify SpecSpace in hosted mode with `specspace product-smoke` and +`--expect-managed-mode backend_managed_ready`. Reboot the host, rerun strict +recovery, create `probe-after-reboot.json`, and submit the identical canary +request again as `replay-canary.json`. The replay must preserve request id, +idempotency key, output refs and `attempt=1`. + +Before rollback, audit that no active job or lock remains: + +```bash +docker compose --project-name platform-managed-production \ + --file docker-compose.hosted-managed-production.example.yml \ + exec -T managed-operation-worker \ + python3 scripts/hosted_managed_production_signoff.py queue-audit \ + --database-url-file /run/secrets/managed_operation_database_url \ + --output /workspace/SpecGraph/runs/production-queue-audit.json +``` + +Then disable hosted enqueueing in SpecSpace, stop the worker after drain, and +verify SpecSpace with `--expect-managed-mode read_only`. Do not copy queue rows +to a local SQLite executor. A consume-on-attempt or irreversible request needs +new operator intent after rollback. + +The final sign-off command requires all evidence rather than trusting a single +green queue receipt: + +```bash +.venv/bin/python scripts/hosted_managed_production_signoff.py signoff \ + --preflight /srv/0al/evidence/production-preflight.json \ + --probe-before-reboot /srv/0al/evidence/probe-before-reboot.json \ + --probe-after-reboot /srv/0al/evidence/probe-after-reboot.json \ + --canary /srv/0al/evidence/canary.json \ + --replay-canary /srv/0al/evidence/replay-canary.json \ + --recovery /srv/0al/evidence/recovery.json \ + --backup "/srv/0al/backups/$backup_id/backup-report.json" \ + --restore-smoke "/srv/0al/backups/$backup_id/restore-smoke-report.json" \ + --queue-audit /srv/0al/evidence/queue-audit.json \ + --hosted-specspace-smoke /srv/0al/evidence/specspace-hosted-smoke.json \ + --rollback-specspace-smoke /srv/0al/evidence/specspace-rollback-smoke.json \ + --output /srv/0al/evidence/production-canary-signoff.json +``` + +Only `production_canary_signed_off` permits a later, separate rollout proposal +for `promotion_execute_dry_run`. This sign-off does not enable that operation, +consume-on-attempt work, non-dry-run Git review, or read-model publication. + ## Delivery And Recovery Hosted execution uses **at-least-once** delivery. It must not claim exactly-once diff --git a/docs/ontology-specgraph-specspace-roadmap.md b/docs/ontology-specgraph-specspace-roadmap.md index 973136a..25503a7 100644 --- a/docs/ontology-specgraph-specspace-roadmap.md +++ b/docs/ontology-specgraph-specspace-roadmap.md @@ -282,12 +282,17 @@ The current execution order is: arbitrary execution parameters. SpecSpace hosted mode and execution-backed browser coverage use this boundary, while Platform Compose smoke validates PostgreSQL service/worker health. Existing Platform reports remain lifecycle - authority; queue status remains transport telemetry. Remaining operations - work is deployment recovery rehearsal and SLO/alert definition, not another - execution contract. -5. **Human-friendly candidate aliases.** SpecGraph should keep stable machine - ids for refs and promotion paths, but expose readable aliases for candidate - overview, topology, PR artifacts, and operator-facing diagnostics. + authority; queue status remains transport telemetry. The portable single-node + runtime and clean-VM canary are complete. The tracked production profile now + adds digest-pinned images, TLS-only ingress, host preflight, private + backup/restore verification, runtime probes, queue-drain audit, reboot/replay + evidence, and a fail-closed final sign-off contract. Actual production + sign-off still requires deployment-owned DNS, certificates, secrets, a real + workspace request, and the resulting evidence reports. +5. **Human-friendly candidate aliases.** Implemented. SpecGraph keeps stable + machine ids for refs and promotion paths while exposing deterministic, + privacy-checked display aliases in candidate overview and topology artifacts; + SpecSpace uses them as presentation metadata without changing identity. 6. **Ontology applicability in product review.** Continue compiler-backed layers, `modelApplicability`, and change classification so product candidates can explain which ontology layer and applicability frame each diff --git a/scripts/hosted_managed_production_preflight.py b/scripts/hosted_managed_production_preflight.py new file mode 100644 index 0000000..1c83309 --- /dev/null +++ b/scripts/hosted_managed_production_preflight.py @@ -0,0 +1,293 @@ +"""Fail-closed host preflight for production hosted managed operations.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import stat +from typing import Any +from urllib.parse import urlsplit + + +READ_ONLY_CANARY_OPERATION = "review_status_execute" +DRY_RUN_OPERATION = "promotion_execute_dry_run" +SECRET_SPECS = { + "service_token": (32, None), + "database_password": (24, None), + "database_url": (32, "postgresql"), + "github_token": (20, None), + "tls_certificate": (64, "certificate"), + "tls_private_key": (64, "private_key"), +} + + +class ProductionPreflightError(RuntimeError): + """Production inputs do not satisfy the deployment contract.""" + + +def _digest_pinned_image(value: str) -> bool: + prefix, marker, digest = value.partition("@sha256:") + return bool(prefix and marker and len(digest) == 64) and all( + character in "0123456789abcdef" for character in digest.lower() + ) + + +def _secret_diagnostic( + *, + label: str, + path: Path, + minimum_length: int, + content_kind: str | None, + expected_uid: int, + expected_gid: int, +) -> tuple[list[str], bytes | None]: + diagnostics: list[str] = [] + if not path.is_absolute(): + return [f"{label}_path_not_absolute"], None + if path.is_symlink(): + return [f"{label}_path_is_symlink"], None + try: + metadata = path.stat() + except OSError: + return [f"{label}_file_missing"], None + if not stat.S_ISREG(metadata.st_mode): + diagnostics.append(f"{label}_not_regular_file") + if stat.S_IMODE(metadata.st_mode) != 0o440: + diagnostics.append(f"{label}_mode_not_0440") + if metadata.st_uid != expected_uid: + diagnostics.append(f"{label}_owner_uid_mismatch") + if metadata.st_gid != expected_gid: + diagnostics.append(f"{label}_owner_gid_mismatch") + try: + content = path.read_bytes().strip() + except OSError: + diagnostics.append(f"{label}_unreadable") + return diagnostics, None + if len(content) < minimum_length: + diagnostics.append(f"{label}_too_short") + if b"\x00" in content or b"\r" in content: + diagnostics.append(f"{label}_invalid_bytes") + if content_kind == "postgresql" and not content.startswith( + (b"postgresql://", b"postgres://") + ): + diagnostics.append("database_url_scheme_invalid") + if content_kind == "certificate" and b"BEGIN CERTIFICATE" not in content: + diagnostics.append("tls_certificate_pem_invalid") + if content_kind == "private_key" and b"PRIVATE KEY" not in content: + diagnostics.append("tls_private_key_pem_invalid") + return diagnostics, content + + +def _directory_diagnostics( + *, + label: str, + path: Path, + expected_uid: int, + expected_gid: int, + require_owner_write: bool, +) -> list[str]: + if not path.is_absolute(): + return [f"{label}_path_not_absolute"] + if path.is_symlink(): + return [f"{label}_path_is_symlink"] + try: + metadata = path.stat() + except OSError: + return [f"{label}_missing"] + diagnostics: list[str] = [] + if not stat.S_ISDIR(metadata.st_mode): + diagnostics.append(f"{label}_not_directory") + if metadata.st_uid != expected_uid: + diagnostics.append(f"{label}_owner_uid_mismatch") + if metadata.st_gid != expected_gid: + diagnostics.append(f"{label}_owner_gid_mismatch") + if require_owner_write and not metadata.st_mode & stat.S_IWUSR: + diagnostics.append(f"{label}_owner_write_missing") + return diagnostics + + +def run_preflight( + *, + service_url: str, + allowlist: str, + image_refs: dict[str, str], + secret_paths: dict[str, Path], + artifact_root: Path, + state_dir: Path, + expected_secret_uid: int = 0, + runtime_uid: int = 1000, + runtime_gid: int = 1000, + allow_dry_run: bool = False, +) -> dict[str, Any]: + diagnostics: list[str] = [] + parsed = urlsplit(service_url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + ): + diagnostics.append("service_url_not_private_https_endpoint") + + enabled = [item.strip() for item in allowlist.split(",") if item.strip()] + expected = [READ_ONLY_CANARY_OPERATION] + if allow_dry_run: + expected.append(DRY_RUN_OPERATION) + if enabled != expected: + diagnostics.append("deployment_allowlist_not_exact_canary_scope") + + for label in ("platform", "postgresql", "ingress"): + if not _digest_pinned_image(image_refs.get(label, "")): + diagnostics.append(f"{label}_image_not_digest_pinned") + + if set(secret_paths) != set(SECRET_SPECS): + diagnostics.append("secret_set_incomplete") + secret_contents: dict[str, bytes] = {} + for label, (minimum_length, content_kind) in SECRET_SPECS.items(): + path = secret_paths.get(label) + if path is None: + continue + findings, content = _secret_diagnostic( + label=label, + path=path, + minimum_length=minimum_length, + content_kind=content_kind, + expected_uid=expected_secret_uid, + expected_gid=runtime_gid, + ) + diagnostics.extend(findings) + if content is not None: + secret_contents[label] = content + if len(secret_contents) != len(set(secret_contents.values())): + diagnostics.append("secret_values_not_distinct") + + diagnostics.extend( + _directory_diagnostics( + label="artifact_root", + path=artifact_root, + expected_uid=runtime_uid, + expected_gid=runtime_gid, + require_owner_write=True, + ) + ) + diagnostics.extend( + _directory_diagnostics( + label="state_dir", + path=state_dir, + expected_uid=runtime_uid, + expected_gid=runtime_gid, + require_owner_write=False, + ) + ) + + diagnostics = sorted(set(diagnostics)) + return { + "artifact_kind": "platform_hosted_managed_production_preflight_report", + "contract_ref": "platform.hosted-managed.production-preflight.v1", + "ok": not diagnostics, + "summary": { + "status": "ready" if not diagnostics else "blocked", + "service_transport": "https" if parsed.scheme == "https" else "invalid", + "enabled_operations": enabled, + "dry_run_enabled": DRY_RUN_OPERATION in enabled, + "image_count": len(image_refs), + "secret_file_count": len(secret_paths), + "artifact_root_ready": not any( + item.startswith("artifact_root_") for item in diagnostics + ), + "state_dir_ready": not any( + item.startswith("state_dir_") for item in diagnostics + ), + }, + "diagnostics": diagnostics, + "privacy_boundary": { + "public_safe": True, + "includes_secret_values": False, + "includes_secret_paths": False, + "includes_local_paths": False, + }, + "authority_boundary": { + "may_execute_platform": False, + "may_enqueue_operations": False, + "may_mutate_specs": False, + "may_write_ontology": False, + "may_create_git_review": False, + }, + } + + +def _env_path(name: str) -> Path: + value = os.environ.get(name, "") + if not value: + raise ProductionPreflightError(f"{name} is required") + return Path(value) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--service-url", required=True) + parser.add_argument("--allow-dry-run", action="store_true") + parser.add_argument("--expected-secret-uid", type=int, default=0) + parser.add_argument("--runtime-uid", type=int, default=1000) + parser.add_argument("--runtime-gid", type=int, default=1000) + parser.add_argument("--output") + args = parser.parse_args(argv) + try: + report = run_preflight( + service_url=args.service_url, + allowlist=os.environ.get("PLATFORM_MANAGED_OPERATION_ALLOWLIST", ""), + image_refs={ + "platform": os.environ.get("PLATFORM_MANAGED_OPERATION_IMAGE", ""), + "postgresql": os.environ.get( + "PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE", "" + ), + "ingress": os.environ.get( + "PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE", "" + ), + }, + secret_paths={ + "service_token": _env_path("PLATFORM_MANAGED_OPERATION_TOKEN_FILE"), + "database_password": _env_path( + "PLATFORM_MANAGED_OPERATION_DB_PASSWORD_FILE" + ), + "database_url": _env_path( + "PLATFORM_MANAGED_OPERATION_DATABASE_URL_FILE" + ), + "github_token": _env_path( + "PLATFORM_MANAGED_OPERATION_GITHUB_TOKEN_FILE" + ), + "tls_certificate": _env_path( + "PLATFORM_MANAGED_OPERATION_TLS_CERTIFICATE_FILE" + ), + "tls_private_key": _env_path( + "PLATFORM_MANAGED_OPERATION_TLS_PRIVATE_KEY_FILE" + ), + }, + artifact_root=_env_path("PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT"), + state_dir=_env_path("PLATFORM_MANAGED_OPERATION_STATE_DIR"), + expected_secret_uid=args.expected_secret_uid, + runtime_uid=args.runtime_uid, + runtime_gid=args.runtime_gid, + allow_dry_run=args.allow_dry_run, + ) + except ProductionPreflightError as exc: + report = { + "artifact_kind": "platform_hosted_managed_production_preflight_report", + "ok": False, + "summary": {"status": "blocked"}, + "diagnostics": [str(exc)], + } + rendered = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + Path(args.output).write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if report.get("ok") is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/hosted_managed_production_probe.py b/scripts/hosted_managed_production_probe.py new file mode 100644 index 0000000..336efb3 --- /dev/null +++ b/scripts/hosted_managed_production_probe.py @@ -0,0 +1,248 @@ +"""Probe a deployed TLS-fronted hosted managed-operation runtime.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +from typing import Any, Callable +from urllib.parse import urlsplit +import urllib.request + + +EXPECTED_SERVICES = { + "managed-operation-postgres", + "managed-operation-service", + "managed-operation-worker", + "managed-operation-ingress", +} +READ_ONLY_CANARY_OPERATION = "review_status_execute" + + +class ProductionProbeError(RuntimeError): + """The deployed runtime did not provide trustworthy health evidence.""" + + +def _load_json_response(url: str, *, timeout: float) -> dict[str, Any]: + try: + with urllib.request.urlopen(url, timeout=timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProductionProbeError("hosted HTTPS health endpoint is unavailable") from exc + if not isinstance(payload, dict): + raise ProductionProbeError("hosted HTTPS health response must be an object") + return payload + + +def _compose_rows(output: str) -> list[dict[str, Any]]: + stripped = output.strip() + if not stripped: + return [] + try: + payload = json.loads(stripped) + except json.JSONDecodeError: + payload = [json.loads(line) for line in stripped.splitlines()] + if isinstance(payload, dict): + payload = [payload] + if not isinstance(payload, list) or not all(isinstance(item, dict) for item in payload): + raise ProductionProbeError("docker compose ps returned an invalid payload") + return payload + + +def _run( + command: list[str], + *, + runner: Callable[..., subprocess.CompletedProcess[str]], +) -> str: + completed = runner(command, capture_output=True, text=True, check=False) + if completed.returncode != 0: + raise ProductionProbeError("docker compose health inspection failed") + return completed.stdout + + +def run_probe( + *, + service_url: str, + compose_file: Path, + project_name: str, + timeout_seconds: float = 10.0, + max_heartbeat_age_seconds: float = 30.0, + now: datetime | None = None, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + fetch_health: Callable[[str], dict[str, Any]] | None = None, +) -> dict[str, Any]: + parsed = urlsplit(service_url) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + ): + raise ProductionProbeError("production probe requires a clean HTTPS service URL") + if not compose_file.is_absolute(): + raise ProductionProbeError("production Compose path must be absolute") + health_url = f"{service_url.rstrip('/')}/v1/health" + health = ( + fetch_health(health_url) + if fetch_health is not None + else _load_json_response(health_url, timeout=timeout_seconds) + ) + ps_output = _run( + [ + "docker", + "compose", + "--project-name", + project_name, + "--file", + str(compose_file), + "ps", + "--format", + "json", + ], + runner=runner, + ) + rows = _compose_rows(ps_output) + service_states: dict[str, dict[str, str]] = {} + for row in rows: + service = row.get("Service") or row.get("service") + if not isinstance(service, str): + continue + service_states[service] = { + "state": str(row.get("State") or row.get("state") or "unknown").lower(), + "health": str(row.get("Health") or row.get("health") or "unknown").lower(), + } + heartbeat_output = _run( + [ + "docker", + "compose", + "--project-name", + project_name, + "--file", + str(compose_file), + "exec", + "-T", + "managed-operation-worker", + "cat", + "/tmp/managed-operation-worker-health.json", + ], + runner=runner, + ) + try: + heartbeat = json.loads(heartbeat_output) + except json.JSONDecodeError as exc: + raise ProductionProbeError("worker heartbeat is invalid") from exc + if not isinstance(heartbeat, dict): + raise ProductionProbeError("worker heartbeat must be an object") + + diagnostics: list[str] = [] + if health.get("ok") is not True or health.get("adapter") != "postgresql": + diagnostics.append("service_health_not_postgresql_ready") + operations = health.get("operations") + operations = operations if isinstance(operations, dict) else {} + enabled = operations.get("enabled_operation_ids") + if enabled != [READ_ONLY_CANARY_OPERATION]: + diagnostics.append("service_allowlist_not_read_only_canary") + if set(service_states) != EXPECTED_SERVICES: + diagnostics.append("compose_service_set_mismatch") + for service in sorted(EXPECTED_SERVICES): + state = service_states.get(service, {}) + if state.get("state") != "running" or state.get("health") != "healthy": + diagnostics.append(f"{service}_not_healthy") + if heartbeat.get("ok") is not True or heartbeat.get("adapter") != "postgresql": + diagnostics.append("worker_heartbeat_not_postgresql_ready") + generated_at = heartbeat.get("generated_at") + heartbeat_age: float | None = None + if isinstance(generated_at, str): + try: + timestamp = datetime.fromisoformat(generated_at.replace("Z", "+00:00")) + current = now or datetime.now(timezone.utc) + heartbeat_age = max(0.0, (current - timestamp).total_seconds()) + except ValueError: + pass + if heartbeat_age is None or heartbeat_age > max_heartbeat_age_seconds: + diagnostics.append("worker_heartbeat_stale") + + diagnostics = sorted(set(diagnostics)) + return { + "artifact_kind": "platform_hosted_managed_production_probe_report", + "contract_ref": "platform.hosted-managed.production-probe.v1", + "generated_at": (now or datetime.now(timezone.utc)).isoformat(), + "ok": not diagnostics, + "service": { + "origin": f"{parsed.scheme}://{parsed.netloc}", + "adapter": health.get("adapter"), + "enabled_operation_ids": enabled if isinstance(enabled, list) else [], + }, + "worker": { + "adapter": heartbeat.get("adapter"), + "heartbeat_sequence": heartbeat.get("heartbeat_sequence"), + "heartbeat_age_seconds": heartbeat_age, + "last_cycle_status": heartbeat.get("last_cycle_status"), + }, + "services": service_states, + "summary": { + "status": "healthy" if not diagnostics else "unhealthy", + "healthy_service_count": sum( + value.get("state") == "running" and value.get("health") == "healthy" + for value in service_states.values() + ), + "expected_service_count": len(EXPECTED_SERVICES), + "read_only_allowlist": enabled == [READ_ONLY_CANARY_OPERATION], + }, + "diagnostics": diagnostics, + "privacy_boundary": { + "public_safe": True, + "includes_secret_values": False, + "includes_local_paths": False, + }, + "authority_boundary": { + "may_enqueue_operations": False, + "may_execute_platform": False, + "may_mutate_specs": False, + "may_create_git_review": False, + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--service-url", required=True) + parser.add_argument( + "--compose-file", + default=str( + Path(__file__).resolve().parents[1] + / "docker-compose.hosted-managed-production.example.yml" + ), + ) + parser.add_argument("--project-name", default="platform-managed-production") + parser.add_argument("--timeout", type=float, default=10.0) + parser.add_argument("--max-heartbeat-age", type=float, default=30.0) + parser.add_argument("--output") + args = parser.parse_args(argv) + try: + report = run_probe( + service_url=args.service_url, + compose_file=Path(args.compose_file).resolve(), + project_name=args.project_name, + timeout_seconds=args.timeout, + max_heartbeat_age_seconds=args.max_heartbeat_age, + ) + except ProductionProbeError as exc: + report = { + "artifact_kind": "platform_hosted_managed_production_probe_report", + "ok": False, + "summary": {"status": "unhealthy"}, + "diagnostics": [str(exc)], + } + rendered = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + Path(args.output).write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if report.get("ok") is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/hosted_managed_production_signoff.py b/scripts/hosted_managed_production_signoff.py new file mode 100644 index 0000000..ee6f7ba --- /dev/null +++ b/scripts/hosted_managed_production_signoff.py @@ -0,0 +1,331 @@ +"""Audit queue drain state and assemble final hosted production canary sign-off.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +from pathlib import Path +from typing import Any + + +ACTIVE_QUEUE_STATUSES = {"queued", "leased", "running"} +EXPECTED_KINDS = { + "preflight": "platform_hosted_managed_production_preflight_report", + "probe_before_reboot": "platform_hosted_managed_production_probe_report", + "probe_after_reboot": "platform_hosted_managed_production_probe_report", + "canary": "platform_hosted_managed_operation_canary_report", + "replay_canary": "platform_hosted_managed_operation_canary_report", + "recovery": "platform_hosted_managed_operation_queue_recovery_report", + "backup": "platform_hosted_managed_runtime_backup_report", + "restore_smoke": "platform_hosted_managed_runtime_restore_smoke_report", + "queue_audit": "platform_hosted_managed_production_queue_audit_report", + "hosted_specspace_smoke": "platform_specspace_product_workspace_production_smoke_report", + "rollback_specspace_smoke": "platform_specspace_product_workspace_production_smoke_report", +} + + +class ProductionSignoffError(RuntimeError): + """Production evidence is missing, stale, or contradictory.""" + + +def _database_url(path: Path) -> str: + try: + value = path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise ProductionSignoffError("database URL secret file is unavailable") from exc + if not value.startswith(("postgresql://", "postgres://")): + raise ProductionSignoffError("database URL must use PostgreSQL") + return value + + +def queue_audit(database_url_file: Path) -> dict[str, Any]: + try: + import psycopg + except ImportError as exc: + raise ProductionSignoffError("psycopg is required for queue audit") from exc + database_url = _database_url(database_url_file) + status_counts: dict[str, int] = {} + try: + with psycopg.connect(database_url, autocommit=True) as connection: + with connection.cursor() as cursor: + cursor.execute( + "SELECT status, COUNT(*) FROM managed_operation_jobs " + "GROUP BY status ORDER BY status" + ) + status_counts = { + str(row[0]): int(row[1]) for row in cursor.fetchall() + } + cursor.execute("SELECT COUNT(*) FROM managed_operation_locks") + lock_count = int(cursor.fetchone()[0]) + cursor.execute("SELECT COUNT(*) FROM managed_operation_events") + event_count = int(cursor.fetchone()[0]) + except Exception as exc: + raise ProductionSignoffError("production queue audit failed") from exc + active_count = sum(status_counts.get(status, 0) for status in ACTIVE_QUEUE_STATUSES) + rollback_ready = active_count == 0 and lock_count == 0 + return { + "artifact_kind": "platform_hosted_managed_production_queue_audit_report", + "contract_ref": "platform.hosted-managed.production-queue-audit.v1", + "generated_at": datetime.now(timezone.utc).isoformat(), + "ok": rollback_ready, + "summary": { + "status": "drained" if rollback_ready else "active", + "rollback_ready": rollback_ready, + "active_job_count": active_count, + "lock_count": lock_count, + "event_count": event_count, + "job_status_counts": status_counts, + }, + "privacy_boundary": { + "public_safe": True, + "includes_request_payloads": False, + "includes_workspace_ids": False, + "includes_database_url": False, + }, + "authority_boundary": { + "may_requeue_operations": False, + "may_execute_platform": False, + "may_mutate_queue": False, + "may_mutate_specs": False, + }, + } + + +def _load_report(path: Path, *, label: str) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProductionSignoffError(f"{label} report is unavailable or invalid") from exc + if not isinstance(payload, dict): + raise ProductionSignoffError(f"{label} report must be an object") + return payload + + +def _canary_identity(report: dict[str, Any]) -> tuple[Any, ...]: + request = report.get("request") + request = request if isinstance(request, dict) else {} + outputs = report.get("authoritative_outputs") + outputs = outputs if isinstance(outputs, dict) else {} + return ( + request.get("request_id"), + request.get("idempotency_key"), + request.get("operation_id"), + request.get("workspace_id"), + tuple(outputs.get("observed_refs") or []), + tuple(outputs.get("verified_refs") or []), + ) + + +def _write_authority_findings(value: Any, *, path: str = "$") -> list[str]: + findings: list[str] = [] + if isinstance(value, dict): + for key, nested in value.items(): + nested_path = f"{path}.{key}" + if isinstance(key, str) and key.startswith("may_") and nested is not False: + findings.append(nested_path) + findings.extend(_write_authority_findings(nested, path=nested_path)) + elif isinstance(value, list): + for index, nested in enumerate(value): + findings.extend(_write_authority_findings(nested, path=f"{path}[{index}]")) + return findings + + +def build_signoff(reports: dict[str, dict[str, Any]]) -> dict[str, Any]: + diagnostics: list[str] = [] + if set(reports) != set(EXPECTED_KINDS): + diagnostics.append("evidence_set_incomplete") + for label, expected_kind in EXPECTED_KINDS.items(): + report = reports.get(label, {}) + if report.get("artifact_kind") != expected_kind: + diagnostics.append(f"{label}_artifact_kind_invalid") + if report.get("ok") is not True: + diagnostics.append(f"{label}_not_ready") + if _write_authority_findings(report): + diagnostics.append(f"{label}_write_authority_expanded") + + preflight_summary = reports.get("preflight", {}).get("summary") + preflight_summary = preflight_summary if isinstance(preflight_summary, dict) else {} + if ( + preflight_summary.get("status") != "ready" + or preflight_summary.get("enabled_operations") != ["review_status_execute"] + or preflight_summary.get("dry_run_enabled") is not False + ): + diagnostics.append("preflight_scope_invalid") + + for label in ("probe_before_reboot", "probe_after_reboot"): + report = reports.get(label, {}) + summary = report.get("summary") + service = report.get("service") + if not isinstance(summary, dict) or summary.get("status") != "healthy": + diagnostics.append(f"{label}_not_healthy") + if not isinstance(service, dict) or service.get("enabled_operation_ids") != [ + "review_status_execute" + ]: + diagnostics.append(f"{label}_allowlist_invalid") + before_service = reports.get("probe_before_reboot", {}).get("service") + before_service = before_service if isinstance(before_service, dict) else {} + after_service = reports.get("probe_after_reboot", {}).get("service") + after_service = after_service if isinstance(after_service, dict) else {} + before_origin = before_service.get("origin") + after_origin = after_service.get("origin") + if not before_origin or before_origin != after_origin: + diagnostics.append("reboot_probe_origin_mismatch") + + for label in ("canary", "replay_canary"): + report = reports.get(label, {}) + summary = report.get("summary") + queue = report.get("queue") + outputs = report.get("authoritative_outputs") + if not isinstance(summary, dict) or summary.get("profile") != "read_only": + diagnostics.append(f"{label}_profile_invalid") + if not isinstance(queue, dict) or queue.get("status") != "succeeded": + diagnostics.append(f"{label}_queue_not_succeeded") + if not isinstance(queue, dict) or queue.get("attempt") != 1: + diagnostics.append(f"{label}_attempt_not_one") + if not isinstance(outputs, dict) or outputs.get("receipt_pins_reports") is not True: + diagnostics.append(f"{label}_outputs_not_pinned") + if not isinstance(outputs, dict) or outputs.get("verified_refs") != outputs.get( + "observed_refs" + ): + diagnostics.append(f"{label}_output_files_not_verified") + if _canary_identity(reports.get("canary", {})) != _canary_identity( + reports.get("replay_canary", {}) + ): + diagnostics.append("canary_replay_identity_mismatch") + + recovery_summary = reports.get("recovery", {}).get("summary") + recovery_summary = recovery_summary if isinstance(recovery_summary, dict) else {} + if ( + recovery_summary.get("strict") is not True + or recovery_summary.get("policy_safe") is not True + or recovery_summary.get("preflight_blocked") is not False + ): + diagnostics.append("strict_recovery_not_safe") + + backup = reports.get("backup", {}) + restore = reports.get("restore_smoke", {}) + backup_summary = backup.get("summary") + backup_summary = backup_summary if isinstance(backup_summary, dict) else {} + restore_summary = restore.get("summary") + restore_summary = restore_summary if isinstance(restore_summary, dict) else {} + if ( + backup_summary.get("status") != "backup_ready" + or backup_summary.get("database_backup_schema_version") != 1 + or restore_summary.get("status") != "restore_smoke_passed" + or restore_summary.get("temporary_database_removed") is not True + ): + diagnostics.append("backup_restore_contract_invalid") + if not backup.get("backup_id") or backup.get("backup_id") != restore.get("backup_id"): + diagnostics.append("backup_restore_identity_mismatch") + queue_summary = reports.get("queue_audit", {}).get("summary") + queue_summary = queue_summary if isinstance(queue_summary, dict) else {} + if queue_summary.get("rollback_ready") is not True: + diagnostics.append("queue_not_drained_for_rollback") + + hosted_smoke = reports.get("hosted_specspace_smoke", {}).get("summary") + hosted_smoke = hosted_smoke if isinstance(hosted_smoke, dict) else {} + if hosted_smoke.get("expected_managed_mode") != "backend_managed_ready": + diagnostics.append("hosted_specspace_mode_not_ready") + rollback_smoke = reports.get("rollback_specspace_smoke", {}).get("summary") + rollback_smoke = rollback_smoke if isinstance(rollback_smoke, dict) else {} + if rollback_smoke.get("expected_managed_mode") != "read_only": + diagnostics.append("rollback_specspace_mode_not_read_only") + + diagnostics = sorted(set(diagnostics)) + canary_request = reports.get("canary", {}).get("request") + canary_request = canary_request if isinstance(canary_request, dict) else {} + return { + "artifact_kind": "platform_hosted_managed_production_canary_signoff_report", + "contract_ref": "platform.hosted-managed.production-canary-signoff.v1", + "generated_at": datetime.now(timezone.utc).isoformat(), + "ok": not diagnostics, + "summary": { + "status": "production_canary_signed_off" + if not diagnostics + else "production_canary_blocked", + "workspace_id": canary_request.get("workspace_id"), + "operation_id": canary_request.get("operation_id"), + "read_only_canary_passed": not any( + item.startswith(("canary_", "replay_canary_")) + for item in diagnostics + ), + "reboot_verified": not any( + item.startswith(("probe_after_reboot_", "reboot_probe_")) + for item in diagnostics + ), + "backup_restore_verified": not any( + item.startswith("backup_restore_") for item in diagnostics + ), + "rollback_verified": not any( + item.startswith(("queue_not_", "rollback_specspace_")) + for item in diagnostics + ), + }, + "diagnostics": diagnostics, + "evidence": { + label: { + "artifact_kind": report.get("artifact_kind"), + "ok": report.get("ok"), + } + for label, report in sorted(reports.items()) + }, + "privacy_boundary": { + "public_safe": True, + "includes_secret_values": False, + "includes_local_paths": False, + "includes_request_payloads": False, + }, + "authority_boundary": { + "may_enqueue_operations": False, + "may_execute_platform": False, + "may_mutate_specs": False, + "may_create_git_review": False, + "may_publish_read_model": False, + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subcommands = parser.add_subparsers(dest="command", required=True) + audit = subcommands.add_parser("queue-audit") + audit.add_argument("--database-url-file", required=True) + audit.add_argument("--output") + signoff = subcommands.add_parser("signoff") + for label in EXPECTED_KINDS: + signoff.add_argument(f"--{label.replace('_', '-')}", required=True) + signoff.add_argument("--output") + args = parser.parse_args(argv) + try: + if args.command == "queue-audit": + report = queue_audit(Path(args.database_url_file)) + else: + reports = { + label: _load_report(Path(getattr(args, label)), label=label) + for label in EXPECTED_KINDS + } + report = build_signoff(reports) + except ProductionSignoffError as exc: + report = { + "artifact_kind": "platform_hosted_managed_production_canary_signoff_report", + "ok": False, + "summary": {"status": "production_canary_blocked"}, + "diagnostics": [str(exc)], + } + except Exception: + report = { + "artifact_kind": "platform_hosted_managed_production_canary_signoff_report", + "ok": False, + "summary": {"status": "production_canary_blocked"}, + "diagnostics": ["production sign-off evaluation failed"], + } + rendered = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + Path(args.output).write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if report.get("ok") is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/hosted_managed_runtime_backup.py b/scripts/hosted_managed_runtime_backup.py new file mode 100644 index 0000000..bdb4142 --- /dev/null +++ b/scripts/hosted_managed_runtime_backup.py @@ -0,0 +1,463 @@ +"""Create and restore-verify private hosted managed-operation backups.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import tarfile +from typing import Any +from urllib.parse import urlsplit, urlunsplit + + +BACKUP_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +QUEUE_TABLES = ( + "managed_operation_jobs", + "managed_operation_events", + "managed_operation_locks", +) +TABLE_COLUMNS = { + "managed_operation_jobs": ( + "request_id", + "idempotency_key", + "operation_id", + "workspace_id", + "request_sha256", + "request_json", + "status", + "attempt", + "available_at", + "lease_owner", + "lease_expires_at", + "receipt_json", + "created_at", + "updated_at", + ), + "managed_operation_events": ( + "event_id", + "request_id", + "status", + "attempt", + "recorded_at", + "receipt_json", + ), + "managed_operation_locks": ( + "lock_scope", + "request_id", + "lease_owner", + "lease_expires_at", + ), +} + + +class HostedBackupError(RuntimeError): + """Backup or restore verification failed without mutating production state.""" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _database_url(path: Path) -> str: + try: + value = path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise HostedBackupError("database URL secret file is unavailable") from exc + if not value.startswith(("postgresql://", "postgres://")): + raise HostedBackupError("database URL must use PostgreSQL") + return value + + +def _driver() -> tuple[Any, Any]: + try: + import psycopg + from psycopg import sql + except ImportError as exc: + raise HostedBackupError("psycopg is required for hosted backup") from exc + return psycopg, sql + + +def _row_counts(database_url: str) -> dict[str, int]: + psycopg, sql = _driver() + counts: dict[str, int] = {} + with psycopg.connect(database_url, autocommit=True) as connection: + with connection.cursor() as cursor: + for table in QUEUE_TABLES: + cursor.execute(sql.SQL("SELECT COUNT(*) FROM {}").format(sql.Identifier(table))) + row = cursor.fetchone() + counts[table] = int(row[0]) + return counts + + +def _replace_database(database_url: str, database: str) -> str: + parsed = urlsplit(database_url) + if not parsed.path.lstrip("/"): + raise HostedBackupError("database URL must include a database name") + return urlunsplit( + (parsed.scheme, parsed.netloc, f"/{database}", parsed.query, parsed.fragment) + ) + + +def _database_export(database_url: str) -> dict[str, Any]: + psycopg, sql = _driver() + tables: dict[str, list[dict[str, Any]]] = {} + with psycopg.connect(database_url, autocommit=False) as connection: + with connection.cursor() as cursor: + cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + for table in QUEUE_TABLES: + columns = TABLE_COLUMNS[table] + cursor.execute( + sql.SQL("SELECT {} FROM {} ORDER BY {}").format( + sql.SQL(", ").join(map(sql.Identifier, columns)), + sql.Identifier(table), + sql.Identifier(columns[0]), + ) + ) + tables[table] = [ + dict(zip(columns, row, strict=True)) for row in cursor.fetchall() + ] + connection.commit() + return { + "artifact_kind": "platform_hosted_managed_queue_backup", + "schema_version": 1, + "tables": tables, + } + + +def _initialize_queue_schema(database_url: str) -> None: + try: + from scripts import hosted_managed_operation_postgres as postgres_module + except ImportError: + import hosted_managed_operation_postgres as postgres_module + queue = postgres_module.PostgreSQLManagedOperationQueue(database_url) + queue.close() + + +def _restore_database_export(database_url: str, export: dict[str, Any]) -> None: + if export.get("artifact_kind") != "platform_hosted_managed_queue_backup": + raise HostedBackupError("queue backup artifact kind is invalid") + if export.get("schema_version") != 1: + raise HostedBackupError("queue backup schema version is unsupported") + tables = export.get("tables") + if not isinstance(tables, dict) or set(tables) != set(QUEUE_TABLES): + raise HostedBackupError("queue backup table set is invalid") + _initialize_queue_schema(database_url) + psycopg, sql = _driver() + with psycopg.connect(database_url, autocommit=True) as connection: + with connection.cursor() as cursor: + for table in QUEUE_TABLES: + rows = tables[table] + columns = TABLE_COLUMNS[table] + if not isinstance(rows, list): + raise HostedBackupError("queue backup rows must be arrays") + for row in rows: + if not isinstance(row, dict) or set(row) != set(columns): + raise HostedBackupError("queue backup row shape is invalid") + cursor.execute( + sql.SQL("INSERT INTO {} ({}) VALUES ({})").format( + sql.Identifier(table), + sql.SQL(", ").join(map(sql.Identifier, columns)), + sql.SQL(", ").join(sql.Placeholder() for _ in columns), + ), + [row[column] for column in columns], + ) + events = tables["managed_operation_events"] + if events: + cursor.execute( + "SELECT setval(pg_get_serial_sequence('managed_operation_events', " + "'event_id'), (SELECT MAX(event_id) FROM managed_operation_events))" + ) + + +def _artifact_inventory(artifact_root: Path) -> list[dict[str, Any]]: + runs_root = artifact_root / "runs" + if not runs_root.is_dir() or runs_root.is_symlink(): + raise HostedBackupError("artifact root must contain a regular runs directory") + inventory: list[dict[str, Any]] = [] + for path in sorted(runs_root.rglob("*")): + if path.is_symlink(): + raise HostedBackupError("artifact backup refuses symbolic links") + if not path.is_file(): + continue + relative = path.relative_to(artifact_root).as_posix() + inventory.append( + { + "logical_ref": relative, + "sha256": sha256_file(path), + "size_bytes": path.stat().st_size, + } + ) + return inventory + + +def _write_artifact_archive( + *, artifact_root: Path, inventory: list[dict[str, Any]], output: Path +) -> None: + with tarfile.open(output, "w:gz") as archive: + for item in inventory: + logical_ref = str(item["logical_ref"]) + archive.add( + artifact_root / logical_ref, + arcname=logical_ref, + recursive=False, + ) + + +def create_backup( + *, + database_url_file: Path, + artifact_root: Path, + backup_root: Path, + backup_id: str, +) -> dict[str, Any]: + if not BACKUP_ID_PATTERN.fullmatch(backup_id): + raise HostedBackupError("backup id is invalid") + if not backup_root.is_absolute() or backup_root.is_symlink(): + raise HostedBackupError("backup root must be an absolute regular directory") + backup_root.mkdir(parents=True, exist_ok=True) + destination = backup_root / backup_id + if destination.exists(): + raise HostedBackupError("backup destination already exists") + destination.mkdir(mode=0o700) + database_url = _database_url(database_url_file) + dump_path = destination / "managed-operations.json" + archive_path = destination / "workspace-artifacts.tar.gz" + try: + database_export = _database_export(database_url) + exported_tables = database_export["tables"] + status_counts: dict[str, int] = {} + for row in exported_tables["managed_operation_jobs"]: + status = str(row["status"]) + status_counts[status] = status_counts.get(status, 0) + 1 + active_count = sum( + status_counts.get(status, 0) for status in ("queued", "leased", "running") + ) + if active_count or exported_tables["managed_operation_locks"]: + raise HostedBackupError("backup requires a drained queue without active locks") + dump_path.write_text( + json.dumps(database_export, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + counts = {table: len(rows) for table, rows in exported_tables.items()} + inventory = _artifact_inventory(artifact_root) + _write_artifact_archive( + artifact_root=artifact_root, + inventory=inventory, + output=archive_path, + ) + _verify_artifact_archive(archive_path, inventory) + if _artifact_inventory(artifact_root) != inventory: + raise HostedBackupError("artifact tree changed during backup") + report = { + "artifact_kind": "platform_hosted_managed_runtime_backup_report", + "contract_ref": "platform.hosted-managed.runtime-backup.v1", + "ok": True, + "backup_id": backup_id, + "summary": { + "status": "backup_ready", + "database_export_sha256": sha256_file(dump_path), + "database_backup_schema_version": 1, + "artifact_archive_sha256": sha256_file(archive_path), + "artifact_file_count": len(inventory), + "database_row_counts": counts, + }, + "artifact_inventory": inventory, + "privacy_boundary": { + "public_safe": False, + "contains_private_workspace_artifacts": True, + "contains_database_rows": True, + "contains_secret_values": False, + }, + "authority_boundary": { + "may_restore_production_database": False, + "may_execute_managed_operations": False, + "may_mutate_specs": False, + "may_create_git_review": False, + }, + } + report_path = destination / "backup-report.json" + report_path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + except Exception: + for path in destination.iterdir(): + path.unlink() + destination.rmdir() + raise + + +def _load_backup(backup_root: Path, backup_id: str) -> tuple[Path, dict[str, Any]]: + if not BACKUP_ID_PATTERN.fullmatch(backup_id): + raise HostedBackupError("backup id is invalid") + destination = backup_root / backup_id + report_path = destination / "backup-report.json" + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise HostedBackupError("backup report is unavailable or invalid") from exc + if not isinstance(report, dict) or report.get("ok") is not True: + raise HostedBackupError("backup report is not ready") + return destination, report + + +def _verify_artifact_archive( + archive_path: Path, inventory: list[dict[str, Any]] +) -> None: + expected = {str(item["logical_ref"]): item for item in inventory} + with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + if len(names) != len(set(names)) or set(names) != set(expected): + raise HostedBackupError("artifact archive inventory mismatch") + for member in members: + logical = PurePosixPath(member.name) + if logical.is_absolute() or ".." in logical.parts or not member.isfile(): + raise HostedBackupError("artifact archive contains unsafe member") + stream = archive.extractfile(member) + if stream is None: + raise HostedBackupError("artifact archive member is unreadable") + content = stream.read() + if hashlib.sha256(content).hexdigest() != expected[member.name]["sha256"]: + raise HostedBackupError("artifact archive digest mismatch") + + +def restore_smoke( + *, database_url_file: Path, backup_root: Path, backup_id: str +) -> dict[str, Any]: + destination, backup = _load_backup(backup_root, backup_id) + dump_path = destination / "managed-operations.json" + archive_path = destination / "workspace-artifacts.tar.gz" + summary = backup.get("summary") + inventory = backup.get("artifact_inventory") + if not isinstance(summary, dict) or not isinstance(inventory, list): + raise HostedBackupError("backup report contract is incomplete") + if sha256_file(dump_path) != summary.get("database_export_sha256"): + raise HostedBackupError("database export digest mismatch") + if sha256_file(archive_path) != summary.get("artifact_archive_sha256"): + raise HostedBackupError("artifact archive digest mismatch") + _verify_artifact_archive(archive_path, inventory) + + source_url = _database_url(database_url_file) + psycopg, sql = _driver() + parsed = urlsplit(source_url) + source_database = parsed.path.lstrip("/") + if not source_database: + raise HostedBackupError("database URL must include a database name") + restore_database = f"platform_restore_smoke_{os.getpid()}" + admin_url = _replace_database(source_url, "postgres") + restore_url = _replace_database(source_url, restore_database) + created = False + try: + with psycopg.connect(admin_url, autocommit=True) as connection: + with connection.cursor() as cursor: + cursor.execute( + sql.SQL("CREATE DATABASE {}").format(sql.Identifier(restore_database)) + ) + created = True + try: + database_export = json.loads(dump_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise HostedBackupError("database export is invalid") from exc + if not isinstance(database_export, dict): + raise HostedBackupError("database export must be an object") + _restore_database_export( + restore_url, + database_export, + ) + restored_counts = _row_counts(restore_url) + expected_counts = summary.get("database_row_counts") + if restored_counts != expected_counts: + raise HostedBackupError("restored database row counts differ from backup") + finally: + if created: + with psycopg.connect(admin_url, autocommit=True) as connection: + with connection.cursor() as cursor: + cursor.execute( + sql.SQL("DROP DATABASE {} WITH (FORCE)").format( + sql.Identifier(restore_database) + ) + ) + + return { + "artifact_kind": "platform_hosted_managed_runtime_restore_smoke_report", + "contract_ref": "platform.hosted-managed.runtime-restore-smoke.v1", + "ok": True, + "backup_id": backup_id, + "summary": { + "status": "restore_smoke_passed", + "database_row_counts_verified": True, + "artifact_inventory_verified": True, + "artifact_file_count": len(inventory), + "temporary_database_removed": True, + }, + "privacy_boundary": { + "public_safe": True, + "contains_secret_values": False, + "contains_local_paths": False, + "contains_workspace_artifact_content": False, + }, + "authority_boundary": { + "may_restore_production_database": False, + "may_execute_managed_operations": False, + "may_mutate_specs": False, + "may_create_git_review": False, + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subcommands = parser.add_subparsers(dest="command", required=True) + for name in ("backup", "restore-smoke"): + command = subcommands.add_parser(name) + command.add_argument("--database-url-file", required=True) + command.add_argument("--backup-root", required=True) + command.add_argument("--backup-id", required=True) + command.add_argument("--output") + if name == "backup": + command.add_argument("--artifact-root", required=True) + args = parser.parse_args(argv) + try: + if args.command == "backup": + report = create_backup( + database_url_file=Path(args.database_url_file), + artifact_root=Path(args.artifact_root), + backup_root=Path(args.backup_root), + backup_id=args.backup_id, + ) + else: + report = restore_smoke( + database_url_file=Path(args.database_url_file), + backup_root=Path(args.backup_root), + backup_id=args.backup_id, + ) + except (HostedBackupError, OSError, tarfile.TarError) as exc: + report = { + "artifact_kind": "platform_hosted_managed_runtime_maintenance_error", + "ok": False, + "diagnostics": [str(exc)], + } + except Exception: + report = { + "artifact_kind": "platform_hosted_managed_runtime_maintenance_error", + "ok": False, + "diagnostics": ["hosted runtime maintenance operation failed"], + } + rendered = json.dumps(report, indent=2, sort_keys=True) + "\n" + if args.output: + Path(args.output).write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if report.get("ok") is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_hosted_managed_production_compose.py b/scripts/validate_hosted_managed_production_compose.py new file mode 100644 index 0000000..5774da9 --- /dev/null +++ b/scripts/validate_hosted_managed_production_compose.py @@ -0,0 +1,288 @@ +"""Validate the TLS-fronted production hosted managed-operation profile.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import tempfile +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[1] +COMPOSE_FILE = REPO_ROOT / "docker-compose.hosted-managed-production.example.yml" +ALLOWLIST_ENV = "PLATFORM_MANAGED_OPERATION_ALLOWLIST" +READ_ONLY_CANARY_OPERATION = "review_status_execute" +RUNTIME_SERVICES = { + "managed-operation-postgres", + "managed-operation-service", + "managed-operation-worker", + "managed-operation-ingress", +} +MAINTENANCE_SERVICE = "managed-operation-maintenance" +SHA256 = "1" * 64 + + +def _command(*, maintenance: bool = False) -> list[str]: + command = [ + "docker", + "compose", + "--file", + str(COMPOSE_FILE), + ] + if maintenance: + command.extend(("--profile", "maintenance")) + command.extend(("config", "--format", "json")) + return command + + +def _environment(temp_root: Path, *, allowlist: str | None) -> dict[str, str]: + temp_root.mkdir(parents=True) + artifact_root = temp_root / "specgraph" + state_dir = temp_root / "state" + secrets_dir = temp_root / "secrets" + artifact_root.mkdir() + state_dir.mkdir() + backup_root = temp_root / "backups" + backup_root.mkdir() + secrets_dir.mkdir() + secret_values = { + "managed-operation-token": "fixture-token", + "managed-operation-db-password": "fixture-password", + "managed-operation-database-url": ( + "postgresql://managed_operations:fixture-password@" + "managed-operation-postgres:5432/managed_operations" + ), + "managed-operation-github-token": "fixture-github-token", + "managed-operation-tls-certificate": "fixture-certificate", + "managed-operation-tls-private-key": "fixture-private-key", + } + for name, value in secret_values.items(): + (secrets_dir / name).write_text(value, encoding="utf-8") + + environment = dict(os.environ) + environment.pop(ALLOWLIST_ENV, None) + environment.update( + { + "PLATFORM_MANAGED_OPERATION_IMAGE": f"ghcr.io/0al/platform@sha256:{SHA256}", + "PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE": f"postgres@sha256:{SHA256}", + "PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE": f"caddy@sha256:{SHA256}", + "PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT": str(artifact_root), + "PLATFORM_MANAGED_OPERATION_STATE_DIR": str(state_dir), + "PLATFORM_MANAGED_OPERATION_BACKUP_ROOT": str(backup_root), + "PLATFORM_MANAGED_OPERATION_TOKEN_FILE": str( + secrets_dir / "managed-operation-token" + ), + "PLATFORM_MANAGED_OPERATION_DB_PASSWORD_FILE": str( + secrets_dir / "managed-operation-db-password" + ), + "PLATFORM_MANAGED_OPERATION_DATABASE_URL_FILE": str( + secrets_dir / "managed-operation-database-url" + ), + "PLATFORM_MANAGED_OPERATION_GITHUB_TOKEN_FILE": str( + secrets_dir / "managed-operation-github-token" + ), + "PLATFORM_MANAGED_OPERATION_TLS_CERTIFICATE_FILE": str( + secrets_dir / "managed-operation-tls-certificate" + ), + "PLATFORM_MANAGED_OPERATION_TLS_PRIVATE_KEY_FILE": str( + secrets_dir / "managed-operation-tls-private-key" + ), + } + ) + if allowlist is not None: + environment[ALLOWLIST_ENV] = allowlist + return environment + + +def _bind_mount(service: dict[str, Any], target: str) -> dict[str, Any]: + volumes = service.get("volumes") + if not isinstance(volumes, list): + raise RuntimeError(f"service omitted volumes for {target}") + for volume in volumes: + if isinstance(volume, dict) and volume.get("target") == target: + return volume + raise RuntimeError(f"service omitted required bind mount {target}") + + +def _assert_hardened(service_name: str, service: dict[str, Any]) -> None: + if service.get("read_only") is not True: + raise RuntimeError(f"{service_name} root filesystem must be read-only") + if service.get("cap_drop") != ["ALL"]: + raise RuntimeError(f"{service_name} must drop all Linux capabilities") + if "no-new-privileges:true" not in service.get("security_opt", []): + raise RuntimeError(f"{service_name} must disable privilege escalation") + + +def _assert_digest_pinned(service_name: str, service: dict[str, Any]) -> None: + image = service.get("image") + if not isinstance(image, str) or "@sha256:" not in image: + raise RuntimeError(f"{service_name} image must be pinned by sha256 digest") + + +def validate_hosted_managed_production_compose() -> dict[str, Any]: + with tempfile.TemporaryDirectory() as temp_dir: + temp_root = Path(temp_dir) + missing = subprocess.run( + _command(), + cwd=REPO_ROOT, + env=_environment(temp_root / "missing", allowlist=None), + capture_output=True, + text=True, + check=False, + ) + if missing.returncode == 0: + raise RuntimeError("production Compose rendered without an allowlist") + if ALLOWLIST_ENV not in f"{missing.stdout}\n{missing.stderr}": + raise RuntimeError("missing allowlist failure omitted the variable name") + + configured = subprocess.run( + _command(), + cwd=REPO_ROOT, + env=_environment( + temp_root / "configured", + allowlist=READ_ONLY_CANARY_OPERATION, + ), + capture_output=True, + text=True, + check=False, + ) + if configured.returncode != 0: + raise RuntimeError( + "configured production Compose did not render: " + f"{configured.stderr.strip()}" + ) + payload = json.loads(configured.stdout) + + services = payload.get("services") + if not isinstance(services, dict) or set(services) != RUNTIME_SERVICES: + raise RuntimeError("production Compose must contain exactly four runtime services") + + with tempfile.TemporaryDirectory() as temp_dir: + maintenance_render = subprocess.run( + _command(maintenance=True), + cwd=REPO_ROOT, + env=_environment( + Path(temp_dir) / "maintenance", + allowlist=READ_ONLY_CANARY_OPERATION, + ), + capture_output=True, + text=True, + check=False, + ) + if maintenance_render.returncode != 0: + raise RuntimeError("maintenance profile did not render") + maintenance_payload = json.loads(maintenance_render.stdout) + maintenance_services = maintenance_payload.get("services") + if not isinstance(maintenance_services, dict) or set(maintenance_services) != ( + RUNTIME_SERVICES | {MAINTENANCE_SERVICE} + ): + raise RuntimeError("maintenance profile must add exactly one service") + maintenance = maintenance_services[MAINTENANCE_SERVICE] + + for service_name, service in services.items(): + _assert_digest_pinned(service_name, service) + _assert_digest_pinned(MAINTENANCE_SERVICE, maintenance) + + for service_name in ( + "managed-operation-service", + "managed-operation-worker", + "managed-operation-ingress", + ): + _assert_hardened(service_name, services[service_name]) + _assert_hardened(MAINTENANCE_SERVICE, maintenance) + + for service_name in ("managed-operation-service", "managed-operation-worker"): + environment = services[service_name].get("environment") + if not isinstance(environment, dict) or environment.get(ALLOWLIST_ENV) != ( + READ_ONLY_CANARY_OPERATION + ): + raise RuntimeError(f"{service_name} did not receive the canary allowlist") + state_mount = _bind_mount(service=services[service_name], target="/data/specspace-state") + if state_mount.get("read_only") is not True: + raise RuntimeError(f"{service_name} state mount must be read-only") + + service = services["managed-operation-service"] + worker = services["managed-operation-worker"] + ingress = services["managed-operation-ingress"] + if service.get("ports"): + raise RuntimeError("managed service must not publish a host port") + service_artifacts = _bind_mount(service, "/workspace/SpecGraph") + worker_artifacts = _bind_mount(worker, "/workspace/SpecGraph") + if service_artifacts.get("read_only") is not True: + raise RuntimeError("HTTP service artifact mount must be read-only") + if worker_artifacts.get("read_only") is True: + raise RuntimeError("worker artifact mount must allow authoritative reports") + + ingress_ports = ingress.get("ports") + if not ( + isinstance(ingress_ports, list) + and len(ingress_ports) == 1 + and ingress_ports[0].get("target") == 8443 + and ingress_ports[0].get("published") == "443" + ): + raise RuntimeError("TLS ingress must be the only published production port") + if ingress.get("user") != "1000:1000": + raise RuntimeError("TLS ingress must run as the unprivileged runtime user") + caddyfile = _bind_mount(ingress, "/etc/caddy/Caddyfile") + if caddyfile.get("read_only") is not True: + raise RuntimeError("TLS ingress Caddyfile must be read-only") + if maintenance.get("profiles") != ["maintenance"]: + raise RuntimeError("backup tooling must remain an explicit maintenance profile") + if maintenance.get("user") != "1000:1000": + raise RuntimeError("backup tooling must run as the unprivileged runtime user") + maintenance_artifacts = _bind_mount(maintenance, "/workspace/SpecGraph") + maintenance_backups = _bind_mount(maintenance, "/backups") + if maintenance_artifacts.get("read_only") is not True: + raise RuntimeError("backup tooling must read artifacts without mutation authority") + if maintenance_backups.get("read_only") is True: + raise RuntimeError("backup tooling requires only its dedicated writable backup root") + + networks = payload.get("networks") + if not isinstance(networks, dict): + raise RuntimeError("production Compose omitted networks") + backend_names = [name for name, value in networks.items() if value.get("internal")] + if len(backend_names) != 1: + raise RuntimeError("production Compose must have one internal backend network") + backend_name = backend_names[0] + for service_name in ( + "managed-operation-postgres", + "managed-operation-service", + "managed-operation-ingress", + ): + if set(services[service_name].get("networks", {})) != {backend_name}: + raise RuntimeError(f"{service_name} must use only the internal network") + worker_networks = set(worker.get("networks", {})) + if backend_name not in worker_networks or len(worker_networks) != 2: + raise RuntimeError("worker must have internal queue access and one egress network") + + return { + "artifact_kind": "platform_hosted_managed_production_compose_contract_report", + "ok": True, + "summary": { + "missing_allowlist_blocked": True, + "read_only_canary_allowlist": True, + "runtime_service_count": len(services), + "digest_pinned_images": True, + "service_direct_port_absent": True, + "tls_ingress_only": True, + "internal_backend_network": True, + "worker_egress_only": True, + "maintenance_profile_isolated": True, + }, + } + + +def main() -> int: + try: + report = validate_hosted_managed_production_compose() + except (json.JSONDecodeError, OSError, RuntimeError) as exc: + print(json.dumps({"ok": False, "diagnostic": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_hosted_managed_operation_postgres.py b/tests/test_hosted_managed_operation_postgres.py index 1540523..01c6523 100644 --- a/tests/test_hosted_managed_operation_postgres.py +++ b/tests/test_hosted_managed_operation_postgres.py @@ -20,6 +20,8 @@ import hosted_managed_operation_postgres as postgres_module import hosted_managed_operation_queue as queue_module import hosted_managed_operation_service as service_module +import hosted_managed_runtime_backup as backup_module +import hosted_managed_production_signoff as signoff_module from tests.test_hosted_managed_operation_executor import ( ExecutorFixture, RecordingRunner, @@ -79,6 +81,68 @@ def test_real_postgresql_queue_lifecycle(self) -> None: ["queued", "leased", "running", "succeeded"], ) + @unittest.skipUnless( + os.environ.get("PLATFORM_TEST_POSTGRES_URL"), + "set PLATFORM_TEST_POSTGRES_URL for PostgreSQL queue integration", + ) + def test_real_postgresql_backup_and_restore_smoke(self) -> None: + database_url = os.environ["PLATFORM_TEST_POSTGRES_URL"] + queue = postgres_module.PostgreSQLManagedOperationQueue(database_url) + try: + with queue.connection.cursor() as cursor: + cursor.execute( + "TRUNCATE managed_operation_events, managed_operation_locks, " + "managed_operation_jobs RESTART IDENTITY CASCADE" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + request = request_for(root, "review_status_execute") + queue.enqueue( + request, + now_epoch=100, + now_iso="2026-07-10T00:00:00Z", + ) + worker = queue_module.HostedManagedOperationWorker( + queue, + SuccessfulExecutor(), + worker_id="postgres-backup-worker", + ) + receipt = worker.run_once( + now_epoch=101, + now_iso="2026-07-10T00:00:01Z", + ) + self.assertEqual(receipt["status"], "succeeded") + artifact_root = root / "specgraph" + report_dir = artifact_root / "runs" / "workspace-a" + report_dir.mkdir(parents=True) + (report_dir / "review-status.json").write_text( + '{"ok":true}\n', encoding="utf-8" + ) + database_url_file = root / "database-url" + database_url_file.write_text(database_url, encoding="utf-8") + backup_root = root / "backups" + backup_root.mkdir() + backup_report = backup_module.create_backup( + database_url_file=database_url_file, + artifact_root=artifact_root, + backup_root=backup_root, + backup_id="postgres-integration", + ) + restore_report = backup_module.restore_smoke( + database_url_file=database_url_file, + backup_root=backup_root, + backup_id="postgres-integration", + ) + queue_audit_report = signoff_module.queue_audit(database_url_file) + finally: + queue.close() + + self.assertTrue(backup_report["ok"]) + self.assertEqual(backup_report["summary"]["artifact_file_count"], 1) + self.assertTrue(restore_report["ok"]) + self.assertTrue(restore_report["summary"]["temporary_database_removed"]) + self.assertTrue(queue_audit_report["ok"]) + @unittest.skipUnless( os.environ.get("PLATFORM_TEST_POSTGRES_URL"), "set PLATFORM_TEST_POSTGRES_URL for PostgreSQL queue integration", diff --git a/tests/test_hosted_managed_production.py b/tests/test_hosted_managed_production.py new file mode 100644 index 0000000..e0adcad --- /dev/null +++ b/tests/test_hosted_managed_production.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import copy +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +from scripts import hosted_managed_production_preflight as preflight +from scripts import hosted_managed_production_probe as probe +from scripts import hosted_managed_production_signoff as signoff +from scripts import hosted_managed_runtime_backup as backup + + +class HostedManagedProductionPreflightTests(unittest.TestCase): + def fixture(self, root: Path) -> dict: + uid = os.getuid() + gid = os.getgid() + artifact_root = root / "artifacts" + state_dir = root / "state" + artifact_root.mkdir() + state_dir.mkdir() + secrets = root / "secrets" + secrets.mkdir() + values = { + "service_token": b"service-token-0123456789abcdef0123456789abcdef", + "database_password": b"database-password-0123456789abcdef", + "database_url": ( + b"postgresql://managed:password@managed-operation-postgres/managed" + ), + "github_token": b"github-token-0123456789abcdef", + "tls_certificate": ( + b"-----BEGIN CERTIFICATE-----\n" + + b"A" * 80 + + b"\n-----END CERTIFICATE-----" + ), + "tls_private_key": ( + b"-----BEGIN PRIVATE KEY-----\n" + + b"B" * 80 + + b"\n-----END PRIVATE KEY-----" + ), + } + secret_paths = {} + for label, value in values.items(): + path = secrets / label + path.write_bytes(value) + path.chmod(0o440) + secret_paths[label] = path + digest = "a" * 64 + return { + "service_url": "https://managed.example.test", + "allowlist": "review_status_execute", + "image_refs": { + "platform": f"ghcr.io/0al/platform@sha256:{digest}", + "postgresql": f"postgres@sha256:{digest}", + "ingress": f"caddy@sha256:{digest}", + }, + "secret_paths": secret_paths, + "artifact_root": artifact_root, + "state_dir": state_dir, + "expected_secret_uid": uid, + "runtime_uid": uid, + "runtime_gid": gid, + } + + def test_ready_preflight_is_public_safe(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + fixture = self.fixture(Path(temp_dir)) + report = preflight.run_preflight(**fixture) + self.assertTrue(report["ok"], report["diagnostics"]) + self.assertEqual(report["summary"]["enabled_operations"], ["review_status_execute"]) + rendered = str(report) + self.assertNotIn(temp_dir, rendered) + self.assertNotIn("service-token", rendered) + + def test_preflight_rejects_unsafe_transport_scope_and_secret_mode(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + fixture = self.fixture(Path(temp_dir)) + fixture["service_url"] = "http://managed.example.test?token=value" + fixture["allowlist"] = "review_status_execute,promotion_review_execute" + fixture["secret_paths"]["service_token"].chmod(0o644) + report = preflight.run_preflight(**fixture) + self.assertFalse(report["ok"]) + self.assertIn("service_url_not_private_https_endpoint", report["diagnostics"]) + self.assertIn("deployment_allowlist_not_exact_canary_scope", report["diagnostics"]) + self.assertIn("service_token_mode_not_0440", report["diagnostics"]) + + def test_preflight_requires_distinct_secret_values_and_digest_images(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + fixture = self.fixture(Path(temp_dir)) + shared = fixture["secret_paths"]["service_token"].read_bytes() + fixture["secret_paths"]["github_token"].chmod(0o600) + fixture["secret_paths"]["github_token"].write_bytes(shared) + fixture["secret_paths"]["github_token"].chmod(0o440) + fixture["image_refs"]["platform"] = "ghcr.io/0al/platform:latest" + report = preflight.run_preflight(**fixture) + self.assertFalse(report["ok"]) + self.assertIn("secret_values_not_distinct", report["diagnostics"]) + self.assertIn("platform_image_not_digest_pinned", report["diagnostics"]) + + def test_dry_run_requires_explicit_preflight_mode(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + fixture = self.fixture(Path(temp_dir)) + fixture["allowlist"] = ( + "review_status_execute,promotion_execute_dry_run" + ) + blocked = preflight.run_preflight(**fixture) + fixture["allow_dry_run"] = True + ready = preflight.run_preflight(**fixture) + self.assertFalse(blocked["ok"]) + self.assertTrue(ready["ok"], ready["diagnostics"]) + + +class HostedManagedRuntimeBackupTests(unittest.TestCase): + def test_backup_archives_workspace_artifacts_without_paths_in_summary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + artifact_root = root / "specgraph" + run_dir = artifact_root / "runs" / "workspace-a" + run_dir.mkdir(parents=True) + (run_dir / "report.json").write_text('{"ok":true}\n', encoding="utf-8") + database_url = root / "database-url" + database_url.write_text( + "postgresql://managed:password@postgres/managed\n", + encoding="utf-8", + ) + backup_root = root / "backups" + backup_root.mkdir() + database_export = { + "artifact_kind": "platform_hosted_managed_queue_backup", + "schema_version": 1, + "tables": {table: [] for table in backup.QUEUE_TABLES}, + } + with mock.patch.object( + backup, "_database_export", return_value=database_export + ), mock.patch.object( + backup, + "_row_counts", + return_value={table: 0 for table in backup.QUEUE_TABLES}, + ): + report = backup.create_backup( + database_url_file=database_url, + artifact_root=artifact_root, + backup_root=backup_root, + backup_id="test-backup", + ) + + destination = backup_root / "test-backup" + backup._verify_artifact_archive( + destination / "workspace-artifacts.tar.gz", + report["artifact_inventory"], + ) + self.assertTrue(report["ok"]) + self.assertEqual(report["summary"]["artifact_file_count"], 1) + self.assertFalse(report["privacy_boundary"]["public_safe"]) + self.assertNotIn(temp_dir, str(report)) + + def test_backup_refuses_symbolic_links_in_artifact_tree(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + runs = root / "runs" + runs.mkdir() + target = root / "private.json" + target.write_text("private", encoding="utf-8") + (runs / "linked.json").symlink_to(target) + with self.assertRaisesRegex(backup.HostedBackupError, "symbolic links"): + backup._artifact_inventory(root) + + def test_restore_smoke_rejects_tampered_database_export_before_connecting(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + artifact_root = root / "specgraph" + (artifact_root / "runs").mkdir(parents=True) + database_url = root / "database-url" + database_url.write_text( + "postgresql://managed:password@postgres/managed\n", + encoding="utf-8", + ) + backup_root = root / "backups" + backup_root.mkdir() + database_export = { + "artifact_kind": "platform_hosted_managed_queue_backup", + "schema_version": 1, + "tables": {table: [] for table in backup.QUEUE_TABLES}, + } + with mock.patch.object( + backup, "_database_export", return_value=database_export + ), mock.patch.object( + backup, + "_row_counts", + return_value={table: 0 for table in backup.QUEUE_TABLES}, + ): + backup.create_backup( + database_url_file=database_url, + artifact_root=artifact_root, + backup_root=backup_root, + backup_id="tampered", + ) + export_path = backup_root / "tampered" / "managed-operations.json" + export_path.write_text("{}\n", encoding="utf-8") + with mock.patch.object(backup, "_driver") as driver: + with self.assertRaisesRegex(backup.HostedBackupError, "digest mismatch"): + backup.restore_smoke( + database_url_file=database_url, + backup_root=backup_root, + backup_id="tampered", + ) + driver.assert_not_called() + + +class HostedManagedProductionProbeTests(unittest.TestCase): + def fixture_runner(self, *, heartbeat_generated_at: str): + rows = [ + { + "Service": service, + "State": "running", + "Health": "healthy", + } + for service in sorted(probe.EXPECTED_SERVICES) + ] + heartbeat = { + "artifact_kind": "platform_hosted_managed_operation_worker_health", + "ok": True, + "adapter": "postgresql", + "generated_at": heartbeat_generated_at, + "heartbeat_sequence": 12, + "last_cycle_status": "hosted_managed_operation_worker_idle", + } + + def runner(command, **kwargs): + output = json.dumps(heartbeat) if "exec" in command else json.dumps(rows) + return subprocess.CompletedProcess(command, 0, stdout=output, stderr="") + + return runner + + def test_probe_requires_healthy_tls_runtime_and_exact_allowlist(self) -> None: + now = datetime(2026, 7, 13, tzinfo=timezone.utc) + report = probe.run_probe( + service_url="https://managed.example.test", + compose_file=Path("/srv/platform/compose.yml"), + project_name="platform-managed", + now=now, + runner=self.fixture_runner(heartbeat_generated_at=now.isoformat()), + fetch_health=lambda _: { + "ok": True, + "adapter": "postgresql", + "operations": {"enabled_operation_ids": ["review_status_execute"]}, + }, + ) + self.assertTrue(report["ok"], report["diagnostics"]) + self.assertEqual(report["summary"]["healthy_service_count"], 4) + self.assertNotIn("/srv/platform", str(report)) + + def test_probe_blocks_stale_heartbeat_and_expanded_allowlist(self) -> None: + now = datetime(2026, 7, 13, tzinfo=timezone.utc) + report = probe.run_probe( + service_url="https://managed.example.test", + compose_file=Path("/srv/platform/compose.yml"), + project_name="platform-managed", + now=now, + runner=self.fixture_runner( + heartbeat_generated_at="2026-07-12T23:58:00+00:00" + ), + fetch_health=lambda _: { + "ok": True, + "adapter": "postgresql", + "operations": { + "enabled_operation_ids": [ + "review_status_execute", + "promotion_execute_dry_run", + ] + }, + }, + ) + self.assertFalse(report["ok"]) + self.assertIn("worker_heartbeat_stale", report["diagnostics"]) + self.assertIn("service_allowlist_not_read_only_canary", report["diagnostics"]) + + +class HostedManagedProductionSignoffTests(unittest.TestCase): + def evidence(self) -> dict[str, dict]: + reports = { + label: {"artifact_kind": kind, "ok": True} + for label, kind in signoff.EXPECTED_KINDS.items() + } + for label in ("probe_before_reboot", "probe_after_reboot"): + reports[label].update( + { + "summary": {"status": "healthy"}, + "service": { + "origin": "https://managed.example.test", + "enabled_operation_ids": ["review_status_execute"], + }, + } + ) + canary = { + "summary": {"profile": "read_only"}, + "queue": {"status": "succeeded", "attempt": 1}, + "request": { + "request_id": "managed-operation://request-1", + "idempotency_key": "idempotency-1", + "operation_id": "review_status_execute", + "workspace_id": "hosted-operation-canary", + }, + "authoritative_outputs": { + "observed_refs": ["runs/workspace/review.json"], + "verified_refs": ["runs/workspace/review.json"], + "receipt_pins_reports": True, + }, + } + reports["canary"].update(copy.deepcopy(canary)) + reports["replay_canary"].update(copy.deepcopy(canary)) + reports["recovery"]["summary"] = { + "strict": True, + "policy_safe": True, + "preflight_blocked": False, + } + reports["preflight"]["summary"] = { + "status": "ready", + "enabled_operations": ["review_status_execute"], + "dry_run_enabled": False, + } + reports["backup"]["backup_id"] = "production-1" + reports["backup"]["summary"] = { + "status": "backup_ready", + "database_backup_schema_version": 1, + } + reports["restore_smoke"]["backup_id"] = "production-1" + reports["restore_smoke"]["summary"] = { + "status": "restore_smoke_passed", + "temporary_database_removed": True, + } + reports["queue_audit"]["summary"] = {"rollback_ready": True} + reports["hosted_specspace_smoke"]["summary"] = { + "expected_managed_mode": "backend_managed_ready" + } + reports["rollback_specspace_smoke"]["summary"] = { + "expected_managed_mode": "read_only" + } + return reports + + def test_signoff_requires_complete_reboot_replay_backup_and_rollback_evidence(self) -> None: + report = signoff.build_signoff(self.evidence()) + self.assertTrue(report["ok"], report["diagnostics"]) + self.assertEqual(report["summary"]["status"], "production_canary_signed_off") + self.assertTrue(report["summary"]["rollback_verified"]) + + def test_signoff_blocks_reexecution_and_expanded_allowlist(self) -> None: + evidence = self.evidence() + evidence["replay_canary"]["queue"]["attempt"] = 2 + evidence["probe_after_reboot"]["service"]["enabled_operation_ids"] = [ + "review_status_execute", + "promotion_execute_dry_run", + ] + report = signoff.build_signoff(evidence) + self.assertFalse(report["ok"]) + self.assertIn("replay_canary_attempt_not_one", report["diagnostics"]) + self.assertIn("probe_after_reboot_allowlist_invalid", report["diagnostics"]) + + def test_signoff_rejects_write_capable_evidence(self) -> None: + evidence = self.evidence() + evidence["canary"]["authority_boundary"] = {"may_execute_platform": True} + report = signoff.build_signoff(evidence) + self.assertFalse(report["ok"]) + self.assertIn("canary_write_authority_expanded", report["diagnostics"]) + + +if __name__ == "__main__": + unittest.main() From 5c7ab98e35a1ffecdcd067460b1874c9e64151f4 Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:26:15 +0300 Subject: [PATCH 2/9] Exercise production TLS Compose profile --- .github/workflows/deploy-bundle.yml | 3 + Makefile | 5 +- docs/hosted-managed-operations.md | 6 + ...hosted_managed_production_compose_smoke.py | 303 ++++++++++++++++++ 4 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 scripts/hosted_managed_production_compose_smoke.py diff --git a/.github/workflows/deploy-bundle.yml b/.github/workflows/deploy-bundle.yml index 910c890..5f8fa43 100644 --- a/.github/workflows/deploy-bundle.yml +++ b/.github/workflows/deploy-bundle.yml @@ -88,6 +88,9 @@ jobs: - name: Start hosted service and worker profile run: make hosted-managed-compose-smoke PYTHON=python3 + - name: Start TLS-fronted production hosted profile + run: make hosted-managed-production-compose-smoke PYTHON=python3 + platform-deploy-bundle: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index fda62ab..8214177 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) -.PHONY: python-quality test hosted-managed-contract hosted-managed-compose-contract hosted-managed-runtime-compose-contract hosted-managed-production-compose-contract hosted-managed-production-contract hosted-managed-postgres-integration hosted-managed-compose-smoke +.PHONY: python-quality test hosted-managed-contract hosted-managed-compose-contract hosted-managed-runtime-compose-contract hosted-managed-production-compose-contract hosted-managed-production-contract hosted-managed-postgres-integration hosted-managed-compose-smoke hosted-managed-production-compose-smoke python-quality: $(PYTHON) -m unittest discover -s tests @@ -33,3 +33,6 @@ hosted-managed-postgres-integration: hosted-managed-compose-smoke: $(PYTHON) scripts/hosted_managed_compose_smoke.py + +hosted-managed-production-compose-smoke: + $(PYTHON) scripts/hosted_managed_production_compose_smoke.py diff --git a/docs/hosted-managed-operations.md b/docs/hosted-managed-operations.md index ec8ba00..1299bd5 100644 --- a/docs/hosted-managed-operations.md +++ b/docs/hosted-managed-operations.md @@ -364,6 +364,7 @@ Validate and start the production profile: ```bash make hosted-managed-production-contract +make hosted-managed-production-compose-smoke docker compose --project-name platform-managed-production \ --file docker-compose.hosted-managed-production.example.yml config >/dev/null docker compose --project-name platform-managed-production \ @@ -376,6 +377,11 @@ docker compose --project-name platform-managed-production \ --output /srv/0al/evidence/probe-before-reboot.json ``` +The bounded Compose smoke starts the real Caddy, PostgreSQL, service, and worker +profile with a one-day self-signed fixture certificate and a temporary local +registry so even the test image is addressed by digest. It enqueues no managed +request and removes containers, registry, and volumes afterward. + The probe requires all four runtime services to be healthy, PostgreSQL as the queue adapter, a fresh worker heartbeat, and exactly `review_status_execute` in service health. The report is public-safe and omits diff --git a/scripts/hosted_managed_production_compose_smoke.py b/scripts/hosted_managed_production_compose_smoke.py new file mode 100644 index 0000000..51d1ff1 --- /dev/null +++ b/scripts/hosted_managed_production_compose_smoke.py @@ -0,0 +1,303 @@ +"""Run the TLS-fronted production Compose profile without enqueueing work.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import socket +import ssl +import subprocess +import tempfile +import time +from typing import Any +import urllib.request + + +REPO_ROOT = Path(__file__).resolve().parents[1] +COMPOSE_FILE = REPO_ROOT / "docker-compose.hosted-managed-production.example.yml" +SERVICES = ( + "managed-operation-postgres", + "managed-operation-service", + "managed-operation-worker", + "managed-operation-ingress", +) + + +def _port() -> int: + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return int(listener.getsockname()[1]) + + +def _run( + command: list[str], + *, + environment: dict[str, str] | None = None, + timeout_seconds: int = 420, +) -> str: + completed = subprocess.run( + command, + cwd=REPO_ROOT, + env=environment, + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + ) + if completed.returncode != 0: + diagnostic = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(f"production Compose smoke command failed: {diagnostic[-500:]}") + return completed.stdout.strip() + + +def _repo_digest(image: str) -> str: + payload = json.loads(_run(["docker", "image", "inspect", image])) + digests = payload[0].get("RepoDigests") if isinstance(payload, list) and payload else [] + if not isinstance(digests, list) or not digests: + raise RuntimeError(f"image {image} did not expose a repository digest") + return str(digests[0]) + + +def _wait_registry(port: int) -> None: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/v2/", timeout=1): + return + except OSError: + time.sleep(0.25) + raise RuntimeError("temporary image registry did not become ready") + + +def _fixture(root: Path, *, platform_image: str, ingress_port: int) -> dict[str, str]: + artifact_root = root / "specgraph" + state_dir = root / "state" + backup_root = root / "backups" + secrets = root / "secrets" + for directory in (artifact_root / "runs", state_dir, backup_root, secrets): + directory.mkdir(parents=True) + (artifact_root / "Makefile").write_text("test:\n\t@true\n", encoding="utf-8") + + password = "production-compose-smoke-password" + secret_values = { + "service-token": "production-compose-smoke-token-0123456789abcdef", + "database-password": password, + "database-url": ( + "postgresql://managed_operations:" + f"{password}@managed-operation-postgres:5432/managed_operations" + ), + "github-token": "production-compose-smoke-github-token", + } + for name, value in secret_values.items(): + path = secrets / name + path.write_text(value + "\n", encoding="utf-8") + path.chmod(0o444) + + certificate = secrets / "tls-certificate.pem" + private_key = secrets / "tls-private-key.pem" + _run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=127.0.0.1", + "-addext", + "subjectAltName=IP:127.0.0.1", + "-keyout", + str(private_key), + "-out", + str(certificate), + ] + ) + private_key.chmod(0o444) + certificate.chmod(0o444) + + environment = dict(os.environ) + environment.update( + { + "PLATFORM_MANAGED_OPERATION_IMAGE": platform_image, + "PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE": _repo_digest( + "postgres:16-alpine" + ), + "PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE": _repo_digest("caddy:2-alpine"), + "PLATFORM_MANAGED_OPERATION_ALLOWLIST": "review_status_execute", + "PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT": str(artifact_root), + "PLATFORM_MANAGED_OPERATION_STATE_DIR": str(state_dir), + "PLATFORM_MANAGED_OPERATION_BACKUP_ROOT": str(backup_root), + "PLATFORM_MANAGED_OPERATION_INGRESS_BIND_IP": "127.0.0.1", + "PLATFORM_MANAGED_OPERATION_INGRESS_PORT": str(ingress_port), + "PLATFORM_MANAGED_OPERATION_TOKEN_FILE": str(secrets / "service-token"), + "PLATFORM_MANAGED_OPERATION_DB_PASSWORD_FILE": str( + secrets / "database-password" + ), + "PLATFORM_MANAGED_OPERATION_DATABASE_URL_FILE": str( + secrets / "database-url" + ), + "PLATFORM_MANAGED_OPERATION_GITHUB_TOKEN_FILE": str( + secrets / "github-token" + ), + "PLATFORM_MANAGED_OPERATION_TLS_CERTIFICATE_FILE": str(certificate), + "PLATFORM_MANAGED_OPERATION_TLS_PRIVATE_KEY_FILE": str(private_key), + } + ) + return environment + + +def run_smoke() -> dict[str, Any]: + registry_port = _port() + ingress_port = _port() + registry_name = f"platform-production-smoke-registry-{os.getpid()}" + project_name = f"platform-production-smoke-{os.getpid()}" + registry_image = f"127.0.0.1:{registry_port}/platform:smoke" + compose = [ + "docker", + "compose", + "--project-name", + project_name, + "--file", + str(COMPOSE_FILE), + ] + environment: dict[str, str] | None = None + try: + for image in ("registry:2", "postgres:16-alpine", "caddy:2-alpine"): + _run(["docker", "pull", image]) + _run( + [ + "docker", + "run", + "--detach", + "--rm", + "--name", + registry_name, + "--publish", + f"127.0.0.1:{registry_port}:5000", + "registry:2", + ] + ) + _wait_registry(registry_port) + _run( + [ + "docker", + "build", + "--file", + str(REPO_ROOT / "Dockerfile.hosted-managed"), + "--tag", + registry_image, + ".", + ] + ) + _run(["docker", "push", registry_image]) + platform_image = _repo_digest(registry_image) + + with tempfile.TemporaryDirectory( + prefix=".hosted-managed-production-smoke-", dir=REPO_ROOT + ) as temp_dir: + environment = _fixture( + Path(temp_dir), + platform_image=platform_image, + ingress_port=ingress_port, + ) + _run( + [ + *compose, + "up", + "--detach", + "--wait", + "--wait-timeout", + "300", + *SERVICES, + ], + environment=environment, + ) + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with urllib.request.urlopen( + f"https://127.0.0.1:{ingress_port}/v1/health", + context=context, + timeout=5, + ) as response: + ingress_health = json.loads(response.read().decode("utf-8")) + worker_health = json.loads( + _run( + [ + *compose, + "exec", + "--no-TTY", + "managed-operation-worker", + "cat", + "/tmp/managed-operation-worker-health.json", + ], + environment=environment, + ) + ) + finally: + if environment is not None: + try: + _run( + [*compose, "down", "--volumes", "--remove-orphans"], + environment=environment, + timeout_seconds=120, + ) + except RuntimeError: + pass + subprocess.run( + ["docker", "rm", "--force", registry_name], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + if ingress_health.get("ok") is not True or ingress_health.get("adapter") != "postgresql": + raise RuntimeError("TLS ingress did not expose PostgreSQL-backed service health") + operations = ingress_health.get("operations") + operations = operations if isinstance(operations, dict) else {} + if operations.get("enabled_operation_ids") != ["review_status_execute"]: + raise RuntimeError("TLS ingress exposed an expanded operation allowlist") + if worker_health.get("ok") is not True or worker_health.get("adapter") != "postgresql": + raise RuntimeError("production worker heartbeat was not PostgreSQL-ready") + return { + "artifact_kind": "platform_hosted_managed_production_compose_smoke_report", + "ok": True, + "summary": { + "tls_ingress_ready": True, + "postgresql_ready": True, + "service_ready": True, + "worker_ready": True, + "enabled_operation_ids": ["review_status_execute"], + "managed_requests_executed": 0, + }, + "authority_boundary": { + "executes_managed_operations": False, + "opens_pull_requests": False, + "publishes_read_model": False, + }, + } + + +def main() -> int: + try: + report = run_smoke() + except ( + json.JSONDecodeError, + OSError, + RuntimeError, + subprocess.SubprocessError, + ) as exc: + print(json.dumps({"ok": False, "diagnostic": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 9cb01bedca8017db2da52fcec024ac798e81b232 Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:29:27 +0300 Subject: [PATCH 3/9] Preserve production ingress smoke diagnostics --- ...hosted_managed_production_compose_smoke.py | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/scripts/hosted_managed_production_compose_smoke.py b/scripts/hosted_managed_production_compose_smoke.py index 51d1ff1..9a60fd2 100644 --- a/scripts/hosted_managed_production_compose_smoke.py +++ b/scripts/hosted_managed_production_compose_smoke.py @@ -204,18 +204,36 @@ def run_smoke() -> dict[str, Any]: platform_image=platform_image, ingress_port=ingress_port, ) - _run( - [ - *compose, - "up", - "--detach", - "--wait", - "--wait-timeout", - "300", - *SERVICES, - ], - environment=environment, - ) + try: + _run( + [ + *compose, + "up", + "--detach", + "--wait", + "--wait-timeout", + "300", + *SERVICES, + ], + environment=environment, + ) + except RuntimeError as exc: + try: + logs = _run( + [ + *compose, + "logs", + "--no-color", + "--tail", + "80", + "managed-operation-ingress", + ], + environment=environment, + timeout_seconds=30, + ) + except RuntimeError: + logs = "ingress logs unavailable" + raise RuntimeError(f"{exc}; ingress logs: {logs[-1500:]}") from exc context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE From ce1ff3de40cbd4d7482dc32b49a4f3c72da8b1a1 Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:32:16 +0300 Subject: [PATCH 4/9] Run production ingress without file capabilities --- ...-compose.hosted-managed-production.example.yml | 15 +++++++++------ docs/hosted-managed-operations.md | 3 +++ .../validate_hosted_managed_production_compose.py | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docker-compose.hosted-managed-production.example.yml b/docker-compose.hosted-managed-production.example.yml index 6efe422..97cb648 100644 --- a/docker-compose.hosted-managed-production.example.yml +++ b/docker-compose.hosted-managed-production.example.yml @@ -141,13 +141,16 @@ services: - ALL security_opt: - no-new-privileges:true + entrypoint: + - /bin/sh + - -ec command: - - caddy - - run - - --config - - /etc/caddy/Caddyfile - - --adapter - - caddyfile + - >- + cp /usr/bin/caddy /tmp/caddy-runtime; + chmod 0500 /tmp/caddy-runtime; + exec /tmp/caddy-runtime run + --config /etc/caddy/Caddyfile + --adapter caddyfile ports: - "${PLATFORM_MANAGED_OPERATION_INGRESS_BIND_IP:-0.0.0.0}:${PLATFORM_MANAGED_OPERATION_INGRESS_PORT:-443}:8443" volumes: diff --git a/docs/hosted-managed-operations.md b/docs/hosted-managed-operations.md index 1299bd5..3b88806 100644 --- a/docs/hosted-managed-operations.md +++ b/docs/hosted-managed-operations.md @@ -381,6 +381,9 @@ The bounded Compose smoke starts the real Caddy, PostgreSQL, service, and worker profile with a one-day self-signed fixture certificate and a temporary local registry so even the test image is addressed by digest. It enqueues no managed request and removes containers, registry, and volumes afterward. +The ingress copies the immutable upstream Caddy binary into tmpfs before exec; +this deliberately removes its unused low-port file capability so the container +can keep `cap_drop: ALL` and `no-new-privileges` while listening on port `8443`. The probe requires all four runtime services to be healthy, PostgreSQL as the queue adapter, a fresh worker heartbeat, and exactly diff --git a/scripts/validate_hosted_managed_production_compose.py b/scripts/validate_hosted_managed_production_compose.py index 5774da9..cc2c7f8 100644 --- a/scripts/validate_hosted_managed_production_compose.py +++ b/scripts/validate_hosted_managed_production_compose.py @@ -225,6 +225,20 @@ def validate_hosted_managed_production_compose() -> dict[str, Any]: raise RuntimeError("TLS ingress must be the only published production port") if ingress.get("user") != "1000:1000": raise RuntimeError("TLS ingress must run as the unprivileged runtime user") + if ingress.get("entrypoint") != ["/bin/sh", "-ec"]: + raise RuntimeError("TLS ingress must strip upstream file capabilities in tmpfs") + ingress_command = ingress.get("command") + ingress_command_text = ( + " ".join(ingress_command) if isinstance(ingress_command, list) else "" + ) + if not all( + token in ingress_command_text + for token in ( + "cp /usr/bin/caddy /tmp/caddy-runtime", + "exec /tmp/caddy-runtime run", + ) + ): + raise RuntimeError("TLS ingress runtime command may retain upstream file capabilities") caddyfile = _bind_mount(ingress, "/etc/caddy/Caddyfile") if caddyfile.get("read_only") is not True: raise RuntimeError("TLS ingress Caddyfile must be read-only") From d9708693e55e7fe855b4d91afdd5d7e525e03718 Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:35:31 +0300 Subject: [PATCH 5/9] Build capability-free hosted ingress --- .github/workflows/deploy-bundle.yml | 8 +++++ Dockerfile.hosted-managed-ingress | 11 +++++++ deploy/hosted-managed/production.env.example | 2 +- ...pose.hosted-managed-production.example.yml | 14 ++++----- docs/hosted-managed-operations.md | 9 +++--- ...hosted_managed_production_compose_smoke.py | 28 ++++++++++++++++-- ...idate_hosted_managed_production_compose.py | 29 ++++++++++--------- 7 files changed, 72 insertions(+), 29 deletions(-) create mode 100644 Dockerfile.hosted-managed-ingress diff --git a/.github/workflows/deploy-bundle.yml b/.github/workflows/deploy-bundle.yml index 5f8fa43..49fcd6c 100644 --- a/.github/workflows/deploy-bundle.yml +++ b/.github/workflows/deploy-bundle.yml @@ -39,6 +39,14 @@ jobs: docker build --file Dockerfile.hosted-managed --tag 0al/platform-hosted-managed:ci . + - name: Build capability-free hosted ingress image + run: | + docker pull caddy:2-alpine + caddy_base="$(docker image inspect --format '{{index .RepoDigests 0}}' caddy:2-alpine)" + docker build --file Dockerfile.hosted-managed-ingress \ + --build-arg "CADDY_BASE_IMAGE=${caddy_base}" \ + --tag 0al/platform-hosted-managed-ingress:ci . + hosted-managed-postgres: runs-on: ubuntu-latest services: diff --git a/Dockerfile.hosted-managed-ingress b/Dockerfile.hosted-managed-ingress new file mode 100644 index 0000000..f9985fe --- /dev/null +++ b/Dockerfile.hosted-managed-ingress @@ -0,0 +1,11 @@ +ARG CADDY_BASE_IMAGE +FROM ${CADDY_BASE_IMAGE} + +USER root +RUN apk add --no-cache libcap \ + && setcap -r /usr/bin/caddy \ + && apk del libcap + +USER 1000:1000 +ENTRYPOINT ["caddy"] +CMD ["run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] diff --git a/deploy/hosted-managed/production.env.example b/deploy/hosted-managed/production.env.example index 19c9afb..c464867 100644 --- a/deploy/hosted-managed/production.env.example +++ b/deploy/hosted-managed/production.env.example @@ -1,7 +1,7 @@ # Non-secret deployment inputs only. Keep secret values in the referenced files. PLATFORM_MANAGED_OPERATION_IMAGE=ghcr.io/0al-spec/platform@sha256: PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE=postgres@sha256: -PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE=caddy@sha256: +PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE=ghcr.io/0al-spec/platform-hosted-managed-ingress@sha256: PLATFORM_MANAGED_OPERATION_ALLOWLIST=review_status_execute PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT=/srv/0al/specgraph diff --git a/docker-compose.hosted-managed-production.example.yml b/docker-compose.hosted-managed-production.example.yml index 97cb648..dfd4947 100644 --- a/docker-compose.hosted-managed-production.example.yml +++ b/docker-compose.hosted-managed-production.example.yml @@ -141,16 +141,12 @@ services: - ALL security_opt: - no-new-privileges:true - entrypoint: - - /bin/sh - - -ec command: - - >- - cp /usr/bin/caddy /tmp/caddy-runtime; - chmod 0500 /tmp/caddy-runtime; - exec /tmp/caddy-runtime run - --config /etc/caddy/Caddyfile - --adapter caddyfile + - run + - --config + - /etc/caddy/Caddyfile + - --adapter + - caddyfile ports: - "${PLATFORM_MANAGED_OPERATION_INGRESS_BIND_IP:-0.0.0.0}:${PLATFORM_MANAGED_OPERATION_INGRESS_PORT:-443}:8443" volumes: diff --git a/docs/hosted-managed-operations.md b/docs/hosted-managed-operations.md index 3b88806..a0a62fa 100644 --- a/docs/hosted-managed-operations.md +++ b/docs/hosted-managed-operations.md @@ -302,7 +302,7 @@ All runtime images must be immutable digest refs: ```bash export PLATFORM_MANAGED_OPERATION_IMAGE='ghcr.io/0al-spec/platform@sha256:' export PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE='postgres@sha256:' -export PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE='caddy@sha256:' +export PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE='ghcr.io/0al-spec/platform-hosted-managed-ingress@sha256:' export PLATFORM_MANAGED_OPERATION_ALLOWLIST=review_status_execute export PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT=/srv/0al/specgraph export PLATFORM_MANAGED_OPERATION_STATE_DIR=/srv/0al/specspace-state @@ -381,9 +381,10 @@ The bounded Compose smoke starts the real Caddy, PostgreSQL, service, and worker profile with a one-day self-signed fixture certificate and a temporary local registry so even the test image is addressed by digest. It enqueues no managed request and removes containers, registry, and volumes afterward. -The ingress copies the immutable upstream Caddy binary into tmpfs before exec; -this deliberately removes its unused low-port file capability so the container -can keep `cap_drop: ALL` and `no-new-privileges` while listening on port `8443`. +Build `Dockerfile.hosted-managed-ingress` from a digest-pinned Caddy base and +publish that image by digest. The build removes Caddy's unused low-port file +capability, allowing the non-root ingress container to keep `cap_drop: ALL` and +`no-new-privileges` while listening on port `8443`. The probe requires all four runtime services to be healthy, PostgreSQL as the queue adapter, a fresh worker heartbeat, and exactly diff --git a/scripts/hosted_managed_production_compose_smoke.py b/scripts/hosted_managed_production_compose_smoke.py index 9a60fd2..5438512 100644 --- a/scripts/hosted_managed_production_compose_smoke.py +++ b/scripts/hosted_managed_production_compose_smoke.py @@ -70,7 +70,13 @@ def _wait_registry(port: int) -> None: raise RuntimeError("temporary image registry did not become ready") -def _fixture(root: Path, *, platform_image: str, ingress_port: int) -> dict[str, str]: +def _fixture( + root: Path, + *, + platform_image: str, + ingress_image: str, + ingress_port: int, +) -> dict[str, str]: artifact_root = root / "specgraph" state_dir = root / "state" backup_root = root / "backups" @@ -126,7 +132,7 @@ def _fixture(root: Path, *, platform_image: str, ingress_port: int) -> dict[str, "PLATFORM_MANAGED_OPERATION_POSTGRES_IMAGE": _repo_digest( "postgres:16-alpine" ), - "PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE": _repo_digest("caddy:2-alpine"), + "PLATFORM_MANAGED_OPERATION_INGRESS_IMAGE": ingress_image, "PLATFORM_MANAGED_OPERATION_ALLOWLIST": "review_status_execute", "PLATFORM_MANAGED_OPERATION_ARTIFACT_ROOT": str(artifact_root), "PLATFORM_MANAGED_OPERATION_STATE_DIR": str(state_dir), @@ -156,6 +162,7 @@ def run_smoke() -> dict[str, Any]: registry_name = f"platform-production-smoke-registry-{os.getpid()}" project_name = f"platform-production-smoke-{os.getpid()}" registry_image = f"127.0.0.1:{registry_port}/platform:smoke" + ingress_registry_image = f"127.0.0.1:{registry_port}/ingress:smoke" compose = [ "docker", "compose", @@ -195,6 +202,22 @@ def run_smoke() -> dict[str, Any]: ) _run(["docker", "push", registry_image]) platform_image = _repo_digest(registry_image) + caddy_base_image = _repo_digest("caddy:2-alpine") + _run( + [ + "docker", + "build", + "--file", + str(REPO_ROOT / "Dockerfile.hosted-managed-ingress"), + "--build-arg", + f"CADDY_BASE_IMAGE={caddy_base_image}", + "--tag", + ingress_registry_image, + ".", + ] + ) + _run(["docker", "push", ingress_registry_image]) + ingress_image = _repo_digest(ingress_registry_image) with tempfile.TemporaryDirectory( prefix=".hosted-managed-production-smoke-", dir=REPO_ROOT @@ -202,6 +225,7 @@ def run_smoke() -> dict[str, Any]: environment = _fixture( Path(temp_dir), platform_image=platform_image, + ingress_image=ingress_image, ingress_port=ingress_port, ) try: diff --git a/scripts/validate_hosted_managed_production_compose.py b/scripts/validate_hosted_managed_production_compose.py index cc2c7f8..3328e16 100644 --- a/scripts/validate_hosted_managed_production_compose.py +++ b/scripts/validate_hosted_managed_production_compose.py @@ -12,6 +12,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] COMPOSE_FILE = REPO_ROOT / "docker-compose.hosted-managed-production.example.yml" +INGRESS_DOCKERFILE = REPO_ROOT / "Dockerfile.hosted-managed-ingress" ALLOWLIST_ENV = "PLATFORM_MANAGED_OPERATION_ALLOWLIST" READ_ONLY_CANARY_OPERATION = "review_status_execute" RUNTIME_SERVICES = { @@ -122,6 +123,13 @@ def _assert_digest_pinned(service_name: str, service: dict[str, Any]) -> None: def validate_hosted_managed_production_compose() -> dict[str, Any]: + ingress_dockerfile = INGRESS_DOCKERFILE.read_text(encoding="utf-8") + if "ARG CADDY_BASE_IMAGE\n" not in ingress_dockerfile or ( + "ARG CADDY_BASE_IMAGE=" in ingress_dockerfile + ): + raise RuntimeError("ingress build must require an explicit Caddy base image") + if "setcap -r /usr/bin/caddy" not in ingress_dockerfile: + raise RuntimeError("ingress build must remove the upstream file capability") with tempfile.TemporaryDirectory() as temp_dir: temp_root = Path(temp_dir) missing = subprocess.run( @@ -225,20 +233,15 @@ def validate_hosted_managed_production_compose() -> dict[str, Any]: raise RuntimeError("TLS ingress must be the only published production port") if ingress.get("user") != "1000:1000": raise RuntimeError("TLS ingress must run as the unprivileged runtime user") - if ingress.get("entrypoint") != ["/bin/sh", "-ec"]: - raise RuntimeError("TLS ingress must strip upstream file capabilities in tmpfs") ingress_command = ingress.get("command") - ingress_command_text = ( - " ".join(ingress_command) if isinstance(ingress_command, list) else "" - ) - if not all( - token in ingress_command_text - for token in ( - "cp /usr/bin/caddy /tmp/caddy-runtime", - "exec /tmp/caddy-runtime run", - ) - ): - raise RuntimeError("TLS ingress runtime command may retain upstream file capabilities") + if ingress_command != [ + "run", + "--config", + "/etc/caddy/Caddyfile", + "--adapter", + "caddyfile", + ]: + raise RuntimeError("TLS ingress must use the fixed Caddy entrypoint contract") caddyfile = _bind_mount(ingress, "/etc/caddy/Caddyfile") if caddyfile.get("read_only") is not True: raise RuntimeError("TLS ingress Caddyfile must be read-only") From 88ee21906c805fab880fd61330dfc061dd0bcc9c Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:38:38 +0300 Subject: [PATCH 6/9] Wait for production ingress publication --- ...hosted_managed_production_compose_smoke.py | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/scripts/hosted_managed_production_compose_smoke.py b/scripts/hosted_managed_production_compose_smoke.py index 5438512..2e95d78 100644 --- a/scripts/hosted_managed_production_compose_smoke.py +++ b/scripts/hosted_managed_production_compose_smoke.py @@ -70,6 +70,26 @@ def _wait_registry(port: int) -> None: raise RuntimeError("temporary image registry did not become ready") +def _wait_ingress_health(port: int) -> dict[str, Any]: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen( + f"https://127.0.0.1:{port}/v1/health", + context=context, + timeout=2, + ) as response: + payload = json.loads(response.read().decode("utf-8")) + if isinstance(payload, dict): + return payload + except (OSError, json.JSONDecodeError): + time.sleep(0.25) + raise RuntimeError("TLS ingress health did not become reachable on the host") + + def _fixture( root: Path, *, @@ -258,15 +278,14 @@ def run_smoke() -> dict[str, Any]: except RuntimeError: logs = "ingress logs unavailable" raise RuntimeError(f"{exc}; ingress logs: {logs[-1500:]}") from exc - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - with urllib.request.urlopen( - f"https://127.0.0.1:{ingress_port}/v1/health", - context=context, - timeout=5, - ) as response: - ingress_health = json.loads(response.read().decode("utf-8")) + published = _run( + [*compose, "port", "managed-operation-ingress", "8443"], + environment=environment, + timeout_seconds=30, + ) + if published.rsplit(":", 1)[-1] != str(ingress_port): + raise RuntimeError("TLS ingress published an unexpected host port") + ingress_health = _wait_ingress_health(ingress_port) worker_health = json.loads( _run( [ From aa1a443ea0cf07285848cede7e27425140881670 Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:44:59 +0300 Subject: [PATCH 7/9] Separate production ingress and worker networks --- docker-compose.hosted-managed-production.example.yml | 2 ++ docs/hosted-managed-operations.md | 3 +++ scripts/validate_hosted_managed_production_compose.py | 6 +++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docker-compose.hosted-managed-production.example.yml b/docker-compose.hosted-managed-production.example.yml index dfd4947..0445143 100644 --- a/docker-compose.hosted-managed-production.example.yml +++ b/docker-compose.hosted-managed-production.example.yml @@ -156,6 +156,7 @@ services: - managed_operation_tls_private_key networks: - managed-backend + - managed-ingress depends_on: managed-operation-service: condition: service_healthy @@ -203,6 +204,7 @@ networks: managed-backend: internal: true managed-egress: {} + managed-ingress: {} secrets: managed_operation_token: diff --git a/docs/hosted-managed-operations.md b/docs/hosted-managed-operations.md index a0a62fa..c908924 100644 --- a/docs/hosted-managed-operations.md +++ b/docs/hosted-managed-operations.md @@ -289,6 +289,9 @@ The clean-VM runtime is staging evidence. A production deployment uses `docker-compose.hosted-managed-production.example.yml` and is not signed off until its TLS, backup, reboot, replay, SpecSpace cutover, and rollback evidence passes the final audit. +PostgreSQL and the Platform service use only an internal backend network. The +worker has a distinct outbound network for GitHub, while Caddy has a distinct +published ingress network; worker egress and public ingress are never shared. ### Provisioning boundary diff --git a/scripts/validate_hosted_managed_production_compose.py b/scripts/validate_hosted_managed_production_compose.py index 3328e16..ecfee61 100644 --- a/scripts/validate_hosted_managed_production_compose.py +++ b/scripts/validate_hosted_managed_production_compose.py @@ -266,13 +266,17 @@ def validate_hosted_managed_production_compose() -> dict[str, Any]: for service_name in ( "managed-operation-postgres", "managed-operation-service", - "managed-operation-ingress", ): if set(services[service_name].get("networks", {})) != {backend_name}: raise RuntimeError(f"{service_name} must use only the internal network") worker_networks = set(worker.get("networks", {})) if backend_name not in worker_networks or len(worker_networks) != 2: raise RuntimeError("worker must have internal queue access and one egress network") + ingress_networks = set(ingress.get("networks", {})) + if backend_name not in ingress_networks or len(ingress_networks) != 2: + raise RuntimeError("TLS ingress must have internal service access and one ingress network") + if worker_networks == ingress_networks: + raise RuntimeError("worker egress and public ingress networks must remain separate") return { "artifact_kind": "platform_hosted_managed_production_compose_contract_report", From af1b75af7b4ed84806f78e43a9da3b9007d52bca Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:47:56 +0300 Subject: [PATCH 8/9] Read Platform service allowlist contract --- scripts/hosted_managed_production_compose_smoke.py | 4 +--- scripts/hosted_managed_production_probe.py | 4 +--- tests/test_hosted_managed_production.py | 12 +++++------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/scripts/hosted_managed_production_compose_smoke.py b/scripts/hosted_managed_production_compose_smoke.py index 2e95d78..067f06a 100644 --- a/scripts/hosted_managed_production_compose_smoke.py +++ b/scripts/hosted_managed_production_compose_smoke.py @@ -320,9 +320,7 @@ def run_smoke() -> dict[str, Any]: if ingress_health.get("ok") is not True or ingress_health.get("adapter") != "postgresql": raise RuntimeError("TLS ingress did not expose PostgreSQL-backed service health") - operations = ingress_health.get("operations") - operations = operations if isinstance(operations, dict) else {} - if operations.get("enabled_operation_ids") != ["review_status_execute"]: + if ingress_health.get("enabled_operation_ids") != ["review_status_execute"]: raise RuntimeError("TLS ingress exposed an expanded operation allowlist") if worker_health.get("ok") is not True or worker_health.get("adapter") != "postgresql": raise RuntimeError("production worker heartbeat was not PostgreSQL-ready") diff --git a/scripts/hosted_managed_production_probe.py b/scripts/hosted_managed_production_probe.py index 336efb3..8b14f7d 100644 --- a/scripts/hosted_managed_production_probe.py +++ b/scripts/hosted_managed_production_probe.py @@ -140,9 +140,7 @@ def run_probe( diagnostics: list[str] = [] if health.get("ok") is not True or health.get("adapter") != "postgresql": diagnostics.append("service_health_not_postgresql_ready") - operations = health.get("operations") - operations = operations if isinstance(operations, dict) else {} - enabled = operations.get("enabled_operation_ids") + enabled = health.get("enabled_operation_ids") if enabled != [READ_ONLY_CANARY_OPERATION]: diagnostics.append("service_allowlist_not_read_only_canary") if set(service_states) != EXPECTED_SERVICES: diff --git a/tests/test_hosted_managed_production.py b/tests/test_hosted_managed_production.py index e0adcad..1fe6c93 100644 --- a/tests/test_hosted_managed_production.py +++ b/tests/test_hosted_managed_production.py @@ -248,7 +248,7 @@ def test_probe_requires_healthy_tls_runtime_and_exact_allowlist(self) -> None: fetch_health=lambda _: { "ok": True, "adapter": "postgresql", - "operations": {"enabled_operation_ids": ["review_status_execute"]}, + "enabled_operation_ids": ["review_status_execute"], }, ) self.assertTrue(report["ok"], report["diagnostics"]) @@ -268,12 +268,10 @@ def test_probe_blocks_stale_heartbeat_and_expanded_allowlist(self) -> None: fetch_health=lambda _: { "ok": True, "adapter": "postgresql", - "operations": { - "enabled_operation_ids": [ - "review_status_execute", - "promotion_execute_dry_run", - ] - }, + "enabled_operation_ids": [ + "review_status_execute", + "promotion_execute_dry_run", + ], }, ) self.assertFalse(report["ok"]) From 0ca8356236e73bcd56ca9cfff9e7707aaba37f3c Mon Sep 17 00:00:00 2001 From: Egor Merkushev Date: Mon, 13 Jul 2026 02:53:23 +0300 Subject: [PATCH 9/9] Require fresh production sign-off evidence --- docs/hosted-managed-operations.md | 5 +- .../hosted_managed_production_preflight.py | 2 + scripts/hosted_managed_production_probe.py | 10 ++- scripts/hosted_managed_production_signoff.py | 80 ++++++++++++++++++- scripts/hosted_managed_runtime_backup.py | 3 + tests/test_hosted_managed_production.py | 52 +++++++++++- 6 files changed, 143 insertions(+), 9 deletions(-) diff --git a/docs/hosted-managed-operations.md b/docs/hosted-managed-operations.md index c908924..7dcd893 100644 --- a/docs/hosted-managed-operations.md +++ b/docs/hosted-managed-operations.md @@ -483,7 +483,10 @@ to a local SQLite executor. A consume-on-attempt or irreversible request needs new operator intent after rollback. The final sign-off command requires all evidence rather than trusting a single -green queue receipt: +green queue receipt. By default every evidence report must be no older than 24 +hours and must follow the documented causal order from preflight through +rollback; `--max-evidence-age` may narrow that window but should not be expanded +to reuse evidence from an earlier deployment: ```bash .venv/bin/python scripts/hosted_managed_production_signoff.py signoff \ diff --git a/scripts/hosted_managed_production_preflight.py b/scripts/hosted_managed_production_preflight.py index 1c83309..d407d50 100644 --- a/scripts/hosted_managed_production_preflight.py +++ b/scripts/hosted_managed_production_preflight.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +from datetime import datetime, timezone import json import os from pathlib import Path @@ -189,6 +190,7 @@ def run_preflight( return { "artifact_kind": "platform_hosted_managed_production_preflight_report", "contract_ref": "platform.hosted-managed.production-preflight.v1", + "generated_at": datetime.now(timezone.utc).isoformat(), "ok": not diagnostics, "summary": { "status": "ready" if not diagnostics else "blocked", diff --git a/scripts/hosted_managed_production_probe.py b/scripts/hosted_managed_production_probe.py index 8b14f7d..dea8167 100644 --- a/scripts/hosted_managed_production_probe.py +++ b/scripts/hosted_managed_production_probe.py @@ -77,6 +77,8 @@ def run_probe( if ( parsed.scheme != "https" or not parsed.hostname + or parsed.username is not None + or parsed.password is not None or parsed.path not in ("", "/") or parsed.query or parsed.fragment @@ -164,13 +166,19 @@ def run_probe( diagnostics.append("worker_heartbeat_stale") diagnostics = sorted(set(diagnostics)) + host = parsed.hostname + if ":" in host and not host.startswith("["): + host = f"[{host}]" + origin = f"{parsed.scheme}://{host}" + if parsed.port is not None: + origin = f"{origin}:{parsed.port}" return { "artifact_kind": "platform_hosted_managed_production_probe_report", "contract_ref": "platform.hosted-managed.production-probe.v1", "generated_at": (now or datetime.now(timezone.utc)).isoformat(), "ok": not diagnostics, "service": { - "origin": f"{parsed.scheme}://{parsed.netloc}", + "origin": origin, "adapter": health.get("adapter"), "enabled_operation_ids": enabled if isinstance(enabled, list) else [], }, diff --git a/scripts/hosted_managed_production_signoff.py b/scripts/hosted_managed_production_signoff.py index ee6f7ba..d3c3eee 100644 --- a/scripts/hosted_managed_production_signoff.py +++ b/scripts/hosted_managed_production_signoff.py @@ -10,6 +10,7 @@ ACTIVE_QUEUE_STATUSES = {"queued", "leased", "running"} +DEFAULT_MAX_EVIDENCE_AGE_SECONDS = 86400.0 EXPECTED_KINDS = { "preflight": "platform_hosted_managed_production_preflight_report", "probe_before_reboot": "platform_hosted_managed_production_probe_report", @@ -131,8 +132,73 @@ def _write_authority_findings(value: Any, *, path: str = "$") -> list[str]: return findings -def build_signoff(reports: dict[str, dict[str, Any]]) -> dict[str, Any]: +def _evidence_timestamps( + reports: dict[str, dict[str, Any]], + *, + now: datetime, + max_age_seconds: float, +) -> tuple[dict[str, datetime], list[str]]: + timestamps: dict[str, datetime] = {} diagnostics: list[str] = [] + for label, report in reports.items(): + value = report.get("generated_at") + if not isinstance(value, str): + diagnostics.append(f"{label}_generated_at_missing") + continue + try: + timestamp = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + diagnostics.append(f"{label}_generated_at_invalid") + continue + if timestamp.tzinfo is None: + diagnostics.append(f"{label}_generated_at_not_utc") + continue + timestamp = timestamp.astimezone(timezone.utc) + age = (now - timestamp).total_seconds() + if age < -300: + diagnostics.append(f"{label}_generated_at_in_future") + elif age > max_age_seconds: + diagnostics.append(f"{label}_evidence_stale") + timestamps[label] = timestamp + return timestamps, diagnostics + + +def _ordering_diagnostics(timestamps: dict[str, datetime]) -> list[str]: + ordered_edges = ( + ("preflight", "probe_before_reboot"), + ("probe_before_reboot", "backup"), + ("backup", "restore_smoke"), + ("restore_smoke", "canary"), + ("canary", "recovery"), + ("recovery", "probe_after_reboot"), + ("probe_after_reboot", "replay_canary"), + ("replay_canary", "queue_audit"), + ("queue_audit", "rollback_specspace_smoke"), + ("preflight", "hosted_specspace_smoke"), + ("hosted_specspace_smoke", "rollback_specspace_smoke"), + ) + diagnostics: list[str] = [] + for earlier, later in ordered_edges: + if earlier in timestamps and later in timestamps and timestamps[earlier] > timestamps[later]: + diagnostics.append(f"evidence_order_invalid_{earlier}_after_{later}") + return diagnostics + + +def build_signoff( + reports: dict[str, dict[str, Any]], + *, + now: datetime | None = None, + max_evidence_age_seconds: float = DEFAULT_MAX_EVIDENCE_AGE_SECONDS, +) -> dict[str, Any]: + diagnostics: list[str] = [] + current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) + timestamps, timestamp_diagnostics = _evidence_timestamps( + reports, + now=current, + max_age_seconds=max_evidence_age_seconds, + ) + diagnostics.extend(timestamp_diagnostics) + diagnostics.extend(_ordering_diagnostics(timestamps)) if set(reports) != set(EXPECTED_KINDS): diagnostics.append("evidence_set_incomplete") for label, expected_kind in EXPECTED_KINDS.items(): @@ -238,7 +304,7 @@ def build_signoff(reports: dict[str, dict[str, Any]]) -> dict[str, Any]: return { "artifact_kind": "platform_hosted_managed_production_canary_signoff_report", "contract_ref": "platform.hosted-managed.production-canary-signoff.v1", - "generated_at": datetime.now(timezone.utc).isoformat(), + "generated_at": current.isoformat(), "ok": not diagnostics, "summary": { "status": "production_canary_signed_off" @@ -296,6 +362,11 @@ def main(argv: list[str] | None = None) -> int: for label in EXPECTED_KINDS: signoff.add_argument(f"--{label.replace('_', '-')}", required=True) signoff.add_argument("--output") + signoff.add_argument( + "--max-evidence-age", + type=float, + default=DEFAULT_MAX_EVIDENCE_AGE_SECONDS, + ) args = parser.parse_args(argv) try: if args.command == "queue-audit": @@ -305,7 +376,10 @@ def main(argv: list[str] | None = None) -> int: label: _load_report(Path(getattr(args, label)), label=label) for label in EXPECTED_KINDS } - report = build_signoff(reports) + report = build_signoff( + reports, + max_evidence_age_seconds=args.max_evidence_age, + ) except ProductionSignoffError as exc: report = { "artifact_kind": "platform_hosted_managed_production_canary_signoff_report", diff --git a/scripts/hosted_managed_runtime_backup.py b/scripts/hosted_managed_runtime_backup.py index bdb4142..0ad6998 100644 --- a/scripts/hosted_managed_runtime_backup.py +++ b/scripts/hosted_managed_runtime_backup.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +from datetime import datetime, timezone import hashlib import json import os @@ -258,6 +259,7 @@ def create_backup( report = { "artifact_kind": "platform_hosted_managed_runtime_backup_report", "contract_ref": "platform.hosted-managed.runtime-backup.v1", + "generated_at": datetime.now(timezone.utc).isoformat(), "ok": True, "backup_id": backup_id, "summary": { @@ -390,6 +392,7 @@ def restore_smoke( return { "artifact_kind": "platform_hosted_managed_runtime_restore_smoke_report", "contract_ref": "platform.hosted-managed.runtime-restore-smoke.v1", + "generated_at": datetime.now(timezone.utc).isoformat(), "ok": True, "backup_id": backup_id, "summary": { diff --git a/tests/test_hosted_managed_production.py b/tests/test_hosted_managed_production.py index 1fe6c93..93e0ee5 100644 --- a/tests/test_hosted_managed_production.py +++ b/tests/test_hosted_managed_production.py @@ -278,11 +278,28 @@ def test_probe_blocks_stale_heartbeat_and_expanded_allowlist(self) -> None: self.assertIn("worker_heartbeat_stale", report["diagnostics"]) self.assertIn("service_allowlist_not_read_only_canary", report["diagnostics"]) + def test_probe_rejects_url_userinfo_before_fetching(self) -> None: + with self.assertRaisesRegex( + probe.ProductionProbeError, + "clean HTTPS service URL", + ): + probe.run_probe( + service_url="https://user:password@managed.example.test", + compose_file=Path("/srv/platform/compose.yml"), + project_name="platform-managed", + fetch_health=lambda _: self.fail("health must not be fetched"), + ) + class HostedManagedProductionSignoffTests(unittest.TestCase): def evidence(self) -> dict[str, dict]: + generated_at = "2026-07-13T00:00:00+00:00" reports = { - label: {"artifact_kind": kind, "ok": True} + label: { + "artifact_kind": kind, + "generated_at": generated_at, + "ok": True, + } for label, kind in signoff.EXPECTED_KINDS.items() } for label in ("probe_before_reboot", "probe_after_reboot"): @@ -342,7 +359,10 @@ def evidence(self) -> dict[str, dict]: return reports def test_signoff_requires_complete_reboot_replay_backup_and_rollback_evidence(self) -> None: - report = signoff.build_signoff(self.evidence()) + report = signoff.build_signoff( + self.evidence(), + now=datetime(2026, 7, 13, 1, tzinfo=timezone.utc), + ) self.assertTrue(report["ok"], report["diagnostics"]) self.assertEqual(report["summary"]["status"], "production_canary_signed_off") self.assertTrue(report["summary"]["rollback_verified"]) @@ -354,7 +374,10 @@ def test_signoff_blocks_reexecution_and_expanded_allowlist(self) -> None: "review_status_execute", "promotion_execute_dry_run", ] - report = signoff.build_signoff(evidence) + report = signoff.build_signoff( + evidence, + now=datetime(2026, 7, 13, 1, tzinfo=timezone.utc), + ) self.assertFalse(report["ok"]) self.assertIn("replay_canary_attempt_not_one", report["diagnostics"]) self.assertIn("probe_after_reboot_allowlist_invalid", report["diagnostics"]) @@ -362,10 +385,31 @@ def test_signoff_blocks_reexecution_and_expanded_allowlist(self) -> None: def test_signoff_rejects_write_capable_evidence(self) -> None: evidence = self.evidence() evidence["canary"]["authority_boundary"] = {"may_execute_platform": True} - report = signoff.build_signoff(evidence) + report = signoff.build_signoff( + evidence, + now=datetime(2026, 7, 13, 1, tzinfo=timezone.utc), + ) self.assertFalse(report["ok"]) self.assertIn("canary_write_authority_expanded", report["diagnostics"]) + def test_signoff_rejects_stale_and_out_of_order_evidence(self) -> None: + evidence = self.evidence() + evidence["preflight"]["generated_at"] = "2026-07-10T00:00:00+00:00" + evidence["backup"]["generated_at"] = "2026-07-13T00:30:00+00:00" + evidence["probe_before_reboot"]["generated_at"] = ( + "2026-07-13T00:45:00+00:00" + ) + report = signoff.build_signoff( + evidence, + now=datetime(2026, 7, 13, 1, tzinfo=timezone.utc), + ) + self.assertFalse(report["ok"]) + self.assertIn("preflight_evidence_stale", report["diagnostics"]) + self.assertIn( + "evidence_order_invalid_probe_before_reboot_after_backup", + report["diagnostics"], + ) + if __name__ == "__main__": unittest.main()