From 8008819cf226cfc61367e8ecb7dc98fde14da399 Mon Sep 17 00:00:00 2001 From: Pino de Candia <32303022+pinodeca@users.noreply.github.com> Date: Wed, 20 May 2026 14:44:20 +0000 Subject: [PATCH 1/5] =?UTF-8?q?Switch=20duroxide-pg-opt=20=E2=86=92=20duro?= =?UTF-8?q?xide-pg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the duroxide-pg-opt git submodule and depend on the upstream duroxide-pg crate directly. Adopt the new ProviderConfig / MigrationPolicy API: backends use VerifyOnly (no DDL, no advisory locks), the background worker uses ApplyAll. Temporarily pins the dependency to a fork branch backing microsoft/duroxide-pg#10 until that PR merges and a tag is published. - Remove .gitmodules and duroxide-pg-opt submodule - Cargo.toml: cargo git dep (fork branch, with TEMP comment) - Update all PostgresProvider call sites to new_with_schema_and_config(url, Some(DUROXIDE_SCHEMA), config) - types.rs: backend_provider_config / worker_provider_config helpers - Drop GH_PAT plumbing in CI/devcontainer (no longer needed) - Update docs and prompts to reference duroxide-pg --- .devcontainer/devcontainer.json | 9 - .devcontainer/onCreateCommand.sh | 105 +---- .github/copilot-instructions.md | 4 +- .github/workflows/ci.yml | 5 - .github/workflows/copilot-setup-steps.yml | 8 - .github/workflows/docker.yml | 4 - .github/workflows/prebuild.yml | 1 - .gitmodules | 3 - Cargo.lock | 418 +++++++++++++++--- Cargo.toml | 7 +- Dockerfile | 3 - README.md | 16 +- docs/CODESPACES_PREBUILDS.md | 47 +- docs/bgw-applies-migrations.md | 16 +- docs/dep_issues.md | 20 +- docs/extension_lifecycle.md | 20 +- .../threat-model.dfd-lite.yaml | 2 +- duroxide-pg-opt | 1 - prompts/README.md | 4 +- prompts/pg_durable-check-upstream-fixes.md | 23 +- prompts/pg_durable-release.md | 29 +- src/client.rs | 14 +- src/explain.rs | 12 +- src/lib.rs | 40 +- src/monitoring.rs | 44 +- src/types.rs | 24 +- src/worker.rs | 33 +- 27 files changed, 545 insertions(+), 367 deletions(-) delete mode 100644 .gitmodules delete mode 160000 duroxide-pg-opt diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 08ff98ab..4fdbedef 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -19,15 +19,6 @@ "rust-analyzer.check.command": "clippy", "rust-analyzer.check.extraArgs": ["--features", "pg17"] } - }, - "codespaces": { - "repositories": { - "microsoft/duroxide-pg-opt": { - "permissions": { - "contents": "read" - } - } - } } }, diff --git a/.devcontainer/onCreateCommand.sh b/.devcontainer/onCreateCommand.sh index a2a912e6..2a301fdc 100755 --- a/.devcontainer/onCreateCommand.sh +++ b/.devcontainer/onCreateCommand.sh @@ -56,99 +56,30 @@ else cargo pgrx init --pg17 download fi -# ── Initialize private submodule (duroxide-pg-opt) ────────────────── -# duroxide-pg-opt is a private repo. Two auth mechanisms: -# -# 1. Prebuild phase: GH_PAT Codespace secret provides access. -# We use a temporary git insteadOf rewrite during submodule clone. -# The secret remains available in the Codespace environment, so there -# is no meaningful security benefit to trying to scrub local traces. -# -# 2. Interactive Codespace: devcontainer.json grants the built-in -# GITHUB_TOKEN read access via customizations.codespaces.repositories. -# The Codespace credential helper handles auth automatically. -# -# 3. Local Dev Container: user must have their own credentials. - -SUBMODULE_INITIALIZED=0 - -if [ -n "$GH_PAT" ]; then - echo "GH_PAT detected — initializing submodule with PAT..." - - # Temporarily rewrite GitHub HTTPS URLs to include the token. - PAT_REWRITE_URL="https://x-access-token:${GH_PAT}@github.com/" - - cleanup_pat_rewrite() { - local rc=$? - # GH_PAT is still available in Codespace env vars; cleanup here ensures - # subsequent user git operations prefer devcontainer.json repo permissions - # and Codespaces credential helper instead of forcing PAT rewrite behavior. - git config --global --remove-section "url.${PAT_REWRITE_URL}" 2>/dev/null || true - return $rc - } - - trap cleanup_pat_rewrite EXIT - git config --global url."${PAT_REWRITE_URL}".insteadOf "https://github.com/" - - if [ "$SMOKE_MODE" = "1" ]; then - echo "Smoke mode: skipping git submodule update" - if [ -f "duroxide-pg-opt/Cargo.toml" ]; then - SUBMODULE_INITIALIZED=1 - fi - elif git submodule update --init --recursive; then - echo "✅ Submodule initialized successfully (via PAT)" - SUBMODULE_INITIALIZED=1 - else - echo "⚠️ Submodule initialization failed with PAT" - fi -else - echo "GH_PAT not set — trying submodule init with default credentials..." - if [ "$SMOKE_MODE" = "1" ]; then - echo "Smoke mode: skipping git submodule update" - if [ -f "duroxide-pg-opt/Cargo.toml" ]; then - SUBMODULE_INITIALIZED=1 - fi - elif git submodule update --init --recursive; then - echo "✅ Submodule initialized successfully" - SUBMODULE_INITIALIZED=1 - else - echo "⚠️ Submodule initialization failed — skipping" - echo " Set GH_PAT secret or ensure credentials for microsoft/duroxide-pg-opt" - fi -fi - # ── Build pg_durable ──────────────────────────────────────────────── -# Only build if the submodule is present (needed for compilation) -if [ "$SUBMODULE_INITIALIZED" = "1" ] && [ -f "duroxide-pg-opt/Cargo.toml" ]; then - echo "Building pg_durable..." - if [ "$SMOKE_MODE" = "1" ]; then - echo "Smoke mode: skipping cargo build" - else - cargo build --features pg17,http-allow-test-domains - echo "✅ pg_durable built successfully" - fi +# duroxide-pg is pulled as a cargo git dependency (see Cargo.toml). +echo "Building pg_durable..." +if [ "$SMOKE_MODE" = "1" ]; then + echo "Smoke mode: skipping cargo build" +else + cargo build --features pg17,http-allow-test-domains + echo "✅ pg_durable built successfully" echo "Installing pg_durable into PostgreSQL ${PG_MAJOR}..." - if [ "$SMOKE_MODE" = "1" ]; then - echo "Smoke mode: skipping install/cluster bootstrap" - else - resolve_pgrx_environment "$PG_MAJOR" - cargo pgrx install --release --pg-config "$PG_CONFIG" + resolve_pgrx_environment "$PG_MAJOR" + cargo pgrx install --release --pg-config "$PG_CONFIG" - echo "Preparing PostgreSQL ${PG_MAJOR} cluster..." - recreate_local_cluster - start_local_postgres - ensure_compatible_roles - ensure_pg_durable_extension + echo "Preparing PostgreSQL ${PG_MAJOR} cluster..." + recreate_local_cluster + start_local_postgres + ensure_compatible_roles + ensure_pg_durable_extension - VERSION=$(pg_durable_version) - echo "✅ pg_durable ${VERSION} installed and verified" + VERSION=$(pg_durable_version) + echo "✅ pg_durable ${VERSION} installed and verified" - echo "Stopping PostgreSQL ${PG_MAJOR} after prebuild verification..." - stop_local_postgres - fi -else - echo "⚠️ Submodule not available — skipping pg_durable build" + echo "Stopping PostgreSQL ${PG_MAJOR} after prebuild verification..." + stop_local_postgres fi echo "" diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 6408e59a..fdaeed9a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -94,7 +94,7 @@ The new `.so` must work against **all** previous versions' schemas (same major v For Scenario A, treat the upgrade path as the contract for already-shipped versions: before release, fresh install for the new version should match what an existing customer gets by installing the previous version and applying the upgrade chain. -**Updating duroxide-pg-opt dependency**: Update the submodule (`cd duroxide-pg-opt && git fetch && git checkout `), then rebuild. The BGW's embedded migration files update automatically via `include_dir!`. No changes to extension SQL, upgrade scripts, or any checked-in SQL copies are needed. +**Updating duroxide-pg dependency**: Update the `tag = "..."` value for `duroxide-pg` in [`Cargo.toml`](../Cargo.toml), then `cargo update -p duroxide-pg` and rebuild. The BGW's embedded migration files update automatically via `include_dir!`. No changes to extension SQL, upgrade scripts, or any checked-in SQL copies are needed. **Writing a spec or design doc:** Include an "Upgrade & Migration" section covering: backward compatibility impact (B1 — will the new `.so` work against all previous schemas?), upgrade script DDL needed, and any runtime schema detection required. See [docs/upgrade-testing.md](../docs/upgrade-testing.md) for the full upgrade testing strategy. @@ -102,7 +102,7 @@ For Scenario A, treat the upgrade path as the contract for already-shipped versi - **pgrx 0.16.1**: PostgreSQL extension framework (pinned version) - **duroxide**: Durable execution runtime -- **duroxide-pg-opt**: duroxide provider/stores engine state in PostgreSQL (git submodule) +- **duroxide-pg**: duroxide provider/stores engine state in PostgreSQL (cargo git dependency, pinned by tag in [`Cargo.toml`](../Cargo.toml)) - **sqlx**: Async PostgreSQL from background worker - **tokio**: Async runtime for background worker diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acec9d82..4436c347 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,6 @@ on: env: CARGO_TERM_COLOR: always - GITHUB_TOKEN: ${{ secrets.GH_PAT }} FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true concurrency: @@ -59,7 +58,6 @@ jobs: rustup default nightly - name: Check formatting - # --all includes path dependencies; limit to our package to skip submodules run: cargo fmt --package pg_durable -- --check prepare: @@ -96,9 +94,6 @@ jobs: continue-on-error: ${{ matrix.pg_version == 18 }} steps: - uses: actions/checkout@v4 - with: - submodules: true - token: ${{ secrets.GH_PAT }} - name: Free disk space and show usage run: | diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 4420b373..e4705ea1 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -24,14 +24,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v5 - with: - submodules: false - - - name: Initialize private submodule duroxide-pg-opt - run: | - test -n "${{ secrets.DUROXIDE_PG_OPT_TOKEN }}" - git -c url."https://x-access-token:${{ secrets.DUROXIDE_PG_OPT_TOKEN }}@github.com/".insteadOf="https://github.com/" \ - submodule update --init --recursive - name: Install Rust toolchain (stable) uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a21218b8..6a9579e0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,9 +43,6 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: true - token: ${{ secrets.GH_PAT }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -101,7 +98,6 @@ jobs: - name: Run E2E tests in Docker env: - GITHUB_TOKEN: ${{ secrets.GH_PAT }} PG_DURABLE_LOG_DIR: /tmp/docker-logs run: ./scripts/test-e2e-docker.sh diff --git a/.github/workflows/prebuild.yml b/.github/workflows/prebuild.yml index 410374d4..8d654e44 100644 --- a/.github/workflows/prebuild.yml +++ b/.github/workflows/prebuild.yml @@ -76,6 +76,5 @@ jobs: # Exercise script entrypoints without running heavy setup. export SKIP_APT_UPDATE=1 export PG_DURABLE_SMOKE=1 - export GH_PAT="smoke-test-token" bash .devcontainer/onCreateCommand.sh diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 464560fa..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "duroxide-pg-opt"] - path = duroxide-pg-opt - url = https://github.com/microsoft/duroxide-pg-opt.git diff --git a/Cargo.lock b/Cargo.lock index 66357e56..ad9d38df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -60,6 +66,29 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -92,6 +121,58 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "azure_core" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966f75417a6779ea221531960d9d46fe630879606501aa5277312ad4ae38587" +dependencies = [ + "async-lock", + "async-trait", + "azure_core_macros", + "bytes", + "futures", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "tokio", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_core_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70066e34b8db2c3f0b852c56a99333bd25fdedfb8850cd95dddf930928b47b80" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "tracing", +] + +[[package]] +name = "azure_identity" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385e4426c01965149a002a72c54c43d6f1b9d2244179e70647df48e3b28f8c32" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "futures", + "pin-project", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "url", +] + [[package]] name = "base64" version = "0.22.1" @@ -384,6 +465,23 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -463,6 +561,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "cron" version = "0.13.0" @@ -528,6 +635,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + [[package]] name = "digest" version = "0.10.7" @@ -571,9 +688,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "duroxide" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f77ea9fc6f0bc9ff65a95ff7c3e4ea8a2bb274fda44ee9d3393f2116825200f5" +checksum = "9377f5bf81d9a8ce56a13f913f60ca0e0ba8a9464349c407e7d11c326488bf0c" dependencies = [ "async-trait", "futures", @@ -588,15 +705,20 @@ dependencies = [ ] [[package]] -name = "duroxide-pg-opt" -version = "0.1.26" +name = "duroxide-pg" +version = "0.1.33" +source = "git+https://github.com/pinodeca/duroxide-pg.git?branch=feat%2Fmigration-policy#715ec60abcd61c06be7c2de77b79a52e27a05f6a" dependencies = [ "anyhow", "async-trait", + "azure_core", + "azure_identity", "chrono", "dotenvy", "duroxide", + "futures-util", "include_dir", + "reqwest 0.13.3", "semver", "serde", "serde_json", @@ -607,6 +729,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -683,6 +811,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "eyre" version = "0.6.12" @@ -717,6 +855,16 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -1121,7 +1269,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -1465,6 +1613,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.0" @@ -1547,6 +1705,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -1734,10 +1898,10 @@ dependencies = [ "chrono", "cron", "duroxide", - "duroxide-pg-opt", + "duroxide-pg", "pgrx", "pgrx-tests", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "sqlx", @@ -1894,6 +2058,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1991,6 +2175,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2239,6 +2429,42 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "rsa" version = "0.9.10" @@ -2265,6 +2491,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -2517,6 +2752,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "siphasher" version = "1.0.2" @@ -2857,7 +3098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2912,6 +3153,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -3085,13 +3357,18 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "iri-string", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -3196,6 +3473,57 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "typespec" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "247afbeabe0c383f630d0fdcb4fb92818c31cd3fe5d293335daff8cac79d4e0f" +dependencies = [ + "base64", + "bytes", + "futures", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4714a935f6aa74724775f07291390e8f07df0a1804544cd4106ac3b75c0478d2" +dependencies = [ + "async-trait", + "base64", + "bytes", + "dyn-clone", + "futures", + "pin-project", + "rand 0.10.1", + "reqwest 0.13.3", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17153fde258b3c862cecffad95e185c0eb83e619e76542c4b3ed9828af840f0" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "unarray" version = "0.1.4" @@ -3454,6 +3782,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -3536,7 +3877,7 @@ version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ - "windows-core 0.57.0", + "windows-core", "windows-targets 0.52.6", ] @@ -3546,25 +3887,12 @@ version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" dependencies = [ - "windows-implement 0.57.0", - "windows-interface 0.57.0", - "windows-result 0.1.2", + "windows-implement", + "windows-interface", + "windows-result", "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement 0.60.2", - "windows-interface 0.59.3", - "windows-link", - "windows-result 0.4.1", - "windows-strings", -] - [[package]] name = "windows-implement" version = "0.57.0" @@ -3576,17 +3904,6 @@ dependencies = [ "syn", ] -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-interface" version = "0.57.0" @@ -3598,17 +3915,6 @@ dependencies = [ "syn", ] -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-link" version = "0.2.1" @@ -3624,24 +3930,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 89eb3fa6..b44d43f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,8 +34,11 @@ serde_json = { version = "1.0", features = ["preserve_order"] } uuid = { version = "1.0", features = ["v4", "serde"] } # duroxide integration -duroxide = "=0.1.28" -duroxide-pg-opt = { path = "duroxide-pg-opt", package = "duroxide-pg-opt" } +duroxide = "0.1.29" +# TEMP: pinned to the fork branch backing https://github.com/microsoft/duroxide-pg/pull/10. +# Revert to the git+tag form below once the PR merges and a tag is published upstream. +duroxide-pg = { git = "https://github.com/pinodeca/duroxide-pg.git", branch = "feat/migration-policy", package = "duroxide-pg" } +# duroxide-pg = { git = "https://github.com/microsoft/duroxide-pg.git", tag = "v0.1.33", package = "duroxide-pg" } tokio = { version = "1", features = ["rt-multi-thread", "sync", "time"] } # For ExecuteSQL activity (connecting back to PostgreSQL) diff --git a/Dockerfile b/Dockerfile index d87f24ed..ebb17ae3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,9 +36,6 @@ COPY Cargo.toml Cargo.lock* ./ COPY .cargo .cargo COPY build.rs ./ -# Copy duroxide-pg-opt submodule (path dependency) -COPY duroxide-pg-opt ./duroxide-pg-opt - # Copy source code COPY src ./src COPY pg_durable.control ./ diff --git a/README.md b/README.md index 03d27e00..317275c2 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,6 @@ SELECT df.start( - PostgreSQL 17 - Rust (nightly) - [cargo-pgrx](https://github.com/pgcentralfoundation/pgrx) 0.16.1 -- Access to `microsoft/duroxide-pg-opt` (private submodule; handled automatically in Codespaces) ## Development Installation @@ -49,27 +48,14 @@ The main branch prebuild installs PostgreSQL 17, builds `pg_durable`, and prepar ~/.pgrx/17.*/pgrx-install/bin/psql -h localhost -p 28817 -d postgres ``` -On a branch without a ready prebuild, initialize the submodule first, then run `pg-start.sh` — it will build and install the extension on first run (expect a few minutes): +On a branch without a ready prebuild, run `pg-start.sh` — it will build and install the extension on first run (expect a few minutes): ```bash -git submodule update --init --recursive ./scripts/pg-start.sh ``` ### Other environments -#### Submodule Access (Prerequisite) - -This project requires access to `microsoft/duroxide-pg-opt`, a private submodule: - -1. **Create a fine-grained GitHub PAT** with read-only `Contents` and `Metadata` access scoped to `microsoft/duroxide-pg-opt`: [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) -2. **Configure git** and initialize the submodule: - -```bash -git config --global url."https://@github.com/".insteadOf "https://github.com/" - -git submodule update --init --recursive -``` #### Local and Dev Container A VS Code Dev Container (`.devcontainer/`) provides Rust, cargo-pgrx, and PostgreSQL 17 pre-installed. For a bare local machine, install the toolchain first by following the steps in `.devcontainer/onCreateCommand.sh`. diff --git a/docs/CODESPACES_PREBUILDS.md b/docs/CODESPACES_PREBUILDS.md index 2c77125c..77018805 100644 --- a/docs/CODESPACES_PREBUILDS.md +++ b/docs/CODESPACES_PREBUILDS.md @@ -19,42 +19,11 @@ Pre-builds must be enabled by a repository administrator: - **Reduce prebuild available to specific regions**: Optional 4. Click **Create** -### Private Submodule Access +### Submodule Access -The `duroxide-pg-opt` submodule is a **private repository**. There are two relevant access paths: +The `duroxide-pg` submodule is a **public repository**, so no PAT or Codespaces secret is required for prebuilds or interactive Codespaces. The default Codespaces credentials are sufficient for `git submodule update --init --recursive`. -**1. Prebuild phase** — A GitHub PAT stored as a Codespaces secret is used during `onCreateCommand.sh`: - -1. Create a **fine-grained PAT** with **read-only** access to `microsoft/duroxide-pg-opt`: - - Repository access: only `microsoft/duroxide-pg-opt` - - Permissions: `Contents: Read`, `Metadata: Read` -2. Go to repository **Settings** → **Secrets and variables** → **Codespaces** -3. Click **New repository secret** -4. Name: `GH_PAT`, Value: the PAT from step 1 -5. Click **Add secret** - -`onCreateCommand.sh` uses that PAT via a git `insteadOf` rewrite so `git submodule update --init --recursive` can fetch the private submodule during prebuild. - -**2. Interactive Codespaces** — `devcontainer.json` also grants the built-in Codespaces token read access: - -```json -"codespaces": { - "repositories": { - "microsoft/duroxide-pg-opt": { - "permissions": { "contents": "read" } - } - } -} -``` - -This is still useful when users open a Codespace directly, especially on branches without a warm prebuild, because the built-in Codespaces token can satisfy normal repository access without depending on PAT-based git configuration. - -**Security notes:** -- The `GH_PAT` Codespaces secret is exposed as an environment variable to Codespaces, including existing Codespaces after a reload. Because of that, removing temporary git config entries during `onCreateCommand.sh` does not meaningfully hide the token from the user environment. -- `onCreateCommand.sh` still removes the temporary PAT-based `insteadOf` rewrite after submodule initialization. This avoids forcing PAT-based URL rewriting for later interactive git usage, so post-start interactions can rely on `devcontainer.json` repository permissions and the default Codespaces credential helper. -- The prebuild image is still a **filesystem snapshot**. The secret itself is not baked into the image just because it was present in the environment during prebuild. -- Users who open a Codespace from the prebuild get the submodule files already present, and the same `GH_PAT` secret is available in their environment if the repository is configured with it. -- Use a fine-grained PAT scoped only to `duroxide-pg-opt` with read-only `Contents` and `Metadata` permissions to minimize exposure. +> If you maintain a fork that points the submodule at a private alternative (for example `microsoft/duroxide-pg-opt`), you will need to add your own auth (Codespaces secret + `insteadOf` rewrite, or repository permission in `devcontainer.json`). ## How It Works @@ -70,7 +39,7 @@ Codespaces has two distinct phases: - System dependencies (libssl, clang, bison, etc.) - cargo-pgrx 0.16.1 - PostgreSQL 17 (downloaded and compiled via pgrx) - - `duroxide-pg-opt` submodule (via `GH_PAT` Codespace secret) + - `duroxide-pg` submodule (public, no auth required) - Builds and installs pg_durable - Recreates the local `~/.pgrx/data-17` cluster with `initdb -U postgres` - Pre-creates the `pg_durable` extension and verifies it @@ -162,13 +131,7 @@ This means the prebuild did not run or failed. There is no automatic fallback ### User Can See `GH_PAT` In Their Codespace Environment -This is expected for a repository-level Codespaces secret. - -- Repository Codespaces secrets are made available to Codespaces as environment variables. -- That includes existing Codespaces after a reload. -- Because the PAT is already present in the user environment, removing temporary git config entries during prebuild does not materially change visibility. - -The mitigation here is scope, not concealment: keep `GH_PAT` fine-grained, repository-scoped to `microsoft/duroxide-pg-opt`, and read-only. +No longer applicable — the `duroxide-pg` submodule is public and `GH_PAT` is no longer used by the prebuild. If you see a `GH_PAT` secret configured at the repo level, you can safely remove it. ## Cost Considerations diff --git a/docs/bgw-applies-migrations.md b/docs/bgw-applies-migrations.md index 61d659dd..8e69f278 100644 --- a/docs/bgw-applies-migrations.md +++ b/docs/bgw-applies-migrations.md @@ -5,7 +5,7 @@ ## Motivation -Previously, `CREATE EXTENSION pg_durable` included the full duroxide-pg-opt schema DDL via `extension_sql_file!("../sql/duroxide_install.sql")`, creating all duroxide tables, functions, indexes, and triggers as extension-owned objects. This change moved schema migration responsibility from the extension to the background worker (BGW). +Previously, `CREATE EXTENSION pg_durable` included the full duroxide-pg schema DDL via `extension_sql_file!("../sql/duroxide_install.sql")`, creating all duroxide tables, functions, indexes, and triggers as extension-owned objects. This change moved schema migration responsibility from the extension to the background worker (BGW). Three concrete motivations: @@ -13,7 +13,7 @@ Three concrete motivations: 2. **Future provider flexibility**: pg_durable may use a different duroxide provider in the future. If DDL lives in the BGW rather than the extension SQL, swapping providers does not require touching extension install/upgrade scripts. -3. **Backward compatibility relief**: duroxide-pg-opt does not offer binary compatibility with old schema versions. With BGW-managed migrations, new duroxide migrations are applied automatically at BGW startup — no extension upgrade involvement needed, no `.so` compatibility burden. +3. **Backward compatibility relief**: duroxide-pg does not offer binary compatibility with old schema versions. With BGW-managed migrations, new duroxide migrations are applied automatically at BGW startup — no extension upgrade involvement needed, no `.so` compatibility burden. ## Architecture @@ -74,9 +74,9 @@ After the ownership check passes, the BGW releases extension ownership of any ob - **Fresh 0.2.0 install**: schema is empty → BGW creates all duroxide tables, functions, indexes, triggers, and records all migrations in `_duroxide_migrations` (5 as of 0.2.0). - **Upgraded from 0.1.1**: all migrations already recorded in `_duroxide_migrations` → `ApplyAll` detects no pending work → no-ops → starts runtime normally. -- **Future duroxide-pg-opt upgrade**: new migration files embedded in the binary are applied automatically without any `ALTER EXTENSION` involvement. +- **Future duroxide-pg upgrade**: new migration files embedded in the binary are applied automatically without any `ALTER EXTENSION` involvement. -Unknown-migration rejection is always enforced: `duroxide-pg-opt` unconditionally calls `check_no_unknown_migrations()` after both `ApplyAll` and `VerifyOnly`. If the database has a migration row the binary does not recognize, initialization fails with an error (schema is newer than code — indicates a downgrade scenario). +Unknown-migration rejection is always enforced: `duroxide-pg` unconditionally calls `check_no_unknown_migrations()` after both `ApplyAll` and `VerifyOnly`. If the database has a migration row the binary does not recognize, initialization fails with an error (schema is newer than code — indicates a downgrade scenario). #### Step 5: readiness record via `duroxide._worker_ready` @@ -98,7 +98,7 @@ After `ApplyAll` succeeds, the BGW writes a row with the current `WORKER_SCHEMA_ ```rust /// Monotonically increasing schema version written to duroxide._worker_ready /// after successful BGW initialization. Increment whenever a new binary -/// introduces new duroxide-pg-opt migration scripts or any other BGW-applied +/// introduces new duroxide-pg migration scripts or any other BGW-applied /// duroxide schema change. const WORKER_SCHEMA_VERSION: i32 = 1; ``` @@ -217,13 +217,13 @@ Call `wait_for_ready` after any `CREATE EXTENSION` in scenarios that subsequentl | `docs/upgrade-testing.md` | Add duroxide ownership entry to the v0.1.1→v0.2.0 version-specific changes section. Note that both paths converge (objects not extension-owned after BGW runs). Note `wait_for_ready()` requirement in upgrade test infrastructure. | | `USER_GUIDE.md` | Add note that `DROP EXTENSION pg_durable CASCADE` is always required. Update readiness polling to use `duroxide._worker_ready` directly. | -The `duroxide-pg-opt/` submodule and `submodules: true` in CI remain — the submodule is still a Rust code dependency. +The `duroxide-pg/` submodule and `submodules: true` in CI remain — the submodule is still a Rust code dependency. ## What this enables going forward To summarize the benefits outlined in Motivation: -- **Duroxide upgrades decouple from pg_durable releases**: adding a new migration to `duroxide-pg-opt` requires no changes to pg_durable extension SQL, upgrade scripts, or the migration-copy sync scripts. The BGW applies it on next startup. -- **Provider swap path**: swapping `duroxide-pg-opt` for a different provider means changing BGW initialization code, not extension DDL. +- **Duroxide upgrades decouple from pg_durable releases**: adding a new migration to `duroxide-pg` requires no changes to pg_durable extension SQL, upgrade scripts, or the migration-copy sync scripts. The BGW applies it on next startup. +- **Provider swap path**: swapping `duroxide-pg` for a different provider means changing BGW initialization code, not extension DDL. - **No extension upgrade required for engine fixes**: duroxide bug fixes that involve schema changes are applied automatically by the BGW after the `.so` is updated, even for customers who never run `ALTER EXTENSION UPDATE`. - **Cleaner `pg_dump`**: duroxide schema objects are not extension-owned (on any install path), so they do not appear as extension members in `pg_dump` output. diff --git a/docs/dep_issues.md b/docs/dep_issues.md index 3d00ce3f..26c29226 100644 --- a/docs/dep_issues.md +++ b/docs/dep_issues.md @@ -1,23 +1,23 @@ # Dependency Issues & Blockers -**Purpose:** Track duroxide-pg-opt issues/limitations that require workarounds in pg_durable. +**Purpose:** Track duroxide-pg issues/limitations that require workarounds in pg_durable. **Last Updated:** 2026-01-06 -**GitHub Query:** [All pg_durable issues in duroxide-pg-opt](https://github.com/microsoft/duroxide-pg-opt/issues?q=is%3Aissue+label%3Apg_durable) +**GitHub Query:** [All pg_durable issues in duroxide-pg](https://github.com/microsoft/duroxide-pg/issues?q=is%3Aissue+label%3Apg_durable) --- ## How to Check for Fixes -1. **Check duroxide-pg-opt releases:** +1. **Check duroxide-pg releases:** ```bash - gh release list --repo microsoft/duroxide-pg-opt --limit 10 + gh release list --repo microsoft/duroxide-pg --limit 10 ``` 2. **Check specific issue status:** ```bash - gh issue view --repo microsoft/duroxide-pg-opt + gh issue view --repo microsoft/duroxide-pg ``` 3. **Check current duroxide version in use:** @@ -44,21 +44,21 @@ _No active blockers at this time._ | Field | Value | |-------|-------| -| **Issue** | [microsoft/duroxide-pg-opt#6](https://github.com/microsoft/duroxide-pg-opt/issues/6) | +| **Issue** | [microsoft/duroxide-pg#6](https://github.com/microsoft/duroxide-pg/issues/6) | | **Also filed** | [microsoft/duroxide-pg#1](https://github.com/microsoft/duroxide-pg/issues/1) (FYI only) | | **Status** | ✅ Resolved | -| **Fixed In** | duroxide-pg-opt v0.1.9 (requires duroxide 0.1.11) | +| **Fixed In** | duroxide-pg v0.1.9 (requires duroxide 0.1.11) | **Resolution Date:** 2026-01-06 **Problem (was):** -When upgrading `duroxide` or `duroxide-pg-opt` versions, the PostgreSQL schema in the `duroxide` schema may change (new columns, changed function signatures, etc.). This caused runtime errors: +When upgrading `duroxide` or `duroxide-pg` versions, the PostgreSQL schema in the `duroxide` schema may change (new columns, changed function signatures, etc.). This caused runtime errors: - `function duroxide.XXX does not exist` (function signature changed) - `column index out of bounds` (table columns changed) - `cached plan must not change result type` (prepared statement cache invalidated) **Resolution:** -The duroxide-pg-opt v0.1.9 release includes ProviderAdmin lifecycle management which handles schema versioning. No workarounds were needed in pg_durable codebase at the time of the fix. +The duroxide-pg v0.1.9 release includes ProviderAdmin lifecycle management which handles schema versioning. No workarounds were needed in pg_durable codebase at the time of the fix. --- @@ -80,7 +80,7 @@ When updating the duroxide dependency, run through this checklist: ## Version Compatibility Matrix -| pg_durable | duroxide | duroxide-pg-opt | Notes | +| pg_durable | duroxide | duroxide-pg | Notes | |------------|----------|-----------------|-------| | 0.1.1 | 0.1.11 | 0.1.9 | Current - schema versioning fix | | 0.1.0 | 0.1.6 | 0.1.6 | Legacy | diff --git a/docs/extension_lifecycle.md b/docs/extension_lifecycle.md index 4f530e46..6f5213b3 100644 --- a/docs/extension_lifecycle.md +++ b/docs/extension_lifecycle.md @@ -2,9 +2,9 @@ **Status:** Implemented **Last Updated:** 2026-03-01 -**Dependencies:** duroxide-pg-opt (git submodule) +**Dependencies:** duroxide-pg (git submodule) -> **Note:** This document describes the extension lifecycle management, background worker behavior, and duroxide-pg-opt schema integration in pg_durable. +> **Note:** This document describes the extension lifecycle management, background worker behavior, and duroxide-pg schema integration in pg_durable. ## Problem Statement @@ -109,7 +109,7 @@ See [bgw-applies-migrations.md](bgw-applies-migrations.md) for the full design. ### 2. Background Worker: `MigrationPolicy::ApplyAll` -duroxide-pg-opt provides `MigrationPolicy::ApplyAll` which: +duroxide-pg provides `MigrationPolicy::ApplyAll` which: - Applies pending migrations from the embedded migration files - Creates the schema tables if they don't yet exist - Records applied migrations in `_duroxide_migrations` @@ -396,7 +396,7 @@ Because the Duroxide schema DDL runs inside `CREATE EXTENSION` as extension SQL, ### Where this is intentionally non-idiomatic (trade-off) -- Duroxide provider DDL is applied by the BGW at startup (`ApplyAll`) rather than embedded in extension SQL. This decouples the duroxide schema lifecycle from `ALTER EXTENSION UPDATE`, allowing duroxide-pg-opt upgrades without extension upgrade scripts. +- Duroxide provider DDL is applied by the BGW at startup (`ApplyAll`) rather than embedded in extension SQL. This decouples the duroxide schema lifecycle from `ALTER EXTENSION UPDATE`, allowing duroxide-pg upgrades without extension upgrade scripts. ### Rationale for Polling in MVP @@ -406,13 +406,13 @@ While processUtility hooks are the "correct" PostgreSQL approach, polling is acc - Simpler implementation = faster time to value - Can be upgraded to hooks post-MVP without changing client-facing behavior -### Trade-offs with duroxide-pg-opt +### Trade-offs with duroxide-pg The duroxide schema is extension-owned but its contents are BGW-managed: - ✅ PostgreSQL-native lifecycle: install/upgrade/drop go through extension scripts (for `df.*` schema) and BGW-applied migrations (for `duroxide.*` schema). - ✅ Clear ownership: the `duroxide` schema is extension-owned; objects inside it are BGW-managed. -- ✅ Decoupled: duroxide-pg-opt upgrades do not require changes to extension SQL or upgrade scripts. +- ✅ Decoupled: duroxide-pg upgrades do not require changes to extension SQL or upgrade scripts. ### Known Limitations (future work) @@ -449,7 +449,7 @@ The duroxide schema is extension-owned but its contents are BGW-managed: - **New behavior:** BGW waits for extension existence, and also stays in "waiting" when migrations are missing/behind (VerifyOnly fails). 3. **Backend sessions disable long-polling** - - **Default behavior (upstream):** `duroxide_pg_opt::PostgresProvider` can enable long-polling by default. + - **Default behavior (upstream):** `duroxide_pg::PostgresProvider` can enable long-polling by default. - **pg_durable behavior:** For backend request/response operations (start/cancel/signal, monitoring), we disable long-polling to avoid a dedicated listener connection and notifier task. - **Impact:** Resource savings for installations with many backends. - **Compatibility:** No user-visible changes expected. @@ -458,8 +458,8 @@ The duroxide schema is extension-owned but its contents are BGW-managed: ### Resolved -1. **Should duroxide-pg-opt provide a "don't create schema" mode?** - - ✅ **RESOLVED:** duroxide-pg-opt 0.1.18 provides `MigrationPolicy::VerifyOnly` which never executes DDL +1. **Should duroxide-pg provide a "don't create schema" mode?** + - ✅ **RESOLVED:** duroxide-pg 0.1.18 provides `MigrationPolicy::VerifyOnly` which never executes DDL - No need for upstream changes or manual schema checks 2. **How should pg_durable avoid implicit schema creation?** @@ -504,7 +504,7 @@ The duroxide schema is extension-owned but its contents are BGW-managed: - PostgreSQL Extension Documentation: https://www.postgresql.org/docs/current/extend-extensions.html - pgrx Background Worker Documentation: https://github.com/pgcentralfoundation/pgrx/blob/develop/pgrx-examples/bgworker/src/lib.rs -- duroxide-pg-opt: https://github.com/microsoft/duroxide-pg-opt +- duroxide-pg: https://github.com/microsoft/duroxide-pg - pg_durable current architecture: [ARCHITECTURE.md](ARCHITECTURE.md) ## Timeline Estimate diff --git a/docs/security-review/threat-model.dfd-lite.yaml b/docs/security-review/threat-model.dfd-lite.yaml index 886e7618..e9388b3c 100644 --- a/docs/security-review/threat-model.dfd-lite.yaml +++ b/docs/security-review/threat-model.dfd-lite.yaml @@ -16,7 +16,7 @@ model: externalDependencies: - "PostgreSQL 17 (host database engine)" - "duroxide (durable execution runtime, embedded Rust crate)" - - "duroxide-pg-opt (PostgreSQL persistence provider for duroxide)" + - "duroxide-pg (PostgreSQL persistence provider for duroxide)" - "sqlx (async PostgreSQL driver for background worker connections)" - "reqwest (HTTP client for df.http() activity)" - "pgrx 0.16.1 (PostgreSQL extension framework)" diff --git a/duroxide-pg-opt b/duroxide-pg-opt deleted file mode 160000 index 8a753201..00000000 --- a/duroxide-pg-opt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8a753201838495b125381c96ee8ace9e9b5df6f1 diff --git a/prompts/README.md b/prompts/README.md index 3421db15..10a7481e 100644 --- a/prompts/README.md +++ b/prompts/README.md @@ -36,7 +36,7 @@ This directory contains structured prompts for LLMs working on the pg_durable co ### Release 5. **[pg_durable-release.md](pg_durable-release.md)** - - Check for duroxide/duroxide-pg-opt dependency updates + - Check for duroxide/duroxide-pg dependency updates - Build, clippy, and clean warnings - Update documentation and tests - Run unit and E2E tests @@ -92,7 +92,7 @@ These prompts follow consistent patterns: ### Release Workflow ``` 1. @pg_durable-release.md - Full release checklist: - - Check dependency updates (duroxide, duroxide-pg-opt) + - Check dependency updates (duroxide, duroxide-pg) - Build, clippy, clean warnings - Update docs and tests - Run unit + E2E tests diff --git a/prompts/pg_durable-check-upstream-fixes.md b/prompts/pg_durable-check-upstream-fixes.md index 58fd4191..3e379c81 100644 --- a/prompts/pg_durable-check-upstream-fixes.md +++ b/prompts/pg_durable-check-upstream-fixes.md @@ -1,6 +1,6 @@ # Check Upstream Fixes and Update Dependencies -**Purpose:** Check if fixes for tracked blockers have been released in upstream dependencies (duroxide-pg-opt) and guide updating pg_durable. +**Purpose:** Check if fixes for tracked blockers have been released in upstream dependencies (duroxide-pg) and guide updating pg_durable. --- @@ -16,7 +16,7 @@ For each active blocker, check if the GitHub issue has been closed/resolved: ```bash # Check issue status (replace ISSUE_NUMBER with actual number) -gh issue view --repo microsoft/duroxide-pg-opt --json state,title,closedAt +gh issue view --repo microsoft/duroxide-pg --json state,title,closedAt ``` If the issue is still open, stop here - no action needed. @@ -27,10 +27,10 @@ If an issue is closed, check if it's included in a release: ```bash # List recent releases -gh release list --repo microsoft/duroxide-pg-opt --limit 10 +gh release list --repo microsoft/duroxide-pg --limit 10 # Check what version we currently use -grep 'duroxide-pg-opt' Cargo.toml +grep 'duroxide-pg' Cargo.toml ``` Compare the release date with the issue close date. If there's a release after the issue was closed, the fix is likely available. @@ -39,7 +39,7 @@ Compare the release date with the issue close date. If there's a release after t ```bash # View specific release notes (replace TAG with version like v0.1.7) -gh release view --repo microsoft/duroxide-pg-opt +gh release view --repo microsoft/duroxide-pg ``` Confirm the fix is mentioned in the release notes. @@ -48,9 +48,10 @@ Confirm the fix is mentioned in the release notes. If a fix is available in a new release: -1. **Update submodule** - point to the new version: +1. **Bump the `duroxide-pg` tag** in `Cargo.toml` to the new version, then refresh the lockfile: ```bash - cd duroxide-pg-opt && git fetch && git checkout v0.1.X && cd .. + # Edit Cargo.toml: duroxide-pg = { git = "...", tag = "v0.1.X", package = "duroxide-pg" } + cargo update -p duroxide-pg ``` 2. **Also update duroxide if needed** (check compatibility): @@ -101,10 +102,10 @@ Present the changes to the user for review before committing. Include: ```bash # Check all tracked issues at once -gh issue view 6 --repo microsoft/duroxide-pg-opt --json state,title +gh issue view 6 --repo microsoft/duroxide-pg --json state,title # Current dependency versions -grep -E 'duroxide|duroxide-pg-opt' Cargo.toml +grep -E 'duroxide|duroxide-pg' Cargo.toml # Find all workarounds in codebase grep -rn "STOPGAP\|BLOCKED on duroxide\|TODO.*duroxide" --include="*.rs" . @@ -117,6 +118,6 @@ grep -rn "STOPGAP\|BLOCKED on duroxide\|TODO.*duroxide" --include="*.rs" . ## Notes -- We depend on `microsoft/duroxide-pg-opt`, not `microsoft/duroxide-pg`. Only act on fixes released in duroxide-pg-opt. +- We depend on `microsoft/duroxide-pg`. Only act on fixes released in duroxide-pg. - The `--clean` flag is important when testing dependency updates to ensure fresh schema creation. -- Always check the Version Compatibility Matrix to ensure duroxide and duroxide-pg-opt versions are compatible. +- Always check the Version Compatibility Matrix to ensure duroxide and duroxide-pg versions are compatible. diff --git a/prompts/pg_durable-release.md b/prompts/pg_durable-release.md index 2726b30e..090fcda0 100644 --- a/prompts/pg_durable-release.md +++ b/prompts/pg_durable-release.md @@ -9,7 +9,7 @@ Prepare and release a new version of pg_durable with quality checks, documentati The project uses these duroxide dependencies from `Cargo.toml`: - `duroxide` (crates.io version) -- `duroxide-pg-opt` (GitHub tag) +- `duroxide-pg` (GitHub tag) **Check for new duroxide version:** ```bash @@ -20,10 +20,10 @@ grep "duroxide" Cargo.toml cargo search duroxide --limit 5 ``` -**Check for new duroxide-pg-opt tag:** +**Check for new duroxide-pg tag:** ```bash -# List recent tags from the duroxide-pg-opt submodule -cd duroxide-pg-opt && git fetch --tags && git tag --sort=-v:refname | head -10 && cd .. +# List recent tags from the duroxide-pg repo +gh api repos/microsoft/duroxide-pg/tags --jq '.[].name' | head -10 ``` ### 1.2 Ask User About Updates @@ -35,29 +35,26 @@ If new versions are available, present them to the user: Current versions: - duroxide: 0.1.6 - - duroxide-pg-opt: v0.1.1 + - duroxide-pg: v0.1.1 New versions available: - duroxide: [new_version] ✨ - - duroxide-pg-opt: [new_tag] ✨ + - duroxide-pg: [new_tag] ✨ Would you like to update to the new versions? (y/n) ``` **If user approves updates:** -Update `Cargo.toml` duroxide version and update the submodule: -```bash -# Update duroxide in Cargo.toml +Update the dependency versions in `Cargo.toml`: +```toml # duroxide = "NEW_VERSION" - -# Update submodule to new tag -cd duroxide-pg-opt && git checkout NEW_TAG && cd .. +# duroxide-pg = { git = "...", tag = "NEW_TAG", package = "duroxide-pg" } ``` -Then run: +Then refresh `Cargo.lock`: ```bash -cargo update -p duroxide +cargo update -p duroxide -p duroxide-pg ``` ## Step 2: Update Package Version (if releasing) @@ -323,7 +320,7 @@ Suggested message based on changes: - Bump version from X.Y.Z to X.Y.Z+1 - Upgrade duroxide from vA.B.C to vX.Y.Z - - Upgrade duroxide-pg-opt from vA.B.C to vX.Y.Z + - Upgrade duroxide-pg from vA.B.C to vX.Y.Z - [other changes]" Would you like to: @@ -432,7 +429,7 @@ docker inspect pg_durable:latest --format '{{.Created}}' ## Checklist Summary ### Pre-Release Checklist -- [ ] Check for dependency updates (duroxide, duroxide-pg-opt) +- [ ] Check for dependency updates (duroxide, duroxide-pg) - [ ] Update dependencies if user approves - [ ] Bump version in Cargo.toml if releasing - [ ] `cargo build --features pg17` - no errors diff --git a/src/client.rs b/src/client.rs index 61ff293f..44adeb3b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -6,11 +6,11 @@ use std::sync::{Arc, OnceLock}; use duroxide::Client; -use duroxide_pg_opt::PostgresProvider; +use duroxide_pg::PostgresProvider; use pgrx::prelude::*; use tokio::runtime::Runtime; -use crate::types::{backend_provider_config, postgres_connection_string}; +use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; /// Cached tokio runtime for client operations. static CLIENT_RUNTIME: OnceLock = OnceLock::new(); @@ -82,9 +82,13 @@ fn get_duroxide_client() -> Result<&'static Client, String> { std::env::set_var("DUROXIDE_PG_POOL_MAX", "1"); let store = Arc::new( - PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await - .map_err(|e| format!("Failed to connect to duroxide store: {e}"))?, + PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await + .map_err(|e| format!("Failed to connect to duroxide store: {e}"))?, ); let _ = DUROXIDE_CLIENT.set(Client::new(store)); diff --git a/src/explain.rs b/src/explain.rs index 9b474d40..254e41c5 100644 --- a/src/explain.rs +++ b/src/explain.rs @@ -121,9 +121,9 @@ fn explain_instance(instance_id: &str) -> String { /// Get instance info from Duroxide store fn get_duroxide_instance_info(instance_id: &str) -> (String, Option) { - use crate::types::{backend_provider_config, postgres_connection_string}; + use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; use duroxide::Client; - use duroxide_pg_opt::PostgresProvider; + use duroxide_pg::PostgresProvider; use std::sync::Arc; let pg_conn_str = postgres_connection_string(); @@ -137,8 +137,12 @@ fn get_duroxide_instance_info(instance_id: &str) -> (String, Option) { }; rt.block_on(async { - let store = match PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await + let store = match PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await { Ok(s) => Arc::new(s), Err(_) => return (String::new(), None), diff --git a/src/lib.rs b/src/lib.rs index 4b297db1..98d7902a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,7 +43,7 @@ pub use types::Durofut; /// Monotonically increasing schema version written to `duroxide._worker_ready` /// by the background worker after successful initialization. Increment whenever -/// a new binary introduces new duroxide-pg-opt migration scripts or any other +/// a new binary introduces new duroxide-pg migration scripts or any other /// BGW-applied duroxide schema change. pub const WORKER_SCHEMA_VERSION: i32 = 1; @@ -794,7 +794,7 @@ mod tests { /// Ensure the Duroxide store exists and is ready fn ensure_store_ready() -> Result { use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; - use duroxide_pg_opt::PostgresProvider; + use duroxide_pg::PostgresProvider; use std::time::{Duration, Instant}; let pg_conn_str = postgres_connection_string(); @@ -812,7 +812,13 @@ mod tests { let config = backend_provider_config(); loop { - match PostgresProvider::new_with_config(&pg_conn_str, config.clone()).await { + match PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + config.clone(), + ) + .await + { Ok(_) => return Ok(format!("{pg_conn_str} (schema: {DUROXIDE_SCHEMA})")), Err(e) => { if start.elapsed() > timeout { @@ -831,9 +837,9 @@ mod tests { /// Wait for a durable function to complete, polling Duroxide status fn wait_for_completion(instance_id: &str, timeout_secs: u64) -> Result { - use crate::types::{backend_provider_config, postgres_connection_string}; + use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; use duroxide::Client; - use duroxide_pg_opt::PostgresProvider; + use duroxide_pg::PostgresProvider; use std::time::{Duration, Instant}; // Ensure store is ready first @@ -850,9 +856,13 @@ mod tests { rt.block_on(async { let store = Arc::new( - PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await - .map_err(|e| format!("Failed to connect to store: {e}"))?, + PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await + .map_err(|e| format!("Failed to connect to store: {e}"))?, ); let client = Client::new(store); @@ -893,9 +903,9 @@ mod tests { /// Get the current status from Duroxide fn get_duroxide_status(instance_id: &str) -> Option { - use crate::types::{backend_provider_config, postgres_connection_string}; + use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; use duroxide::Client; - use duroxide_pg_opt::PostgresProvider; + use duroxide_pg::PostgresProvider; let _ = ensure_store_ready().ok()?; let pg_conn_str = postgres_connection_string(); @@ -907,9 +917,13 @@ mod tests { rt.block_on(async { let store = Arc::new( - PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await - .ok()?, + PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await + .ok()?, ); let client = Client::new(store); client diff --git a/src/monitoring.rs b/src/monitoring.rs index 5434f0fa..66ba4d86 100644 --- a/src/monitoring.rs +++ b/src/monitoring.rs @@ -6,8 +6,8 @@ use duroxide::Client; use pgrx::prelude::*; use std::sync::Arc; -use crate::types::{backend_provider_config, postgres_connection_string}; -use duroxide_pg_opt::PostgresProvider; +use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; +use duroxide_pg::PostgresProvider; // ============================================================================ // Monitoring Functions @@ -81,8 +81,12 @@ pub fn list_instances( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await + let store = match PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await { Ok(s) => Arc::new(s), Err(_) => return vec![], @@ -166,8 +170,12 @@ pub fn instance_info( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await + let store = match PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await { Ok(s) => Arc::new(s), Err(_) => return vec![], @@ -232,8 +240,12 @@ pub fn instance_executions( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await + let store = match PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await { Ok(s) => Arc::new(s), Err(_) => return vec![], @@ -297,8 +309,12 @@ pub fn metrics() -> TableIterator< }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await + let store = match PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await { Ok(s) => Arc::new(s), Err(_) => return vec![], @@ -399,8 +415,12 @@ pub fn instance_nodes( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_config(&pg_conn_str, backend_provider_config()) - .await + let store = match PostgresProvider::new_with_schema_and_config( + &pg_conn_str, + Some(DUROXIDE_SCHEMA), + backend_provider_config(), + ) + .await { Ok(s) => Arc::new(s), Err(_) => return vec![], diff --git a/src/types.rs b/src/types.rs index 0f15e962..1b290017 100644 --- a/src/types.rs +++ b/src/types.rs @@ -219,13 +219,11 @@ pub const DUROXIDE_SCHEMA: &str = "duroxide"; /// Create a `ProviderConfig` for backend (request/response) operations. /// -/// - `VerifyOnly`: never create schema/tables, reject unknown migrations -/// - `long_poll` disabled: avoid a dedicated listener connection per backend session -pub fn backend_provider_config() -> duroxide_pg_opt::ProviderConfig { - let mut config = duroxide_pg_opt::ProviderConfig::default(); - config.schema_name = Some(DUROXIDE_SCHEMA.to_string()); - config.migration_policy = duroxide_pg_opt::MigrationPolicy::VerifyOnly; - config.long_poll.enabled = false; +/// - `VerifyOnly`: never create schema/tables, reject unknown migrations. +/// Backend sessions must not run DDL — the BGW owns schema lifecycle. +pub fn backend_provider_config() -> duroxide_pg::ProviderConfig { + let mut config = duroxide_pg::ProviderConfig::default(); + config.migration_policy = duroxide_pg::MigrationPolicy::VerifyOnly; config } @@ -233,13 +231,11 @@ pub fn backend_provider_config() -> duroxide_pg_opt::ProviderConfig { /// /// - `ApplyAll`: applies pending duroxide migrations at startup; creates tables /// inside the extension-owned `duroxide` schema. Safe because the BGW verifies -/// schema ownership via `pg_depend` before calling `PostgresProvider::new_with_config`. -/// - Long-polling intentionally left enabled (default) for the BGW runtime, -/// unlike backend sessions where it's disabled to save resources. -pub fn worker_provider_config() -> duroxide_pg_opt::ProviderConfig { - let mut config = duroxide_pg_opt::ProviderConfig::default(); - config.schema_name = Some(DUROXIDE_SCHEMA.to_string()); - config.migration_policy = duroxide_pg_opt::MigrationPolicy::ApplyAll; +/// schema ownership via `pg_depend` before calling +/// `PostgresProvider::new_with_schema_and_config`. +pub fn worker_provider_config() -> duroxide_pg::ProviderConfig { + let mut config = duroxide_pg::ProviderConfig::default(); + config.migration_policy = duroxide_pg::MigrationPolicy::ApplyAll; config } diff --git a/src/worker.rs b/src/worker.rs index 944ef135..80888c6b 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use std::time::Duration; use duroxide::runtime; -use duroxide_pg_opt::PostgresProvider; +use duroxide_pg::PostgresProvider; use tracing_subscriber::EnvFilter; use crate::registry::{create_activity_registry, create_orchestration_registry}; @@ -407,7 +407,7 @@ async fn initialize_duroxide_runtime( log!("pg_durable: initializing duroxide runtime..."); // Control duroxide provider pool size via env var (the only mechanism - // without modifying duroxide-pg-opt). BGW is single-threaded so no + // without modifying duroxide-pg). BGW is single-threaded so no // concurrent readers. Note: std::env::set_var becomes unsafe in Rust 2024 edition. std::env::set_var( "DUROXIDE_PG_POOL_MAX", @@ -457,18 +457,23 @@ async fn initialize_duroxide_runtime( } } - let store = - match PostgresProvider::new_with_config(pg_conn_str, worker_provider_config()).await { - Ok(s) => Arc::new(s), - Err(e) => { - log!( - "pg_durable: failed to create PostgreSQL store (will retry): {}", - e - ); - tokio::time::sleep(retry_interval).await; - continue; - } - }; + let store = match PostgresProvider::new_with_schema_and_config( + pg_conn_str, + Some(DUROXIDE_SCHEMA), + worker_provider_config(), + ) + .await + { + Ok(s) => Arc::new(s), + Err(e) => { + log!( + "pg_durable: failed to create PostgreSQL store (will retry): {}", + e + ); + tokio::time::sleep(retry_interval).await; + continue; + } + }; // Reuse the management pool for activities (graph loading, status updates). // The former dedicated activity pool with its df.in_workflow hook is no From e096af470277f0f7712ba0117974ba60be61f1ea Mon Sep 17 00:00:00 2001 From: Pino de Candia <32303022+pinodeca@users.noreply.github.com> Date: Wed, 20 May 2026 20:23:37 +0000 Subject: [PATCH 2/5] Adapt to duroxide-pg ProviderConfig API change new_with_schema_and_config was removed upstream in favor of new_with_config(ProviderConfig). ProviderConfig is now constructed via ProviderConfig::url(...) with schema_name and migration_policy fields set explicitly. - types.rs: backend_provider_config / worker_provider_config now take a database URL and build the full ProviderConfig (URL + schema + migration policy). - All call sites (worker, client, explain, monitoring, lib.rs test helpers) switched to PostgresProvider::new_with_config(...). - Drop now-unused DUROXIDE_SCHEMA imports in client/explain/monitoring. - cargo update -p duroxide-pg to pick up the new branch HEAD. --- Cargo.lock | 4 +-- src/client.rs | 12 +++----- src/explain.rs | 17 ++++------- src/lib.rs | 30 +++++------------- src/monitoring.rs | 77 ++++++++++++++++------------------------------- src/types.rs | 12 +++++--- src/worker.rs | 29 ++++++++---------- 7 files changed, 65 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad9d38df..7b9ced65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -707,7 +707,7 @@ dependencies = [ [[package]] name = "duroxide-pg" version = "0.1.33" -source = "git+https://github.com/pinodeca/duroxide-pg.git?branch=feat%2Fmigration-policy#715ec60abcd61c06be7c2de77b79a52e27a05f6a" +source = "git+https://github.com/pinodeca/duroxide-pg.git?branch=feat%2Fmigration-policy#d5d16192a0d09dfbc4c2c48c195f6dca33d0a9d2" dependencies = [ "anyhow", "async-trait", @@ -3098,7 +3098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/src/client.rs b/src/client.rs index 44adeb3b..7b3c1675 100644 --- a/src/client.rs +++ b/src/client.rs @@ -10,7 +10,7 @@ use duroxide_pg::PostgresProvider; use pgrx::prelude::*; use tokio::runtime::Runtime; -use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; +use crate::types::{backend_provider_config, postgres_connection_string}; /// Cached tokio runtime for client operations. static CLIENT_RUNTIME: OnceLock = OnceLock::new(); @@ -82,13 +82,9 @@ fn get_duroxide_client() -> Result<&'static Client, String> { std::env::set_var("DUROXIDE_PG_POOL_MAX", "1"); let store = Arc::new( - PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - .map_err(|e| format!("Failed to connect to duroxide store: {e}"))?, + PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)) + .await + .map_err(|e| format!("Failed to connect to duroxide store: {e}"))?, ); let _ = DUROXIDE_CLIENT.set(Client::new(store)); diff --git a/src/explain.rs b/src/explain.rs index 254e41c5..2dcc60df 100644 --- a/src/explain.rs +++ b/src/explain.rs @@ -121,7 +121,7 @@ fn explain_instance(instance_id: &str) -> String { /// Get instance info from Duroxide store fn get_duroxide_instance_info(instance_id: &str) -> (String, Option) { - use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; + use crate::types::{backend_provider_config, postgres_connection_string}; use duroxide::Client; use duroxide_pg::PostgresProvider; use std::sync::Arc; @@ -137,16 +137,11 @@ fn get_duroxide_instance_info(instance_id: &str) -> (String, Option) { }; rt.block_on(async { - let store = match PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - { - Ok(s) => Arc::new(s), - Err(_) => return (String::new(), None), - }; + let store = + match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { + Ok(s) => Arc::new(s), + Err(_) => return (String::new(), None), + }; let client = Client::new(store); diff --git a/src/lib.rs b/src/lib.rs index 98d7902a..1ce60da6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -809,16 +809,10 @@ mod tests { let start = Instant::now(); let timeout = Duration::from_secs(10); - let config = backend_provider_config(); + let config = backend_provider_config(&pg_conn_str); loop { - match PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - config.clone(), - ) - .await - { + match PostgresProvider::new_with_config(config.clone()).await { Ok(_) => return Ok(format!("{pg_conn_str} (schema: {DUROXIDE_SCHEMA})")), Err(e) => { if start.elapsed() > timeout { @@ -856,13 +850,9 @@ mod tests { rt.block_on(async { let store = Arc::new( - PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - .map_err(|e| format!("Failed to connect to store: {e}"))?, + PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)) + .await + .map_err(|e| format!("Failed to connect to store: {e}"))?, ); let client = Client::new(store); @@ -917,13 +907,9 @@ mod tests { rt.block_on(async { let store = Arc::new( - PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - .ok()?, + PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)) + .await + .ok()?, ); let client = Client::new(store); client diff --git a/src/monitoring.rs b/src/monitoring.rs index 66ba4d86..3d76b42d 100644 --- a/src/monitoring.rs +++ b/src/monitoring.rs @@ -6,7 +6,7 @@ use duroxide::Client; use pgrx::prelude::*; use std::sync::Arc; -use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; +use crate::types::{backend_provider_config, postgres_connection_string}; use duroxide_pg::PostgresProvider; // ============================================================================ @@ -81,16 +81,11 @@ pub fn list_instances( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = + match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { + Ok(s) => Arc::new(s), + Err(_) => return vec![], + }; let client = Client::new(store); @@ -170,16 +165,11 @@ pub fn instance_info( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = + match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { + Ok(s) => Arc::new(s), + Err(_) => return vec![], + }; let client = Client::new(store); @@ -240,16 +230,11 @@ pub fn instance_executions( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = + match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { + Ok(s) => Arc::new(s), + Err(_) => return vec![], + }; let client = Client::new(store); @@ -309,16 +294,11 @@ pub fn metrics() -> TableIterator< }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = + match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { + Ok(s) => Arc::new(s), + Err(_) => return vec![], + }; let client = Client::new(store); @@ -415,16 +395,11 @@ pub fn instance_nodes( }; let results = rt.block_on(async { - let store = match PostgresProvider::new_with_schema_and_config( - &pg_conn_str, - Some(DUROXIDE_SCHEMA), - backend_provider_config(), - ) - .await - { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = + match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { + Ok(s) => Arc::new(s), + Err(_) => return vec![], + }; let client = Client::new(store); diff --git a/src/types.rs b/src/types.rs index 1b290017..e10368e5 100644 --- a/src/types.rs +++ b/src/types.rs @@ -221,8 +221,9 @@ pub const DUROXIDE_SCHEMA: &str = "duroxide"; /// /// - `VerifyOnly`: never create schema/tables, reject unknown migrations. /// Backend sessions must not run DDL — the BGW owns schema lifecycle. -pub fn backend_provider_config() -> duroxide_pg::ProviderConfig { - let mut config = duroxide_pg::ProviderConfig::default(); +pub fn backend_provider_config(database_url: &str) -> duroxide_pg::ProviderConfig { + let mut config = duroxide_pg::ProviderConfig::url(database_url); + config.schema_name = Some(DUROXIDE_SCHEMA.to_string()); config.migration_policy = duroxide_pg::MigrationPolicy::VerifyOnly; config } @@ -232,9 +233,10 @@ pub fn backend_provider_config() -> duroxide_pg::ProviderConfig { /// - `ApplyAll`: applies pending duroxide migrations at startup; creates tables /// inside the extension-owned `duroxide` schema. Safe because the BGW verifies /// schema ownership via `pg_depend` before calling -/// `PostgresProvider::new_with_schema_and_config`. -pub fn worker_provider_config() -> duroxide_pg::ProviderConfig { - let mut config = duroxide_pg::ProviderConfig::default(); +/// `PostgresProvider::new_with_config`. +pub fn worker_provider_config(database_url: &str) -> duroxide_pg::ProviderConfig { + let mut config = duroxide_pg::ProviderConfig::url(database_url); + config.schema_name = Some(DUROXIDE_SCHEMA.to_string()); config.migration_policy = duroxide_pg::MigrationPolicy::ApplyAll; config } diff --git a/src/worker.rs b/src/worker.rs index 80888c6b..3a434c0b 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -457,23 +457,18 @@ async fn initialize_duroxide_runtime( } } - let store = match PostgresProvider::new_with_schema_and_config( - pg_conn_str, - Some(DUROXIDE_SCHEMA), - worker_provider_config(), - ) - .await - { - Ok(s) => Arc::new(s), - Err(e) => { - log!( - "pg_durable: failed to create PostgreSQL store (will retry): {}", - e - ); - tokio::time::sleep(retry_interval).await; - continue; - } - }; + let store = + match PostgresProvider::new_with_config(worker_provider_config(pg_conn_str)).await { + Ok(s) => Arc::new(s), + Err(e) => { + log!( + "pg_durable: failed to create PostgreSQL store (will retry): {}", + e + ); + tokio::time::sleep(retry_interval).await; + continue; + } + }; // Reuse the management pool for activities (graph loading, status updates). // The former dedicated activity pool with its df.in_workflow hook is no From a98e0ea42a2ffac25838d75da0410604094d94f0 Mon Sep 17 00:00:00 2001 From: Pino de Candia <32303022+pinodeca@users.noreply.github.com> Date: Tue, 26 May 2026 13:52:14 +0000 Subject: [PATCH 3/5] Use crates.io duroxide-pg release --- .github/copilot-instructions.md | 6 +++--- Cargo.lock | 5 +++-- Cargo.toml | 5 +---- prompts/pg_durable-check-upstream-fixes.md | 9 ++++++--- prompts/pg_durable-release.md | 18 ++++++++++++------ 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index fdaeed9a..dffd867e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -94,15 +94,15 @@ The new `.so` must work against **all** previous versions' schemas (same major v For Scenario A, treat the upgrade path as the contract for already-shipped versions: before release, fresh install for the new version should match what an existing customer gets by installing the previous version and applying the upgrade chain. -**Updating duroxide-pg dependency**: Update the `tag = "..."` value for `duroxide-pg` in [`Cargo.toml`](../Cargo.toml), then `cargo update -p duroxide-pg` and rebuild. The BGW's embedded migration files update automatically via `include_dir!`. No changes to extension SQL, upgrade scripts, or any checked-in SQL copies are needed. +**Updating duroxide-pg dependency**: Treat `duroxide` and `duroxide-pg` as a compatible pair. Before changing `duroxide-pg`, check the `duroxide-pg` release notes or compatibility matrix to determine whether `duroxide` must also be updated. Update the crates.io version(s) in [`Cargo.toml`](../Cargo.toml), then run `cargo update -p duroxide-pg` or `cargo update -p duroxide -p duroxide-pg` as appropriate and rebuild. The BGW's embedded migration files update automatically via `include_dir!`. No changes to extension SQL, upgrade scripts, or any checked-in SQL copies are needed. **Writing a spec or design doc:** Include an "Upgrade & Migration" section covering: backward compatibility impact (B1 — will the new `.so` work against all previous schemas?), upgrade script DDL needed, and any runtime schema detection required. See [docs/upgrade-testing.md](../docs/upgrade-testing.md) for the full upgrade testing strategy. ## Dependencies - **pgrx 0.16.1**: PostgreSQL extension framework (pinned version) -- **duroxide**: Durable execution runtime -- **duroxide-pg**: duroxide provider/stores engine state in PostgreSQL (cargo git dependency, pinned by tag in [`Cargo.toml`](../Cargo.toml)) +- **duroxide**: Durable execution runtime (crates.io dependency pinned in [`Cargo.toml`](../Cargo.toml)) +- **duroxide-pg**: duroxide provider/stores engine state in PostgreSQL (crates.io dependency pinned in [`Cargo.toml`](../Cargo.toml)); keep pinned with `duroxide` as a compatible pair - **sqlx**: Async PostgreSQL from background worker - **tokio**: Async runtime for background worker diff --git a/Cargo.lock b/Cargo.lock index 7b9ced65..7a1e07bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -706,8 +706,9 @@ dependencies = [ [[package]] name = "duroxide-pg" -version = "0.1.33" -source = "git+https://github.com/pinodeca/duroxide-pg.git?branch=feat%2Fmigration-policy#d5d16192a0d09dfbc4c2c48c195f6dca33d0a9d2" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd3d51000bd6b0433be07bc2f4ba5dd7d8fc4336fc90ff2321f527cd51a3eac" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b44d43f5..6522c488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,10 +35,7 @@ uuid = { version = "1.0", features = ["v4", "serde"] } # duroxide integration duroxide = "0.1.29" -# TEMP: pinned to the fork branch backing https://github.com/microsoft/duroxide-pg/pull/10. -# Revert to the git+tag form below once the PR merges and a tag is published upstream. -duroxide-pg = { git = "https://github.com/pinodeca/duroxide-pg.git", branch = "feat/migration-policy", package = "duroxide-pg" } -# duroxide-pg = { git = "https://github.com/microsoft/duroxide-pg.git", tag = "v0.1.33", package = "duroxide-pg" } +duroxide-pg = "0.1.34" tokio = { version = "1", features = ["rt-multi-thread", "sync", "time"] } # For ExecuteSQL activity (connecting back to PostgreSQL) diff --git a/prompts/pg_durable-check-upstream-fixes.md b/prompts/pg_durable-check-upstream-fixes.md index 3e379c81..88a630af 100644 --- a/prompts/pg_durable-check-upstream-fixes.md +++ b/prompts/pg_durable-check-upstream-fixes.md @@ -48,13 +48,16 @@ Confirm the fix is mentioned in the release notes. If a fix is available in a new release: -1. **Bump the `duroxide-pg` tag** in `Cargo.toml` to the new version, then refresh the lockfile: +1. **Check duroxide compatibility** before bumping `duroxide-pg`. Use the release notes or compatibility matrix to determine whether `duroxide` must also be updated, then refresh the lockfile for the package set you changed: ```bash - # Edit Cargo.toml: duroxide-pg = { git = "...", tag = "v0.1.X", package = "duroxide-pg" } + # Edit Cargo.toml: duroxide-pg = "0.1.X" + # If required, also edit Cargo.toml: duroxide = "0.1.Y" cargo update -p duroxide-pg + # Or, when both changed: + cargo update -p duroxide -p duroxide-pg ``` -2. **Also update duroxide if needed** (check compatibility): +2. **If duroxide also needs to be updated**: ```toml duroxide = "0.1.X" ``` diff --git a/prompts/pg_durable-release.md b/prompts/pg_durable-release.md index 090fcda0..45c686d3 100644 --- a/prompts/pg_durable-release.md +++ b/prompts/pg_durable-release.md @@ -9,7 +9,7 @@ Prepare and release a new version of pg_durable with quality checks, documentati The project uses these duroxide dependencies from `Cargo.toml`: - `duroxide` (crates.io version) -- `duroxide-pg` (GitHub tag) +- `duroxide-pg` (crates.io version) **Check for new duroxide version:** ```bash @@ -20,12 +20,14 @@ grep "duroxide" Cargo.toml cargo search duroxide --limit 5 ``` -**Check for new duroxide-pg tag:** +**Check for new duroxide-pg version:** ```bash -# List recent tags from the duroxide-pg repo -gh api repos/microsoft/duroxide-pg/tags --jq '.[].name' | head -10 +# Check latest version on crates.io +cargo search duroxide-pg --limit 5 ``` +Before proposing a `duroxide-pg` update, check its release notes or compatibility matrix to determine whether the `duroxide` crate must also be updated. Treat the two versions as a compatible pair, not independent choices. + ### 1.2 Ask User About Updates If new versions are available, present them to the user: @@ -39,7 +41,7 @@ Current versions: New versions available: - duroxide: [new_version] ✨ - - duroxide-pg: [new_tag] ✨ + - duroxide-pg: [new_version] ✨ Would you like to update to the new versions? (y/n) ``` @@ -49,11 +51,15 @@ Would you like to update to the new versions? (y/n) Update the dependency versions in `Cargo.toml`: ```toml # duroxide = "NEW_VERSION" -# duroxide-pg = { git = "...", tag = "NEW_TAG", package = "duroxide-pg" } +# duroxide-pg = "NEW_VERSION" ``` Then refresh `Cargo.lock`: ```bash +# If only duroxide-pg changed: +cargo update -p duroxide-pg + +# If both changed: cargo update -p duroxide -p duroxide-pg ``` From 591a9706f66244961961edce872c420543b59f83 Mon Sep 17 00:00:00 2001 From: Pino de Candia <32303022+pinodeca@users.noreply.github.com> Date: Tue, 26 May 2026 16:43:19 +0000 Subject: [PATCH 4/5] Clean up duroxide-pg migration follow-ups --- .devcontainer/onCreateCommand.sh | 2 +- Cargo.toml | 4 +- TODO.md | 2 +- docs/CODESPACES_PREBUILDS.md | 10 ++-- docs/bgw-applies-migrations.md | 2 +- docs/extension_lifecycle.md | 23 +++------ docs/security-review/ThreatModelDFD.md | 2 +- .../threat-model.dfd-lite.yaml | 4 +- docs/upgrade-testing.md | 10 +++- src/client.rs | 11 ++--- src/explain.rs | 13 ++--- src/lib.rs | 26 +++------- src/monitoring.rs | 49 ++++++++----------- src/types.rs | 11 +++++ 14 files changed, 73 insertions(+), 96 deletions(-) diff --git a/.devcontainer/onCreateCommand.sh b/.devcontainer/onCreateCommand.sh index 2a301fdc..22e9488e 100755 --- a/.devcontainer/onCreateCommand.sh +++ b/.devcontainer/onCreateCommand.sh @@ -57,7 +57,7 @@ else fi # ── Build pg_durable ──────────────────────────────────────────────── -# duroxide-pg is pulled as a cargo git dependency (see Cargo.toml). +# duroxide-pg is pulled as a crates.io dependency (see Cargo.toml). echo "Building pg_durable..." if [ "$SMOKE_MODE" = "1" ]; then echo "Smoke mode: skipping cargo build" diff --git a/Cargo.toml b/Cargo.toml index 6522c488..cb769e6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,8 +34,8 @@ serde_json = { version = "1.0", features = ["preserve_order"] } uuid = { version = "1.0", features = ["v4", "serde"] } # duroxide integration -duroxide = "0.1.29" -duroxide-pg = "0.1.34" +duroxide = "=0.1.29" +duroxide-pg = "=0.1.34" tokio = { version = "1", features = ["rt-multi-thread", "sync", "time"] } # For ExecuteSQL activity (connecting back to PostgreSQL) diff --git a/TODO.md b/TODO.md index ae2a15b2..dc6e7432 100644 --- a/TODO.md +++ b/TODO.md @@ -23,7 +23,7 @@ - figure out the right security model with least possible priveleges - resource constraining the duroxide runtime - variable logging/tracing levels? -- update to long polling PG provider +- evaluate provider work-dispatch latency under sustained load - error handling stratgy, impl and tests - rename ExecuteWorkflow orchestration to DurableFunction, add a version to list_instances. - think through SQL error handling in details diff --git a/docs/CODESPACES_PREBUILDS.md b/docs/CODESPACES_PREBUILDS.md index 77018805..d9ebe408 100644 --- a/docs/CODESPACES_PREBUILDS.md +++ b/docs/CODESPACES_PREBUILDS.md @@ -19,11 +19,9 @@ Pre-builds must be enabled by a repository administrator: - **Reduce prebuild available to specific regions**: Optional 4. Click **Create** -### Submodule Access +### Provider Dependencies -The `duroxide-pg` submodule is a **public repository**, so no PAT or Codespaces secret is required for prebuilds or interactive Codespaces. The default Codespaces credentials are sufficient for `git submodule update --init --recursive`. - -> If you maintain a fork that points the submodule at a private alternative (for example `microsoft/duroxide-pg-opt`), you will need to add your own auth (Codespaces secret + `insteadOf` rewrite, or repository permission in `devcontainer.json`). +`duroxide` and `duroxide-pg` are crates.io dependencies. No provider checkout, provider PAT, or extra Codespaces repository permission is required. ## How It Works @@ -39,7 +37,7 @@ Codespaces has two distinct phases: - System dependencies (libssl, clang, bison, etc.) - cargo-pgrx 0.16.1 - PostgreSQL 17 (downloaded and compiled via pgrx) - - `duroxide-pg` submodule (public, no auth required) + - Rust dependencies from crates.io - Builds and installs pg_durable - Recreates the local `~/.pgrx/data-17` cluster with `initdb -U postgres` - Pre-creates the `pg_durable` extension and verifies it @@ -131,7 +129,7 @@ This means the prebuild did not run or failed. There is no automatic fallback ### User Can See `GH_PAT` In Their Codespace Environment -No longer applicable — the `duroxide-pg` submodule is public and `GH_PAT` is no longer used by the prebuild. If you see a `GH_PAT` secret configured at the repo level, you can safely remove it. +No longer applicable — provider dependencies come from crates.io and `GH_PAT` is not used by the prebuild. If you see a `GH_PAT` secret configured at the repo level, you can safely remove it. ## Cost Considerations diff --git a/docs/bgw-applies-migrations.md b/docs/bgw-applies-migrations.md index 8e69f278..2f5d6b63 100644 --- a/docs/bgw-applies-migrations.md +++ b/docs/bgw-applies-migrations.md @@ -217,7 +217,7 @@ Call `wait_for_ready` after any `CREATE EXTENSION` in scenarios that subsequentl | `docs/upgrade-testing.md` | Add duroxide ownership entry to the v0.1.1→v0.2.0 version-specific changes section. Note that both paths converge (objects not extension-owned after BGW runs). Note `wait_for_ready()` requirement in upgrade test infrastructure. | | `USER_GUIDE.md` | Add note that `DROP EXTENSION pg_durable CASCADE` is always required. Update readiness polling to use `duroxide._worker_ready` directly. | -The `duroxide-pg/` submodule and `submodules: true` in CI remain — the submodule is still a Rust code dependency. +`duroxide-pg` is a crates.io dependency. No extra provider checkout or CI repository-recursion configuration is required. ## What this enables going forward diff --git a/docs/extension_lifecycle.md b/docs/extension_lifecycle.md index 6f5213b3..5deaa459 100644 --- a/docs/extension_lifecycle.md +++ b/docs/extension_lifecycle.md @@ -2,7 +2,7 @@ **Status:** Implemented **Last Updated:** 2026-03-01 -**Dependencies:** duroxide-pg (git submodule) +**Dependencies:** duroxide-pg (crates.io) > **Note:** This document describes the extension lifecycle management, background worker behavior, and duroxide-pg schema integration in pg_durable. @@ -99,7 +99,7 @@ fn get_duroxide_client() -> Result<&'static Client, String> { The duroxide provider tables, functions, indexes, and triggers are **not** created by extension SQL. Instead, the background worker (BGW) populates the `duroxide` schema at startup via `MigrationPolicy::ApplyAll` (see section 2). This decouples the duroxide engine schema from the extension lifecycle: -- Duroxide-pg-opt upgrades require no changes to extension SQL or upgrade scripts — the BGW applies new migrations automatically. +- duroxide-pg dependency upgrades require no changes to extension SQL or upgrade scripts — the BGW applies new migrations automatically. - The duroxide schema can evolve independently of pg_durable releases. - Swapping the duroxide provider only requires changes to BGW initialization code, not extension DDL. @@ -187,7 +187,7 @@ All backend sessions (`df.*` functions) continue to use `MigrationPolicy::Verify See `src/worker.rs` for the full implementation. The main loop in `run_duroxide_runtime()` drives the state machine: 1. `wait_for_extension_creation()` polls `pg_extension` via a reusable `sqlx::PgPool` (max 1 connection) every 5 seconds. -2. `initialize_duroxide_runtime()` first checks that the `duroxide` schema is owned by the `pg_durable` extension (via `pg_depend`). If not, it logs a warning and retries after the poll interval. It then releases extension ownership of any duroxide objects (no-op on fresh installs; needed for upgrades from ≤0.1.1). Then it creates a `PostgresProvider` using `worker_provider_config()` (from `src/types.rs`) which sets `ApplyAll`, with long-polling intentionally left enabled for work dispatch. Unknown-migration rejection is always enforced unconditionally by the provider. +2. `initialize_duroxide_runtime()` first checks that the `duroxide` schema is owned by the `pg_durable` extension (via `pg_depend`). If not, it logs a warning and retries after the poll interval. It then releases extension ownership of any duroxide objects (no-op on fresh installs; needed for upgrades from ≤0.1.1). Then it creates a `PostgresProvider` using `worker_provider_config()` (from `src/types.rs`) which sets `ApplyAll`. Unknown-migration rejection is always enforced unconditionally by the provider. 3. After the runtime starts, the BGW writes a **readiness record** (`duroxide._worker_ready`) with the current `WORKER_SCHEMA_VERSION`, then writes an **epoch sentinel** (`df._worker_epoch`) with a fresh UUID. The readiness record signals backend sessions that the duroxide schema is fully initialized; the epoch sentinel detects drop+recreate scenarios (see below). 4. `run_until_extension_dropped_or_shutdown()` uses `tokio::select!` to interleave shutdown checks (every 1 second, via direct volatile read) with epoch-sentinel checks (every 5 seconds via the shared polling pool). @@ -211,7 +211,7 @@ Key design choices vs. the original sketch: - **Epoch sentinel**: Eliminates the drop+recreate blind spot; no need for explicit sleeps in tests between DROP and CREATE EXTENSION. - **Reusable polling pool**: A single `PgPool(max_connections=1)` is created once and shared across all polling calls, avoiding the overhead of opening/closing a TCP connection on every poll. - **Direct shutdown check**: `is_shutdown_requested()` reads a volatile atomic and does not need `spawn_blocking`; the check runs every 1 second rather than every 100ms. -- **Config helpers**: `worker_provider_config()` and `backend_provider_config()` in `src/types.rs` centralize `ProviderConfig` construction, eliminating duplication across ~10 call sites. +- **Config helpers**: `worker_provider_config()`, `backend_provider_config()`, and `new_backend_provider()` in `src/types.rs` centralize provider construction and lifecycle policy choices. ### 4. Prevent df Functions from Triggering Schema Creation @@ -219,13 +219,10 @@ Key design choices vs. the original sketch: **Implemented:** all call sites use `MigrationPolicy::VerifyOnly`, so they will not execute DDL. -**Implemented:** request/response-style backend calls disable Duroxide long-polling to avoid a dedicated listener connection per backend. - #### Client Functions (src/client.rs) -All backend call sites (client, monitoring, explain) use `backend_provider_config()` from `src/types.rs`, which sets: +All backend call sites (client, monitoring, explain) use `new_backend_provider()` from `src/types.rs`, which applies `backend_provider_config()` settings: - `VerifyOnly`: never create schema/tables -- `long_poll.enabled = false`: avoid dedicated listener connection per backend session Unknown-migration rejection is enforced unconditionally by the provider (not a config flag). @@ -236,13 +233,12 @@ See `src/client.rs::get_duroxide_client()` for the cached-client implementation. **Rationale:** - (Future) Check extension existence before attempting duroxide connection - Use `VerifyOnly` policy to ensure no schema creation from client code -- Disable long-polling for backend request/response operations to save a dedicated listener connection per backend - Fail with clear, actionable error messages for different failure scenarios - Note on caching: after `DROP EXTENSION`, the `df.*` entrypoints disappear, so cached clients are not reachable through SQL. However, if the extension is later re-created while a backend session remains alive, a previously cached client may be stale; this can be handled later by recreating the client on specific errors. #### Monitoring Functions (src/monitoring.rs, src/explain.rs) -Apply the same pattern: use `backend_provider_config()` from `src/types.rs`. See `src/monitoring.rs` and `src/explain.rs` for the full implementations. +Apply the same pattern: use `new_backend_provider()` from `src/types.rs`. See `src/monitoring.rs` and `src/explain.rs` for the full implementations. **Rationale:** - Same benefits as client functions: no schema creation @@ -345,7 +341,6 @@ Because the Duroxide schema DDL runs inside `CREATE EXTENSION` as extension SQL, 1. **ProviderConfig integration** - Test that `MigrationPolicy::VerifyOnly` correctly errors when schema missing - Test that `MigrationPolicy::VerifyOnly` succeeds when schema exists and is current - - Test that disabling long-polling in backend sessions doesn't create PgListener connections ### E2E Tests @@ -448,12 +443,6 @@ The duroxide schema is extension-owned but its contents are BGW-managed: 2. **Background worker wait behavior** - **New behavior:** BGW waits for extension existence, and also stays in "waiting" when migrations are missing/behind (VerifyOnly fails). -3. **Backend sessions disable long-polling** - - **Default behavior (upstream):** `duroxide_pg::PostgresProvider` can enable long-polling by default. - - **pg_durable behavior:** For backend request/response operations (start/cancel/signal, monitoring), we disable long-polling to avoid a dedicated listener connection and notifier task. - - **Impact:** Resource savings for installations with many backends. - - **Compatibility:** No user-visible changes expected. - ## Open Questions ### Resolved diff --git a/docs/security-review/ThreatModelDFD.md b/docs/security-review/ThreatModelDFD.md index 9f9cfd99..670825aa 100644 --- a/docs/security-review/ThreatModelDFD.md +++ b/docs/security-review/ThreatModelDFD.md @@ -250,7 +250,7 @@ P-WORKER ──[sqlx pool (TCP localhost)]──> DS-DUROXIDE |---|---|---|---| | **T** Tampering | Poisoned work items cause code execution | Worker reads from duroxide tables it owns; data provenance is trusted | ✅ Mitigated | | **I** Information Disclosure | Worker role exposes all duroxide state | Worker role is superuser — acceptable for single-tenant; duroxide schema not granted to users | ✅ Mitigated | -| **D** Denial of Service | Large backlog starves worker connections | Fixed pool of 5 connections; long-poll reduces overhead | ⚠️ Partial | +| **D** Denial of Service | Large backlog starves worker connections | Fixed worker connection pool limits concurrent database work | ⚠️ Partial | ### DF-6: Graph Loading diff --git a/docs/security-review/threat-model.dfd-lite.yaml b/docs/security-review/threat-model.dfd-lite.yaml index e9388b3c..ff2a94b7 100644 --- a/docs/security-review/threat-model.dfd-lite.yaml +++ b/docs/security-review/threat-model.dfd-lite.yaml @@ -198,8 +198,8 @@ diagrams: to: "DS-DUROXIDE" description: > Background worker polls duroxide tables for pending orchestration - and activity work items. Uses long-polling with sqlx pool - (5 connections as worker role). + and activity work items using the worker sqlx pool (5 connections + as worker role). properties: Source Authenticated: "Yes" Provides Integrity: "Yes" diff --git a/docs/upgrade-testing.md b/docs/upgrade-testing.md index 64e1666d..cd5d49cc 100644 --- a/docs/upgrade-testing.md +++ b/docs/upgrade-testing.md @@ -235,7 +235,15 @@ what the upgrade script handles, and any backward compatibility considerations. - **Scenario A considerations:** The `df` schema equivalence contract is unchanged. The `duroxide` schema is excluded from snapshot diffs — fresh installs start with an empty `duroxide` schema (BGW fills it in at runtime) while upgrades carry forward the fully-populated schema from v0.1.1. This is expected and acceptable. - **Scenario B1 considerations:** The BGW uses `MigrationPolicy::ApplyAll`. A database that has only migrations 0001–0005 is handled gracefully: the BGW detects the gap and applies 0006–0010 at startup. No manual intervention is needed. - **Scenario B2 considerations:** All five new migrations are additive (new tables and columns with defaults or nullable). Existing `df.vars`, `df.nodes`, `df.instances`, and `df.graphs` data is untouched. -- **Current status:** Implemented — submodule at `4a6bf6b`, `Cargo.toml` pinned to `duroxide = "=0.1.26"`. +- **Historical status:** Implemented with the provider source at `4a6bf6b` and `Cargo.toml` pinned to `duroxide = "=0.1.26"`. + +#### Switch to crates.io duroxide-pg 0.1.34 + duroxide 0.1.29 +- **DDL change (df schema):** None. This is a provider source and version update only. +- **DDL change (duroxide schema):** No extension upgrade script DDL is required. The BGW continues to own provider migrations through `MigrationPolicy::ApplyAll`. +- **Scenario A considerations:** The `df` schema equivalence contract is unchanged. The `duroxide` schema remains excluded from snapshot diffs because it is populated at runtime by the BGW. +- **Scenario B1 considerations:** The new `.so` must continue to initialize against older provider schemas; any missing provider migrations are applied by the BGW at startup. +- **Scenario B2 considerations:** No user data migration is needed. Existing `df.*` metadata and workflow state are unaffected by changing the provider dependency source from the local provider tree to crates.io. +- **Current status:** Implemented — `Cargo.toml` exactly pins `duroxide = "=0.1.29"` and `duroxide-pg = "=0.1.34"`. #### Named Results v2 — df.if_rows - **DDL change:** Upgrade script adds `CREATE FUNCTION df.if_rows(result_name text, then_branch text, else_branch text)` — a new C-language function backed by the pgrx `#[pg_extern]` `if_rows_fn_wrapper` symbol. diff --git a/src/client.rs b/src/client.rs index 7b3c1675..6be29d72 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3,14 +3,13 @@ //! This module provides cached Tokio runtime and Duroxide client for efficient //! df.start(), df.signal(), and df.cancel() calls from user sessions. -use std::sync::{Arc, OnceLock}; +use std::sync::OnceLock; use duroxide::Client; -use duroxide_pg::PostgresProvider; use pgrx::prelude::*; use tokio::runtime::Runtime; -use crate::types::{backend_provider_config, postgres_connection_string}; +use crate::types::{new_backend_provider, postgres_connection_string}; /// Cached tokio runtime for client operations. static CLIENT_RUNTIME: OnceLock = OnceLock::new(); @@ -81,11 +80,7 @@ fn get_duroxide_client() -> Result<&'static Client, String> { // (new_current_thread). Note: std::env::set_var becomes unsafe in Rust 2024 edition. std::env::set_var("DUROXIDE_PG_POOL_MAX", "1"); - let store = Arc::new( - PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)) - .await - .map_err(|e| format!("Failed to connect to duroxide store: {e}"))?, - ); + let store = new_backend_provider(&pg_conn_str).await?; let _ = DUROXIDE_CLIENT.set(Client::new(store)); DUROXIDE_CLIENT diff --git a/src/explain.rs b/src/explain.rs index 2dcc60df..ad974cb6 100644 --- a/src/explain.rs +++ b/src/explain.rs @@ -121,10 +121,8 @@ fn explain_instance(instance_id: &str) -> String { /// Get instance info from Duroxide store fn get_duroxide_instance_info(instance_id: &str) -> (String, Option) { - use crate::types::{backend_provider_config, postgres_connection_string}; + use crate::types::{new_backend_provider, postgres_connection_string}; use duroxide::Client; - use duroxide_pg::PostgresProvider; - use std::sync::Arc; let pg_conn_str = postgres_connection_string(); @@ -137,11 +135,10 @@ fn get_duroxide_instance_info(instance_id: &str) -> (String, Option) { }; rt.block_on(async { - let store = - match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { - Ok(s) => Arc::new(s), - Err(_) => return (String::new(), None), - }; + let store = match new_backend_provider(&pg_conn_str).await { + Ok(s) => s, + Err(_) => return (String::new(), None), + }; let client = Client::new(store); diff --git a/src/lib.rs b/src/lib.rs index 1ce60da6..7042064e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -785,7 +785,6 @@ CREATE OPERATOR @> ( mod tests { use crate::Durofut; use pgrx::prelude::*; - use std::sync::Arc; // ======================================================================== // Test Helpers for Integration Tests @@ -793,8 +792,7 @@ mod tests { /// Ensure the Duroxide store exists and is ready fn ensure_store_ready() -> Result { - use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; - use duroxide_pg::PostgresProvider; + use crate::types::{new_backend_provider, postgres_connection_string, DUROXIDE_SCHEMA}; use std::time::{Duration, Instant}; let pg_conn_str = postgres_connection_string(); @@ -809,10 +807,8 @@ mod tests { let start = Instant::now(); let timeout = Duration::from_secs(10); - let config = backend_provider_config(&pg_conn_str); - loop { - match PostgresProvider::new_with_config(config.clone()).await { + match new_backend_provider(&pg_conn_str).await { Ok(_) => return Ok(format!("{pg_conn_str} (schema: {DUROXIDE_SCHEMA})")), Err(e) => { if start.elapsed() > timeout { @@ -831,9 +827,8 @@ mod tests { /// Wait for a durable function to complete, polling Duroxide status fn wait_for_completion(instance_id: &str, timeout_secs: u64) -> Result { - use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; + use crate::types::{new_backend_provider, postgres_connection_string, DUROXIDE_SCHEMA}; use duroxide::Client; - use duroxide_pg::PostgresProvider; use std::time::{Duration, Instant}; // Ensure store is ready first @@ -849,11 +844,7 @@ mod tests { .map_err(|e| format!("Failed to create runtime: {e}"))?; rt.block_on(async { - let store = Arc::new( - PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)) - .await - .map_err(|e| format!("Failed to connect to store: {e}"))?, - ); + let store = new_backend_provider(&pg_conn_str).await?; let client = Client::new(store); loop { @@ -893,9 +884,8 @@ mod tests { /// Get the current status from Duroxide fn get_duroxide_status(instance_id: &str) -> Option { - use crate::types::{backend_provider_config, postgres_connection_string, DUROXIDE_SCHEMA}; + use crate::types::{new_backend_provider, postgres_connection_string, DUROXIDE_SCHEMA}; use duroxide::Client; - use duroxide_pg::PostgresProvider; let _ = ensure_store_ready().ok()?; let pg_conn_str = postgres_connection_string(); @@ -906,11 +896,7 @@ mod tests { .ok()?; rt.block_on(async { - let store = Arc::new( - PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)) - .await - .ok()?, - ); + let store = new_backend_provider(&pg_conn_str).await.ok()?; let client = Client::new(store); client .get_instance_info(instance_id) diff --git a/src/monitoring.rs b/src/monitoring.rs index 3d76b42d..fabd7599 100644 --- a/src/monitoring.rs +++ b/src/monitoring.rs @@ -4,10 +4,8 @@ use duroxide::Client; use pgrx::prelude::*; -use std::sync::Arc; -use crate::types::{backend_provider_config, postgres_connection_string}; -use duroxide_pg::PostgresProvider; +use crate::types::{new_backend_provider, postgres_connection_string}; // ============================================================================ // Monitoring Functions @@ -81,11 +79,10 @@ pub fn list_instances( }; let results = rt.block_on(async { - let store = - match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = match new_backend_provider(&pg_conn_str).await { + Ok(s) => s, + Err(_) => return vec![], + }; let client = Client::new(store); @@ -165,11 +162,10 @@ pub fn instance_info( }; let results = rt.block_on(async { - let store = - match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = match new_backend_provider(&pg_conn_str).await { + Ok(s) => s, + Err(_) => return vec![], + }; let client = Client::new(store); @@ -230,11 +226,10 @@ pub fn instance_executions( }; let results = rt.block_on(async { - let store = - match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = match new_backend_provider(&pg_conn_str).await { + Ok(s) => s, + Err(_) => return vec![], + }; let client = Client::new(store); @@ -294,11 +289,10 @@ pub fn metrics() -> TableIterator< }; let results = rt.block_on(async { - let store = - match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = match new_backend_provider(&pg_conn_str).await { + Ok(s) => s, + Err(_) => return vec![], + }; let client = Client::new(store); @@ -395,11 +389,10 @@ pub fn instance_nodes( }; let results = rt.block_on(async { - let store = - match PostgresProvider::new_with_config(backend_provider_config(&pg_conn_str)).await { - Ok(s) => Arc::new(s), - Err(_) => return vec![], - }; + let store = match new_backend_provider(&pg_conn_str).await { + Ok(s) => s, + Err(_) => return vec![], + }; let client = Client::new(store); diff --git a/src/types.rs b/src/types.rs index e10368e5..ec0a7179 100644 --- a/src/types.rs +++ b/src/types.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::ffi::CString; use std::str::FromStr; +use std::sync::Arc; use std::time::Duration; use uuid::Uuid; @@ -228,6 +229,16 @@ pub fn backend_provider_config(database_url: &str) -> duroxide_pg::ProviderConfi config } +/// Create a backend provider for request/response operations. +pub async fn new_backend_provider( + database_url: &str, +) -> Result, String> { + duroxide_pg::PostgresProvider::new_with_config(backend_provider_config(database_url)) + .await + .map(Arc::new) + .map_err(|e| format!("Failed to connect to duroxide store: {e}")) +} + /// Create a `ProviderConfig` for the background worker runtime. /// /// - `ApplyAll`: applies pending duroxide migrations at startup; creates tables From 5c1e54633ee513c604d4fe393aadd14bd7bb78e3 Mon Sep 17 00:00:00 2001 From: Pino de Candia <32303022+pinodeca@users.noreply.github.com> Date: Tue, 26 May 2026 17:22:10 +0000 Subject: [PATCH 5/5] Handle provider-line upgrade test boundary --- docs/upgrade-testing.md | 52 +++++++++++++++++++---------------- scripts/test-upgrade.sh | 60 ++++++++++++++++++++++++++++++++--------- 2 files changed, 77 insertions(+), 35 deletions(-) diff --git a/docs/upgrade-testing.md b/docs/upgrade-testing.md index cd5d49cc..567b831d 100644 --- a/docs/upgrade-testing.md +++ b/docs/upgrade-testing.md @@ -9,27 +9,31 @@ pg_durable follows a two-phase upgrade model: This means the new `.so` **must be backward compatible** with the previous version's schema. The `.so` and the upgrade script are not atomic — there is an extended period where the new binary runs against the old schema. +Compatibility is scoped to a **provider compatibility line**. A provider line is the set of pg_durable versions that use the same durable-state provider family and are expected to upgrade in place. The open-source line starts at v0.2.2, where pg_durable switches from `duroxide-pg-opt` to crates.io `duroxide-pg`. Versions before v0.2.2 used `duroxide-pg-opt`; they are not upgrade sources for the `duroxide-pg` line because the provider schemas and runtime state are different. Azure's fork owns upgrade testing for the `duroxide-pg-opt` line. + We never downgrade. Downgrade scripts are not needed. ## Test Scenarios ### Chain tests vs. direct-contact tests -Scenarios A and B2 are **chain tests**: PostgreSQL applies upgrade scripts sequentially (v0.1.1→v0.2.0→v0.3.0), so each step is validated transitively by its own version's CI. Testing the current upgrade script against the immediately previous version is sufficient. +Scenarios A and B2 are **chain tests**: PostgreSQL applies upgrade scripts sequentially (v0.2.2→v0.2.3→v0.3.0 within the current provider line), so each step is validated transitively by its own version's CI. Testing the current upgrade script against the immediately previous compatible version is sufficient. + +Scenario B1 is a **direct-contact test**: the `.so` faces whatever raw schema the customer has, with no intermediate transformation. There is no chain — a customer on v0.2.2 who receives the v0.5.0 binary without ever upgrading has a v0.2.2 schema with a v0.5.0 `.so`. That's why B1 must test against all previous compatible versions in the same provider line. -Scenario B1 is a **direct-contact test**: the `.so` faces whatever raw schema the customer has, with no intermediate transformation. There is no chain — a customer on v0.1.1 who receives the v0.5.0 binary without ever upgrading has a v0.1.1 schema with a v0.5.0 `.so`. That's why B1 must test against all previous versions. +### Compatibility boundaries -### Major version boundaries +All three scenarios scope to versions within the same provider compatibility line. A provider-line boundary is stronger than a major-version boundary: the new `.so` does not need to execute against provider state from another line, and the upgrade tests should not treat that crossing as a required customer path. -All three scenarios scope to versions within the same major version: -- **B1**: A major version bump is the boundary where binary backward compatibility may be dropped. The new `.so` does not need to work with schemas from a previous major version. -- **A and B2**: Upgrade scripts still need to work across a major version bump — customers must be able to upgrade their schema. However, the transitive chain property means testing only the immediately previous version is still sufficient. +- **B1**: Tests all previous schemas in the current provider line. It skips versions before `PROVIDER_COMPAT_START_VERSION`. +- **A and B2**: Test the immediately previous version only when that previous version is in the current provider line. If the current version is the first version in a provider line, A and B2 are skipped because there is no valid previous upgrade source for that line. +- **Major versions**: A major version bump can still be used as a compatibility boundary. When no provider-line split is involved, the previous same-major rules continue to apply. ### Scenario A: Schema Upgrade Correctness **Goal:** Verify that `ALTER EXTENSION UPDATE` produces an identical schema to a fresh `CREATE EXTENSION`. -**Contract:** For a not-yet-released version, the fresh-install schema is expected to match what an existing customer would get by starting from the immediately previous shipped version and applying the shipped upgrade chain to the new version. In other words, Scenario A treats the upgrade result as the reference shape for already-shipped versions. If fresh install and upgrade differ before release, prefer aligning the new version's fresh-install DDL with the upgrade path unless there is a deliberate reason to change the contract. +**Contract:** For a not-yet-released version, the fresh-install schema is expected to match what an existing customer would get by starting from the immediately previous compatible shipped version and applying the shipped upgrade chain to the new version. In other words, Scenario A treats the upgrade result as the reference shape for already-shipped versions in the current provider line. If fresh install and upgrade differ before release, prefer aligning the new version's fresh-install DDL with the upgrade path unless there is a deliberate reason to change the contract. **Method:** 1. Install current `.so` and all upgrade SQL files @@ -42,21 +46,21 @@ All three scenarios scope to versions within the same major version: - Wrong column types, defaults, or constraint names - Ordering issues in upgrade SQL -**Why only the immediately previous version?** Upgrade scripts are frozen once shipped. The chain of upgrades (v0.1.1 → v0.2.0 → v0.3.0) is validated transitively — each version's CI tests its own upgrade script. Only the current work-in-progress upgrade script might introduce an inconsistency, so testing it against the immediately previous version is sufficient. +**Why only the immediately previous compatible version?** Upgrade scripts are frozen once shipped. The chain of upgrades (v0.2.2 → v0.2.3 → v0.3.0 within a provider line) is validated transitively — each version's CI tests its own upgrade script. Only the current work-in-progress upgrade script might introduce an inconsistency, so testing it against the immediately previous compatible version is sufficient. **Priority:** High — foundational test, catches the most common class of upgrade bugs. ### Scenario B1: Binary Backward Compatibility -**Goal:** Verify that the new `.so` works correctly against **all** previous versions' schemas, not just the immediately previous one. Customers may never run `ALTER EXTENSION UPDATE`, so the new binary must work against any older schema. +**Goal:** Verify that the new `.so` works correctly against **all** previous compatible versions' schemas, not just the immediately previous one. Customers may never run `ALTER EXTENSION UPDATE`, so the new binary must work against any older schema in the same provider line. -This is the **most deployment-critical test** because the new binary may run against any older schema indefinitely. A customer on v0.1.1 who receives the v0.5.0 binary without ever upgrading must still be able to use the extension. +This is the **most deployment-critical test** because the new binary may run against any older compatible schema indefinitely. A customer on v0.2.2 who receives the v0.5.0 binary without ever upgrading must still be able to use the extension. -We test against all previous versions within the same major version. A major version bump is the boundary where backward compatibility may be dropped. +We test against all previous versions in the same provider compatibility line. The line starts at `PROVIDER_COMPAT_START_VERSION` in `scripts/test-upgrade.sh`, which can be overridden by downstream forks or CI environments. **Method:** 1. Install the new `.so` -2. For each previous version (same major): create the extension with that version's install SQL +2. For each previous compatible version: create the extension with that version's install SQL 3. Exercise all SQL-callable functions against each schema 4. Verify: no errors, correct results @@ -80,9 +84,9 @@ We test against all previous versions within the same major version. A major ver ### Scenario B2: Data Compatibility After Upgrade -**Goal:** Verify that data created under the previous version remains accessible and functional after `ALTER EXTENSION UPDATE`. +**Goal:** Verify that data created under the previous compatible version remains accessible and functional after `ALTER EXTENSION UPDATE`. -This is a **chain test** (like Scenario A) — upgrade scripts are applied sequentially, so testing against the immediately previous version is sufficient. Each intermediate upgrade was validated by its own version's CI. +This is a **chain test** (like Scenario A) — upgrade scripts are applied sequentially within the provider compatibility line, so testing against the immediately previous compatible version is sufficient. Each intermediate upgrade was validated by its own version's CI. **Method:** 1. Create extension at previous version @@ -145,7 +149,7 @@ Returns the version that was last installed/updated. Compare against known thres - `sql/pg_durable--0.1.1.sql` — first install SQL for the current major version (only the first version per major needs a fixture; intermediate versions are reconstructed by chaining upgrade scripts) - `sql/pg_durable--0.1.1--0.2.0.sql` — upgrade script (initially empty, populated by subsequent PRs) -- `scripts/test-upgrade.sh` — runs Scenarios A, B1, and B2 +- `scripts/test-upgrade.sh` — runs Scenarios A, B1, and B2. The `PROVIDER_COMPAT_START_VERSION` environment variable/default controls the first version in the current provider compatibility line. Versions before that boundary are excluded from B1 and cannot be used as A/B2 upgrade sources. - CI step in `.github/workflows/ci.yml` ### Per-version checklist @@ -153,7 +157,7 @@ Returns the version that was last installed/updated. Compare against known thres Each PR that changes the extension schema or modifies SQL queries in Rust code should: 1. Add the necessary DDL to the upgrade script (`sql/pg_durable----.sql`) -2. Ensure the `.so` is backward compatible with **all** previous schemas within the same major version (Scenario B1) +2. Ensure the `.so` is backward compatible with **all** previous schemas in the same provider compatibility line (Scenario B1) 3. Add version-specific notes to this document under "Version-Specific Changes" below 4. Pass upgrade tests in CI @@ -166,11 +170,12 @@ Each PR that changes the extension schema or modifies SQL queries in Rust code s **Minor release** (e.g. 0.2.0 → 0.3.0): 1. Create empty `sql/pg_durable----.sql` upgrade script 2. Bump `Cargo.toml` version to `` +3. If this release starts a new provider compatibility line, update the `PROVIDER_COMPAT_START_VERSION` default in `scripts/test-upgrade.sh` and document the boundary under "Version-Specific Changes". Downstream forks can instead override `PROVIDER_COMPAT_START_VERSION` in CI to keep the script shared. If this is the first minor after a new major (e.g. 1.0.0 → 1.1.0), also: -3. Check in `sql/pg_durable--.sql` as the first install SQL fixture for the new major (e.g. copy the generated `pg_durable--1.0.0.sql` from the extension directory) -4. Optionally delete the previous major's install SQL fixture and upgrade scripts — they are no longer needed by any of A, B1, or B2 +4. Check in `sql/pg_durable--.sql` as the first install SQL fixture for the new major (e.g. copy the generated `pg_durable--1.0.0.sql` from the extension directory) +5. Optionally delete the previous major's install SQL fixture and upgrade scripts — they are no longer needed by any of A, B1, or B2 No additional fixture is needed for subsequent minors — intermediate versions are reconstructed by chaining `ALTER EXTENSION UPDATE` from the first version's install SQL. @@ -178,7 +183,7 @@ No additional fixture is needed for subsequent minors — intermediate versions 1. Create empty `sql/pg_durable----<1.0.0>.sql` upgrade script 2. Bump `Cargo.toml` version to `<1.0.0>` -`cargo pgrx package` generates the new major's install SQL. The previous major's install SQL and upgrade scripts are still needed for the A/B2 upgrade chain. B1 will be a no-op — there are no previous versions within the new major to test backward compatibility against. +`cargo pgrx package` generates the new major's install SQL. The previous major's install SQL and upgrade scripts are still needed for the A/B2 upgrade chain when the provider line continues across the major bump. B1 will be a no-op if there are no previous compatible versions within the new major, or if `PROVIDER_COMPAT_START_VERSION` marks the new major as the start of a new provider line. --- @@ -240,10 +245,11 @@ what the upgrade script handles, and any backward compatibility considerations. #### Switch to crates.io duroxide-pg 0.1.34 + duroxide 0.1.29 - **DDL change (df schema):** None. This is a provider source and version update only. - **DDL change (duroxide schema):** No extension upgrade script DDL is required. The BGW continues to own provider migrations through `MigrationPolicy::ApplyAll`. -- **Scenario A considerations:** The `df` schema equivalence contract is unchanged. The `duroxide` schema remains excluded from snapshot diffs because it is populated at runtime by the BGW. -- **Scenario B1 considerations:** The new `.so` must continue to initialize against older provider schemas; any missing provider migrations are applied by the BGW at startup. -- **Scenario B2 considerations:** No user data migration is needed. Existing `df.*` metadata and workflow state are unaffected by changing the provider dependency source from the local provider tree to crates.io. -- **Current status:** Implemented — `Cargo.toml` exactly pins `duroxide = "=0.1.29"` and `duroxide-pg = "=0.1.34"`. +- **Provider compatibility boundary:** v0.2.2 is the first version in the open-source `duroxide-pg` provider line. Earlier pg_durable versions used `duroxide-pg-opt`, whose SQL migrations and runtime state are not an upgrade source for this line. GitHub CI therefore sets `PROVIDER_COMPAT_START_VERSION=0.2.2` by default and skips A/B1/B2 coverage that would cross from `duroxide-pg-opt` to `duroxide-pg`. Azure's fork owns upgrade testing for the `duroxide-pg-opt` line. +- **Scenario A considerations:** Skipped for the v0.2.1 → v0.2.2 boundary in GitHub CI because v0.2.1 is before the provider compatibility start. Future `duroxide-pg`-line releases resume the normal fresh-vs-upgraded `df` schema comparison against the immediately previous compatible version. +- **Scenario B1 considerations:** The new `.so` is not required to execute against pre-v0.2.2 `duroxide-pg-opt` provider state. A failure pattern where basic `df.*` functions work but provider-backed execution remains pending is expected across that boundary and should not be treated as a GitHub CI regression. Future `duroxide-pg`-line releases must remain binary-compatible with v0.2.2+ schemas unless a later provider-line or major-version boundary explicitly changes that contract. +- **Scenario B2 considerations:** Data compatibility is not tested across the `duroxide-pg-opt` → `duroxide-pg` split. Future `duroxide-pg`-line releases must preserve data created under the immediately previous compatible version. +- **Current status:** Implemented — `Cargo.toml` exactly pins `duroxide = "=0.1.29"` and `duroxide-pg = "=0.1.34"`; `scripts/test-upgrade.sh` defaults `PROVIDER_COMPAT_START_VERSION` to `0.2.2` while allowing forks/CI to override it. #### Named Results v2 — df.if_rows - **DDL change:** Upgrade script adds `CREATE FUNCTION df.if_rows(result_name text, then_branch text, else_branch text)` — a new C-language function backed by the pgrx `#[pg_extern]` `if_rows_fn_wrapper` symbol. diff --git a/scripts/test-upgrade.sh b/scripts/test-upgrade.sh index 2d1b00d5..a84463fe 100755 --- a/scripts/test-upgrade.sh +++ b/scripts/test-upgrade.sh @@ -79,6 +79,19 @@ EXTENSION_DIR=$("$PG_CONFIG" --sharedir)/extension CURRENT_VERSION=$(grep '^version' "$PROJECT_DIR/Cargo.toml" | head -1 | sed 's/.*"\(.*\)".*/\1/') CURRENT_MAJOR=$(echo "$CURRENT_VERSION" | cut -d. -f1) +# First pg_durable version in the current provider compatibility line. +# Versions before this are not valid B1/B2 upgrade sources for this line. +# Keep this environment-overridable so downstream forks can reuse this script +# while testing their own provider lineage. +PROVIDER_COMPAT_START_VERSION="${PROVIDER_COMPAT_START_VERSION:-0.2.2}" + +version_ge() { + local left="$1" + local right="$2" + + [ "$(printf '%s\n%s\n' "$right" "$left" | sort -V | head -1)" = "$right" ] +} + first_fixture_for_major() { local target_major="$1" local version="" @@ -125,20 +138,26 @@ fi # Discover all previous versions from upgrade scripts (for B1 generalized testing). # Each upgrade script pg_durable--FROM--TO.sql tells us FROM is a previous version. -# B1 tests the current .so against ALL previous schemas, not just the immediately previous one. +# B1 tests the current .so against all previous schemas in the current provider +# compatibility line, not just the immediately previous one. ALL_PREV_VERSIONS=() for f in "$PROJECT_DIR"/sql/pg_durable--*--*.sql; do fname=$(basename "$f") if [[ "$fname" =~ ^pg_durable--([0-9]+\.[0-9]+\.[0-9]+)--([0-9]+\.[0-9]+\.[0-9]+)\.sql$ ]]; then from_ver="${BASH_REMATCH[1]}" from_major=$(echo "$from_ver" | cut -d. -f1) - if [ "$from_major" = "$CURRENT_MAJOR" ]; then + if [ "$from_major" = "$CURRENT_MAJOR" ] && version_ge "$from_ver" "$PROVIDER_COMPAT_START_VERSION"; then ALL_PREV_VERSIONS+=("$from_ver") fi fi done IFS=$'\n' ALL_PREV_VERSIONS=($(sort -V -u <<< "${ALL_PREV_VERSIONS[*]}")); unset IFS +HAS_COMPAT_PREV=false +if version_ge "$PREV_VERSION" "$PROVIDER_COMPAT_START_VERSION"; then + HAS_COMPAT_PREV=true +fi + # Test databases — must use the pg_durable.database (default: postgres) # since the extension enforces it can only be created in that database. # Tests run sequentially: create → snapshot → drop → next test. @@ -155,11 +174,16 @@ echo "================================================" echo "pg_durable Upgrade Tests" echo -e "PostgreSQL: ${CYAN}PG${PG_VERSION}${NC} (port ${PG_PORT})" echo -e "First version (major ${CURRENT_MAJOR}): ${CYAN}${FIRST_VERSION}${NC}" -echo -e "Scenario A upgrade path: ${CYAN}${PREV_VERSION} → ${CURRENT_VERSION}${NC}" +echo -e "Provider compat start: ${CYAN}${PROVIDER_COMPAT_START_VERSION}${NC}" +if [ "$HAS_COMPAT_PREV" = true ]; then + echo -e "Scenario A upgrade path: ${CYAN}${PREV_VERSION} → ${CURRENT_VERSION}${NC}" +else + echo -e "Scenario A upgrade path: ${YELLOW}(previous version ${PREV_VERSION} is before provider compat start; skipped)${NC}" +fi if [ ${#ALL_PREV_VERSIONS[@]} -gt 0 ]; then echo -e "Scenario B1 compat versions: ${CYAN}${ALL_PREV_VERSIONS[*]}${NC}" else - echo -e "Scenario B1 compat versions: ${YELLOW}(none in major ${CURRENT_MAJOR}; B1 skipped)${NC}" + echo -e "Scenario B1 compat versions: ${YELLOW}(none in provider compat line; B1 skipped)${NC}" fi echo "================================================" echo "" @@ -575,7 +599,11 @@ snapshot_schema() { echo "" echo -e "${CYAN}Scenario A: Schema Upgrade Correctness${NC}" -echo " Testing: CREATE EXTENSION VERSION '$PREV_VERSION' + ALTER EXTENSION UPDATE = fresh CREATE EXTENSION" +if [ "$HAS_COMPAT_PREV" = true ]; then + echo " Testing: CREATE EXTENSION VERSION '$PREV_VERSION' + ALTER EXTENSION UPDATE = fresh CREATE EXTENSION" +else + echo " Skipping: previous version $PREV_VERSION is before provider compatibility start $PROVIDER_COMPAT_START_VERSION" +fi echo "" test_schema_upgrade() { @@ -634,7 +662,9 @@ test_schema_upgrade() { fi } -run_test "Schema comparison (upgrade vs fresh install)" test_schema_upgrade +if [ "$HAS_COMPAT_PREV" = true ]; then + run_test "Schema comparison (upgrade vs fresh install)" test_schema_upgrade +fi # ============================================================================ # Scenario B1: Binary backward compatibility @@ -778,7 +808,7 @@ test_b1_instance_info() { if [ ${#ALL_PREV_VERSIONS[@]} -eq 0 ]; then echo "" echo -e "${CYAN}Scenario B1: Binary Backward Compatibility${NC}" - echo " No previous versions within major ${CURRENT_MAJOR}; skipping direct-contact compatibility checks" + echo " No previous versions in provider compatibility line ${PROVIDER_COMPAT_START_VERSION}+; skipping direct-contact compatibility checks" else for B1_VERSION in "${ALL_PREV_VERSIONS[@]}"; do echo "" @@ -815,7 +845,11 @@ fi echo "" echo -e "${CYAN}Scenario B2: Data Compatibility After Upgrade${NC}" -echo " Testing: data created under v${PREV_VERSION} remains accessible after ALTER EXTENSION UPDATE" +if [ "$HAS_COMPAT_PREV" = true ]; then + echo " Testing: data created under v${PREV_VERSION} remains accessible after ALTER EXTENSION UPDATE" +else + echo " Skipping: previous version $PREV_VERSION is before provider compatibility start $PROVIDER_COMPAT_START_VERSION" +fi echo "" B2_PRE_INSTANCE_ID="" @@ -875,10 +909,12 @@ test_b2_new_data_after_upgrade() { assert_sql_equals "SELECT msg FROM test_upgrade_b2_log WHERE kind = 'post' ORDER BY id DESC LIMIT 1;" "new_value" } -run_test "B2: Pre-upgrade data survives ALTER EXTENSION UPDATE" test_b2_data_survives_upgrade -run_test "B2: Pre-upgrade instance remains queryable" test_b2_pre_upgrade_instance_after_upgrade -run_test "B2: In-flight work completes after upgrade" test_b2_inflight_work_after_upgrade -run_test "B2: New data and execution after upgrade" test_b2_new_data_after_upgrade +if [ "$HAS_COMPAT_PREV" = true ]; then + run_test "B2: Pre-upgrade data survives ALTER EXTENSION UPDATE" test_b2_data_survives_upgrade + run_test "B2: Pre-upgrade instance remains queryable" test_b2_pre_upgrade_instance_after_upgrade + run_test "B2: In-flight work completes after upgrade" test_b2_inflight_work_after_upgrade + run_test "B2: New data and execution after upgrade" test_b2_new_data_after_upgrade +fi # ============================================================================ # Results