Merge upstream Vortex 0.76.0 into spiceai-54

.github/actions/check-rebuild/action.yml

@@ -2,8 +2,12 @@ name: "Build and verify rebuild is a no-op"
 description: "Run a cargo command, then re-run it to verify all artifacts are cached"
 inputs:
   command:
-    description: "The cargo build command to run and verify (--message-format json is appended on the verification run)"
+    description: "The cargo build command to run and verify"
     required: true
+  message-format-flag:
+    description: "The flag used to request Cargo compiler-artifact JSON during the verification run"
+    required: false
+    default: "--message-format"
 runs:
   using: "composite"
   steps:
@@ -14,7 +18,7 @@ runs:
       shell: bash
       run: |
         stale=$(${{ inputs.command }} \
-          --message-format json 2>/dev/null \
+          ${{ inputs.message-format-flag }} json 2>/dev/null \
           | jq -r 'select(.reason == "compiler-artifact" and .fresh == false) | .target.name')
         if [ -n "$stale" ]; then
           echo "ERROR: Rebuild recompiled crates that should have been cached:"

.github/pull_request_template.md

@@ -1,55 +1,23 @@
-<!--
-Thank you for submitting a pull request! We appreciate your time and effort.
-
-Please make sure to provide enough information so that we can review your pull
-request. The Summary and Testing sections below contain guidance on what to
-include.
--->
-
-## Summary
+## Rationale for this change
 
 <!--
-If this PR is related to a tracked effort, please link to the relevant issue
-here (e.g., `Closes: #123`). Otherwise, feel free to ignore / delete this.
-
-In this section, please:
-
-1. Explain the rationale for this change.
-2. Summarize the changes included in this PR.
+Why are you proposing this change, and what is its impact? 
+Is it part of a long term effort, or a bigger change?
 
-A general rule of thumb is that larger PRs should have larger summaries. If
-there are a lot of changes, please help us review the code by explaining what
-was changed and why.
-
-If there is an issue or discussion attached, there is no need to duplicate all
-the details, but clarity is always preferred over brevity.
+If this PR is related to a tracked effort or an open issue, please link to the relevant issue.
 -->
 
-Closes: #000
+- Closes: #000
 
+## What changes are included in this PR?
 <!--
-## API Changes
-
-Uncomment this section if there are any user-facing changes.
-
-Consider whether the change affects users in one of the following ways:
-
-1. Breaks public APIs in some way.
-2. Changes the underlying behavior of one of the engine integrations.
-3. Should some documentation be updated to reflect this change?
-
-If a public API is changed in a breaking manner, make sure to add the
-appropriate label.
+No need to duplicate information from the previous section, but if you're touching many
+parts of the code base, its worth explicitly noting the important changes or how they are tested.
 -->
 
-## Testing
+## What APIs are changed? Are there any user-facing changes?
 
 <!--
-Please describe how this change was tested. Here are some common categories for
-testing in Vortex:
-
-1. Verifying existing behavior is maintained.
-2. Verifying new behavior and functionality works correctly.
-3. Serialization compatibility (backwards and forwards) should be maintained or
-   explicitly broken.
+Are there any user-facing changes that might require documentation updates
+Is any public API changed?
 -->

.github/workflows/bench-dispatch.yml

@@ -47,3 +47,21 @@ jobs:
     needs: remove-sql-label
     uses: ./.github/workflows/sql-pr.yml
     secrets: inherit
+
+  remove-sql-full-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/benchmark-sql-full'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/benchmark-sql-full
+          fail_on_error: true
+
+  sql-full-bench:
+    needs: remove-sql-full-label
+    uses: ./.github/workflows/sql-pr.yml
+    secrets: inherit
+    with:
+      benchmark_profile: "full"

.github/workflows/bench-pr.yml

@@ -38,7 +38,7 @@ jobs:
         if: github.event.pull_request.head.repo.fork == false
         with:
           sccache: s3
-      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Setup benchmark environment
@@ -72,7 +72,18 @@ jobs:
           profiling_frequency: 199
           extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz
 
+      - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
+        if: matrix.benchmark.id == 'random-access-bench'
+        shell: bash
+        env:
+          RUST_BACKTRACE: full
+          VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
+          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
+        run: |
+          python3 scripts/random-access-split.py
+
       - name: Run ${{ matrix.benchmark.name }} benchmark
+        if: matrix.benchmark.id != 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full

.github/workflows/bench.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b  # v6
         with:
@@ -54,7 +54,7 @@ jobs:
         if: github.repository == 'vortex-data/vortex'
         with:
           sccache: s3
-      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
       - name: Setup benchmark environment
         run: sudo bash scripts/setup-benchmark.sh
       - uses: ./.github/actions/setup-rust
@@ -85,7 +85,18 @@ jobs:
           profiling_frequency: 199
           extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz
 
+      - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
+        if: matrix.benchmark.id == 'random-access-bench'
+        shell: bash
+        env:
+          RUST_BACKTRACE: full
+          VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
+          FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
+        run: |
+          python3 scripts/random-access-split.py --v3
+
       - name: Run ${{ matrix.benchmark.name }} benchmark
+        if: matrix.benchmark.id != 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full
@@ -117,6 +128,53 @@ jobs:
             --benchmark-id "${{ matrix.benchmark.id }}" \
             --repo-url "${{ github.server_url }}/${{ github.repository }}"
 
+      # v4 (Postgres) dual-write -- BEST-EFFORT during the migration soak. The
+      # proven v3 step above is hard-required; a v4 failure must NOT fail the job
+      # (v4 is promoted to required at cutover, PR-5.1). Gated on the ingest-role
+      # ARN var (the assume-role input that MUST exist for OIDC to succeed) so it
+      # no-ops until v4 infra is wired, and every step is
+      # `continue-on-error` so an OIDC / uv / connect hiccup never breaks the v3
+      # pipeline. post-ingest.py mints the RDS IAM token internally (boto3) from
+      # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert.
+      #
+      # ORDER MATTERS: "Install uv" runs BEFORE "Configure AWS credentials".
+      # configure-aws-credentials persists the assumed ingest-role (rds-db:connect
+      # only) as the job's ambient AWS creds; the uv setup compiles via sccache
+      # (S3-backed), so running it after the role switch fails with S3 AccessDenied.
+      # Installing uv first keeps sccache on the original S3-capable creds; the role
+      # is assumed immediately before the ingest, which needs only rds-db:connect.
+      - name: Install uv for v4 ingest
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        continue-on-error: true
+        uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
+      - name: Configure AWS credentials for v4 ingest (OIDC)
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        continue-on-error: true
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b  # v6
+        with:
+          role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }}
+          aws-region: ${{ vars.RDS_BENCH_REGION }}
+      - name: Ingest results to v4 Postgres (best-effort)
+        if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
+        continue-on-error: true
+        shell: bash
+        env:
+          RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
+          RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }}
+          AWS_REGION: ${{ vars.RDS_BENCH_REGION }}
+          BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }}
+          BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}
+        run: |
+          set -Eeuo pipefail
+          curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
+            -o "${RUNNER_TEMP}/rds-global-bundle.pem"
+          DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem"
+          uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \
+            scripts/post-ingest.py results.v3.jsonl \
+            --postgres "${DSN}" \
+            --commit-sha "${{ github.sha }}" \
+            --region "${AWS_REGION}"
+
       - name: Alert incident.io
         if: failure()
         uses: ./.github/actions/alert-incident-io