diff --git a/.github/workflows/delta_spark_ut.yml b/.github/workflows/delta_spark_ut.yml new file mode 100644 index 00000000000..580cb95fbd9 --- /dev/null +++ b/.github/workflows/delta_spark_ut.yml @@ -0,0 +1,459 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runs Delta Lake's `spark` sbt module unit tests against a Gluten Velox bundle +# that is built from the source in this repository. The pipeline: +# +# 1. Builds the Velox/Gluten native libraries (centos-7 + vcpkg, x86_64). +# 2. Builds the Gluten Java/Scala jars and assembles the +# `gluten-velox-bundle-spark_-linux_amd64-.jar` +# fat jar for Spark 4.1 + Scala 2.13 + Java 17 with the Delta profile. +# 3. Clones delta-io/delta at the requested release tag (default `v4.2.0`), +# drops the bundle jar into `spark-unified/lib/` only (NOT `spark/lib/` +# -- see setup-delta.sh for the unmanagedJars scoping rationale), +# patches Delta's `DeltaSQLCommandTest` to register the Gluten plugin, +# and runs `sbt spark/test` sharded across the matrix. +# +# Limited to Velox + x86 to keep the matrix simple, per the pipeline's purpose +# of validating Gluten changes against the latest Delta release. + +name: Delta Spark UT (Gluten) + +on: + # Reusable workflow. velox_backend_x86.yml calls this (gated on Delta-relevant + # changes) and passes the native-lib artifact it already built, so the expensive + # native C++ build is NOT duplicated. That artifact lives in the CALLER's run (a + # called workflow runs as part of the caller run), so the jobs below download it + # by name. See velox_backend_x86.yml `delta-spark-ut`. + # + # NOTE: the `pull_request` trigger was removed so this no longer runs as its own + # workflow on PRs (which would double-run the Delta suite). velox_backend_x86.yml + # is now the single PR entry point; `workflow_dispatch` keeps manual standalone + # runs working (those build the native lib themselves -- see build-native-lib). + workflow_call: + inputs: + native_lib_artifact: + description: 'Name of the cpp/build artifact uploaded by the caller' + type: string + required: true + delta_ref: + type: string + required: false + default: 'v4.2.0' + spark_version: + description: 'Spark version driving both the Gluten bundle profile (-Pspark-) and Delta -DsparkVersion.' + type: string + required: false + default: '4.1' + test_parallelism: + type: string + required: false + default: '4' + update_baseline: + type: boolean + required: false + default: false + fail_on_fixed: + type: boolean + required: false + default: true + workflow_dispatch: + inputs: + delta_ref: + description: 'delta-io/delta git ref (tag/branch/SHA) to test against' + required: true + default: 'v4.2.0' + spark_version: + description: 'Spark version: drives the Gluten bundle profile (-Pspark-) and Delta -DsparkVersion together. Scala 2.13 + JDK 17 are assumed, so pair a non-4.1 value with a compatible delta_ref.' + required: true + default: '4.1' + test_parallelism: + description: 'Forked test JVMs per shard (TEST_PARALLELISM_COUNT)' + required: true + default: '4' + update_baseline: + description: 'Seed/refresh the known-failures baseline instead of enforcing it' + type: boolean + required: false + default: false + fail_on_fixed: + description: 'Fail when a baseline test now passes (keeps the baseline honest)' + type: boolean + required: false + default: true + # Nightly full run against the latest default branch. The per-PR entry point + # (velox_backend_x86.yml) now runs the Delta suite only when a PR touches + # Delta-relevant paths (or carries the opt-in label), to save GHA minutes; this + # scheduled run keeps full coverage once a day so rarer regressions are still + # caught. It builds its own native lib (build-native-lib-centos-7 below) since + # there is no caller to provide one, and uses the workflow's default inputs. + schedule: + - cron: '0 5 * * *' + +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + MVN_CMD: 'build/mvn -ntp' + CCACHE_DIR: "${{ github.workspace }}/.ccache" + # Gluten profile / bundle naming for the build-gluten-bundle and + # delta-spark-test jobs. `spark_version` is the single source of truth for the + # Spark version: it drives the Gluten bundle profile (-Pspark-), the bundle + # jar name, and Delta's -DsparkVersion, so the tests always run against a bundle + # built for the same Spark version (no separate value to keep in sync). Scala + # 2.13 + JDK 17 are pinned -- they match Delta v4.2.0's default Spark 4.1.0 from + # project/CrossSparkVersions.scala -- so pair a non-default spark_version with a + # compatible delta_ref. + GLUTEN_SPARK_PROFILE: spark-${{ inputs.spark_version || '4.1' }} + GLUTEN_SCALA_PROFILE: 'scala-2.13' + GLUTEN_JAVA_PROFILE: 'java-17' + GLUTEN_BUNDLE_SPARK_VERSION: ${{ inputs.spark_version || '4.1' }} + GLUTEN_BUNDLE_SCALA_VERSION: '2.13' + DELTA_SCALA_VERSION: '2.13.16' + # Number of shards in the delta-spark-test matrix. Must equal the length of + # the `shard` matrix below. + # + # 4 shards x TEST_PARALLELISM_COUNT=4 gives ~16-way parallelism packed into 4 + # runner jobs (4 forks each) rather than 16 single-fork jobs -- fewer concurrent + # runners for the same throughput. Sharding is by SUITE; total work + # (~1250 shard-minutes) is fixed. Each forked test JVM uses ~4G (2G heap + 2G + # off-heap), so 4 forks plus the sbt launcher sit close to the ~16G runner limit; + # this fits because the worst memory hog (DeletionVectorsSuite 2B-row) is + # force-failed in setup-delta.sh. + DELTA_NUM_SHARDS: '4' + +# No `concurrency:` here on purpose. As a reusable workflow this runs inside the +# caller's run, where `github.workflow` resolves to the CALLER's name -- a group +# keyed on it would collide with the caller's own group and, with +# cancel-in-progress, could cancel the parent run. The caller's concurrency +# already governs cancellation. (A standalone workflow_dispatch run just won't +# auto-cancel, which is fine for infrequent manual runs.) + +jobs: + build-native-lib-centos-7: + # Standalone runs (workflow_dispatch + nightly schedule) build the native lib + # here. When called by velox_backend_x86.yml the caller already built it and + # passes it as an input, so this job is skipped and the duplicate build avoided. + if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Get Ccache + uses: actions/cache/restore@v4 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-delta-spark-ut-centos7-release-default-${{github.sha}} + restore-keys: | + ccache-delta-spark-ut-centos7-release-default + ccache-centos7-release-default + - name: Build Gluten native libraries + run: | + docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7-gcc13 bash -c " + set -e + yum install tzdata -y + df -a + cd /work + export CCACHE_DIR=/work/.ccache + export CCACHE_MAXSIZE=1G + mkdir -p /work/.ccache + ccache -sz + bash dev/ci-velox-buildstatic-centos-7.sh + ccache -s + " + - name: Save Ccache + if: always() + uses: actions/cache/save@v4 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-delta-spark-ut-centos7-release-default-${{github.sha}} + - uses: actions/upload-artifact@v4 + with: + name: delta-spark-ut-native-lib-centos-7-${{github.sha}} + path: ./cpp/build/ + if-no-files-found: error + + build-gluten-bundle: + needs: build-native-lib-centos-7 + # Run whether the native lib was built here (dispatch -> success) or provided + # by the caller (workflow_call -> build-native-lib-centos-7 skipped). + if: ${{ always() && needs.build-native-lib-centos-7.result != 'failure' && needs.build-native-lib-centos-7.result != 'cancelled' }} + runs-on: ubuntu-22.04 + container: apache/gluten:centos-9-jdk17 + steps: + - uses: actions/checkout@v4 + - name: Download native artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.native_lib_artifact || format('delta-spark-ut-native-lib-centos-7-{0}', github.sha) }} + path: ./cpp/build/ + - name: Cache Maven repository + uses: actions/cache@v4 + with: + path: /root/.m2/repository + key: m2-delta-spark-ut-bundle-${{ env.GLUTEN_SPARK_PROFILE }}-${{ env.GLUTEN_SCALA_PROFILE }}-${{ hashFiles('pom.xml', '**/pom.xml') }} + restore-keys: | + m2-delta-spark-ut-bundle-${{ env.GLUTEN_SPARK_PROFILE }}-${{ env.GLUTEN_SCALA_PROFILE }}- + m2-delta-spark-ut-bundle- + - name: Build Gluten Velox + Delta bundle + run: | + set -euo pipefail + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + java -version + cd "$GITHUB_WORKSPACE" + # `install` (not `package`) so the gluten-delta artifact is in the local + # m2 repo before the `package/` shaded jar is built. `Dmaven.compiler.release=17` + # overrides any user settings.xml that may pin release=1.8 for Java 17 builds. + $MVN_CMD clean install \ + -P${{ env.GLUTEN_SPARK_PROFILE }} \ + -P${{ env.GLUTEN_SCALA_PROFILE }} \ + -P${{ env.GLUTEN_JAVA_PROFILE }} \ + -Pbackends-velox -Pdelta \ + -DskipTests -Dmaven.compiler.release=17 + - name: Stage bundle jar + run: | + set -euo pipefail + mkdir -p bundle-out + # Match the renamed fat jar produced by package/pom.xml's copy-fat-jar + # exec. The version part may bump (e.g. 1.7.0-SNAPSHOT -> 1.8.0-SNAPSHOT), + # so glob the version suffix. `2>/dev/null ... || true` keeps a no-match + # `ls` from aborting the step under `set -o pipefail`, so the explicit + # check below runs instead of dying with a generic "cannot access". + jar=$(ls package/target/gluten-velox-bundle-spark${{ env.GLUTEN_BUNDLE_SPARK_VERSION }}_${{ env.GLUTEN_BUNDLE_SCALA_VERSION }}-linux_amd64-*.jar 2>/dev/null | head -n 1 || true) + if [ -z "$jar" ] || [ ! -f "$jar" ]; then + echo "ERROR: Could not find Gluten bundle jar under package/target/" >&2 + ls -la package/target/ || true + exit 1 + fi + cp "$jar" bundle-out/ + ls -lh bundle-out/ + - uses: actions/upload-artifact@v4 + with: + name: delta-spark-ut-gluten-bundle-${{github.sha}} + path: bundle-out/gluten-velox-bundle-spark*_*-linux_amd64-*.jar + if-no-files-found: error + + delta-spark-test: + needs: build-gluten-bundle + # build-gluten-bundle runs via `if: always()` (its build-native-lib-centos-7 need + # is skipped on workflow_call), so this job needs an explicit condition too -- + # otherwise GitHub's transitive skip propagation, seeing the skipped + # build-native-lib-centos-7 ancestor, would skip the whole shard matrix. + if: ${{ !cancelled() && needs.build-gluten-bundle.result == 'success' }} + runs-on: ubuntu-22.04 + container: apache/gluten:centos-9-jdk17 + # 350-min safety cap. With 4 forks per shard the per-shard suites run + # 4-at-a-time, so a shard finishes well under this. + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + # Length of this list MUST equal env.DELTA_NUM_SHARDS. + shard: [0, 1, 2, 3] + env: + # Mirror Delta's spark_test.yaml env vars used by run-tests.py / + # TestParallelization.scala. + SHARD_ID: ${{ matrix.shard }} + steps: + - uses: actions/checkout@v4 + + - name: Resolve workflow inputs + id: resolve + # Surface the inputs as step outputs. workflow_call / workflow_dispatch + # supply them (with defaults); the nightly `schedule` event supplies NONE, + # so fall back to the same defaults here. The boolean inputs are rendered + # as explicit 'true'/'false' strings (never empty) via the `&&/||` form so + # a schedule run resolves cleanly: update_baseline=false (enforce) and + # fail_on_fixed=true (so a now-passing baseline test turns the nightly red + # -- our signal that the committed baseline needs refreshing). + env: + DELTA_REF: ${{ inputs.delta_ref || 'v4.2.0' }} + SPARK_VERSION: ${{ inputs.spark_version || '4.1' }} + TEST_PARALLELISM: ${{ inputs.test_parallelism || '4' }} + UPDATE_BASELINE: ${{ inputs.update_baseline && 'true' || 'false' }} + FAIL_ON_FIXED: ${{ github.event_name == 'schedule' && 'true' || (inputs.fail_on_fixed && 'true' || 'false') }} + run: | + set -euo pipefail + { + echo "delta_ref=${DELTA_REF}" + echo "spark_version=${SPARK_VERSION}" + echo "test_parallelism=${TEST_PARALLELISM}" + echo "update_baseline=${UPDATE_BASELINE}" + echo "fail_on_fixed=${FAIL_ON_FIXED}" + } | tee -a "$GITHUB_OUTPUT" + + - name: Download Gluten bundle jar + uses: actions/download-artifact@v4 + with: + name: delta-spark-ut-gluten-bundle-${{github.sha}} + path: gluten-bundle + + - name: Install minimal tools + run: | + set -euo pipefail + # apache/gluten:centos-9-jdk17 already has java-17, git, tar, a POSIX + # shell, and curl-minimal (which provides the `curl` command sbt's + # launcher needs). Install the rest of what Delta's build/sbt and the + # tests may need. We deliberately do NOT install the full `curl` + # package -- it conflicts with the pre-installed curl-minimal. + yum install -y java-17-openjdk-devel which findutils gzip python3 + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + java -version + git --version + curl --version | head -n 1 + + - name: Cache sbt / Ivy / Coursier + uses: actions/cache@v4 + with: + path: | + /root/.sbt + /root/.ivy2 + /root/.cache/coursier + # Intentionally NOT keyed by ${{ matrix.shard }} -- all shards share + # the same dependency tree, so a single shared cache (with parallel + # save races resolved by GH on a first-write-wins basis) gives the + # best storage / hit-rate tradeoff. + key: delta-spark-ut-sbt-${{ steps.resolve.outputs.delta_ref }}-${{ steps.resolve.outputs.spark_version }}-${{ env.DELTA_SCALA_VERSION }} + restore-keys: | + delta-spark-ut-sbt-${{ steps.resolve.outputs.delta_ref }}-${{ steps.resolve.outputs.spark_version }}- + delta-spark-ut-sbt-${{ steps.resolve.outputs.delta_ref }}- + + - name: Clone and patch Delta + run: | + set -euo pipefail + # `2>/dev/null ... || true` keeps a no-match `ls` from aborting the step + # under `set -o pipefail`, so the explicit check below emits a clear + # error instead of a generic "cannot access". + GLUTEN_JAR=$(ls "$GITHUB_WORKSPACE"/gluten-bundle/gluten-velox-bundle-spark*_*-linux_amd64-*.jar 2>/dev/null | head -n 1 || true) + if [ -z "$GLUTEN_JAR" ] || [ ! -f "$GLUTEN_JAR" ]; then + echo "ERROR: No Gluten bundle jar found under $GITHUB_WORKSPACE/gluten-bundle/" >&2 + ls -la "$GITHUB_WORKSPACE/gluten-bundle/" || true + exit 1 + fi + echo "Using Gluten bundle: $GLUTEN_JAR" + bash "$GITHUB_WORKSPACE/.github/workflows/util/delta-spark-ut/setup-delta.sh" \ + "${{ steps.resolve.outputs.delta_ref }}" \ + "$GITHUB_WORKSPACE/delta" \ + "$GLUTEN_JAR" \ + "$GITHUB_WORKSPACE" + + - name: Run Delta spark module tests (shard ${{ matrix.shard }} / ${{ env.DELTA_NUM_SHARDS }}) + env: + NUM_SHARDS: ${{ env.DELTA_NUM_SHARDS }} + TEST_PARALLELISM_COUNT: ${{ steps.resolve.outputs.test_parallelism }} + SPARK_VERSION: ${{ steps.resolve.outputs.spark_version }} + UPDATE_BASELINE: ${{ steps.resolve.outputs.update_baseline }} + FAIL_ON_FIXED: ${{ steps.resolve.outputs.fail_on_fixed }} + # Required by Delta to enable testing-only code paths + # (see delta build.sbt: "Test / envVars += DELTA_TESTING -> 1"). + DELTA_TESTING: '1' + # NOTE: the Gluten/JDK17 test JVM flags (JAVA_TOOL_OPTIONS) live in + # util/delta-spark-ut/java-test-args.sh, which run-delta-tests.sh sources, + # so CI and local dev runs share one definition. + run: | + set -euo pipefail + # Run the shard's Delta tests + hang watchdog + memory forensics, then + # gate against the baseline. See util/delta-spark-ut/run-delta-tests.sh. + bash "$GITHUB_WORKSPACE/.github/workflows/util/delta-spark-ut/run-delta-tests.sh" + + - name: Compress heap dumps (if any) + if: ${{ failure() }} + run: | + set -euo pipefail + if compgen -G "/tmp/*.hprof" > /dev/null; then + echo "Found heap dump(s); compressing..." + ls -lh /tmp/*.hprof + # gzip is single-threaded and slow on multi-GB heaps but is + # always present in the centos image. Heap dumps compress ~10x. + gzip -1 /tmp/*.hprof + ls -lh /tmp/*.hprof.gz + else + echo "No heap dumps found in /tmp/." + fi + + - name: Upload per-shard gate lists + if: always() + uses: actions/upload-artifact@v4 + with: + name: delta-spark-ut-gate-lists-shard-${{ matrix.shard }} + path: gate-out/*.txt + if-no-files-found: warn + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: delta-spark-ut-reports-shard-${{ matrix.shard }} + path: | + delta/**/target/test-reports/**/*.xml + delta/**/target/surefire-reports/**/*.xml + if-no-files-found: warn + + - name: Upload hang watchdog thread dumps + if: always() + uses: actions/upload-artifact@v4 + with: + name: delta-spark-ut-threaddumps-shard-${{ matrix.shard }} + path: /tmp/threaddump-shard-${{ matrix.shard }}-*.txt + if-no-files-found: ignore + + - name: Upload JVM crash logs and other failure artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: delta-spark-ut-failure-logs-shard-${{ matrix.shard }} + path: | + delta/**/target/*.log + delta/**/hs_err_pid*.log + delta/**/core.* + /tmp/*.hprof + /tmp/*.hprof.gz + if-no-files-found: ignore + + # Merges every shard's failure/ran lists into a single, sorted, ready-to-commit + # known-failures.txt and reports global regressions / now-passing / stale + # entries. Runs even when some shards went red (if: always()) so the refreshed + # baseline artifact is always available -- this is what you download and commit + # to bootstrap or refresh the baseline (see util/delta-spark-ut/README.md). + delta-spark-aggregate: + needs: delta-spark-test + if: always() + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Download per-shard gate lists + uses: actions/download-artifact@v4 + continue-on-error: true + with: + pattern: delta-spark-ut-gate-lists-shard-* + path: gate-lists + merge-multiple: true + - name: Aggregate known failures + run: | + set -euo pipefail + python3 .github/workflows/util/delta-spark-ut/compare-test-results.py \ + --mode aggregate \ + --inputs-dir gate-lists \ + --expected-shards "${{ env.DELTA_NUM_SHARDS }}" \ + --known-failures .github/workflows/util/delta-spark-ut/known-failures.txt \ + --flaky-tests .github/workflows/util/delta-spark-ut/flaky-tests.txt \ + --baseline-out aggregated/known-failures.txt + - name: Upload refreshed baseline + if: always() + uses: actions/upload-artifact@v4 + with: + name: delta-spark-ut-known-failures + path: aggregated/known-failures.txt + if-no-files-found: warn diff --git a/.github/workflows/util/delta-spark-ut/README.md b/.github/workflows/util/delta-spark-ut/README.md new file mode 100644 index 00000000000..3111050f687 --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/README.md @@ -0,0 +1,208 @@ + + +# Delta Spark UT (Gluten) — managing expected failures + +Running delta-io/delta's `spark` ScalaTest suite against the Gluten Velox +bundle produces **many expected failures**: Gluten does not yet offload every +Delta code path, and falls back or behaves differently in places. If CI simply +went red on any failure, the signal would be useless and we could never tell a +*new* breakage from the hundreds of already-known ones. + +To make this manageable we keep a **baseline of known failures** and gate each +run against it. The build is green when the only failing tests are ones already +recorded in the baseline; it goes red the moment a **previously-passing test +starts failing** (a regression). + +## Files + +| File | Purpose | +|---|---| +| `known-failures.txt` | Committed baseline: the tests currently expected to fail. One `#` per line. | +| `flaky-tests.txt` | Quarantine list by test name: tests whose pass/fail is non-deterministic. Ignored by the gate whether they pass or fail. `#` per line. | +| `flaky-error-patterns.txt` | Quarantine list by error signature: regex patterns matched against a failure's text, for bugs that surface on a different test each run (e.g. the native DV bitmap row-index error). | +| `compare-test-results.py` | Parses the JUnit XML from `sbt spark/test` and gates / seeds / aggregates against the baseline. Standard-library only. | +| `run-delta-tests.sh` | The shard step's body: runs `sbt spark/test` (tuned JVM/heap flags) under a hang watchdog, prints memory forensics, then gates the results against the baseline via `compare-test-results.py`. | +| `java-test-args.sh` | Shared JVM flags (`--add-opens` + Netty property) needed to run the suite on JDK 17 with the Gluten bundle. Sourced by `run-delta-tests.sh` and by local runs. | +| `setup-delta.sh` | Clones Delta, drops in the Gluten bundle, and patches `DeltaSQLCommandTest`. | + +## How the gate works + +Each test shard: + +1. Runs `sbt spark/test` with ScalaTest's JUnit XML reporter enabled + (`-u target/test-reports`), so every suite writes per-test results. (Delta + itself only configures the console reporter, so the workflow injects this.) +2. Runs `compare-test-results.py --mode enforce`, which classifies every test: + - **regression** — failed, but not in the baseline → **fails the shard**. + - **expected** — failed and in the baseline → ignored. + - **now-passing** — in the baseline but passed this run → fails the shard + (so the baseline is kept honest), unless `fail_on_fixed=false`. + - **quarantined** — matches an entry in `flaky-tests.txt` → always ignored, + whether it passed or failed (see [Flaky tests](#flaky-tests) below). + +A final `aggregate` job merges every shard's results into a single, sorted, +ready-to-commit `known-failures.txt` artifact and reports **stale** baseline +entries (tests no longer present in any shard, e.g. after a Delta version bump). + +Because Delta shards **by suite**, every suite (and therefore every test) runs +in exactly one shard, so per-shard enforcement sees complete suites and never +double-counts. + +## When it runs + +To keep GitHub Actions usage in check, the suite does **not** run on every PR: + +- **Per PR** — `velox_backend_x86.yml` runs the Delta suite only when the PR + touches a **high-signal Delta path**: the Delta integration code + (`backends-velox/src-delta*`), the `gluten-delta` module, or this pipeline's + own files (`delta_spark_ut.yml`, `util/delta-spark-ut/**`, + `velox_backend_x86.yml`). Changes to general Velox/core/native code can also + affect Delta offload, but they're touched on most PRs, so per-PR they skip the + suite — the nightly run and the opt-in label are the safety nets. Add the + **`run-delta-ci`** label to force the suite on any PR (the label is read from + the triggering event, so apply it before/with a push). +- **Nightly** — `delta_spark_ut.yml` runs the **full** suite against the latest + default branch on a `schedule` (05:00 UTC), so rarer regressions are still + caught daily. The nightly run enforces the baseline **and** fails on + now-passing tests (`fail_on_fixed=true`), so baseline drift surfaces as a red + nightly — the signal to refresh `known-failures.txt`. +- **Manually** — **Actions → Delta Spark UT (Gluten) → Run workflow** + (`workflow_dispatch`), e.g. to refresh the baseline (see below). + +## Bootstrapping the baseline (first time) + +While `known-failures.txt` has no entries the gate auto-runs in **seed mode** +(it never fails — it only records failures). To create the initial baseline: + +1. Trigger **Actions → Delta Spark UT (Gluten) → Run workflow** with + `update_baseline = true`. +2. When it finishes, download the **`delta-spark-ut-known-failures`** artifact. +3. Replace `known-failures.txt` with the file from that artifact and commit it. + +From the next run onward the gate enforces the baseline. + +## Day-to-day: fixing tests incrementally + +- **You fixed Gluten and some Delta tests now pass.** CI will flag them as + *now-passing*. Delete those lines from `known-failures.txt` in your PR. That + is the whole point — the baseline only ever shrinks as coverage improves. +- **You intentionally added a new expected failure** (e.g. a Delta path Gluten + can't offload yet). Add the exact `Suite#test` line(s) the gate prints under + *Regressions* to `known-failures.txt`, ideally with a comment explaining why. +- **A genuine regression.** Fix it; do **not** add it to the baseline. + +The error log prints copy-pasteable `Suite#test` lines for both regressions and +now-passing tests, and each run's job summary shows the full breakdown. + +## Regenerating / refreshing the whole baseline + +After a Delta version bump or a large Gluten change, regenerate from scratch the +same way as bootstrapping: run the workflow with `update_baseline=true`, download +the `delta-spark-ut-known-failures` artifact, and commit it. The aggregate job +also lists **stale** entries you can prune. + +The aggregate job passes `--expected-shards` (the shard count), so if a shard +dies before writing its gate lists (or its artifact fails to download) the +aggregate **fails** instead of emitting a baseline that silently omits that +shard's failures — which would otherwise shrink `known-failures.txt` and red the +next run. Re-run the workflow if this happens. + +## Flaky tests + +Some tests are genuinely non-deterministic (e.g. the Delta MERGE-with-deletion-vector +suites that intermittently hit a native row-index bug). Such a test would otherwise +red the gate as a **regression** when it flakes to a failure, or as **now-passing** +when it flakes to a pass — noise either way. + +List these in **`flaky-tests.txt`** to **quarantine** them: the gate ignores a +quarantined test whether it passes or fails, and never writes it into the +regenerated baseline. Format is one `#` per line, `#`-comments +and blank lines allowed: + +``` +# suite portion is an fnmatch glob; test portion is matched exactly. +*DVs*Suite#matched only merge - enabled - with update and delete - isPartitioned: true +``` + +- The **suite** portion is an `fnmatch` glob, so `*DVs*Suite` covers every + generated deletion-vector merge variant in one line. Use the narrowest glob + that still covers the root-cause family. +- The **test** portion is matched **exactly** (test names are freeform and may + contain glob metacharacters), so a same-named test in a non-matching suite is + still gated normally. + +### Quarantine by error signature + +Some bugs surface on a **different test each run** — for example the native Delta +DV bitmap row-index error (the aggregator gets a garbage row index during a MERGE +that writes deletion vectors and aborts, e.g. `Delta RoaringBitmapArray row index +... exceeds max representable value` or `Delta bitmap row index cannot be +negative: ...`) lands on a different `*DVs*Suite` MERGE test every time. Chasing +those by name is whack-a-mole, so quarantine them by **root cause** in +**`flaky-error-patterns.txt`** instead: each line is a regex matched against a +failed test's ``/`` text. Any failure that matches is treated as +flaky regardless of which test it hit (and is dropped from the shard's failures +list so it can't leak into the baseline): + +``` +# regex matched against the failure message + stack (enforce mode). +# one explicit pattern per known error, deliberately specific. +Delta RoaringBitmapArray row index \d+ exceeds max representable value +Delta bitmap row index cannot be negative: -?\d+ +``` + +This is more precise than a name glob: a *different* real failure in the same +suite is still caught, because only failures carrying the signature are ignored. + +Quarantining (either kind) is an **interim** measure — it hides a real bug from +CI. Each entry should reference the tracking issue, and be removed once the +underlying bug is fixed so the test is enforced again. + +## Caveats + +- **Known failures still execute** (and fail) — they are gated *after* the run, + not skipped — so they still consume CI time. This keeps us decoupled from + Delta's sources; skipping them at runtime would require patching Delta. + +## Running the comparison locally + +```bash +# after an sbt spark/test run that wrote delta/**/target/test-reports/*.xml +python3 .github/workflows/util/delta-spark-ut/compare-test-results.py \ + --mode enforce \ + --reports-dir delta \ + --known-failures .github/workflows/util/delta-spark-ut/known-failures.txt \ + --flaky-tests .github/workflows/util/delta-spark-ut/flaky-tests.txt \ + --failures-out /tmp/failures.txt --ran-out /tmp/ran.txt +``` + +## Running the suite locally + +`sbt spark/test` needs extra JDK-17 JVM flags to run the Delta suite against the +Gluten bundle (`--add-opens` + the Netty reflection property). CI and local runs +share one definition in `java-test-args.sh` — `source` it before invoking sbt so +the flags reach the sbt launcher and the forked test JVM: + +```bash +# from the Delta clone prepared by setup-delta.sh (which has the Gluten bundle): +source /.github/workflows/util/delta-spark-ut/java-test-args.sh +./build/sbt "++ 2.13.16" spark/test # or a single suite via testOnly +``` + +`run-delta-tests.sh` sources the same file, so CI and local runs use identical +flags. diff --git a/.github/workflows/util/delta-spark-ut/compare-test-results.py b/.github/workflows/util/delta-spark-ut/compare-test-results.py new file mode 100644 index 00000000000..e3b6b9b2404 --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/compare-test-results.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gate / seed / aggregate the Delta-on-Gluten unit test results. + +Running delta-io/delta's ScalaTest suite against the Gluten Velox bundle +produces many *expected* failures (Gluten does not yet support every Delta +code path). To keep the red/green signal meaningful while we fix those +failures incrementally, we maintain a committed baseline of known failing +tests (``known-failures.txt``) and compare each CI run against it. + +This script has three modes: + +``enforce`` (default, per shard) + Parse the JUnit XML produced by ``sbt spark/test`` (ScalaTest ``-u`` + reporter) and compare against the baseline: + + * regression -- a test that FAILED but is NOT in the baseline. These + fail the build: a previously-passing test just started failing. + * expected -- a test that failed and IS in the baseline. Ignored. + * fixed -- a baseline test that now PASSES. By default these also + fail the build (``--fail-on-fixed true``) so the baseline stays honest + and contributors remove entries as they fix them. + + If the baseline file exists but is empty (not yet bootstrapped) the mode + automatically degrades to ``seed`` so the first run is never spuriously red. + A *missing* ``--known-failures`` file is treated as a configuration error + (the gate fails) so a mis-referenced path can't silently pass. + +``seed`` (bootstrap / ``update_baseline``) + Never fails. Just writes the current shard's failing tests so the baseline + can be (re)generated from a real run. + +``aggregate`` (final job) + Merge every shard's ``--failures-out`` / ``--ran-out`` file into a single, + sorted, ready-to-commit ``known-failures.txt`` and report stale baseline + entries (tests no longer present in any shard). Pass ``--expected-shards N`` + to fail when fewer than ``N`` shards contributed gate lists (a shard that + died before writing them), so an incomplete baseline is never produced. + +Flaky quarantine (``--flaky-tests``) + Some Delta-on-Gluten failures are non-deterministic (e.g. a native bug that + only triggers on certain runtime plans), so they are neither a stable pass + nor a stable failure and cannot live in the baseline: baselining them turns + the gate red on every run where they pass, and leaving them out turns it red + on every run where they fail. ``flaky-tests.txt`` quarantines them -- a + quarantined test never counts as a regression (when it fails) nor as + now-passing (when it passes), and is excluded from the regenerated baseline. + Its SUITE is an fnmatch glob (so one line covers a root-cause family across + generated suite variants); its TEST name is matched exactly. + +Flaky quarantine by error signature (``--flaky-error-patterns``) + When a failure is caused by a known nondeterministic bug that surfaces on a + *different test each run* (e.g. the native Delta DV bitmap row-index error), + matching by test name is whack-a-mole. ``flaky-error-patterns.txt`` instead + quarantines by root cause: each line is a regex matched against a failed + test's / text, and any failure that matches is treated as + flaky regardless of which test it landed on. + +Baseline file format (``known-failures.txt``):: + + # comment lines start with '#' + # + +The suite is always a JVM class name (dot-separated, never starts with '#'), +so a line whose first non-space character is '#' is unambiguously a comment, +and the FIRST '#' after the suite separates suite from the (possibly +'#'-containing) test name. + +Only the Python standard library is used so the script runs in the bare +centos image used by the Delta UT pipeline with no ``pip install``. +""" + +import argparse +import fnmatch +import glob +import os +import re +import sys +import xml.etree.ElementTree as ET + +# Synthetic "test name" recorded when a whole suite aborts (e.g. beforeAll +# throws) so that the JUnit XML reports a suite-level error with no per-test +# . Without this, a suite that used to pass but now aborts entirely +# would record zero failing testcases and the regression would be missed. +SUITE_ABORTED = "" + + +class NoReportsError(RuntimeError): + """Raised when no JUnit elements are found under reports_dir.""" + + +class CorruptReportError(NoReportsError): + """Raised when an expected JUnit report file (TEST-*.xml) fails to parse. + + Subclasses NoReportsError so the enforce/seed handler treats a truncated + report as a hard data error (exit 2) instead of silently dropping the + suite's results and letting the gate pass on partial data. + """ + + +SEP = "#" + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +# --------------------------------------------------------------------------- # +# Baseline (known-failures.txt) parsing / formatting +# --------------------------------------------------------------------------- # +def format_entry(suite, test): + return "{}{}{}".format(suite, SEP, test) + + +def parse_entry(line): + """Parse a 'suite#test' line into (suite, test) or return None for blanks/comments.""" + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + idx = stripped.find(SEP) + if idx < 0: + # No separator: treat the whole line as a suite-level entry. + return (stripped, SUITE_ABORTED) + return (stripped[:idx], stripped[idx + len(SEP) :]) + + +def normalize_key(suite, test): + """Normalize a (suite, test) key parsed from JUnit XML to match baseline keys. + + Baseline/flaky entries round-trip through write_entries (which collapses CR/LF + in the test name to spaces) and parse_entry (which strips the whole + ``suite#test`` line). A raw XML name carrying a trailing newline or + surrounding whitespace would therefore never equal its normalized baseline + entry: the gate would keep reporting it as a REGRESSION, and the + copy-pasteable line it prints could never suppress it (load strips it back). + Delta test names are freeform, so a version bump could introduce exactly that. + Applying the identical format+parse round-trip here keeps the two sides in + sync (and is a no-op for the normal, whitespace-free names). + """ + safe_test = (test or "").replace("\r", " ").replace("\n", " ") + normalized = parse_entry(format_entry(suite or "", safe_test)) + # parse_entry only returns None for a blank/comment line, which a real + # testcase key is not; fall back to a bare strip to keep this total. + if normalized is None: + return ((suite or "").strip(), safe_test.strip()) + return normalized + + +def load_entries(path): + """Load a set of (suite, test) tuples from a baseline/shard-list file.""" + entries = set() + if not path or not os.path.exists(path): + return entries + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + parsed = parse_entry(line) + if parsed is not None: + entries.add(parsed) + return entries + + +def make_is_flaky(flaky_entries): + """Build a predicate that matches a (suite, test) tuple against flaky entries. + + A flaky entry quarantines a test whose failure is known to be non-deterministic + (see flaky-tests.txt). The entry's SUITE is treated as an fnmatch glob so a + single line can cover a root-cause family across generated suite variants + (e.g. ``*DVs*Suite`` matches every deletion-vector merge suite, ``*`` matches + any suite); the TEST name is matched exactly (test names are freeform and may + contain glob metacharacters, so they are never globbed). + """ + exact = set() + globbed = [] + for suite, test in flaky_entries: + if any(ch in suite for ch in "*?["): + globbed.append((suite, test)) + else: + exact.add((suite, test)) + + def is_flaky(entry): + if entry in exact: + return True + suite, test = entry + for glob_suite, glob_test in globbed: + if test == glob_test and fnmatch.fnmatchcase(suite, glob_suite): + return True + return False + + return is_flaky + + +def load_patterns(path): + """Load flaky-error regex patterns from a file (one per line). + + Blank lines and ``#`` comments are ignored. Each remaining line is compiled + as a case-sensitive regex. These match the FAILURE TEXT of a failed test + (its JUnit / message + stack), so a test that fails with a + known-nondeterministic native error (e.g. the Delta DV bitmap row-index bug) + can be quarantined by root cause instead of by exact test name. + """ + patterns = [] + if not path or not os.path.exists(path): + return patterns + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + if not line.strip() or line.lstrip().startswith("#"): + continue + patterns.append(re.compile(line)) + return patterns + + +def make_signature_matcher(patterns): + """Return a predicate matching a failure text against any flaky-error pattern.""" + + def matches(text): + if not text: + return False + return any(p.search(text) for p in patterns) + + return matches + + +def write_entries(path, entries, header=None): + """Write a sorted set of (suite, test) tuples to a file.""" + os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + if header: + for hl in header.splitlines(): + fh.write(hl.rstrip() + "\n") + for suite, test in sorted(entries): + # Defensive: collapse any stray newlines so each entry stays on one line. + safe_test = test.replace("\r", " ").replace("\n", " ") + fh.write(format_entry(suite, safe_test) + "\n") + + +# --------------------------------------------------------------------------- # +# JUnit XML parsing +# --------------------------------------------------------------------------- # +def _iter_testsuites(root): + """Yield every element regardless of whether the file root is + (wrapper) or a single .""" + tag = root.tag.split("}")[-1] # strip any namespace + if tag == "testsuites": + for child in root: + if child.tag.split("}")[-1] == "testsuite": + yield child + elif tag == "testsuite": + yield root + + +def _child_local_tags(elem): + return {c.tag.split("}")[-1] for c in elem} + + +def _failure_text(tc): + """Concatenate the message attribute + body text of a testcase's + / children, for error-signature matching.""" + parts = [] + for c in tc: + if c.tag.split("}")[-1] in ("failure", "error"): + msg = c.get("message") + if msg: + parts.append(msg) + if c.text: + parts.append(c.text) + return "\n".join(parts) + + +def parse_reports(reports_dir): + """Walk reports_dir for JUnit XML and classify every test. + + Returns (passed, failed, skipped, fail_texts). The first three are sets of + (suite, test) tuples; fail_texts maps each failed (suite, test) to its + combined / message + stack text (used for error-signature + quarantine). A test is 'failed' if its has a or + child, 'skipped' if it has a child, otherwise 'passed'. Suite-level + aborts (a reporting errors/failures with no failing ) + are recorded as a synthetic (suite, SUITE_ABORTED) failure. + """ + passed, failed, skipped = set(), set(), set() + fail_texts = {} + + xml_files = [] + # ScalaTest's -u reporter and Maven surefire both write `TEST-.xml` + # under a `target/.../*-reports/` dir. Restrict the secondary glob to + # `target/` so we never parse Delta's own XML *test resources* (which live + # under src/test/resources and are not reports). The -root guard + # below is a final safety net. + for pattern in ("**/TEST-*.xml", "**/target/**/*.xml"): + xml_files.extend(glob.glob(os.path.join(reports_dir, pattern), recursive=True)) + xml_files = sorted(set(xml_files)) + + parsed_any = False + for xml_file in xml_files: + try: + tree = ET.parse(xml_file) + except ET.ParseError as exc: + # A TEST-*.xml that fails to parse is almost always a report truncated + # when a forked test JVM was killed mid-write (e.g. OOM). Silently + # skipping it drops that suite's results and could let the gate go + # green on partial data, so fail hard for report files. Other XML that + # merely matched the broad `target/**` glob is still skipped. + if os.path.basename(xml_file).startswith("TEST-"): + raise CorruptReportError( + "corrupt or truncated JUnit report {}: {}. Refusing to " + "evaluate the gate on partial data.".format(xml_file, exc) + ) + eprint("WARNING: could not parse {}: {}".format(xml_file, exc)) + continue + root = tree.getroot() + root_tag = root.tag.split("}")[-1] + if root_tag not in ("testsuites", "testsuite"): + continue # not a JUnit report + + for ts in _iter_testsuites(root): + parsed_any = True + suite_name = ts.get("name") or "" + suite_has_failing_tc = False + for tc in ts: + if tc.tag.split("}")[-1] != "testcase": + continue + suite = tc.get("classname") or suite_name + name = tc.get("name") or "" + key = normalize_key(suite, name) + tags = _child_local_tags(tc) + if "failure" in tags or "error" in tags: + failed.add(key) + suite_has_failing_tc = True + fail_texts[key] = _failure_text(tc) + elif "skipped" in tags: + skipped.add(key) + else: + passed.add(key) + + # Suite-level abort: counters say something failed but no testcase + # carried the failure (the suite blew up in beforeAll/constructor). + # Record a + # synthetic entry so the regression is visible. + try: + errors = int(ts.get("errors", "0") or "0") + failures = int(ts.get("failures", "0") or "0") + except ValueError: + errors = failures = 0 + if (errors + failures) > 0 and not suite_has_failing_tc: + failed.add(normalize_key(suite_name, SUITE_ABORTED)) + + if not parsed_any: + raise NoReportsError( + "No JUnit elements found under {}. The test reports are " + "missing or in an unexpected format -- refusing to evaluate the gate " + "on an empty result set (this would otherwise pass silently).".format( + reports_dir + ) + ) + + # A test can't be both passed and failed; failure wins. Skipped only counts + # if the test was not otherwise seen (e.g. retried). + passed -= failed + skipped -= failed + skipped -= passed + return passed, failed, skipped, fail_texts + + +# --------------------------------------------------------------------------- # +# Reporting helpers +# --------------------------------------------------------------------------- # +def _summary_sink(): + """Return a writer that mirrors to GITHUB_STEP_SUMMARY when available.""" + path = os.environ.get("GITHUB_STEP_SUMMARY") + handle = open(path, "a", encoding="utf-8") if path else None + + def write(line=""): + print(line) + if handle: + handle.write(line + "\n") + + return write, handle + + +def _print_block(write, title, entries, limit=50): + write("") + write("### {} ({})".format(title, len(entries))) + if not entries: + return + write("") + write("```") + for i, (suite, test) in enumerate(sorted(entries)): + if i >= limit: + write("... and {} more".format(len(entries) - limit)) + break + write(format_entry(suite, test)) + write("```") + + +# --------------------------------------------------------------------------- # +# Modes +# --------------------------------------------------------------------------- # +def run_enforce(args): + # In enforce mode a missing baseline file would make load_entries() return an + # empty set, silently degrading to seed mode and passing the gate without + # enforcing anything. Treat a missing path as a configuration error; an + # existing-but-empty file is still allowed (it legitimately seeds). + if args.mode == "enforce" and ( + not args.known_failures or not os.path.exists(args.known_failures) + ): + eprint( + "ERROR: --known-failures '{}' does not exist. In enforce mode the " + "baseline file must exist (an existing-but-empty file is allowed and " + "triggers seed mode). Refusing to silently pass.".format( + args.known_failures + ) + ) + return 2 + baseline = load_entries(args.known_failures) + name_flaky = make_is_flaky(load_entries(args.flaky_tests)) + sig_matches = make_signature_matcher(load_patterns(args.flaky_error_patterns)) + try: + passed, failed, skipped, fail_texts = parse_reports(args.reports_dir) + except NoReportsError as exc: + eprint("ERROR: {}".format(exc)) + return 2 + + # A test is quarantined if its NAME is in flaky-tests.txt, or its failure + # TEXT matches a flaky-error signature (e.g. the native DV bitmap row-index + # bug that hits a different DV-merge test each run). + def sig_flaky(e): + return e in fail_texts and sig_matches(fail_texts[e]) + + def flaky_is(e): + return name_flaky(e) or sig_flaky(e) + + # Always emit this shard's artifacts for the aggregation job. Signature-flaky + # failures are dropped from failures-out: the aggregate job works off these + # text-less lists and cannot re-derive the signature match, so excluding them + # here keeps the regenerated baseline from absorbing a flaky failure. Name- + # flaky entries stay (the aggregate re-filters them via flaky-tests.txt). + if args.failures_out: + write_entries(args.failures_out, {e for e in failed if not sig_flaky(e)}) + if args.ran_out: + write_entries(args.ran_out, passed | failed) + + write, handle = _summary_sink() + try: + seeding = args.mode == "seed" or not baseline + if seeding and args.mode != "seed": + write( + "> NOTE: baseline `{}` is empty -- running in SEED mode " + "(no failures will be enforced). Bootstrap the baseline from " + "the aggregated artifact, commit it, then enforcement begins.".format( + args.known_failures + ) + ) + + write( + "## Delta-on-Gluten test gate -- shard {}".format( + os.environ.get("SHARD_ID", "?") + ) + ) + write("") + write("| Category | Count |") + write("|---|---:|") + write("| Ran (pass+fail) | {} |".format(len(passed) + len(failed))) + write("| Passed | {} |".format(len(passed))) + write("| Failed | {} |".format(len(failed))) + write("| Skipped | {} |".format(len(skipped))) + write("| Baseline (known failures) | {} |".format(len(baseline))) + + if seeding: + write("") + write( + "Seed mode: recorded {} failing test(s) for this shard. " + "Nothing enforced.".format(len(failed)) + ) + return 0 + + regressions = {e for e in (failed - baseline) if not flaky_is(e)} + quarantined = {e for e in (failed - baseline) if flaky_is(e)} + # `- flaky` on fixed is defensive: flaky tests are excluded from the + # regenerated baseline (aggregate mode), so a flaky test should never be in + # `baseline` in the first place -- but if one slips in, don't let its + # non-deterministic pass trip the now-passing gate. + fixed = {e for e in (baseline & passed) if not flaky_is(e)} + expected = failed & baseline + + write("") + write("| Gate result | Count |") + write("|---|---:|") + write("| Expected failures (in baseline) | {} |".format(len(expected))) + write("| **Regressions (new failures)** | {} |".format(len(regressions))) + write("| Now-passing (remove from baseline) | {} |".format(len(fixed))) + write("| Quarantined flaky failures (ignored) | {} |".format(len(quarantined))) + + _print_block( + write, "Regressions -- new failures NOT in the baseline", regressions + ) + if regressions: + write("") + write( + "These tests were not previously known to fail. Either fix " + "the regression, or (if it is a genuinely new expected " + "failure) add the lines above to `known-failures.txt`." + ) + + _print_block( + write, + "Quarantined flaky failures -- ignored (flaky-tests.txt + flaky-error-patterns.txt)", + quarantined, + ) + + if args.fail_on_fixed: + _print_block( + write, "Now-passing -- delete these lines from the baseline", fixed + ) + + exit_code = 0 + if regressions: + for suite, test in sorted(regressions): + eprint("::error::REGRESSION {}".format(format_entry(suite, test))) + exit_code = 1 + if args.fail_on_fixed and fixed: + for suite, test in sorted(fixed): + eprint( + "::error::NOW-PASSING (remove from baseline) {}".format( + format_entry(suite, test) + ) + ) + exit_code = 1 + + if exit_code == 0: + write("") + write("All failures are expected (in the baseline). Gate passed.") + return exit_code + finally: + if handle: + handle.close() + + +def _shard_ids(files, prefix): + """Return the set of shard ids from gate-list filenames. + + Gate lists are named ``.txt`` (e.g. ``failures-shard-0.txt``, + ``ran-shard-0.txt``); this extracts the ```` token so aggregate mode + can count how many shards contributed. Matching is on the basename, so nested + download dirs are fine. + """ + ids = set() + pat = re.compile(r"^" + re.escape(prefix) + r"(.+)\.txt$") + for f in files: + m = pat.match(os.path.basename(f)) + if m: + ids.add(m.group(1)) + return ids + + +def run_aggregate(args): + failure_files = sorted( + glob.glob(os.path.join(args.inputs_dir, "**", "failures-*.txt"), recursive=True) + ) + ran_files = sorted( + glob.glob(os.path.join(args.inputs_dir, "**", "ran-*.txt"), recursive=True) + ) + + # No per-shard gate lists means the artifacts were never produced or the + # download failed (the workflow's download step is continue-on-error). Bail + # out before writing an empty baseline-out, which could otherwise be committed + # and wipe the entire known-failures.txt. + if not failure_files and not ran_files: + eprint( + "ERROR: no per-shard failures-*.txt / ran-*.txt files found under " + "{}. Refusing to aggregate an empty baseline (gate-list artifacts are " + "missing or were not downloaded).".format(args.inputs_dir) + ) + return 2 + + # Completeness guard: each shard writes its failures-.txt and ran-.txt + # together (see run_enforce), so a shard that died before the gate step -- or + # whose artifact failed to download -- contributes neither. The empty-inputs + # check above only catches losing *all* shards; without this a partial set + # would silently regenerate a baseline missing that shard's failures, wrongly + # shrinking known-failures.txt and reddening the next run. A shard counts as + # complete only when BOTH its files are present (robust to partial downloads). + if args.expected_shards: + complete = _shard_ids(failure_files, "failures-") & _shard_ids( + ran_files, "ran-" + ) + if len(complete) != args.expected_shards: + eprint( + "ERROR: expected {} shard gate-list set(s) but found {} complete " + "(shards: {}). A shard's failures-*/ran-* artifact is missing -- it " + "likely died before writing its gate lists or its artifact failed " + "to download -- so the regenerated baseline would be incomplete and " + "could wrongly shrink known-failures.txt. Refusing to aggregate.".format( + args.expected_shards, + len(complete), + ", ".join(sorted(complete)) or "none", + ) + ) + return 2 + + union_failed = set() + for f in failure_files: + union_failed |= load_entries(f) + union_ran = set() + for f in ran_files: + union_ran |= load_entries(f) + + flaky_is = make_is_flaky(load_entries(args.flaky_tests)) + # Exclude quarantined flaky tests from the regenerated baseline: a flaky test + # that happened to fail this run must never be baked into known-failures.txt + # (otherwise it would trip the now-passing gate on the next run where it + # passes). Flaky failures are tracked in flaky-tests.txt, not the baseline. + baseline_body = {e for e in union_failed if not flaky_is(e)} + + header = ( + "# Known Delta-on-Gluten unit test failures.\n" + "#\n" + "# Auto-generated by compare-test-results.py --mode aggregate.\n" + "# Format: #\n" + "# Lines starting with '#' are comments.\n" + "#\n" + "# Regenerate by running the 'Delta Spark UT (Gluten)' workflow with\n" + "# update_baseline=true and committing the produced artifact.\n" + ) + if args.baseline_out: + write_entries(args.baseline_out, baseline_body, header=header) + + write, handle = _summary_sink() + try: + write("## Delta-on-Gluten aggregated results") + write("") + write("| Metric | Count |") + write("|---|---:|") + write("| Shards with failure lists | {} |".format(len(failure_files))) + write("| Distinct failing tests | {} |".format(len(union_failed))) + write("| Distinct tests run | {} |".format(len(union_ran))) + + exit_code = 0 + if args.known_failures and os.path.exists(args.known_failures): + baseline = load_entries(args.known_failures) + if baseline: + regressions = {e for e in (union_failed - baseline) if not flaky_is(e)} + quarantined = {e for e in (union_failed - baseline) if flaky_is(e)} + fixed = { + e + for e in (baseline & (union_ran - union_failed)) + if not flaky_is(e) + } + stale = baseline - union_ran + write("| Baseline entries | {} |".format(len(baseline))) + write("| Regressions (global) | {} |".format(len(regressions))) + write("| Now-passing (global) | {} |".format(len(fixed))) + write( + "| Quarantined flaky failures (ignored) | {} |".format( + len(quarantined) + ) + ) + write("| Stale (not seen this run) | {} |".format(len(stale))) + _print_block(write, "Regressions (global)", regressions) + _print_block(write, "Now-passing (global)", fixed) + _print_block( + write, + "Quarantined flaky failures -- ignored (flaky-tests.txt + flaky-error-patterns.txt)", + quarantined, + ) + _print_block(write, "Stale baseline entries (suite/test gone)", stale) + if args.fail_on_regression and regressions: + exit_code = 1 + return exit_code + finally: + if handle: + handle.close() + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # +def str2bool(value): + return str(value).strip().lower() in ("1", "true", "yes", "y", "on") + + +def main(argv=None): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--mode", choices=("enforce", "seed", "aggregate"), default="enforce" + ) + parser.add_argument( + "--known-failures", help="Path to the committed known-failures.txt baseline." + ) + parser.add_argument( + "--flaky-tests", + help="Path to flaky-tests.txt: tests quarantined as non-deterministic. A " + "flaky test is neither counted as a regression when it fails nor as " + "now-passing when it passes, and is excluded from the regenerated baseline " + "(aggregate mode). Optional; omitting it disables quarantining.", + ) + parser.add_argument( + "--flaky-error-patterns", + help="Path to flaky-error-patterns.txt: regex patterns matched against a " + "failed test's / text (enforce mode). A failure that " + "matches is quarantined by root cause -- neither a regression nor written " + "to this shard's failures list -- so a nondeterministic native error (e.g. " + "the DV bitmap row-index bug) is ignored on whichever test it lands. " + "Optional.", + ) + parser.add_argument( + "--reports-dir", help="Root dir to search for JUnit XML (enforce/seed)." + ) + parser.add_argument( + "--failures-out", help="Write this shard's failing tests here (enforce/seed)." + ) + parser.add_argument( + "--ran-out", help="Write this shard's run tests (pass+fail) here." + ) + parser.add_argument( + "--fail-on-fixed", + type=str2bool, + default=True, + help="Fail when a baseline test now passes (default true).", + ) + parser.add_argument( + "--inputs-dir", help="Dir with per-shard failures-*/ran-* files (aggregate)." + ) + parser.add_argument( + "--expected-shards", + type=int, + default=0, + help="In aggregate mode, the number of shards expected to contribute gate " + "lists. When >0, fail if fewer complete shard gate-list pairs " + "(failures-*/ran-*) are found -- e.g. a shard died before writing them -- so " + "an incomplete baseline is never produced. 0 (default) disables the check.", + ) + parser.add_argument( + "--baseline-out", help="Write the merged baseline here (aggregate)." + ) + parser.add_argument( + "--fail-on-regression", + type=str2bool, + default=False, + help="In aggregate mode, fail if global regressions exist.", + ) + args = parser.parse_args(argv) + + if args.mode in ("enforce", "seed"): + if not args.reports_dir: + parser.error("--reports-dir is required for --mode {}".format(args.mode)) + return run_enforce(args) + return run_aggregate(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/util/delta-spark-ut/flaky-error-patterns.txt b/.github/workflows/util/delta-spark-ut/flaky-error-patterns.txt new file mode 100644 index 00000000000..4bc29122915 --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/flaky-error-patterns.txt @@ -0,0 +1,13 @@ +# Flaky-error signatures for the Delta Spark UT (Gluten) gate. +# +# Each non-comment line is a Python regex matched (re.search, case-sensitive) +# against a failed test's JUnit / text (message + stack). A +# failure that matches is QUARANTINED by root cause: it never counts as a +# regression, and is dropped from the shard's failures list so it can't leak +# into the regenerated baseline -- regardless of WHICH test it landed on. +# +# Use this (instead of flaky-tests.txt) when a known nondeterministic bug +# surfaces on a different test each run, so matching by test name is +# whack-a-mole. Prefer fixing the underlying bug and REMOVING the entry. +# +# --------------------------------------------------------------------------- diff --git a/.github/workflows/util/delta-spark-ut/flaky-tests.txt b/.github/workflows/util/delta-spark-ut/flaky-tests.txt new file mode 100644 index 00000000000..e42348f0b0d --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/flaky-tests.txt @@ -0,0 +1,24 @@ +# Quarantined flaky Delta-on-Gluten tests. +# +# These tests fail NON-DETERMINISTICALLY under the Gluten Velox bundle: the same +# byte-for-byte bundle passes them on one CI run and fails them on the next. They +# therefore cannot live in known-failures.txt -- baselining them turns the gate +# red on every run where they PASS (fail-on-fixed), while leaving them out turns +# it red on every run where they FAIL. The Delta Spark UT (Gluten) gate treats a +# quarantined test as NEUTRAL: it never counts as a regression when it fails, nor +# as now-passing when it passes, and it is excluded from the regenerated baseline. +# +# Format: #. The SUITE part is an fnmatch glob so a +# single line covers a root-cause family across generated suite variants (e.g. +# `*DVs*Suite` = every deletion-vector merge suite); the TEST name is matched +# exactly. Lines starting with '#' are comments. +# +# Prefer fixing the underlying bug and REMOVING the entry over growing this list. +# Every entry should reference a tracking issue for the root cause. +# +# NOTE: when a bug surfaces on a DIFFERENT test each run (so matching by name is +# whack-a-mole), quarantine it by ERROR SIGNATURE in flaky-error-patterns.txt +# instead. The native Delta DV bitmap row-index bug (RoaringBitmapArray +# Long.MAX_VALUE) is handled there, which is why no `*DVs*Suite` MERGE entries +# are listed below. + diff --git a/.github/workflows/util/delta-spark-ut/java-test-args.sh b/.github/workflows/util/delta-spark-ut/java-test-args.sh new file mode 100755 index 00000000000..94b3246a9fa --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/java-test-args.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Shared JVM options for running delta-io/delta's `spark` ScalaTest suite against +# the Gluten Velox bundle. SOURCE this file (don't execute it) before invoking +# `sbt spark/test`, in CI (run-delta-tests.sh) and locally: +# +# source .github/workflows/util/delta-spark-ut/java-test-args.sh +# ./build/sbt ... spark/test # in the Delta clone +# +# Why these are needed: JDK 17 + Gluten/Arrow/Netty requires extra --add-opens and +# the `io.netty.tryReflectionSetAccessible` property; otherwise the forked test JVM +# fails with "sun.misc.Unsafe or java.nio.DirectByteBuffer.(long, int) not +# available" as soon as Gluten's bundled Arrow allocator initializes Netty direct +# buffers. Delta's own `Test / javaOptions` (project/CrossSparkVersions.scala +# `java17TestSettings`) sets the base add-opens but NOT the Netty property. This set +# mirrors `extraJavaTestArgs` in Gluten's root pom.xml. +# +# Exported via JAVA_TOOL_OPTIONS (not sbt's .jvmopts/.sbtopts, which only configure +# the sbt LAUNCHER JVM) so the flags reach BOTH the launcher and the forked test JVM +# -- the forked child inherits the parent env. +# +# NOTE: no -Xmx here on purpose. JAVA_TOOL_OPTIONS is processed BEFORE the JVM +# command line, so Delta's own -Xmx (build.sbt) would win; run-delta-tests.sh bumps +# the forked-test-JVM heap via `set spark / Test / javaOptions ++= ...` instead. + +export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:+${JAVA_TOOL_OPTIONS} }\ +-XX:+IgnoreUnrecognizedVMOptions \ +--add-opens=java.base/java.lang=ALL-UNNAMED \ +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ +--add-opens=java.base/java.io=ALL-UNNAMED \ +--add-opens=java.base/java.net=ALL-UNNAMED \ +--add-opens=java.base/java.nio=ALL-UNNAMED \ +--add-opens=java.base/java.util=ALL-UNNAMED \ +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \ +--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ +--add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ +--add-opens=java.base/sun.nio.cs=ALL-UNNAMED \ +--add-opens=java.base/sun.security.action=ALL-UNNAMED \ +--add-opens=java.base/sun.util.calendar=ALL-UNNAMED \ +-Djdk.reflect.useDirectMethodHandle=false \ +-Dio.netty.tryReflectionSetAccessible=true \ +-Dfile.encoding=UTF-8" diff --git a/.github/workflows/util/delta-spark-ut/known-failures.txt b/.github/workflows/util/delta-spark-ut/known-failures.txt new file mode 100644 index 00000000000..0999a1a389c --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/known-failures.txt @@ -0,0 +1,756 @@ +# Known Delta-on-Gluten unit test failures. +# +# Baseline of delta-io/delta `spark` ScalaTest tests EXPECTED to fail under the +# Gluten Velox bundle. The Delta Spark UT (Gluten) workflow enforces this list: +# a failing test NOT listed here is a regression (fails CI); a listed test that +# now passes should be removed. Format: #. +# Lines starting with '#' are comments. See README.md in this directory. +# +# --------------------------------------------------------------------------- +# Baseline for the committed 4-shard x 4-fork config. Originally seeded from run +# 27490052632; refreshed from run 28318129710 (4 x 4) after the FileSourceScanLike +# test fixes (delta-io/delta #7104 + #7105). The authoritative failure count is +# the number of entries below (and the delta-spark-aggregate job summary); it is +# deliberately not restated here so it can't drift as the baseline is refreshed. +# +# IMPORTANT: regenerate this baseline under the SAME NUM_SHARDS x +# TEST_PARALLELISM_COUNT the gate runs (here 4 x 4). ~34 of these failures are +# fork-parallelism-sensitive -- they pass with 1 fork per JVM but fail with 4 (4 +# Velox forks x ~4G run close to the ~16G runner limit) -- so a baseline captured +# at a different parallelism spuriously flags them as regressions. +# --------------------------------------------------------------------------- +io.delta.sql.DeltaExtensionAndCatalogSuite#activate Delta SQL parser using SQL conf +io.delta.sql.DeltaExtensionAndCatalogSuite#activate Delta SQL parser using withExtensions +io.delta.sql.JavaDeltaSparkSessionExtensionSuite#testSQLConf +io.delta.tables.DeltaTableHadoopOptionsSuite#delete - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#details - with filesystem options. +io.delta.tables.DeltaTableHadoopOptionsSuite#forPath - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#forPath error out without filesystem options passed in. +io.delta.tables.DeltaTableHadoopOptionsSuite#forPath with unsupported options +io.delta.tables.DeltaTableHadoopOptionsSuite#forPath: as/alias/toDF with filesystem options. +io.delta.tables.DeltaTableHadoopOptionsSuite#generate - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#history - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#merge - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#optimize - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#restoreTable - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#update - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#updateExpr - with filesystem options +io.delta.tables.DeltaTableHadoopOptionsSuite#vacuum - with filesystem options +org.apache.spark.sql.delta.AutoCompactExecutionIdColumnMappingSuite#auto-compact-enabled-conf: auto compact should kick in when enabled - session config - column mapping id mode +org.apache.spark.sql.delta.AutoCompactExecutionIdColumnMappingSuite#auto-compact-enabled-property: auto compact should kick in when enabled - table config - column mapping id mode +org.apache.spark.sql.delta.AutoCompactExecutionIdColumnMappingSuite#auto-compact-enabled-property: auto compact should not kick in when session config is off - column mapping id mode +org.apache.spark.sql.delta.AutoCompactExecutionIdColumnMappingSuite#variant auto compact kicks in when enabled - session config - column mapping id mode +org.apache.spark.sql.delta.AutoCompactExecutionIdColumnMappingSuite#variant auto compact kicks in when enabled - table config - column mapping id mode +org.apache.spark.sql.delta.AutoCompactExecutionNameColumnMappingSuite#auto-compact-enabled-conf: auto compact should kick in when enabled - session config - column mapping name mode +org.apache.spark.sql.delta.AutoCompactExecutionNameColumnMappingSuite#auto-compact-enabled-property: auto compact should kick in when enabled - table config - column mapping name mode +org.apache.spark.sql.delta.AutoCompactExecutionNameColumnMappingSuite#auto-compact-enabled-property: auto compact should not kick in when session config is off - column mapping name mode +org.apache.spark.sql.delta.AutoCompactExecutionNameColumnMappingSuite#variant auto compact kicks in when enabled - session config - column mapping name mode +org.apache.spark.sql.delta.AutoCompactExecutionNameColumnMappingSuite#variant auto compact kicks in when enabled - table config - column mapping name mode +org.apache.spark.sql.delta.AutoCompactExecutionSuite#auto-compact-enabled-conf: auto compact should kick in when enabled - session config +org.apache.spark.sql.delta.AutoCompactExecutionSuite#auto-compact-enabled-property: auto compact should kick in when enabled - table config +org.apache.spark.sql.delta.AutoCompactExecutionSuite#auto-compact-enabled-property: auto compact should not kick in when session config is off +org.apache.spark.sql.delta.AutoCompactExecutionSuite#variant auto compact kicks in when enabled - session config +org.apache.spark.sql.delta.AutoCompactExecutionSuite#variant auto compact kicks in when enabled - table config +org.apache.spark.sql.delta.CheckpointsSuite#SC-86940: writing a GCS checkpoint should happen in a new thread +org.apache.spark.sql.delta.CheckpointsWithCatalogOwnedBatch100Suite#SC-86940: writing a GCS checkpoint should happen in a new thread +org.apache.spark.sql.delta.CheckpointsWithCatalogOwnedBatch1Suite#SC-86940: writing a GCS checkpoint should happen in a new thread +org.apache.spark.sql.delta.CheckpointsWithCatalogOwnedBatch2Suite#SC-86940: writing a GCS checkpoint should happen in a new thread +org.apache.spark.sql.delta.CloneTableSQLSuite#shallow clone across file systems +org.apache.spark.sql.delta.CloneTableSQLWithCatalogOwnedBatch100Suite#shallow clone across file systems +org.apache.spark.sql.delta.CloneTableSQLWithCatalogOwnedBatch1Suite#shallow clone across file systems +org.apache.spark.sql.delta.CloneTableSQLWithCatalogOwnedBatch2Suite#shallow clone across file systems +org.apache.spark.sql.delta.CloneTableScalaDeletionVectorSuite#Cloning table with persistent DVs and absolute parquet paths +org.apache.spark.sql.delta.CloneTableScalaDeletionVectorSuite#Shallow clone round-trip with DVs +org.apache.spark.sql.delta.CloneTableScalaDeletionVectorSuite#shallow clone across file systems +org.apache.spark.sql.delta.CloneTableScalaSuite#shallow clone across file systems +org.apache.spark.sql.delta.ConvertToDeltaSQLSuite#external tables use correct path scheme +org.apache.spark.sql.delta.ConvertToDeltaScalaSuite#external tables use correct path scheme +org.apache.spark.sql.delta.DeleteMetricsSuite#delete-metrics: delete one row per file - Partitioned = false, cdfEnabled = false +org.apache.spark.sql.delta.DeleteMetricsSuite#delete-metrics: delete one row per file - Partitioned = false, cdfEnabled = true +org.apache.spark.sql.delta.DeltaAllFilesInCrcSuite#test all-files-in-crc verification failure also triggers and logs incremental-commit verification result +org.apache.spark.sql.delta.DeltaAlterTableByNameIdColumnMappingSuite#CHANGE COLUMN - case insensitive - column mapping id mode +org.apache.spark.sql.delta.DeltaAlterTableByNameIdColumnMappingSuite#CHANGE COLUMN - move to first (nested) - column mapping id mode +org.apache.spark.sql.delta.DeltaAlterTableByNameNameColumnMappingSuite#CHANGE COLUMN - case insensitive - column mapping name mode +org.apache.spark.sql.delta.DeltaAlterTableByNameNameColumnMappingSuite#CHANGE COLUMN - move to first (nested) - column mapping name mode +org.apache.spark.sql.delta.DeltaArbitraryColumnNameSuite#create table +org.apache.spark.sql.delta.DeltaCDCStreamDeletionVectorSuite#cdc streams with noop merge +org.apache.spark.sql.delta.DeltaCDCStreamSuite#cdc streams with noop merge +org.apache.spark.sql.delta.DeltaCDCStreamWithCatalogManagedBatch100Suite#cdc streams with noop merge +org.apache.spark.sql.delta.DeltaCDCStreamWithCatalogManagedBatch1Suite#cdc streams with noop merge +org.apache.spark.sql.delta.DeltaCDCStreamWithCatalogManagedBatch2Suite#cdc streams with noop merge +org.apache.spark.sql.delta.DeltaColumnMappingSuite#add nested column in schema on new protocol +org.apache.spark.sql.delta.DeltaColumnMappingSuite#alter column order in schema on new protocol +org.apache.spark.sql.delta.DeltaColumnMappingSuite#explicit id matching +org.apache.spark.sql.delta.DeltaColumnMappingSuite#id and name mode should write field_id in parquet schema +org.apache.spark.sql.delta.DeltaColumnMappingSuite#try modifying restricted max id property should fail +org.apache.spark.sql.delta.DeltaDataFrameHadoopOptionsSuite#SC-86916: Delta log cache should respect options +org.apache.spark.sql.delta.DeltaDataFrameHadoopOptionsSuite#SC-86916: checkpoint should pick up Hadoop file system options +org.apache.spark.sql.delta.DeltaDataFrameHadoopOptionsSuite#SC-86916: invalidateCache should invalidate all DeltaLogs of the given path +org.apache.spark.sql.delta.DeltaDataFrameHadoopOptionsSuite#SC-86916: read/write Delta paths using DataFrame should pick up Hadoop file system options +org.apache.spark.sql.delta.DeltaDataFrameHadoopOptionsSuite#all operations should propagate Hadoop file system options +org.apache.spark.sql.delta.DeltaDataFrameHadoopOptionsSuite#operations without Hadoop options should fail for fake:// filesystem +org.apache.spark.sql.delta.DeltaFastDropFeatureSuite#Vacuum does not delete deletion vector files.generateDVTombstones: false +org.apache.spark.sql.delta.DeltaGenerateSymlinkManifestSuite#incremental manifest: failure to generate manifest throws exception +org.apache.spark.sql.delta.DeltaGenerateSymlinkManifestSuite#special partition column values +org.apache.spark.sql.delta.DeltaHistoryManagerSuite#data skipping still works with time travel +org.apache.spark.sql.delta.DeltaHistoryManagerWithCatalogOwnedBatch100Suite#data skipping still works with time travel +org.apache.spark.sql.delta.DeltaHistoryManagerWithCatalogOwnedBatch1Suite#data skipping still works with time travel +org.apache.spark.sql.delta.DeltaHistoryManagerWithCatalogOwnedBatch2Suite#data skipping still works with time travel +org.apache.spark.sql.delta.DeltaInsertIntoDataFrameByPathSuite#insertInto: timestamp partition values with different precisions +org.apache.spark.sql.delta.DeltaInsertIntoDataFrameSuite#insertInto: timestamp partition values with different precisions +org.apache.spark.sql.delta.DeltaInsertIntoSQLByPathSuite#insertInto: timestamp partition values with different precisions +org.apache.spark.sql.delta.DeltaInsertIntoSQLSuite#insertInto: timestamp partition values with different precisions +org.apache.spark.sql.delta.DeltaLiteVacuumSuite#vacuum for cdc - delete tombstones +org.apache.spark.sql.delta.DeltaLiteVacuumSuite#vacuum for cdc - update/merge +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=false, with rowIndexCol=false, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=false, with rowIndexCol=false, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=false, with rowIndexCol=false, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=false, with rowIndexCol=true, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=false, with rowIndexCol=true, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=false, with rowIndexCol=true, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=false, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=false, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=false, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=true, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=true, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=false, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=true, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=true, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=false, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=true, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=false, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=true, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=false, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=true, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=true, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=true, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=true, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatSuite#isDeletionVectorsEnabled=true, read DV metadata columns: with isRowDeletedCol=true, with rowIndexCol=true, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatWithPredicatePushdownSuite#read DV metadata columns: with rowIndexFilterType=IF_CONTAINED, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatWithPredicatePushdownSuite#read DV metadata columns: with rowIndexFilterType=IF_CONTAINED, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatWithPredicatePushdownSuite#read DV metadata columns: with rowIndexFilterType=IF_CONTAINED, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatWithPredicatePushdownSuite#read DV metadata columns: with rowIndexFilterType=IF_NOT_CONTAINED, with vectorized Parquet reader=false, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaParquetFileFormatWithPredicatePushdownSuite#read DV metadata columns: with rowIndexFilterType=IF_NOT_CONTAINED, with vectorized Parquet reader=true, with readColumnarBatchAsRows=false +org.apache.spark.sql.delta.DeltaParquetFileFormatWithPredicatePushdownSuite#read DV metadata columns: with rowIndexFilterType=IF_NOT_CONTAINED, with vectorized Parquet reader=true, with readColumnarBatchAsRows=true +org.apache.spark.sql.delta.DeltaSinkIdColumnMappingSuite#partitioned writing and batch reading - column mapping id mode +org.apache.spark.sql.delta.DeltaSinkNameColumnMappingSuite#partitioned writing and batch reading - column mapping name mode +org.apache.spark.sql.delta.DeltaSuite#SC-8810: skip deleted file +org.apache.spark.sql.delta.DeltaSuite#SC-8810: skipping deleted file still throws on corrupted file +org.apache.spark.sql.delta.DeltaSuite#all operations with special characters in path +org.apache.spark.sql.delta.DeltaSuite#deleted files cause failure by default +org.apache.spark.sql.delta.DeltaSuite#invalid replaceWhere +org.apache.spark.sql.delta.DeltaSuite#replaceArbitrary should enforce proper usage of backtick +org.apache.spark.sql.delta.DeltaTableCreationSuite#Default column values: CONVERT TO DELTA keeps EXISTS_DEFAULT +org.apache.spark.sql.delta.DeltaUpdateCatalogSuite#convert to delta with partitioning change +org.apache.spark.sql.delta.DeltaUpdateCatalogSuite#partitioned convert to delta with schema change +org.apache.spark.sql.delta.DeltaVacuumSuite#vacuum for cdc - delete tombstones +org.apache.spark.sql.delta.DeltaVacuumSuite#vacuum for cdc - update/merge +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch100Suite#SC-8810: skip deleted file +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch100Suite#SC-8810: skipping deleted file still throws on corrupted file +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch100Suite#deleted files cause failure by default +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch100Suite#invalid replaceWhere +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch100Suite#replaceArbitrary should enforce proper usage of backtick +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch1Suite#SC-8810: skip deleted file +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch1Suite#SC-8810: skipping deleted file still throws on corrupted file +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch1Suite#deleted files cause failure by default +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch1Suite#invalid replaceWhere +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch1Suite#replaceArbitrary should enforce proper usage of backtick +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch2Suite#SC-8810: skip deleted file +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch2Suite#SC-8810: skipping deleted file still throws on corrupted file +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch2Suite#deleted files cause failure by default +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch2Suite#invalid replaceWhere +org.apache.spark.sql.delta.DeltaWithCatalogOwnedBatch2Suite#replaceArbitrary should enforce proper usage of backtick +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only delete all rows - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only delete all rows - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only delete all rows - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only delete all rows - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with duplicates - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with duplicates - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with duplicates - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with duplicates - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with skipping - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with skipping - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with skipping - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only with skipping - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join with source with 1 row - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join with source with 1 row - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join with source with 1 row - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: delete-only without join with source with 1 row - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with skipping - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with skipping - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with skipping - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with skipping - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: not matched by source update only - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: not matched by source update only - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: not matched by source update only - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: not matched by source update only - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: replace target with source - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: replace target with source - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: replace target with source - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: replace target with source - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert and delete with conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert and delete with conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert and delete with conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#merge-metrics: upsert and delete with conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistorySuite#operation metrics - merge +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only delete all rows - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only delete all rows - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only delete all rows - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only delete all rows - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with condition on delete and insert with no matching rows - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with duplicates - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with duplicates - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with duplicates - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with duplicates - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with skipping - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with skipping - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with skipping - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only with skipping - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join with source with 1 row - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join with source with 1 row - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join with source with 1 row - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: delete-only without join with source with 1 row - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: insert-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with skipping - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with skipping - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with skipping - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with skipping - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: match-only with update/delete with unsatisfied conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: not matched by source update only - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: not matched by source update only - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: not matched by source update only - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: not matched by source update only - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: replace target with source - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: replace target with source - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: replace target with source - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: replace target with source - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: update/delete/insert with some unsatisfied conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert and delete with conditions - Partitioned = false, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert and delete with conditions - Partitioned = false, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert and delete with conditions - Partitioned = true, CDF = false +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#merge-metrics: upsert and delete with conditions - Partitioned = true, CDF = true +org.apache.spark.sql.delta.DescribeDeltaHistoryWithCatalogOwnedBatch100Suite#operation metrics - merge +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Disable Deletion Vectors feature - withUnset: false +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Disable Deletion Vectors feature - withUnset: true +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Disable row tracking feature - withUnset: false +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Disable row tracking feature - withUnset: true +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Enable column mapping feature - txnInterleaved: true +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Enable deletion vectors feature +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Enable row tracking feature concurrent txn: delete +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Removing column mapping mode produces conflict - startMode: id +org.apache.spark.sql.delta.FeatureEnablementConcurrencySuite#Removing column mapping mode produces conflict - startMode: name +org.apache.spark.sql.delta.FileSizeHistogramSuite#check CommitStats with deletes +org.apache.spark.sql.delta.FileSizeHistogramSuite#histogram is re-calculated when files are removed +org.apache.spark.sql.delta.GeneratedColumnSuite#update_generated_column_with_incorrect_value +org.apache.spark.sql.delta.GeneratedColumnSuite#update_source_and_generated_columns_with_incorrect_value +org.apache.spark.sql.delta.HDFSLogStoreSuite#No AbstractFileSystem - end to end test using data frame +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#Convert a partitioned parquet table with partition schema autofill +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#can convert a partition-like table path +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#can convert table with partition overwrite +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#catalog partition values contain special characters +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a Hive based external parquet table +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a Hive based parquet table +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a delta table where metadata does not reflect that the table is already converted should update the metadata +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a parquet path to delta while database called parquet exists +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a parquet table to delta with database name as parquet +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a parquet table using table name +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a parquet table with catalog schema - false +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert a parquet table with catalog schema - true +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert an external parquet table +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert partitioned parquet table with catalog partitions - false +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert partitioned parquet table with catalog partitions - true +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert to delta using table name without database name +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert two external tables pointing to same underlying files with differing table properties should error if conf enabled otherwise merge properties +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert with statistics +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#convert without statistics +org.apache.spark.sql.delta.HiveConvertToDeltaSuite#negative case: convert parquet path to delta when there is a database called parquet but no table or path exists +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: ARRAY, targetType: ARRAY followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: ARRAY, targetType: ARRAY followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: ARRAY, targetType: ARRAY followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: ARRAY, targetType: ARRAY followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: DECIMAL(7,2) followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: DECIMAL(7,2) followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: DECIMAL(7,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: DECIMAL(7,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: INT followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: INT followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: BIGINT, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DECIMAL(3,1), targetType: DECIMAL(3,2) followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DECIMAL(3,1), targetType: DECIMAL(3,2) followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DECIMAL(3,1), targetType: DECIMAL(3,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DECIMAL(3,1), targetType: DECIMAL(3,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DOUBLE, targetType: BIGINT followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DOUBLE, targetType: BIGINT followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DOUBLE, targetType: BIGINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: DOUBLE, targetType: BIGINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: SMALLINT followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: SMALLINT followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: SMALLINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: SMALLINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: TINYINT followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: TINYINT followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: TINYINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: INT, targetType: TINYINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: MAP, targetType: MAP followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: MAP, targetType: MAP followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: MAP, targetType: MAP followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: MAP, targetType: MAP followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: STRING, targetType: INT followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: STRING, targetType: INT followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: STRING, targetType: INT followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: STRING, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: STRING, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: Struct, targetType: Struct followAnsiEnabled: false, ansiEnabled: false, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: Struct, targetType: Struct followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: Struct, targetType: Struct followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitMergeCastingSuite#MERGE overflow in WHEN MATCHED THEN UPDATE SET t.value = s.value sourceType: Struct, targetType: Struct followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: ARRAY, targetType: ARRAY followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: ARRAY, targetType: ARRAY followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: BIGINT, targetType: DECIMAL(7,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: BIGINT, targetType: DECIMAL(7,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: BIGINT, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: BIGINT, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: DECIMAL(3,1), targetType: DECIMAL(3,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: DECIMAL(3,1), targetType: DECIMAL(3,2) followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: DOUBLE, targetType: BIGINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: DOUBLE, targetType: BIGINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: INT, targetType: SMALLINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: INT, targetType: SMALLINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: INT, targetType: TINYINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: INT, targetType: TINYINT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: MAP, targetType: MAP followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: MAP, targetType: MAP followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: STRING, targetType: INT followAnsiEnabled: false, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: STRING, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: STRING, targetType: INT followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: Struct, targetType: Struct followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: ANSI +org.apache.spark.sql.delta.ImplicitStreamingMergeCastingSuite#Streaming MERGE overflow sourceType: Struct, targetType: Struct followAnsiEnabled: true, ansiEnabled: true, storeAssignmentPolicy: LEGACY +org.apache.spark.sql.delta.PublicHDFSLogStoreSuite#No AbstractFileSystem - end to end test using data frame +org.apache.spark.sql.delta.RestoreTableSQLSuite#cdf + RESTORE +org.apache.spark.sql.delta.RestoreTableSQLSuite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableSQLSuite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableSQLWithCatalogOwnedBatch100Suite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableSQLWithCatalogOwnedBatch100Suite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableSQLWithCatalogOwnedBatch1Suite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableSQLWithCatalogOwnedBatch1Suite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableSQLWithCatalogOwnedBatch2Suite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableSQLWithCatalogOwnedBatch2Suite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableScalaDeletionVectorSuite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableScalaDeletionVectorSuite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableScalaSuite#cdf + RESTORE +org.apache.spark.sql.delta.RestoreTableScalaSuite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableScalaSuite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableScalaWithCatalogOwnedBatch100Suite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableScalaWithCatalogOwnedBatch100Suite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableScalaWithCatalogOwnedBatch1Suite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableScalaWithCatalogOwnedBatch1Suite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.RestoreTableScalaWithCatalogOwnedBatch2Suite#restore command output metrics +org.apache.spark.sql.delta.RestoreTableScalaWithCatalogOwnedBatch2Suite#restore operation metrics in Delta table history +org.apache.spark.sql.delta.SnapshotManagementSuite#recover from a corrupt checkpoint: previous checkpoint doesn't exist +org.apache.spark.sql.delta.SnapshotManagementSuite#should not recover when both the current and previous checkpoints are broken +org.apache.spark.sql.delta.SnapshotManagementWithCoordinatedCommitsBatch100Suite#recover from a corrupt checkpoint: previous checkpoint doesn't exist +org.apache.spark.sql.delta.SnapshotManagementWithCoordinatedCommitsBatch100Suite#should not recover when both the current and previous checkpoints are broken +org.apache.spark.sql.delta.SnapshotManagementWithCoordinatedCommitsBatch1Suite#recover from a corrupt checkpoint: previous checkpoint doesn't exist +org.apache.spark.sql.delta.SnapshotManagementWithCoordinatedCommitsBatch1Suite#should not recover when both the current and previous checkpoints are broken +org.apache.spark.sql.delta.SnapshotManagementWithCoordinatedCommitsBatch2Suite#recover from a corrupt checkpoint: previous checkpoint doesn't exist +org.apache.spark.sql.delta.SnapshotManagementWithCoordinatedCommitsBatch2Suite#should not recover when both the current and previous checkpoints are broken +org.apache.spark.sql.delta.UpdateMetricsSuite#update-metrics: update one row per file - Partitioned = false, cdfEnabled = false +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#DELETE - Scenario 1 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#DELETE - Scenario 2 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#DELETE - Scenario 3 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#DELETE - Scenario 4 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#DELETE - Scenario 5 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#DELETE - Scenario 6 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#DELETE - Scenario 7 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#OPTIMIZE - Scenario 1 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#OPTIMIZE - Scenario 2 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#OPTIMIZE - Scenario 3 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#OPTIMIZE - Scenario 4 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#OPTIMIZE - Scenario 5 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#OPTIMIZE - Scenario 6 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#OPTIMIZE - Scenario 7 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#UPDATE - Scenario 1 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#UPDATE - Scenario 2 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#UPDATE - Scenario 3 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#UPDATE - Scenario 4 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#UPDATE - Scenario 5 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#UPDATE - Scenario 6 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsDVSuite#UPDATE - Scenario 7 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#DELETE - Scenario 1 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#DELETE - Scenario 2 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#DELETE - Scenario 3 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#DELETE - Scenario 4 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#DELETE - Scenario 5 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#DELETE - Scenario 6 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#DELETE - Scenario 7 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#OPTIMIZE - Scenario 1 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#OPTIMIZE - Scenario 2 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#OPTIMIZE - Scenario 3 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#OPTIMIZE - Scenario 4 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#OPTIMIZE - Scenario 5 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#OPTIMIZE - Scenario 6 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#OPTIMIZE - Scenario 7 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#UPDATE - Scenario 1 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#UPDATE - Scenario 2 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#UPDATE - Scenario 3 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#UPDATE - Scenario 4 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#UPDATE - Scenario 5 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#UPDATE - Scenario 6 +org.apache.spark.sql.delta.commands.backfill.RowTrackingBackfillConflictsSuite#UPDATE - Scenario 7 +org.apache.spark.sql.delta.concurrency.TransactionExecutionObserverSuite#Phase Locking - delete command +org.apache.spark.sql.delta.coordinatedcommits.CoordinatedCommitsSuite#Incomplete backfills are handled properly by next commit after CC to FS conversion +org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite#DELETE with DVs with column mapping mode=id +org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite#huge table: delete a small number of rows from tables of 2B rows with DVs +org.apache.spark.sql.delta.deletionvectors.DeletionVectorsSuite#huge table: read from tables of 2B rows with existing DV of many zeros +org.apache.spark.sql.delta.deletionvectors.DeletionVectorsWithPredicatePushdownSuite#(It is not a test it is a sbt.testing.SuiteSelector) +org.apache.spark.sql.delta.deletionvectors.DeletionVectorsWithPredicatePushdownSuite# +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLNameBasedSuite#test delete on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLNameBasedSuite#test delete on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedCDCOnSuite#test delete on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedCDCOnSuite#test delete on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedDVPredPushOffSuite#test delete on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedDVPredPushOffSuite#test delete on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedDVPredPushOnSuite#test delete on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedDVPredPushOnSuite#test delete on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedSuite#test delete on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.DeleteTempViewSQLPathBasedSuite#test delete on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.MergeCDCSQLPathBasedCDCOnSuite#merge CDC - all conditions failed for all rows +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOffSuite#Merge with DVs metrics - Incremental Updates +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOffSuite#Merge with DVs metrics - delete entire file +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOffSuite#Verify error is produced when paths are not joined correctly +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOnSuite#Merge with DVs metrics - Incremental Updates +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOnSuite#Merge with DVs metrics - delete entire file +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedCDCOnDVsPredPushOnSuite#Verify error is produced when paths are not joined correctly +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedDVsPredPushOffSuite#Merge with DVs metrics - Incremental Updates +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedDVsPredPushOffSuite#Merge with DVs metrics - delete entire file +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedDVsPredPushOffSuite#Verify error is produced when paths are not joined correctly +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedDVsPredPushOnSuite#Merge with DVs metrics - Incremental Updates +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedDVsPredPushOnSuite#Merge with DVs metrics - delete entire file +org.apache.spark.sql.delta.generatedsuites.MergeIntoDVsSQLPathBasedDVsPredPushOnSuite#Verify error is produced when paths are not joined correctly +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, non-null target leaves, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, non-null target leaves, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null source nested map, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null source nested map, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null target col, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null target col, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null target leaves, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null target leaves, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null target nested struct, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedMapStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested map-of-struct - non-null source leaves, null target nested struct, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLNameBasedSuite#schema evolution - struct in different order +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLNameBasedSuite#schema evolution - struct in different order - with evolution disabled +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - struct in different order +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - struct in different order - with evolution disabled +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - struct in different order +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - struct in different order - with evolution disabled +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnSuite#schema evolution - struct in different order +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedCDCOnSuite#schema evolution - struct in different order - with evolution disabled +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedSuite#schema evolution - struct in different order +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertSQLPathBasedSuite#schema evolution - struct in different order - with evolution disabled +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertScalaSuite#schema evolution - struct in different order +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionInsertScalaSuite#schema evolution - struct in different order - with evolution disabled +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, non-null target leaves, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, non-null target leaves, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, null target col, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, null target col, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, null target leaves, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, null target leaves, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, null target nested struct, UPDATE * +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionNullnessSQLNameBasedSuite#schema evolution - nested struct - non-null source leaves, null target nested struct, UPDATE t.col = s.col +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLNameBasedSuite#schema evolution - extra nested column in source - update, isPartitioned=false +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLNameBasedSuite#schema evolution - extra nested column in source - update, isPartitioned=true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLNameBasedSuite#schema evolution - extra nested column in source - update, partition on unused column +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - extra nested column in source - update, isPartitioned=false +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - extra nested column in source - update, isPartitioned=true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - extra nested column in source - update, partition on unused column +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - extra nested column in source - update, isPartitioned=false +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - extra nested column in source - update, isPartitioned=true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - extra nested column in source - update, partition on unused column +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnSuite#schema evolution - extra nested column in source - update, isPartitioned=false +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnSuite#schema evolution - extra nested column in source - update, isPartitioned=true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedCDCOnSuite#schema evolution - extra nested column in source - update, partition on unused column +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedSuite#schema evolution - extra nested column in source - update, isPartitioned=false +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedSuite#schema evolution - extra nested column in source - update, isPartitioned=true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlySQLPathBasedSuite#schema evolution - extra nested column in source - update, partition on unused column +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlyScalaSuite#schema evolution - extra nested column in source - update, isPartitioned=false +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlyScalaSuite#schema evolution - extra nested column in source - update, isPartitioned=true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructEvolutionUpdateOnlyScalaSuite#schema evolution - extra nested column in source - update, partition on unused column +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLNameBasedSuite#schema evolution - new source column in map struct key +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLNameBasedSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - new source column in map struct key +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - new source column in map struct key +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnSuite#schema evolution - new source column in map struct key +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedCDCOnSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedDVsPredPushOffSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedDVsPredPushOnSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedSuite#schema evolution - new source column in map struct key +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionSQLPathBasedSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionScalaSuite#schema evolution - new source column in map struct key +org.apache.spark.sql.delta.generatedsuites.MergeIntoNestedStructInMapEvolutionScalaSuite#schema evolution - source nested map struct key contains less columns than target +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2SQLNameBasedSuite#not matched by source - all 3 clauses - no changes - isPartitioned: false - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2SQLNameBasedSuite#not matched by source - all 3 clauses - no changes - isPartitioned: true - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedCDCOnSuite#not matched by source - all 3 clauses - no changes - isPartitioned: false - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedCDCOnSuite#not matched by source - all 3 clauses - no changes - isPartitioned: true - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedSuite#not matched by source - all 3 clauses - no changes - isPartitioned: false - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2SQLPathBasedSuite#not matched by source - all 3 clauses - no changes - isPartitioned: true - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2ScalaSuite#not matched by source - all 3 clauses - no changes - isPartitioned: false - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoNotMatchedBySourceCDCPart2ScalaSuite#not matched by source - all 3 clauses - no changes - isPartitioned: true - cdcEnabled: true +org.apache.spark.sql.delta.generatedsuites.MergeIntoSQLSQLNameBasedSuite#CTE as a source in MERGE +org.apache.spark.sql.delta.generatedsuites.MergeIntoSQLSQLPathBasedCDCOnDVsPredPushOffSuite#CTE as a source in MERGE +org.apache.spark.sql.delta.generatedsuites.MergeIntoSQLSQLPathBasedCDCOnDVsPredPushOnSuite#CTE as a source in MERGE +org.apache.spark.sql.delta.generatedsuites.MergeIntoSQLSQLPathBasedCDCOnSuite#CTE as a source in MERGE +org.apache.spark.sql.delta.generatedsuites.MergeIntoSQLSQLPathBasedDVsPredPushOffSuite#CTE as a source in MERGE +org.apache.spark.sql.delta.generatedsuites.MergeIntoSQLSQLPathBasedDVsPredPushOnSuite#CTE as a source in MERGE +org.apache.spark.sql.delta.generatedsuites.MergeIntoSQLSQLPathBasedSuite#CTE as a source in MERGE +org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLNameBasedSuite#schema evolution - extra nested column in source - update - single target partition +org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnDVsPredPushOffSuite#schema evolution - extra nested column in source - update - single target partition +org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnDVsPredPushOnSuite#schema evolution - extra nested column in source - update - single target partition +org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedCDCOnSuite#schema evolution - extra nested column in source - update - single target partition +org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnSQLPathBasedSuite#schema evolution - extra nested column in source - update - single target partition +org.apache.spark.sql.delta.generatedsuites.MergeIntoSchemaEvolutionBaseNewColumnScalaSuite#schema evolution - extra nested column in source - update - single target partition +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLNameBasedSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLNameBasedSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOffSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOffSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOffSuite#merge with repartition - partition on multiple columns +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOnSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOnSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnDVsPredPushOnSuite#merge with repartition - partition on multiple columns +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedCDCOnSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOffSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOffSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOffSuite#merge with repartition - partition on multiple columns +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOnSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOnSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedDVsPredPushOnSuite#merge with repartition - partition on multiple columns +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscSQLPathBasedSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscScalaSuite#data skipping with matched predicates - with insert clause +org.apache.spark.sql.delta.generatedsuites.MergeIntoSuiteBaseMiscScalaSuite#merge with repartition - insert only merge +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLNameBasedSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLNameBasedSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedCDCOnDVSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedCDCOnDVSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedCDCOnRowTrackingOffSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedCDCOnRowTrackingOffSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedCDCOnSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedCDCOnSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedDVPredPushOffSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedDVPredPushOffSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedDVPredPushOnSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedDVPredPushOnSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedRowTrackingOffSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedRowTrackingOffSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedSuite#test update on temp view - nontrivial projection - Dataset TempView +org.apache.spark.sql.delta.generatedsuites.UpdateBaseTempViewSQLPathBasedSuite#test update on temp view - nontrivial projection - SQL TempView +org.apache.spark.sql.delta.optimize.OptimizeCompactionSQLSuite#optimize - multiple jobs start executing at once +org.apache.spark.sql.delta.optimize.OptimizeCompactionScalaSuite#optimize - multiple jobs start executing at once +org.apache.spark.sql.delta.optimize.OptimizeConflictSuite#conflict handling between Optimize and Business Txn +org.apache.spark.sql.delta.optimize.OptimizeMetricsSuite#optimize ZOrderBy operation metrics in Delta table history +org.apache.spark.sql.delta.optimize.OptimizeMetricsSuite#optimize metrics on idempotent operations +org.apache.spark.sql.delta.perf.OptimizeGeneratedColumnSuite#end-to-end test of behaviors of write/read null on partition column +org.apache.spark.sql.delta.perf.OptimizeGeneratedColumnSuite#five digits year in a date_format yyyy-MM partition column +org.apache.spark.sql.delta.perf.OptimizeGeneratedColumnSuite#five digits year in a date_format yyyy-MM-dd-HH partition column +org.apache.spark.sql.delta.rowid.ConflictCheckerRowIdSuite#Re-added files keep their row IDs after conflict with txn not updating high watermark +org.apache.spark.sql.delta.rowid.ConflictCheckerRowIdSuite#concurrent transactions do not assign overlapping row IDs +org.apache.spark.sql.delta.rowid.ConflictCheckerRowIdSuite#re-added files keep their row ids +org.apache.spark.sql.delta.rowid.RowIdSuite#Filter by base Row IDs +org.apache.spark.sql.delta.rowid.RowIdSuite#Filter by base Row IDs in subquery +org.apache.spark.sql.delta.rowid.RowIdSuite#No dictionary filtering on _metadata.row_id +org.apache.spark.sql.delta.rowid.RowIdSuite#missing base row ids and default row commit versions +org.apache.spark.sql.delta.rowid.RowIdSuite#row ids can be read back +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencySuite#Interleaved delete right after protocol downgrade should abort due to protocol change +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencySuite#Interleaved update right after protocol downgrade should abort due to protocol change +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencySuite#Single Unbackfill batch interleaves delete +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencySuite#Single Unbackfill batch interleaves update +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencyWithoutDVsSuite#Interleaved delete right after protocol downgrade should abort due to protocol change +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencyWithoutDVsSuite#Interleaved update right after protocol downgrade should abort due to protocol change +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencyWithoutDVsSuite#Single Unbackfill batch interleaves delete +org.apache.spark.sql.delta.rowid.RowTrackingRemovalConcurrencyWithoutDVsSuite#Single Unbackfill batch interleaves update +org.apache.spark.sql.delta.rowtracking.RowTrackingReadWriteSuite#write and read table with all-null materialized columns +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#Data skipping handles aliasing for _metadata fields +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#Data skipping handles aliasing for _metadata fields - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping flags +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping flags - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping on TIMESTAMP_NTZ +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping on TIMESTAMP_NTZ - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping stats before and after optimize +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1JsonCheckpointV2Suite#data skipping stats before and after optimize - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - double nested, single 1 - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - double nested, single 1 - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - backtick escapes work as expected - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - backtick escapes work as expected - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - index only a subset of leaf columns - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - index only a subset of leaf columns - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - naming a nested column allows nested complex types - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - naming a nested column allows nested complex types - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - naming a nested column indexes all leaf fields of that column - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - indexed column names - naming a nested column indexes all leaf fields of that column - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested schema - # indexed column = 3 - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested schema - # indexed column = 3 - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested schema - # indexed column = 6 - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested schema - # indexed column = 6 - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested schema - # indexed column = 9 - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested schema - # indexed column = 9 - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested, single 1 - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - nested, single 1 - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - starts with, nested - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping by stats - starts with, nested - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping flags - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping flags - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP_NTZ - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP_NTZ - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping stats before and after optimize - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping stats before and after optimize - column mapping name mode - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping with a different DataFrame schema order and nested columns - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1NameColumnMappingSuite#data skipping with missing columns in DataFrame - column mapping name mode +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#Data skipping handles aliasing for _metadata fields +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#Data skipping handles aliasing for _metadata fields - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping flags +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping flags - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping on TIMESTAMP_NTZ +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping on TIMESTAMP_NTZ - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping stats before and after optimize +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1ParquetCheckpointV2Suite#data skipping stats before and after optimize - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#Data skipping handles aliasing for _metadata fields +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#Data skipping handles aliasing for _metadata fields - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping flags +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping flags - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping on TIMESTAMP_NTZ +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping on TIMESTAMP_NTZ - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping stats before and after optimize +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1Suite#data skipping stats before and after optimize - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#Data skipping handles aliasing for _metadata fields +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#Data skipping handles aliasing for _metadata fields - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping flags +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping flags - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping on TIMESTAMP_NTZ +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping on TIMESTAMP_NTZ - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch100Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#Data skipping handles aliasing for _metadata fields +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#Data skipping handles aliasing for _metadata fields - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping flags +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping flags - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping on TIMESTAMP_NTZ +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping on TIMESTAMP_NTZ - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch1Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#Data skipping handles aliasing for _metadata fields +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#Data skipping handles aliasing for _metadata fields - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping flags +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping flags - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping on TIMESTAMP_NTZ +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping on TIMESTAMP_NTZ - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping on TIMESTAMP_NTZ near Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue +org.apache.spark.sql.delta.stats.DataSkippingDeltaV1WithCatalogOwnedBatch2Suite#data skipping on TIMESTAMP_NTZ with Long.MaxValue - old behavior with DataFrame schema +org.apache.spark.sql.delta.stats.PartitionLikeDataSkippingColumnMappingSuite#partition-like data skipping for expression COALESCE: COALESCE(TO_DATE(S.b), c) = '1976-07-03' - column mapping id mode +org.apache.spark.sql.delta.stats.StatsCollectionSuite#recompute stats multiple columns and files +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#type widening BIGINT -> DECIMAL(20,0), partitioned=true +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#type widening DATE -> TIMESTAMP_NTZ, partitioned=false +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#type widening DATE -> TIMESTAMP_NTZ, partitioned=true +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#type widening DECIMAL(9,2) -> DECIMAL(19,3), partitioned=true +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#type widening FLOAT -> DOUBLE, partitioned=true +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#type widening INT -> DOUBLE, partitioned=true +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#unsupported type changes DOUBLE -> FLOAT, partitioned=true +org.apache.spark.sql.delta.typewidening.TypeWideningAlterTableSuite#unsupported type changes TIMESTAMP_NTZ -> DATE, partitioned=false +org.apache.spark.sql.delta.typewidening.TypeWideningInsertSchemaEvolutionBasicSuite#INSERT - always automatic type widening DATE -> TIMESTAMP_NTZ +org.apache.spark.sql.delta.typewidening.TypeWideningInsertSchemaEvolutionBasicSuite#INSERT - automatic type widening DATE -> TIMESTAMP_NTZ +org.apache.spark.sql.delta.typewidening.TypeWideningInsertSchemaEvolutionBasicSuite#INSERT - unsupported automatic type widening TIMESTAMP_NTZ -> DATE +org.apache.spark.sql.delta.typewidening.TypeWideningMergeIntoSchemaEvolutionSuite#MERGE - automatic type widening DATE -> TIMESTAMP_NTZ +org.apache.spark.sql.delta.typewidening.TypeWideningMergeIntoSchemaEvolutionSuite#MERGE - unsupported automatic type widening TIMESTAMP_NTZ -> DATE +org.apache.spark.sql.delta.typewidening.TypeWideningTableFeatureAdvancedSuite#drop feature after type change DATE -> TIMESTAMP_NTZ +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate no duplicates - Native +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate no duplicates - Portable +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate no duplicates - invalid Int ids - Native +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate no duplicates - invalid Int ids - Portable +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate no duplicates - invalid unsigned Int ids - Native +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate no duplicates - invalid unsigned Int ids - Portable +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate with duplicates - Native +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate with duplicates - Portable +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate with duplicates - invalid Int ids - Native +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate with duplicates - invalid Int ids - Portable +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate with duplicates - invalid unsigned Int ids - Native +org.apache.spark.sql.delta.util.BitmapAggregatorE2ESuite#DataFrame bitmap groupBy aggregate with duplicates - invalid unsigned Int ids - Portable diff --git a/.github/workflows/util/delta-spark-ut/run-delta-tests.sh b/.github/workflows/util/delta-spark-ut/run-delta-tests.sh new file mode 100755 index 00000000000..79238505a43 --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/run-delta-tests.sh @@ -0,0 +1,274 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Runs the Delta `spark` module tests for one shard under the Gluten bundle: +# arms a hang watchdog (thread-dumps + kills a wedged fork), invokes sbt +# spark/test with the tuned JVM/heap flags, prints cgroup memory forensics, and +# then gates the results against the baseline (compare-test-results.py). Extracted +# from delta_spark_ut.yml so the workflow step stays readable. +# +# Driven by environment (set by the workflow step / job): +# SHARD_ID - this shard's id (matrix.shard) +# SPARK_VERSION - Delta -DsparkVersion value +# UPDATE_BASELINE - 'true' -> gate seed mode; else enforce +# FAIL_ON_FIXED - passed through to the gate +# DELTA_SCALA_VERSION, NUM_SHARDS, TEST_PARALLELISM_COUNT, DELTA_TESTING +# - test env (see the workflow step's `env:` block) +# GITHUB_WORKSPACE - repo root (holds the Delta clone + util scripts) +# +# JAVA_TOOL_OPTIONS is set by sourcing java-test-args.sh (below), not the caller. + +set -euo pipefail +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk +export PATH=$JAVA_HOME/bin:$PATH +# Gluten/JDK17 test JVM flags (--add-opens + Netty property), shared with local +# dev runs. Sets JAVA_TOOL_OPTIONS so it reaches the sbt launcher + forked JVMs. +# shellcheck source=./java-test-args.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/java-test-args.sh" +cd "$GITHUB_WORKSPACE/delta" +chmod +x build/sbt +# Only run the unified `spark` sbt project, NOT `sparkGroup/test` -- +# `sparkGroup` aggregates many other projects (sparkV2, contribs, +# sharing, connect*, ...) that are out of scope for this pipeline. +# +# JVM heap layout -- two memory consumers on the ~16G runner: +# * sbt launcher JVM: -J-Xmx4G for the test compile, then forced to +# return idle memory during the (long) test phase via G1 periodic GC +# (G1PeriodicGCInterval=10s; G1PeriodicGCSystemLoadThreshold=0 so the +# busy fork doesn't suppress it; -XX:-G1PeriodicGCInvokesConcurrent +# forces each periodic GC to a full STW collection) that uncommits to a +# tight free ratio (Min/MaxHeapFreeRatio 5/15, JEP 346) above a low +# -Xms512m floor. +# Without this the idle launcher holds ~5.3G for the whole run; with +# it, it drops back to ~1-2G. These flags touch no Gluten/Spark runtime +# config, so they cannot affect the measured pass/fail signal. +# * Forked test JVM: -Xmx2G via the `set ... Test / javaOptions` command +# below. Delta caps its fork at -Xmx1024m in build.sbt; `++=` appends +# so our -Xmx2G comes last and wins. Gluten offloads data to Velox +# off-heap (capped at 2g via spark.memory.offHeap.size in the patched +# DeltaSQLCommandTest), so the fork's heap need is modest. A larger +# fork heap pushed the cgroup peak past the ~16G OOM threshold and the +# kernel OOM-killed the fork mid-shard (no hs_err), wedging sbt -- 2G +# keeps headroom. Keep heap-dump-on-OOM so a real >2G heap OOM is +# analyzable. +# `-u target/test-reports` enables ScalaTest's JUnit XML reporter so +# every suite writes per-test results. Delta itself only configures +# the console reporter (-oDF), so without this we'd have no machine- +# readable results to gate on. The path is relative to the forked +# test JVM's working dir (Test / baseDirectory = spark/), i.e. +# delta/spark/target/test-reports/TEST-*.xml. +# +# We deliberately do NOT let an sbt non-zero exit (which fires on the +# MANY expected Delta-on-Gluten failures) fail this step directly. +# Instead the known-failures gate below decides pass/fail: the build +# is green when the only failures are ones already recorded in the +# baseline, and red on a genuine regression. +set +e +# --- hang watchdog --------------------------------------------------- +# Shard 2 (and occasionally others) hangs indefinitely after a suite's +# last test with no further output. ScalaTest's failAfter only wraps +# individual test BODIES, so a wedge in suite teardown/afterAll -- or in +# a non-interruptible native Velox/JNI call that ignores +# Thread.interrupt() -- has no timeout and stalls until the 350-min job +# limit with zero diagnostics. This watchdog dumps the forked test JVM's +# threads (to the job log, and to a file for the artifact) once the test +# output has been silent for too long, so the deadlock is diagnosable. +SBT_LOG="/tmp/sbt-spark-test-shard-${SHARD_ID}.log" +# Marker the watchdog touches when it KILLS a wedged test fork. The killed fork's +# running suite plus every suite queued behind it never run and never write a +# report, and since we ignore sbt's exit code the gate would only judge the +# suites that DID report -- so the main flow fails the shard when this exists. +WATCHDOG_KILL_MARKER="/tmp/sbt-watchdog-killed-shard-${SHARD_ID}" +: > "$SBT_LOG" +rm -f /tmp/sbt-done "$WATCHDOG_KILL_MARKER" +( + # CRITICAL: the step shell runs with `bash -eo pipefail`, which the + # subshell inherits. Without `set +e` here, ANY non-zero command -- + # e.g. fork detection finding no match, or `kill`/`jps` returning + # non-zero -- silently kills this watchdog. That errexit kill (plus a + # /proc detection miss) once made the watchdog capture ZERO dumps. A + # diagnostic must never abort on a failed probe. + set +e +o pipefail + JSTACK="${JAVA_HOME}/bin/jstack" + JPS="${JAVA_HOME}/bin/jps" + silent_limit=900 # 15 min with no new test output => treat as hung + dumps=0 + fork_pids() { + # The sbt test fork's main class is sbt.ForkMain. Prefer jps (reads + # the main class from hsperfdata, robust to sbt's @argfile launch); + # fall back to scanning /proc cmdline + @argfile. + "$JPS" -l 2>/dev/null | awk '/sbt\.ForkMain/ {print $1}' + local p cl arg + for p in /proc/[0-9]*; do + [ "$(cat "$p/comm" 2>/dev/null)" = "java" ] || continue + cl="$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)" + case "$cl" in *sbt.ForkMain*) echo "${p##*/}"; continue ;; esac + arg="$(printf '%s' "$cl" | tr ' ' '\n' | sed -n 's/^@//p' | head -1)" + [ -n "$arg" ] && [ -f "$arg" ] && grep -qa 'sbt\.ForkMain' "$arg" 2>/dev/null \ + && echo "${p##*/}" + done + } + all_java_pids() { + "$JPS" -q 2>/dev/null + local p + for p in /proc/[0-9]*; do + [ "$(cat "$p/comm" 2>/dev/null)" = "java" ] && echo "${p##*/}" + done + } + echo "HANG WATCHDOG armed: dumps the test JVM after ${silent_limit}s of output silence" + hb=0 + while [ ! -f /tmp/sbt-done ]; do + sleep 60 + [ -f "$SBT_LOG" ] || continue + now=$(date +%s) + mtime=$(stat -c %Y "$SBT_LOG" 2>/dev/null || echo "$now") + silent=$(( now - mtime )) + # Per-minute memory profile: heap tuning proved the ~16G OOM peak is + # NATIVE-driven, so log which JVM (sbt launcher vs fork) actually grows + # toward it -- the last lines before a hang reveal the real hog to cut. + # Read /proc directly (no `ps` dependency in the minimal container). + memnow=$(awk '{printf "%.2fG",$1/1073741824}' /sys/fs/cgroup/memory.current 2>/dev/null) + jvmrss="" + for mp in $(all_java_pids 2>/dev/null | sort -un); do + r=$(awk '/^VmRSS:/{print $2}' "/proc/$mp/status" 2>/dev/null) + [ -n "$r" ] && jvmrss="$jvmrss $(( r / 1024 ))M(p$mp)" + done + echo "MEM cgroup=${memnow} JVMs=[${jvmrss# }]" + hb=$(( hb + 1 )) + # Heartbeat every ~5 min so we can SEE the watchdog is alive (and how + # long the test has been silent) without waiting for a hang. + [ $(( hb % 5 )) -eq 0 ] && echo "HANG WATCHDOG: alive; last test output ${silent}s ago" + # The dump/kill budget below is PER silent-episode: reset it whenever output + # is flowing again. Otherwise a transient pre-fork/compile stall that we dump + # but (correctly) don't kill could exhaust the budget and leave a later real + # fork hang un-dumped and un-killed for the rest of the run. + [ "$silent" -lt "$silent_limit" ] && dumps=0 + if [ "$silent" -ge "$silent_limit" ] && [ "$dumps" -lt 3 ]; then + dumps=$(( dumps + 1 )) + fork_matched="$(fork_pids | sort -un)" + # Dump set: the sbt.ForkMain test fork(s) if we can pinpoint them; if not, + # dump EVERY JVM so a hang is still diagnosable. Diagnostics are harmless on + # any JVM, so the broad fallback stays here. + dump_pids="$fork_matched" + [ -n "$dump_pids" ] || dump_pids="$(all_java_pids | sort -un)" + echo "::group::HANG WATCHDOG: test output silent ${silent}s -- thread dump #${dumps} (pids:$(printf ' %s' $dump_pids))" + [ -n "$dump_pids" ] || echo "HANG WATCHDOG: no java process found to dump" + for pid in $dump_pids; do + # SIGQUIT makes the JVM print a full thread dump to its OWN stderr, + # which sbt relays into the test log via the SAME stream as test + # output -- so it lands in the job log even when a separately + # spawned jstack child's output would be buffered/lost. Also write + # jstack to a file for the per-shard artifact. + echo "----- SIGQUIT + jstack pid ${pid} -----" + kill -QUIT "$pid" 2>/dev/null || echo "HANG WATCHDOG: kill -QUIT failed for pid ${pid}" + timeout 120 "$JSTACK" -l "$pid" > "/tmp/threaddump-shard-${SHARD_ID}-${dumps}-${pid}.txt" 2>&1 \ + || echo "HANG WATCHDOG: jstack failed/timed out for pid ${pid}" + done + echo "::endgroup::" + # The dump is now captured (job log via SIGQUIT + artifact via jstack + # file). A hung fork otherwise stalls the whole shard until the 350-min job + # timeout AND keeps the job log frozen so the dump never becomes reachable. + # So KILL the wedged fork(s): the suite fails fast (acceptable -- errors are + # expected; only an unrecoverable hang blocks CI), the job proceeds/ends, + # and the log + artifacts flush. Give SIGQUIT a moment to print first. + sleep 20 + # Kill ONLY the matched sbt.ForkMain fork(s) -- never the sbt launcher. + # Before any fork exists (dependency resolution or a cold-cache compile of + # the big spark test module) sbt can legitimately go silent for >15 min; + # killing the launcher then would kill the job with a confusing "compile or + # launch failure" and waste the whole slot. A pre-fork hang is left running + # (rare; still bounded by the 350-min job timeout). Touch the marker so the + # main flow fails the shard: the killed fork's queued suites never ran. + if [ -n "$fork_matched" ]; then + echo "HANG WATCHDOG: killing wedged fork JVM(s) to unblock the shard:$(printf ' %s' $fork_matched)" + touch "$WATCHDOG_KILL_MARKER" + for pid in $fork_matched; do kill -KILL "$pid" 2>/dev/null; done + else + echo "HANG WATCHDOG: no sbt.ForkMain fork matched -- dumped all JVMs but leaving sbt running (likely a pre-fork resolve/compile stall, not a wedged test)." + fi + fi + done +) & +WATCHDOG_PID=$! + +./build/sbt \ + -DsparkVersion=${SPARK_VERSION} \ + -v \ + -J-XX:+UseG1GC -J-Xms512m -J-Xmx4G \ + -J-XX:G1PeriodicGCInterval=10000 \ + -J-XX:G1PeriodicGCSystemLoadThreshold=0 \ + -J-XX:-G1PeriodicGCInvokesConcurrent \ + -J-XX:MinHeapFreeRatio=5 -J-XX:MaxHeapFreeRatio=15 \ + "++ ${DELTA_SCALA_VERSION}" \ + 'set spark / Test / javaOptions ++= Seq("-Xmx2G", "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=/tmp/")' \ + 'set spark / Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-u", "target/test-reports")' \ + "spark/test" 2>&1 | tee "$SBT_LOG" +SBT_EXIT=${PIPESTATUS[0]} +touch /tmp/sbt-done +kill "$WATCHDOG_PID" 2>/dev/null || true +set -e +echo "sbt spark/test exited with ${SBT_EXIT}" + +# Memory forensics: a sudden forked-JVM death with no hs_err and no heap +# dump is almost always a kernel/cgroup OOM-kill (Velox off-heap + JVM +# heap exceeding the ~16G runner). Surface the cgroup peak + oom_kill +# count so we can confirm/measure it (cgroup v2 paths; best-effort). +( echo "=== cgroup memory forensics (exit ${SBT_EXIT}) ===" + for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.max \ + /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.events; do + [ -r "$f" ] && { echo "--- $f ---"; cat "$f"; } + done ) || true + +# If the hang watchdog killed a wedged test fork, the suite it was running plus +# every suite QUEUED BEHIND it in that fork never ran and never wrote a report. +# Because we intentionally ignore sbt's exit code, the gate would only judge the +# suites that DID report and could go green with those tests silently unrun. A +# watchdog kill is an abnormal run, so fail the shard outright (re-run to retry). +if [ -f "$WATCHDOG_KILL_MARKER" ]; then + echo "::error::hang watchdog killed a wedged test fork on shard ${SHARD_ID}; suites queued behind it never ran, so results are incomplete. Failing the shard." + exit 1 +fi + +# A compile/launch failure leaves no reports at all. In that case the +# gate would see zero failures and pass spuriously, so fail loudly. +REPORT_COUNT=$(find . -path '*/target/test-reports/*.xml' 2>/dev/null | wc -l || true) +echo "Found ${REPORT_COUNT} JUnit XML report file(s)." +if [ "${REPORT_COUNT}" -eq 0 ]; then + echo "::error::sbt produced no test reports (exit ${SBT_EXIT}) -- likely a compile or launch failure, not test failures." + exit 1 +fi + +# Classify this shard's results against the baseline: seed mode when +# UPDATE_BASELINE=true (record failures, never fail) so the baseline can be +# (re)generated; otherwise enforce against it. Writes this shard's gate-out/*.txt +# for the aggregate job. +UTIL_DIR="$GITHUB_WORKSPACE/.github/workflows/util/delta-spark-ut" +GATE_MODE=enforce +if [ "${UPDATE_BASELINE}" = "true" ]; then + GATE_MODE=seed +fi +mkdir -p "$GITHUB_WORKSPACE/gate-out" +python3 "$UTIL_DIR/compare-test-results.py" \ + --mode "$GATE_MODE" \ + --reports-dir "$GITHUB_WORKSPACE/delta" \ + --known-failures "$UTIL_DIR/known-failures.txt" \ + --flaky-tests "$UTIL_DIR/flaky-tests.txt" \ + --flaky-error-patterns "$UTIL_DIR/flaky-error-patterns.txt" \ + --failures-out "$GITHUB_WORKSPACE/gate-out/failures-shard-${SHARD_ID}.txt" \ + --ran-out "$GITHUB_WORKSPACE/gate-out/ran-shard-${SHARD_ID}.txt" \ + --fail-on-fixed "${FAIL_ON_FIXED}" diff --git a/.github/workflows/util/delta-spark-ut/setup-delta.sh b/.github/workflows/util/delta-spark-ut/setup-delta.sh new file mode 100755 index 00000000000..1d9dcf7f954 --- /dev/null +++ b/.github/workflows/util/delta-spark-ut/setup-delta.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Prepares a delta-io/delta clone for running its `spark` module tests with the +# Gluten (Velox) bundle jar on the classpath. +# +# Usage: +# setup-delta.sh +# +# Arguments: +# delta_ref - git ref (tag/branch/sha) to check out (e.g. v4.2.0) +# delta_dir - destination directory for the Delta clone +# gluten_bundle_jar - path to the gluten-velox-bundle fat jar +# gluten_repo_root - path to the Gluten repository root (used to locate +# backends-velox/src-delta40/.../DeltaSQLCommandTest.scala) +# + +set -euo pipefail + +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +DELTA_REF="$1" +DELTA_DIR="$2" +GLUTEN_BUNDLE_JAR="$3" +GLUTEN_ROOT="$4" + +if [ ! -f "$GLUTEN_BUNDLE_JAR" ]; then + echo "Gluten bundle jar not found: $GLUTEN_BUNDLE_JAR" >&2 + exit 1 +fi + +# Reuse the existing DeltaSQLCommandTest from Gluten's backends-velox module +# rather than maintaining a separate copy. This file is compiled as part of the +# unified `spark` project's Test scope, which has the Gluten bundle on its +# classpath (via spark-unified/lib/), so the typed GlutenConfig / VeloxDeltaConfig +# imports resolve correctly. +PATCH_SOURCE="$GLUTEN_ROOT/backends-velox/src-delta40/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala" +if [ ! -f "$PATCH_SOURCE" ]; then + echo "Gluten DeltaSQLCommandTest not found: $PATCH_SOURCE" >&2 + exit 1 +fi + +echo "::group::Cloning delta-io/delta @ ${DELTA_REF}" +# init + shallow fetch resolves a tag, branch OR commit SHA in a single path +# (`git clone --branch` rejects SHAs). Avoids a full-clone fallback and the +# destructive `rm -rf "$DELTA_DIR"` it required. `--` terminates options so a +# DELTA_REF starting with `-` can't be misread as a git flag (this script is +# workflow_dispatch-runnable with a user-supplied ref). +git init -q "$DELTA_DIR" +git -C "$DELTA_DIR" remote add origin https://github.com/delta-io/delta.git +git -C "$DELTA_DIR" fetch -q --depth 1 origin -- "$DELTA_REF" +git -C "$DELTA_DIR" checkout -q FETCH_HEAD +git -C "$DELTA_DIR" --no-pager log -1 --oneline +echo "::endgroup::" + +echo "::group::Injecting Gluten bundle jar onto the spark project's TEST classpath" +# The Gluten bundle jar must be on the spark project's TEST runtime classpath +# (so DeltaSQLCommandTest can load org.apache.gluten.GlutenPlugin by name) but +# NOT on the COMPILE classpath of `sparkV1`, which is the project that holds +# Delta's main sources. The bundle's transitive contents include extra symbols +# under `org.apache.spark.sql` that collide with Delta's main sources -- e.g. +# MergeOutputGeneration.scala imports both `org.apache.spark.sql._` and +# `org.apache.spark.sql.delta.ClassicColumnConversions._`, and would then fail +# with `reference to expression is ambiguous`. +# +# sbt auto-scans `/lib` via `unmanagedBase`. Two relevant +# projects in Delta v4.2.0 have a `lib/` baseDirectory: +# - sparkV1: `project in file("spark")` -> spark/lib +# - spark : `project in file("spark-unified")` -> spark-unified/lib +# unmanagedJars are project-scoped (NOT inherited by dependents), so dropping +# the bundle into spark-unified/lib/ adds it to the unified `spark` project's +# Compile *and* Test classpaths -- but NOT to sparkV1's. That's exactly what +# we want: +# * sparkV1/Compile sees ONLY Delta's regular deps -> Delta main compiles. +# * spark/Test/fullClasspath sees the bundle -> tests load GlutenPlugin. +# (Verified empirically: with bundle only in spark-unified/lib/, sbt's +# `show sparkV1/Compile/dependencyClasspath` excludes the bundle and +# `show spark/Test/fullClasspath` includes it.) +# +# We deliberately do NOT also drop the bundle into spark/lib/, which is what +# caused the previous compile failure: spark/lib/ is sparkV1's unmanagedBase, +# and putting the bundle there would re-introduce the ambiguity errors. +SPARK_UNIFIED_LIB="$DELTA_DIR/spark-unified/lib" +mkdir -p "$SPARK_UNIFIED_LIB" +cp "$GLUTEN_BUNDLE_JAR" "$SPARK_UNIFIED_LIB/gluten-velox-bundle.jar" +ls -lh "$SPARK_UNIFIED_LIB" +echo "::endgroup::" + +echo "::group::Patching DeltaSQLCommandTest to enable Gluten plugin" +TARGET="$DELTA_DIR/spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala" +if [ ! -f "$TARGET" ]; then + echo "Expected file not found in Delta clone: $TARGET" >&2 + echo "The Delta directory layout for ref '${DELTA_REF}' may have changed." + exit 1 +fi +cp "$PATCH_SOURCE" "$TARGET" +echo "Patched $TARGET" +echo "--- diff vs. upstream ---" +git -C "$DELTA_DIR" --no-pager diff -- "spark/src/test/scala/org/apache/spark/sql/delta/test/DeltaSQLCommandTest.scala" || true +echo "::endgroup::" + +# Delta's tests collect file-source scans by matching the concrete +# `FileSourceScanExec` case class; Gluten offloads the scan to +# DeltaScanTransformer, a `FileSourceScanLike` sibling, so those matches miss +# (`scala.MatchError: List()`, empty partition filters, broken column-pruning / +# scan-metric checks across many suites). delta-io/delta#7104 and #7105 widen the +# matches to the shared `FileSourceScanLike` interface that both the vanilla and +# Gluten scans implement (behavior-preserving for vanilla). Both are merged +# upstream but land after the pinned DELTA_REF (v4.2.0), so apply them here; once +# DELTA_REF includes a commit its cherry-pick is a clean no-op and the call can go. +# +# Depth-2 fetch brings each fix commit and its parent, which cherry-pick needs to +# diff against (a depth-1 fetch grafts the parent away); `-n` stages the change +# without requiring a committer identity. +cherry_pick_delta_fix() { + local sha="$1" pr="$2" + echo "Cherry-picking delta-io/delta${pr}" + git -C "$DELTA_DIR" fetch --quiet --depth 2 origin "$sha" + git -C "$DELTA_DIR" cherry-pick -n "$sha" +} + +echo "::group::Cherry-picking upstream Delta FileSourceScanLike test fixes" +cherry_pick_delta_fix 46bd45d57eadd7e528002a0ae7bd36ce5a456eca "#7104 (ScanReportHelper.collectScans)" +cherry_pick_delta_fix 959e00e15f41f56afc1c9bb95d160c55c6dc7068 "#7105 (9 more test suites)" +echo "::endgroup::" + +echo "::group::Force-failing memory-hog DeletionVectorsSuite 2B-row tests" +# Two DeletionVectorsSuite tests read from / delete from a 2-billion-row table. +# Under the Gluten Velox bundle they balloon the forked test JVM to ~13G of +# NATIVE memory (row-index materialization) and the kernel/cgroup OOM-kills it. +# The dead fork then wedges sbt, hanging the whole shard until the workflow's +# hang-watchdog dumps threads and kills it (~16 min wasted, and every suite +# QUEUED AFTER it in that fork is skipped) -- see delta_spark_ut.yml. +# +# Rather than silently `ignore` these (easy to forget), we make them FAIL FAST +# with a clear message: the gap stays visible in the test reports / baseline +# until the native memory blow-up is fixed, at which point this patch should be +# removed. NOTE: making the suite complete also un-skips the rest of the shard's +# suite queue, so the known-failures baseline must be refreshed after this. +# +# ORDER MATTERS: keep this sed AFTER the cherry-picks above. #7105 also edits +# DeletionVectorsSuite.scala, and git cherry-pick aborts (exit 128) when the work +# tree has uncommitted edits to a file it touches. +DVS="$DELTA_DIR/spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/DeletionVectorsSuite.scala" +if [ ! -f "$DVS" ]; then + echo "Expected file not found in Delta clone: $DVS" >&2 + echo "The Delta directory layout for ref '${DELTA_REF}' may have changed." >&2 + exit 1 +fi +# Inject `fail(...)` as the first statement of each test body (the line ending +# in `) {`). Delta sets no -Xfatal-warnings / dead-code warning, so the now- +# unreachable original body compiles fine. Keep each injected line <100 chars: +# Delta's scalastyle enforces a 100-char line length on test sources. The full +# rationale lives in this comment, so the in-test message stays terse. +sed -i 's#huge table: read from tables of 2B rows with existing DV of many zeros") {#&\n fail("[Gluten CI] Force-failed: 2B-row DV read OOMs the test JVM; see setup-delta.sh")#' "$DVS" +sed -i 's#number of rows from tables of 2B rows with DVs") {#&\n fail("[Gluten CI] Force-failed: 2B-row DV delete OOMs the test JVM; see setup-delta.sh")#' "$DVS" +INJECTED=$(grep -c "Gluten CI] Force-failed" "$DVS" || true) +if [ "$INJECTED" -ne 2 ]; then + echo "ERROR: expected to force-fail 2 DeletionVectorsSuite tests but injected ${INJECTED}." >&2 + echo "Their test names likely changed in Delta ref '${DELTA_REF}'; update setup-delta.sh." >&2 + exit 1 +fi +echo "Force-failed 2 DeletionVectorsSuite 2B-row tests (read + delete)." +git -C "$DELTA_DIR" --no-pager diff -- "spark/src/test/scala/org/apache/spark/sql/delta/deletionvectors/DeletionVectorsSuite.scala" || true +echo "::endgroup::" + +echo "::group::Disabling Delta scalastyle HeaderMatchesChecker" +# Our reused DeltaSQLCommandTest carries Gluten's ASF-only license header, which +# does not match Delta's HeaderMatchesChecker regex (the regex expects either a +# Delta copyright block, or the ASF header followed by a Spark-modifications +# block and the Delta copyright block). HeaderMatchesChecker is a file-level +# checker that does NOT honor `// scalastyle:off` directives, so we instead +# disable it globally in Delta's shared scalastyle-config.xml. The config is +# applied via `ThisBuild / scalastyleConfig` in project/Checkstyle.scala, so a +# single edit covers every sbt sub-project. +SCALASTYLE_CONFIG="$DELTA_DIR/scalastyle-config.xml" +if [ ! -f "$SCALASTYLE_CONFIG" ]; then + echo "Expected scalastyle config not found: $SCALASTYLE_CONFIG" >&2 + exit 1 +fi +sed -i \ + 's|||' \ + "$SCALASTYLE_CONFIG" +if ! grep -q '' "$SCALASTYLE_CONFIG"; then + echo "Failed to disable HeaderMatchesChecker in $SCALASTYLE_CONFIG" >&2 + grep -n 'HeaderMatchesChecker' "$SCALASTYLE_CONFIG" >&2 || true + exit 1 +fi +echo "Disabled HeaderMatchesChecker in $SCALASTYLE_CONFIG" +echo "::endgroup::" diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 9b5d6d12a17..d4e971aa2c8 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -19,6 +19,11 @@ on: pull_request: paths: - '.github/workflows/velox_backend_x86.yml' + # Delta Spark UT runs here too (reusable delta_spark_ut.yml). These extra + # paths make Delta-CI-only changes trigger this workflow; Delta also runs on + # the velox paths below since core/velox changes can affect Delta offload. + - '.github/workflows/delta_spark_ut.yml' + - '.github/workflows/util/delta-spark-ut/**' - '.github/workflows/util/install-spark-deps.sh' #TODO remove after image update - '.github/workflows/util/install-spark-resources.sh' #TODO remove after image update - 'pom.xml' @@ -94,6 +99,70 @@ jobs: path: ./cpp/build/ if-no-files-found: error + # Gate the (expensive) Delta Spark UT suite so per-PR it runs only when the PR + # touches high-signal Delta paths -- the Delta integration code + # (backends-velox/src-delta*), the gluten-delta module, or this pipeline's own + # files -- or carries the `run-delta-ci` opt-in label. Changes to general + # Velox/core/native code can also affect Delta offload but are touched + # constantly, so per-PR they skip it; the nightly full run (delta_spark_ut.yml + # `schedule`) and the opt-in label are the safety nets. This keeps GHA usage + # down. NOTE: the label is read from the event that triggered this run, so add + # it before/with a push; labeling an already-finished run needs a new push. + delta-changes: + runs-on: ubuntu-22.04 + outputs: + run_delta: ${{ steps.filter.outputs.run_delta }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Detect Delta-relevant changes / opt-in label + id: filter + env: + HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'run-delta-ci') }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + # Opt-in label forces the suite even with no Delta-relevant path change. + if [ "$HAS_LABEL" = "true" ]; then + echo "run-delta-ci label present -> running Delta suite" + echo "run_delta=true" >> "$GITHUB_OUTPUT"; exit 0 + fi + # Fail open if we can't determine the PR range (e.g. a non-PR trigger): + # never silently skip coverage. + if [ -z "${BASE_SHA:-}" ] || [ -z "${HEAD_SHA:-}" ]; then + echo "no PR base/head sha -> running Delta suite (fail-open)" + echo "run_delta=true" >> "$GITHUB_OUTPUT"; exit 0 + fi + BASE=$(git merge-base "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "$BASE_SHA") + echo "diff base=$BASE head=$HEAD_SHA" + # High-signal Delta paths only: the Delta integration code + # (backends-velox/src-delta*), the Delta module, and this pipeline's own + # files. A change to general Velox/core/native code can also affect Delta + # offload, but those are touched constantly; per-PR we skip them (the + # nightly full run + the `run-delta-ci` label are the safety nets) to + # keep GHA usage down. + if git diff --name-only "$BASE" "$HEAD_SHA" | grep -Eq \ + '^(\.github/workflows/velox_backend_x86\.yml|\.github/workflows/delta_spark_ut\.yml|\.github/workflows/util/delta-spark-ut/|gluten-delta/|backends-velox/src-delta)'; then + echo "Delta-relevant paths changed -> running Delta suite" + echo "run_delta=true" >> "$GITHUB_OUTPUT" + else + echo "No Delta-relevant paths changed and no opt-in label -> skipping Delta suite" + echo "run_delta=false" >> "$GITHUB_OUTPUT" + fi + + # Run the Delta Spark UT via the reusable workflow, passing the native lib + # built above so it is not rebuilt. Gated by `delta-changes` (Delta-relevant + # paths or the `run-delta-ci` label); the nightly full run lives in + # delta_spark_ut.yml's `schedule` trigger. + delta-spark-ut: + needs: [build-native-lib-centos-7, delta-changes] + if: ${{ needs.delta-changes.outputs.run_delta == 'true' }} + uses: ./.github/workflows/delta_spark_ut.yml + with: + native_lib_artifact: velox-native-lib-centos-7-${{ github.sha }} + tpc-test-ubuntu: needs: build-native-lib-centos-7 strategy: diff --git a/backends-velox/src-delta/main/scala/org/apache/gluten/component/VeloxDeltaComponent.scala b/backends-velox/src-delta/main/scala/org/apache/gluten/component/VeloxDeltaComponent.scala index 164cf528860..175c879c25d 100644 --- a/backends-velox/src-delta/main/scala/org/apache/gluten/component/VeloxDeltaComponent.scala +++ b/backends-velox/src-delta/main/scala/org/apache/gluten/component/VeloxDeltaComponent.scala @@ -17,8 +17,8 @@ package org.apache.gluten.component import org.apache.gluten.backendsapi.velox.VeloxBackend -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.extension.{DeltaPostTransformRules, OffloadDeltaFilter, OffloadDeltaProject, OffloadDeltaScan} +import org.apache.gluten.config.{GlutenConfig, VeloxDeltaConfig} +import org.apache.gluten.extension.{DeltaDeletionVectorDmlUtils, DeltaPostTransformRules, OffloadDeltaFilter, OffloadDeltaProject, OffloadDeltaScan} import org.apache.gluten.extension.columnar.heuristic.HeuristicTransform import org.apache.gluten.extension.columnar.validator.Validators import org.apache.gluten.extension.injector.Injector @@ -40,9 +40,21 @@ class VeloxDeltaComponent extends Component { // PreprocessTableWithDVsStrategy injects the skip-row column and filter during physical // planning, DeltaPostTransformRules.nativeDeletionVectorRule strips them when the scan // offloads, and DeltaScanTransformer materializes the per-file DV payloads for Velox. + // + // For native DELETE/UPDATE/MERGE, the DML target row-index scan is deliberately kept on Spark + // until native row-index execution is proven; tag those scans here so the post-transform rules + // can keep the small subtree off the native path. + legacy.injectPreTransform(_ => DeltaDeletionVectorDmlUtils.tagDmlRowIndexScans) legacy.injectTransform { c => - val offload = Seq(OffloadDeltaScan(), OffloadDeltaProject(), OffloadDeltaFilter()) + val offload = Seq( + OffloadDeltaScan( + enableNativeDeletionVectorDmlRowIndexScanKey = + VeloxDeltaConfig.ENABLE_NATIVE_DML_ROW_INDEX_SCAN.key + ), + OffloadDeltaProject(), + OffloadDeltaFilter() + ) .map(_.toStrcitRule()) HeuristicTransform.Simple( Validators.newValidator(new GlutenConfig(c.sqlConf), offload), diff --git a/backends-velox/src-delta/main/scala/org/apache/gluten/config/VeloxDeltaConfig.scala b/backends-velox/src-delta/main/scala/org/apache/gluten/config/VeloxDeltaConfig.scala index 99c2d2c26a7..481658eda0b 100644 --- a/backends-velox/src-delta/main/scala/org/apache/gluten/config/VeloxDeltaConfig.scala +++ b/backends-velox/src-delta/main/scala/org/apache/gluten/config/VeloxDeltaConfig.scala @@ -22,6 +22,8 @@ class VeloxDeltaConfig(conf: SQLConf) extends GlutenCoreConfig(conf) { import VeloxDeltaConfig._ def enableNativeWrite: Boolean = getConf(ENABLE_NATIVE_WRITE) + + def enableNativeDmlRowIndexScan: Boolean = getConf(ENABLE_NATIVE_DML_ROW_INDEX_SCAN) } object VeloxDeltaConfig extends ConfigRegistry { @@ -40,4 +42,14 @@ object VeloxDeltaConfig extends ConfigRegistry { .doc("Enable native Delta Lake write for Velox backend.") .booleanConf .createWithDefault(false) + + val ENABLE_NATIVE_DML_ROW_INDEX_SCAN: ConfigEntry[Boolean] = + buildConf( + "spark.gluten.sql.columnar.backend.velox.delta.enableNativeDmlRowIndexScan") + .experimental() + .doc( + "Enable the experimental native Delta DELETE/UPDATE/MERGE target row-index scan for " + + "Velox.") + .booleanConf + .createWithDefault(false) } diff --git a/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala b/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala index f5510a95255..484a540aa9d 100644 --- a/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala +++ b/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala @@ -16,24 +16,117 @@ */ package org.apache.spark.sql.delta -import org.apache.gluten.execution.DeltaScanTransformer +import org.apache.gluten.config.VeloxDeltaConfig +import org.apache.gluten.execution.{DeltaScanTransformer, FilterExecTransformerBase, ProjectExecTransformerBase} +import org.apache.gluten.extension.DeltaDeletionVectorDmlUtils import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} +import org.apache.spark.sql.execution.{ColumnarToRowExec, FileSourceScanExec, FilterExec, ProjectExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.tags.ExtendedSQLTest +import org.apache.spark.util.SparkVersionUtil import org.apache.hadoop.fs.Path +import java.io.File + @ExtendedSQLTest class DeltaDeletionVectorHandoffSuite extends QueryTest with SharedSparkSession with DeltaSQLTestUtils - with DeltaSQLCommandTest { + with DeltaSQLCommandTest + with AdaptiveSparkPlanHelper { import testImplicits._ + private def containsDmlFallbackScan(plan: SparkPlan): Boolean = { + // FallbackTags are removed before AQE captures the executed plan. Identify the final fallback + // from the persistent DML row-index marker and the vanilla Spark scan type instead. + collectWithSubqueries(plan) { + case scan: FileSourceScanExec + if DeltaDeletionVectorDmlUtils.isDeletionVectorDmlRowIndexScan(scan) => + scan + }.nonEmpty + } + + private def hasSparkParentOverDmlFallbackScan(plan: SparkPlan): Boolean = { + collectWithSubqueries(plan) { + case project @ ProjectExec(_, child) if isDmlFallbackSubtree(child) => project + case filter @ FilterExec(_, child) if isDmlFallbackSubtree(child) => filter + }.nonEmpty + } + + private def hasNativeParentOverDmlFallbackScan(plan: SparkPlan): Boolean = { + collectWithSubqueries(plan) { + case project: ProjectExecTransformerBase if isDmlFallbackSubtree(project.child) => project + case filter: FilterExecTransformerBase if isDmlFallbackSubtree(filter.child) => filter + }.nonEmpty + } + + private def containsNativeDeltaScan(plan: SparkPlan): Boolean = { + collectWithSubqueries(plan) { case scan: DeltaScanTransformer => scan }.nonEmpty + } + + private def isDmlFallbackSubtree(plan: SparkPlan): Boolean = plan match { + case scan: FileSourceScanExec => containsDmlFallbackScan(scan) + case ColumnarToRowExec(child) => isDmlFallbackSubtree(child) + case ProjectExec(_, child) => isDmlFallbackSubtree(child) + case FilterExec(_, child) => isDmlFallbackSubtree(child) + case project: ProjectExecTransformerBase => isDmlFallbackSubtree(project.child) + case filter: FilterExecTransformerBase => isDmlFallbackSubtree(filter.child) + case _ => false + } + + private def captureDeletePlans( + path: String, + predicate: String, + useMetadataRowIndex: Boolean): Seq[SparkPlan] = { + var executedPlans: Seq[SparkPlan] = Seq.empty + withSQLConf( + DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key -> + useMetadataRowIndex.toString, + VeloxDeltaConfig.ENABLE_NATIVE_DML_ROW_INDEX_SCAN.key -> "false" + ) { + executedPlans = DeltaTestUtils.withAllPlansCaptured(spark) { + spark.sql(s"DELETE FROM delta.`$path` WHERE $predicate").collect() + }.map(_.executedPlan) + } + executedPlans + } + + private def assertSparkDmlFallback(executedPlans: Seq[SparkPlan]): Unit = { + val planText = executedPlans.map(_.treeString).mkString("\n\n") + assert(executedPlans.exists(containsDmlFallbackScan), planText) + assert(executedPlans.exists(hasSparkParentOverDmlFallbackScan), planText) + assert(!executedPlans.exists(hasNativeParentOverDmlFallbackScan), planText) + } + + private def assertReadPlanAfterDmlFallback(path: String, useMetadataRowIndex: Boolean): Unit = { + withSQLConf( + DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key -> useMetadataRowIndex.toString) { + val df = spark.read.format("delta").load(path) + val executedPlan = df.queryExecution.executedPlan + val planText = executedPlan.treeString + if (useMetadataRowIndex) { + assert(containsNativeDeltaScan(executedPlan), planText) + assert(!containsDmlFallbackScan(executedPlan), planText) + } else { + assert(!containsNativeDeltaScan(executedPlan), planText) + } + checkAnswer(df, Seq((1, "a"), (2, "b")).toDF()) + } + } + + private def activeDvCardinality(path: String): Long = { + val log = DeltaLog.forTable(spark, new Path(path)) + log.update().allFiles.collect().flatMap( + file => Option(file.deletionVector).map(_.cardinality)).sum + } + test("Spark 3.5 Delta DV scan handoff should filter deleted rows") { withTempDir { tempDir => @@ -58,11 +151,103 @@ class DeltaDeletionVectorHandoffSuite val df = spark.read.format("delta").load(path) val executedPlan = df.queryExecution.executedPlan - assert(executedPlan.collect { case _: DeltaScanTransformer => true }.nonEmpty) + assert(containsNativeDeltaScan(executedPlan)) val planText = executedPlan.toString() assert(!planText.contains("__delta_internal_is_row_deleted")) assert(!planText.contains("__delta_internal_row_index")) checkAnswer(df, Seq((1, "a"), (2, "b")).toDF()) } } + + test("Delta metadata row-index predicate should not be stripped from a native scan") { + assume(SparkVersionUtil.gteSpark35, "metadata row index is available in Spark 3.5+") + withTempDir { + tempDir => + val path = tempDir.getCanonicalPath + Seq((1, "a"), (2, "b"), (3, "c"), (4, "d")) + .toDF("id", "value") + .coalesce(1) + .write + .format("delta") + .save(path) + + spark.sql( + s"ALTER TABLE delta.`$path` SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)") + + val df = spark.sql( + s"SELECT id, _metadata.row_index AS row_index FROM delta.`$path` " + + "WHERE _metadata.row_index = 2") + val rows = df.collect() + val executedPlan = df.queryExecution.executedPlan + val planText = executedPlan.treeString + assert(containsNativeDeltaScan(executedPlan), planText) + assert(rows.length === 1, planText) + assert(rows.head.getLong(1) === 2L, planText) + } + } + + Seq(true, false).foreach { + useMetadataRowIndex => + test( + "Delta DV DML row-index scan should fall back with Spark project/filter, " + + s"metadata row index=$useMetadataRowIndex") { + assume(SparkVersionUtil.gteSpark35, "DML row-index scan fallback is Spark 3.5+ coverage") + withTempDir { + tempDir => + val path = tempDir.getCanonicalPath + Seq((1, "a"), (2, "b"), (3, "c"), (4, "d")) + .toDF("id", "value") + .coalesce(1) + .write + .format("delta") + .save(path) + + spark.sql( + s"ALTER TABLE delta.`$path` SET TBLPROPERTIES " + + "('delta.enableDeletionVectors' = true)") + + var executedPlans: Seq[SparkPlan] = Seq.empty + withSQLConf( + DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key -> + useMetadataRowIndex.toString, + VeloxDeltaConfig.ENABLE_NATIVE_DML_ROW_INDEX_SCAN.key -> "false" + ) { + executedPlans = DeltaTestUtils.withAllPlansCaptured(spark) { + spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (3, 4)").collect() + }.map(_.executedPlan) + } + assertSparkDmlFallback(executedPlans) + + val log = DeltaLog.forTable(spark, new Path(path)) + assert(log.update().allFiles.collect().exists(_.deletionVector != null)) + assertReadPlanAfterDmlFallback(path, useMetadataRowIndex) + } + } + } + + test("Delta DV DML row-index scan should fall back when updating an existing DV") { + assume(SparkVersionUtil.gteSpark35, "DML row-index scan fallback is Spark 3.5+ coverage") + withTempDir { + tempDir => + val path = new File(tempDir, "delta table with spaces").getCanonicalPath + Seq((1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e"), (6, "f")) + .toDF("id", "value") + .coalesce(1) + .write + .format("delta") + .save(path) + + spark.sql( + s"ALTER TABLE delta.`$path` SET TBLPROPERTIES " + + "('delta.enableDeletionVectors' = true)") + + assertSparkDmlFallback(captureDeletePlans(path, "id IN (5, 6)", useMetadataRowIndex = true)) + assert(activeDvCardinality(path) === 2L) + + assertSparkDmlFallback(captureDeletePlans(path, "id IN (3, 4)", useMetadataRowIndex = true)) + assert(activeDvCardinality(path) === 4L) + + assertReadPlanAfterDmlFallback(path, useMetadataRowIndex = true) + } + } } diff --git a/backends-velox/src-delta40/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala b/backends-velox/src-delta40/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala index dda547b015f..411f01a17ce 100644 --- a/backends-velox/src-delta40/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala +++ b/backends-velox/src-delta40/test/scala/org/apache/spark/sql/delta/DeltaDeletionVectorHandoffSuite.scala @@ -16,25 +16,117 @@ */ package org.apache.spark.sql.delta -import org.apache.gluten.execution.DeltaScanTransformer +import org.apache.gluten.config.VeloxDeltaConfig +import org.apache.gluten.execution.{DeltaScanTransformer, FilterExecTransformerBase, ProjectExecTransformerBase} +import org.apache.gluten.extension.DeltaDeletionVectorDmlUtils import org.apache.spark.sql.QueryTest import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.test.{DeltaSQLCommandTest, DeltaSQLTestUtils} +import org.apache.spark.sql.execution.{ColumnarToRowExec, FileSourceScanExec, FilterExec, ProjectExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.tags.ExtendedSQLTest +import org.apache.spark.util.SparkVersionUtil import org.apache.hadoop.fs.Path +import java.io.File + @ExtendedSQLTest class DeltaDeletionVectorHandoffSuite extends QueryTest with SharedSparkSession with DeltaSQLTestUtils - with DeltaSQLCommandTest { + with DeltaSQLCommandTest + with AdaptiveSparkPlanHelper { import testImplicits._ + private def containsDmlFallbackScan(plan: SparkPlan): Boolean = { + // FallbackTags are removed before AQE captures the executed plan. Identify the final fallback + // from the persistent DML row-index marker and the vanilla Spark scan type instead. + collectWithSubqueries(plan) { + case scan: FileSourceScanExec + if DeltaDeletionVectorDmlUtils.isDeletionVectorDmlRowIndexScan(scan) => + scan + }.nonEmpty + } + + private def hasSparkParentOverDmlFallbackScan(plan: SparkPlan): Boolean = { + collectWithSubqueries(plan) { + case project @ ProjectExec(_, child) if isDmlFallbackSubtree(child) => project + case filter @ FilterExec(_, child) if isDmlFallbackSubtree(child) => filter + }.nonEmpty + } + + private def hasNativeParentOverDmlFallbackScan(plan: SparkPlan): Boolean = { + collectWithSubqueries(plan) { + case project: ProjectExecTransformerBase if isDmlFallbackSubtree(project.child) => project + case filter: FilterExecTransformerBase if isDmlFallbackSubtree(filter.child) => filter + }.nonEmpty + } + + private def containsNativeDeltaScan(plan: SparkPlan): Boolean = { + collectWithSubqueries(plan) { case scan: DeltaScanTransformer => scan }.nonEmpty + } + + private def isDmlFallbackSubtree(plan: SparkPlan): Boolean = plan match { + case scan: FileSourceScanExec => containsDmlFallbackScan(scan) + case ColumnarToRowExec(child) => isDmlFallbackSubtree(child) + case ProjectExec(_, child) => isDmlFallbackSubtree(child) + case FilterExec(_, child) => isDmlFallbackSubtree(child) + case project: ProjectExecTransformerBase => isDmlFallbackSubtree(project.child) + case filter: FilterExecTransformerBase => isDmlFallbackSubtree(filter.child) + case _ => false + } + + private def captureDeletePlans( + path: String, + predicate: String, + useMetadataRowIndex: Boolean): Seq[SparkPlan] = { + var executedPlans: Seq[SparkPlan] = Seq.empty + withSQLConf( + DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key -> + useMetadataRowIndex.toString, + VeloxDeltaConfig.ENABLE_NATIVE_DML_ROW_INDEX_SCAN.key -> "false" + ) { + executedPlans = DeltaTestUtils.withAllPlansCaptured(spark) { + spark.sql(s"DELETE FROM delta.`$path` WHERE $predicate").collect() + }.map(_.executedPlan) + } + executedPlans + } + + private def assertSparkDmlFallback(executedPlans: Seq[SparkPlan]): Unit = { + val planText = executedPlans.map(_.treeString).mkString("\n\n") + assert(executedPlans.exists(containsDmlFallbackScan), planText) + assert(executedPlans.exists(hasSparkParentOverDmlFallbackScan), planText) + assert(!executedPlans.exists(hasNativeParentOverDmlFallbackScan), planText) + } + + private def assertReadPlanAfterDmlFallback(path: String, useMetadataRowIndex: Boolean): Unit = { + withSQLConf( + DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key -> useMetadataRowIndex.toString) { + val df = spark.read.format("delta").load(path) + val executedPlan = df.queryExecution.executedPlan + val planText = executedPlan.treeString + if (useMetadataRowIndex) { + assert(containsNativeDeltaScan(executedPlan), planText) + assert(!containsDmlFallbackScan(executedPlan), planText) + } else { + assert(!containsNativeDeltaScan(executedPlan), planText) + } + checkAnswer(df, Seq((1, "a"), (2, "b")).toDF()) + } + } + + private def activeDvCardinality(path: String): Long = { + val log = DeltaLog.forTable(spark, new Path(path)) + log.update().allFiles.collect().flatMap( + file => Option(file.deletionVector).map(_.cardinality)).sum + } + test("Spark 4 Delta DV scan should fall back when metadata row index is disabled") { withTempDir { tempDir => @@ -58,12 +150,77 @@ class DeltaDeletionVectorHandoffSuite withSQLConf(DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key -> "false") { val df = spark.read.format("delta").load(path) val executedPlan = df.queryExecution.executedPlan - assert(executedPlan.collect { case _: DeltaScanTransformer => true }.isEmpty) + assert(!containsNativeDeltaScan(executedPlan)) checkAnswer(df, Seq((1, "a"), (2, "b")).toDF()) } } } + test("Delta metadata row-index predicate should not be stripped from a native scan") { + withTempDir { + tempDir => + val path = tempDir.getCanonicalPath + Seq((1, "a"), (2, "b"), (3, "c"), (4, "d")) + .toDF("id", "value") + .coalesce(1) + .write + .format("delta") + .save(path) + + spark.sql( + s"ALTER TABLE delta.`$path` SET TBLPROPERTIES ('delta.enableDeletionVectors' = true)") + + val df = spark.sql( + s"SELECT id, _metadata.row_index AS row_index FROM delta.`$path` " + + "WHERE _metadata.row_index = 2") + val rows = df.collect() + val executedPlan = df.queryExecution.executedPlan + val planText = executedPlan.treeString + assert(containsNativeDeltaScan(executedPlan), planText) + assert(rows.length === 1, planText) + assert(rows.head.getLong(1) === 2L, planText) + } + } + + Seq(true, false).foreach { + useMetadataRowIndex => + test( + "Delta DV DML row-index scan should fall back with Spark project/filter, " + + s"metadata row index=$useMetadataRowIndex") { + assume(SparkVersionUtil.gteSpark35, "DML row-index scan fallback is Spark 3.5+ coverage") + withTempDir { + tempDir => + val path = tempDir.getCanonicalPath + Seq((1, "a"), (2, "b"), (3, "c"), (4, "d")) + .toDF("id", "value") + .coalesce(1) + .write + .format("delta") + .save(path) + + spark.sql( + s"ALTER TABLE delta.`$path` SET TBLPROPERTIES " + + "('delta.enableDeletionVectors' = true)") + + var executedPlans: Seq[SparkPlan] = Seq.empty + withSQLConf( + DeltaSQLConf.DELETION_VECTORS_USE_METADATA_ROW_INDEX.key -> + useMetadataRowIndex.toString, + VeloxDeltaConfig.ENABLE_NATIVE_DML_ROW_INDEX_SCAN.key -> "false" + ) { + executedPlans = DeltaTestUtils.withAllPlansCaptured(spark) { + spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (3, 4)").collect() + }.map(_.executedPlan) + } + assertSparkDmlFallback(executedPlans) + + val log = DeltaLog.forTable(spark, new Path(path)) + assert(log.update().allFiles.collect().exists(_.deletionVector != null)) + assertReadPlanAfterDmlFallback(path, useMetadataRowIndex) + } + } + } + test("Spark 4 Delta DV scan handoff should filter deleted rows") { withTempDir { tempDir => @@ -88,11 +245,37 @@ class DeltaDeletionVectorHandoffSuite val df = spark.read.format("delta").load(path) val executedPlan = df.queryExecution.executedPlan - assert(executedPlan.collect { case _: DeltaScanTransformer => true }.nonEmpty) + assert(containsNativeDeltaScan(executedPlan)) val planText = executedPlan.toString() assert(!planText.contains("__delta_internal_is_row_deleted")) assert(!planText.contains("__delta_internal_row_index")) checkAnswer(df, Seq((1, "a"), (2, "b")).toDF()) } } + + test("Delta DV DML row-index scan should fall back when updating an existing DV") { + assume(SparkVersionUtil.gteSpark35, "DML row-index scan fallback is Spark 3.5+ coverage") + withTempDir { + tempDir => + val path = new File(tempDir, "delta table with spaces").getCanonicalPath + Seq((1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e"), (6, "f")) + .toDF("id", "value") + .coalesce(1) + .write + .format("delta") + .save(path) + + spark.sql( + s"ALTER TABLE delta.`$path` SET TBLPROPERTIES " + + "('delta.enableDeletionVectors' = true)") + + assertSparkDmlFallback(captureDeletePlans(path, "id IN (5, 6)", useMetadataRowIndex = true)) + assert(activeDvCardinality(path) === 2L) + + assertSparkDmlFallback(captureDeletePlans(path, "id IN (3, 4)", useMetadataRowIndex = true)) + assert(activeDvCardinality(path) === 4L) + + assertReadPlanAfterDmlFallback(path, useMetadataRowIndex = true) + } + } } diff --git a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaDeletionVectorDmlUtils.scala b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaDeletionVectorDmlUtils.scala new file mode 100644 index 00000000000..7194b1c399f --- /dev/null +++ b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaDeletionVectorDmlUtils.scala @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension + +import org.apache.spark.sql.catalyst.expressions.{Expression, GetStructField, NamedExpression} +import org.apache.spark.sql.catalyst.expressions.aggregation.BitmapAggregator +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.trees.TreeNodeTag +import org.apache.spark.sql.delta.DeltaParquetFileFormat +import org.apache.spark.sql.delta.files.TahoeFileIndex +import org.apache.spark.sql.delta.stats.PreparedDeltaFileIndex +import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} +import org.apache.spark.sql.types.{DataType, StructType} + +object DeltaDeletionVectorDmlUtils { + private val DmlRowIndexScanTag: TreeNodeTag[Boolean] = + TreeNodeTag[Boolean]("org.apache.gluten.delta.dml.row.index.scan") + + // Spark 3.5+ exposes this as ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME. + private val parquetTemporaryRowIndexColumnName = "_tmp_metadata_row_index" + private val deletionVectorRowIndexColumnNames = + Set( + "__delta_internal_row_index", + DeltaParquetFileFormat.ROW_INDEX_COLUMN_NAME, + parquetTemporaryRowIndexColumnName, + "row_index", + "rowIndexCol") + private val filePathColumnNames = Set("file_path", "filePath") + + val tagDmlRowIndexScans: Rule[SparkPlan] = (plan: SparkPlan) => { + def visit( + node: SparkPlan, + hasRowIndexReference: Boolean, + hasFilePathReference: Boolean, + hasBitmapAggregation: Boolean): Unit = { + val nextHasBitmapAggregation = + hasBitmapAggregation || node.expressions.exists(referencesDeletionVectorBitmapAggregator) + // The row-index/file-path signature is only meaningful below Delta's BitmapAggregator. + // Avoid collecting expression references for every node in ordinary (non-DML) queries. + val nextHasRowIndexReference = + nextHasBitmapAggregation && + (hasRowIndexReference || node.expressions.exists(referencesRowIndexColumn)) + val nextHasFilePathReference = + nextHasBitmapAggregation && + (hasFilePathReference || node.expressions.exists(referencesFilePathColumn)) + + node.children.foreach { + case scan: FileSourceScanExec + if nextHasBitmapAggregation && + nextHasRowIndexReference && + nextHasFilePathReference && + isDeletionVectorDmlRowIndexScanCandidate(scan) => + scan.setTagValue(DmlRowIndexScanTag, true) + case child => + visit( + child, + nextHasRowIndexReference, + nextHasFilePathReference, + nextHasBitmapAggregation) + } + } + + visit( + plan, + hasRowIndexReference = false, + hasFilePathReference = false, + hasBitmapAggregation = false) + plan + } + + def copyDmlRowIndexScanTag(from: SparkPlan, to: SparkPlan): Unit = { + if (from.getTagValue(DmlRowIndexScanTag).contains(true)) { + to.setTagValue(DmlRowIndexScanTag, true) + } + } + + def isDeltaScan(scan: FileSourceScanExec): Boolean = { + isDeltaFileIndex(scan) || isDeltaParquetScan(scan) + } + + def isDeltaParquetScan(scan: FileSourceScanExec): Boolean = { + val fileFormatClass = scan.relation.fileFormat.getClass + fileFormatClass == classOf[DeltaParquetFileFormat] || + fileFormatClass.getSimpleName == "GlutenDeltaParquetFileFormat" + } + + def isDeltaFileIndex(scan: FileSourceScanExec): Boolean = { + scan.relation.location.isInstanceOf[TahoeFileIndex] || + scan.relation.location.isInstanceOf[PreparedDeltaFileIndex] + } + + def isDeletionVectorDmlRowIndexScan(scan: FileSourceScanExec): Boolean = { + scan.getTagValue(DmlRowIndexScanTag).contains(true) && + isDeletionVectorDmlRowIndexScanCandidate(scan) + } + + def isDeletionVectorDmlRowIndexScan(plan: SparkPlan): Boolean = { + plan.getTagValue(DmlRowIndexScanTag).contains(true) + } + + private def isDeletionVectorDmlRowIndexScanCandidate(scan: FileSourceScanExec): Boolean = { + if (!isDeltaScan(scan)) { + return false + } + + scanContainsColumnName(scan, deletionVectorRowIndexColumnNames) && + scanContainsColumnName(scan, filePathColumnNames) + } + + private def scanContainsColumnName( + scan: FileSourceScanExec, + columnNames: Set[String]): Boolean = { + def nestedFieldNames(dataType: DataType): Seq[String] = dataType match { + case struct: StructType => + struct.fields.flatMap(field => field.name +: nestedFieldNames(field.dataType)).toSeq + case _ => Seq.empty + } + + val outputColumnNames = + scan.output.flatMap(attribute => attribute.name +: nestedFieldNames(attribute.dataType)) + val requiredColumnNames = scan.requiredSchema.fields.flatMap { + field => field.name +: nestedFieldNames(field.dataType) + } + (outputColumnNames ++ requiredColumnNames).exists(columnNames.contains) + } + + private def referencedColumnNames(expr: Expression): Set[String] = { + val attributeNames = expr.references.iterator.map(_.name) + val nestedFieldNames = expr.collect { + case field: GetStructField => field.name + case named: NamedExpression => Some(named.name) + }.flatten + (attributeNames ++ nestedFieldNames).toSet + } + + private def referencesRowIndexColumn(expr: Expression): Boolean = { + referencedColumnNames(expr).exists(deletionVectorRowIndexColumnNames.contains) + } + + private def referencesFilePathColumn(expr: Expression): Boolean = { + referencedColumnNames(expr).exists(filePathColumnNames.contains) + } + + private def referencesDeletionVectorBitmapAggregator(expr: Expression): Boolean = { + expr.exists(_.isInstanceOf[BitmapAggregator]) + } +} diff --git a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala index fb694c70d9c..ba8c3667064 100644 --- a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala +++ b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala @@ -17,7 +17,8 @@ package org.apache.gluten.extension import org.apache.gluten.backendsapi.BackendsApiManager -import org.apache.gluten.execution.{DeltaScanTransformer, FilterExecTransformerBase, ProjectExecTransformer} +import org.apache.gluten.execution.{DeltaScanTransformer, FilterExecTransformerBase, ProjectExecTransformer, ProjectExecTransformerBase} +import org.apache.gluten.extension.columnar.FallbackTags import org.apache.gluten.extension.columnar.transition.RemoveTransitions import org.apache.spark.sql.SparkSession @@ -26,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{ArrayTransform, TransformKeys, import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreeNodeTag import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaParquetFileFormat, NoMapping} -import org.apache.spark.sql.execution.{FilterExec, ProjectExec, SparkPlan} +import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, ProjectExec, SparkPlan} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType} @@ -36,6 +37,7 @@ import scala.collection.mutable.ListBuffer object DeltaPostTransformRules { def rules: Seq[Rule[SparkPlan]] = RemoveTransitions :: + keepDmlRowIndexFallbackSubtreeOnSpark :: deltaSpecificRules :: Nil @@ -56,6 +58,13 @@ object DeltaPostTransformRules { private val deletionVectorDeletedRowColumnName = "__delta_internal_is_row_deleted" private val deletionVectorRowIndexColumnName = "__delta_internal_row_index" + // Spark 3.5+ exposes this as ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME. + private val parquetTemporaryRowIndexColumnName = "_tmp_metadata_row_index" + private val deletionVectorRowIndexColumnNames = + Set( + deletionVectorRowIndexColumnName, + DeltaParquetFileFormat.ROW_INDEX_COLUMN_NAME, + parquetTemporaryRowIndexColumnName) private val deletionVectorInternalColumnNames = Set(deletionVectorDeletedRowColumnName, deletionVectorRowIndexColumnName) @@ -90,6 +99,25 @@ object DeltaPostTransformRules { child.copy(output = p.output) } + /** + * Native DELETE/UPDATE/MERGE DV support can deliberately keep the target row-index scan in Spark. + * Keep only the contiguous scan-adjacent filter/project chain in Spark, avoiding an isolated + * native island without propagating fallback through joins, exchanges, or aggregations. The + * BitmapAggregator and the rest of the DML plan remain eligible for native execution. + */ + val keepDmlRowIndexFallbackSubtreeOnSpark: Rule[SparkPlan] = (plan: SparkPlan) => + plan.transformUp { + case project: ProjectExecTransformerBase + if isDmlRowIndexFallbackSubtree(project.child) => + val sparkProject = ProjectExec(project.list, project.child) + sparkProject.copyTagsFrom(project) + sparkProject + case filter: FilterExecTransformerBase if isDmlRowIndexFallbackSubtree(filter.child) => + val sparkFilter = FilterExec(filter.cond, filter.child) + sparkFilter.copyTagsFrom(filter) + sparkFilter + } + /** * Spark Delta injects synthetic deletion-vector predicates and columns into the plan (via * `PreprocessTableWithDVsStrategy`). Those drive the JVM reader path; for the native Delta scan @@ -194,6 +222,23 @@ object DeltaPostTransformRules { } } + private def isDmlRowIndexFallbackSubtree(plan: SparkPlan): Boolean = { + plan match { + case scan: FileSourceScanExec => + DeltaDeletionVectorDmlUtils.isDeletionVectorDmlRowIndexScan(scan) && + FallbackTags.nonEmpty(scan) + case project: ProjectExecTransformerBase => + isDmlRowIndexFallbackSubtree(project.child) + case ProjectExec(_, child) => + isDmlRowIndexFallbackSubtree(child) + case filter: FilterExecTransformerBase => + isDmlRowIndexFallbackSubtree(filter.child) + case FilterExec(_, child) => + isDmlRowIndexFallbackSubtree(child) + case _ => false + } + } + private def isDeltaColumnMappingFileFormat(fileFormat: FileFormat): Boolean = fileFormat match { case d: DeltaParquetFileFormat if d.columnMappingMode != NoMapping => true @@ -213,7 +258,7 @@ object DeltaPostTransformRules { } private def referencesDeletionVectorRowIndex(expr: Expression): Boolean = { - expr.references.exists(_.name == deletionVectorRowIndexColumnName) + expr.references.exists(attr => deletionVectorRowIndexColumnNames.contains(attr.name)) } private def tagRowIndexRequiredSubtrees(plan: SparkPlan): Unit = { @@ -235,11 +280,16 @@ object DeltaPostTransformRules { } private def shouldPreserveDeletionVectorRowIndex(plan: SparkPlan): Boolean = { + isDeletionVectorDmlRowIndexScan(plan) || plan.getTagValue(PRESERVE_DELETION_VECTOR_ROW_INDEX_TAG).contains(true) || plan.expressions.exists(containsIncrementMetricExpr) || plan.expressions.exists(referencesDeletionVectorRowIndex) } + private def isDeletionVectorDmlRowIndexScan(plan: SparkPlan): Boolean = { + DeltaDeletionVectorDmlUtils.isDeletionVectorDmlRowIndexScan(plan) + } + private def shouldStripDeletionVectorInternalColumn( columnName: String, preserveRowIndex: Boolean): Boolean = { diff --git a/gluten-delta/src/main/scala/org/apache/gluten/extension/OffloadDeltaScan.scala b/gluten-delta/src/main/scala/org/apache/gluten/extension/OffloadDeltaScan.scala index ebafb0c08c3..0cf6c6e3288 100644 --- a/gluten-delta/src/main/scala/org/apache/gluten/extension/OffloadDeltaScan.scala +++ b/gluten-delta/src/main/scala/org/apache/gluten/extension/OffloadDeltaScan.scala @@ -20,7 +20,6 @@ import org.apache.gluten.execution.DeltaScanTransformer import org.apache.gluten.extension.columnar.FallbackTags import org.apache.gluten.extension.columnar.offload.OffloadSingleNode -import org.apache.spark.sql.delta.DeltaParquetFileFormat import org.apache.spark.sql.delta.SnapshotDescriptor import org.apache.spark.sql.delta.commands.DeletionVectorUtils.deletionVectorsReadable import org.apache.spark.sql.delta.files.TahoeFileIndex @@ -28,7 +27,9 @@ import org.apache.spark.sql.delta.stats.PreparedDeltaFileIndex import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} import org.apache.spark.util.SparkVersionUtil -case class OffloadDeltaScan() extends OffloadSingleNode { +case class OffloadDeltaScan( + enableNativeDeletionVectorDmlRowIndexScanKey: String) + extends OffloadSingleNode { private val DeletionVectorsUseMetadataRowIndexKey = "spark.databricks.delta.deletionVectors.useMetadataRowIndex" @@ -39,28 +40,38 @@ case class OffloadDeltaScan() extends OffloadSingleNode { case scan: FileSourceScanExec if shouldFallbackSpark34DeletionVectorScan(scan) => FallbackTags.add(scan, "fallback Spark 3.4 Delta DV scan") scan + case scan: FileSourceScanExec if shouldFallbackDeletionVectorDmlScan(scan) => + FallbackTags.add(scan, "fallback Delta DV DML row-index scan") + scan case scan: FileSourceScanExec if shouldFallbackDeletionVectorScanWithoutMetadataRowIndex(scan) => FallbackTags.add(scan, "fallback Delta DV scan without metadata row index") scan case scan: FileSourceScanExec if isDeltaScan(scan) => - DeltaScanTransformer(scan) + val transformer = DeltaScanTransformer(scan) + DeltaDeletionVectorDmlUtils.copyDmlRowIndexScanTag(scan, transformer) + transformer case other => other } private def isDeltaScan(scan: FileSourceScanExec): Boolean = { - isDeltaFileIndex(scan) || isDeltaParquetScan(scan) + DeltaDeletionVectorDmlUtils.isDeltaScan(scan) } - private def isDeltaParquetScan(scan: FileSourceScanExec): Boolean = { - val fileFormatClass = scan.relation.fileFormat.getClass - fileFormatClass == classOf[DeltaParquetFileFormat] || - fileFormatClass.getSimpleName == "GlutenDeltaParquetFileFormat" - } + private def shouldFallbackDeletionVectorDmlScan(scan: FileSourceScanExec): Boolean = { + val enableNativeDmlRowIndexScan = + scan.relation.sparkSession.sessionState.conf + .getConfString(enableNativeDeletionVectorDmlRowIndexScanKey, "false") + .toBoolean + if (enableNativeDmlRowIndexScan) { + return false + } - private def isDeltaFileIndex(scan: FileSourceScanExec): Boolean = { - scan.relation.location.isInstanceOf[TahoeFileIndex] || - scan.relation.location.isInstanceOf[PreparedDeltaFileIndex] + // DELETE/UPDATE/MERGE with persistent deletion vectors needs the target scan to expose + // per-file row indexes so Delta can build updated DV bitmaps. Keep this experimental target + // scan on Spark by default until its native row-index correctness is established independently + // of the native BitmapAggregator and write path. + DeltaDeletionVectorDmlUtils.isDeletionVectorDmlRowIndexScan(scan) } private def isDeltaLogScan(scan: FileSourceScanExec): Boolean = {