From 3af54ac4cae6bc289b588a04ab62a944aa2e05ee Mon Sep 17 00:00:00 2001 From: chloex <122268864+xiaoyanxie@users.noreply.github.com> Date: Sat, 18 Jul 2026 22:08:01 -0400 Subject: [PATCH 1/2] Try to reproduce the issue in the GitHub CI pipeline --- .github/workflows/tpcds.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml index d2711dee5..2372a6067 100644 --- a/.github/workflows/tpcds.yml +++ b/.github/workflows/tpcds.yml @@ -112,3 +112,16 @@ jobs: scalaver: '2.13' hadoop-profile: 'hadoop3' sparktests: 'true' + + test-spark-41-jdk21-scala-2-13-with-bloomfilter-optimizer-enabled: + name: Test spark-4.1 JDK21 Scala-2.13 with bloomFilter optimizer enabled + uses: ./.github/workflows/tpcds-reusable.yml + with: + sparkver: spark-4.1 + javaver: '21' + scalaver: '2.13' + hadoop-profile: 'hadoop3' + sparktests: 'true' + extrasparkconf: >- + --conf spark.sql.optimizer.runtime.bloomFilter.enabled=true + --conf spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold=1B From 8335cae4ffc969684a26824be3bdb3004f5a2192 Mon Sep 17 00:00:00 2001 From: chloex <122268864+xiaoyanxie@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:32:02 -0400 Subject: [PATCH 2/2] Force sort-merge join in bloom-filter repro CI job Disable broadcast joins (spark.sql.autoBroadcastJoinThreshold=-1) so the runtime bloom-filter build side becomes a shuffle-backed plan. The ClassCastException only triggers when the injected ScalarSubquery carries a shuffle MapPartitionsRDD lineage; on the sf=1 CI fixture the joins otherwise broadcast and the repro silently passes. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/tpcds.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml index 2372a6067..479cacf85 100644 --- a/.github/workflows/tpcds.yml +++ b/.github/workflows/tpcds.yml @@ -113,15 +113,16 @@ jobs: hadoop-profile: 'hadoop3' sparktests: 'true' - test-spark-41-jdk21-scala-2-13-with-bloomfilter-optimizer-enabled: - name: Test spark-4.1 JDK21 Scala-2.13 with bloomFilter optimizer enabled + test-spark-41-jdk17-scala-2-13-with-bloomfilter-optimizer-enabled: + name: Test spark-4.1 JDK17 Scala-2.13 with bloomFilter optimizer enabled uses: ./.github/workflows/tpcds-reusable.yml with: sparkver: spark-4.1 - javaver: '21' + javaver: '17' scalaver: '2.13' hadoop-profile: 'hadoop3' sparktests: 'true' extrasparkconf: >- --conf spark.sql.optimizer.runtime.bloomFilter.enabled=true --conf spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold=1B + --conf spark.sql.autoBroadcastJoinThreshold=-1