From 4a7a240bf4c5d86593da68c26ae63cc597958023 Mon Sep 17 00:00:00 2001 From: Ke Jia Date: Wed, 18 Mar 2026 08:57:39 -0700 Subject: [PATCH 1/2] Only apply VeloxParquetWriterInjects and NativeWritePostRule for Spark 3.2 and 3.3 --- .../gluten/backendsapi/velox/VeloxListenerApi.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index ba0f1570f802..99dba48d12aa 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -30,6 +30,7 @@ import org.apache.gluten.memory.{MemoryUsageRecorder, SimpleMemoryUsageRecorder} import org.apache.gluten.memory.listener.ReservationListener import org.apache.gluten.memory.memtarget.MemoryTarget import org.apache.gluten.monitor.VeloxMemoryProfiler +import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.udf.UdfJniWrapper import org.apache.gluten.utils._ @@ -238,8 +239,12 @@ class VeloxListenerApi extends ListenerApi with Logging { // Inject backend-specific implementations to override spark classes. GlutenFormatFactory.register(new VeloxParquetWriterInjects) - GlutenFormatFactory.injectPostRuleFactory( - session => GlutenWriterColumnarRules.NativeWritePostRule(session)) + // Only register VeloxParquetWriterInjects and NativeWritePostRule for Spark 3.2 and 3.3 + val sparkVersion = SparkShimLoader.getSparkVersion + if (sparkVersion.startsWith("3.2") || sparkVersion.startsWith("3.3")) { + GlutenFormatFactory.injectPostRuleFactory( + session => GlutenWriterColumnarRules.NativeWritePostRule(session)) + } GlutenFormatFactory.register(new VeloxRowSplitter()) } From 6c1f425505d2dcb3ae722d54eb607ecb313b19a3 Mon Sep 17 00:00:00 2001 From: Ke Jia Date: Thu, 19 Mar 2026 08:57:21 -0700 Subject: [PATCH 2/2] Resolve comments --- .../apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 99dba48d12aa..02392bb886d8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -239,9 +239,8 @@ class VeloxListenerApi extends ListenerApi with Logging { // Inject backend-specific implementations to override spark classes. GlutenFormatFactory.register(new VeloxParquetWriterInjects) - // Only register VeloxParquetWriterInjects and NativeWritePostRule for Spark 3.2 and 3.3 - val sparkVersion = SparkShimLoader.getSparkVersion - if (sparkVersion.startsWith("3.2") || sparkVersion.startsWith("3.3")) { + // Only register NativeWritePostRule for Spark 3.3 + if (SparkShimLoader.getSparkVersion.startsWith("3.3")) { GlutenFormatFactory.injectPostRuleFactory( session => GlutenWriterColumnarRules.NativeWritePostRule(session)) }