From e525dfe8bfa31fb4fb6e38481e091d3f26a052cd Mon Sep 17 00:00:00 2001 From: Smallfu666 Date: Thu, 23 Jul 2026 16:03:09 +0800 Subject: [PATCH 1/3] [GLUTEN-12356][VL] Preserve precision for long decimal to double cast --- .../backendsapi/velox/VeloxSparkPlanExecApi.scala | 11 ++++++++++- .../functions/MathFunctionsValidateSuite.scala | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index d588695cd40..9e303aa82ff 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -1305,7 +1305,16 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with Logging { // ISOControl characters, refer java.lang.Character.isISOControl(int) val isoControlStr = (('\u0000' to '\u001F') ++ ('\u007F' to '\u009F')).toList.mkString // scalastyle:on nonascii - if (VeloxConfig.get.castFromVarcharAddTrimNode && c.child.dataType == StringType) { + val isLongDecimalToDouble = c.dataType == DoubleType && (c.child.dataType match { + case decimalType: DecimalType => decimalType.precision > 18 + case _ => false + }) + if (isLongDecimalToDouble) { + // Velox converts long-decimal unscaled int128 to double before applying the scale factor, + // so precision can be lost. Precision above 18 uses Velox's long-decimal representation; + // parsing its exact string value preserves Spark's correctly rounded conversion. + c.copy(child = Cast(c.child, StringType, c.timeZoneId, c.evalMode)) + } else if (VeloxConfig.get.castFromVarcharAddTrimNode && c.child.dataType == StringType) { val trimStr = c.dataType match { case BinaryType | _: ArrayType | _: MapType | _: StructType | _: UserDefinedType[_] => None diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala index 81a9ad5cdba..b21dde06e24 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala @@ -449,6 +449,19 @@ class MathFunctionsValidateSuite extends FunctionsValidateSuite { } } + test("decimal to double preserves precision after decimal division") { + withSQLConf( + "spark.sql.optimizer.excludedRules" -> + "org.apache.spark.sql.catalyst.optimizer.ConstantFolding", + "spark.sql.decimalOperations.allowPrecisionLoss" -> "false" + ) { + runQueryAndCompare( + "SELECT CAST(2.8 / CAST(0.0130597014925373134 AS DECIMAL(38,19)) AS DOUBLE)") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + testWithMinSparkVersion( "decimal arithmetic respects allowPrecisionLoss captured at view analysis time", "4.1") { From 71bfc1f36286c77c540d33989e81d7e15cca2b69 Mon Sep 17 00:00:00 2001 From: Smallfu666 Date: Thu, 23 Jul 2026 18:56:53 +0800 Subject: [PATCH 2/3] [GLUTEN-12356][VL] Expand decimal-to-double cast test coverage Pin the rewrite scope with a unit test on genCastWithNewChild (precision <= 18 keeps the direct conversion, > 18 goes through the exact string representation), and add a parquet-backed direct-cast matrix across DECIMAL(18,6)/(19,0)/(38,0)/(38,19)/(38,38) covering positive/negative/zero/NULL values, values around 2^53, and near-max 38-digit values, compared against vanilla Spark under both ANSI on and off. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../MathFunctionsValidateSuite.scala | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala index b21dde06e24..28ceb39cd5f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala @@ -16,12 +16,15 @@ */ package org.apache.gluten.functions +import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BatchScanExecTransformer, ProjectExecTransformer} import org.apache.spark.SparkConf import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast} import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{DecimalType, DoubleType, StringType} class MathFunctionsValidateSuiteAnsiOn extends FunctionsValidateSuite { @@ -462,6 +465,78 @@ class MathFunctionsValidateSuite extends FunctionsValidateSuite { } } + test("GLUTEN-12356: long decimal to double rewrite applies only to precision > 18") { + val api = BackendsApiManager.getSparkPlanExecApiInstance + def castOf(precision: Int, scale: Int): Cast = + Cast(AttributeReference("c", DecimalType(precision, scale))(), DoubleType) + // Precision <= 18 keeps Velox's direct conversion. + assert(api.genCastWithNewChild(castOf(18, 6)).child.dataType == DecimalType(18, 6)) + // Precision > 18 goes through the exact string representation. + assert(api.genCastWithNewChild(castOf(19, 0)).child.dataType == StringType) + assert(api.genCastWithNewChild(castOf(38, 19)).child.dataType == StringType) + } + + test("GLUTEN-12356: direct cast of decimal to double matches vanilla Spark") { + withTempView("decimal_double_cast") { + withTempPath { + path => + // Cover short decimal (no rewrite), long decimal at both precision bounds, + // zero/negative/NULL values, values around 2^53, and near-max 38-digit values. + spark + .sql(""" + |SELECT + | CAST(v18 AS DECIMAL(18,6)) AS d18_6, + | CAST(v19 AS DECIMAL(19,0)) AS d19_0, + | CAST(v38_0 AS DECIMAL(38,0)) AS d38_0, + | CAST(v38_19 AS DECIMAL(38,19)) AS d38_19, + | CAST(v38_38 AS DECIMAL(38,38)) AS d38_38 + |FROM VALUES + | ('123456789012.345678', '9007199254740991', + | '12345678901234567890123456789012345678', + | '214.4000000000000006143270301075587414', + | '0.99999999999999999999999999999999999999'), + | ('999999999999.999999', '9007199254740992', + | '9007199254740993', + | '9007199254740993.0000000000000000001', + | '0.00000000000000000000000000000000000001'), + | ('-123456789012.345678', '-9007199254740993', + | '-99999999999999999999999999999999999999', + | '-0.0130597014925373134', + | '-0.5'), + | ('0', '0', '0', '0', '0'), + | (NULL, NULL, NULL, NULL, NULL), + | ('999999999999.999999', '9999999999999999999', + | '99999999999999999999999999999999999999', + | '9999999999999999999.9999999999999999999', + | '0.12345678901234567890123456789012345678') + |AS t(v18, v19, v38_0, v38_19, v38_38) + |""".stripMargin) + .write + .parquet(path.getCanonicalPath) + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("decimal_double_cast") + + Seq("true", "false").foreach { + ansi => + withSQLConf( + SQLConf.ANSI_ENABLED.key -> ansi, + GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key -> "false") { + runQueryAndCompare(""" + |SELECT + | CAST(d18_6 AS DOUBLE), + | CAST(d19_0 AS DOUBLE), + | CAST(d38_0 AS DOUBLE), + | CAST(d38_19 AS DOUBLE), + | CAST(d38_38 AS DOUBLE) + |FROM decimal_double_cast + |""".stripMargin) { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + } + } + } + testWithMinSparkVersion( "decimal arithmetic respects allowPrecisionLoss captured at view analysis time", "4.1") { From 260154370dec31d8150488f121ccd764e510373f Mon Sep 17 00:00:00 2001 From: Smallfu666 Date: Fri, 24 Jul 2026 10:09:08 +0800 Subject: [PATCH 3/3] [GLUTEN-12356][VL] Use 3-arg Cast for cross-version compatibility Cast.evalMode does not exist before Spark 3.4 (it replaced the ansiEnabled constructor arg in SPARK-40389), so the 4-arg Cast(child, StringType, timeZoneId, evalMode) form fails to compile on Spark 3.3. Use the 3-arg Cast instead and keep the outer cast via withNewChildren: decimal-to-string is lossless regardless of eval mode, so the original ANSI semantics are preserved by the outer cast. Verified: Spark 3.3 test-compile passes; MathFunctionsValidateSuite (Spark 3.5, Velox) 46/46. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 9e303aa82ff..b73423f0ec0 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -1313,7 +1313,10 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with Logging { // Velox converts long-decimal unscaled int128 to double before applying the scale factor, // so precision can be lost. Precision above 18 uses Velox's long-decimal representation; // parsing its exact string value preserves Spark's correctly rounded conversion. - c.copy(child = Cast(c.child, StringType, c.timeZoneId, c.evalMode)) + // Use the 3-arg Cast (no evalMode) for cross-version compatibility: Cast.evalMode does + // not exist before Spark 3.4, and decimal-to-string is lossless regardless of eval mode, + // so the outer cast (kept via withNewChildren) still carries the original ANSI semantics. + c.withNewChildren(Seq(Cast(c.child, StringType, c.timeZoneId))).asInstanceOf[Cast] } else if (VeloxConfig.get.castFromVarcharAddTrimNode && c.child.dataType == StringType) { val trimStr = c.dataType match { case BinaryType | _: ArrayType | _: MapType | _: StructType | _: UserDefinedType[_] =>