[WIP] HIVE-29641: Upgrade Calcite to 1.42.0

druid-handler/pom.xml

@@ -409,6 +409,23 @@
                 </configuration>
               </execution>
             </executions>
+            <dependencies>
+              <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm</artifactId>
+                <version>9.9.1</version>
+              </dependency>
+              <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm-commons</artifactId>
+                <version>9.9.1</version>
+              </dependency>
+              <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm-tree</artifactId>
+                <version>9.9.1</version>
+              </dependency>
+            </dependencies>
           </plugin>
         </plugins>
       </build>

hbase-handler/src/test/results/positive/hbase_queries.q.out

@@ -148,7 +148,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: key is not null (type: boolean)
               Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: UDFToDouble(key) (type: double)
@@ -604,7 +604,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: key is not null (type: boolean)
               Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double)

iceberg/iceberg-handler/src/test/results/positive/merge_iceberg_copy_on_write_unpartitioned.q.out

@@ -621,7 +621,7 @@ STAGE PLANS:
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
                 Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE
                 Filter Operator
-                  predicate: ((((_col5 <> _col8) or (_col5 <= 100)) and _col5 is not null) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean)
+                  predicate: ((_col5 <> _col8) or (_col5 <= 100) or (_col5 is null or ((_col5 = _col8) and (_col5 > 100))) is null) (type: boolean)
                   Statistics: Num rows: 8 Data size: 3884 Basic stats: COMPLETE Column stats: COMPLETE
                   Select Operator
                     expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: int)

itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out

@@ -87,7 +87,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')]))
+FROM TABLE(INLINE(ARRAY ((1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505'))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out

@@ -51,7 +51,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1)]))
+FROM TABLE(INLINE(ARRAY ((1))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out

@@ -105,7 +105,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505')]))
+FROM TABLE(INLINE(ARRAY ((1, '101'), (2, '202'), (3, '303'), (4, '404'), (5, '505'))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1

itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out

@@ -59,7 +59,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table1
 OPTIMIZED SQL: SELECT *
-FROM TABLE(INLINE(ARRAY[(1)]))
+FROM TABLE(INLINE(ARRAY ((1))))
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
   Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5

kudu-handler/src/test/results/positive/kudu_complex_queries.q.out

@@ -85,10 +85,10 @@ STAGE PLANS:
             Map Operator Tree:
                 TableScan
                   alias: kv_table
-                  filterExpr: UDFToDouble(key) is not null (type: boolean)
+                  filterExpr: key is not null (type: boolean)
                   Statistics: Num rows: 309 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: UDFToDouble(key) is not null (type: boolean)
+                    predicate: key is not null (type: boolean)
                     Statistics: Num rows: 309 Data size: 1236 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: UDFToDouble(key) (type: double)
@@ -246,7 +246,7 @@ STAGE PLANS:
                   filterExpr: (key > 100) (type: boolean)
                   Statistics: Num rows: 309 Data size: 29355 Basic stats: COMPLETE Column stats: COMPLETE
                   Filter Operator
-                    predicate: (((key % 2) = 0) and UDFToDouble(key) is not null) (type: boolean)
+                    predicate: ((key % 2) = 0) (type: boolean)
                     Statistics: Num rows: 154 Data size: 14630 Basic stats: COMPLETE Column stats: COMPLETE
                     Select Operator
                       expressions: key (type: int), value (type: string), UDFToDouble(key) (type: double)

pom.xml

@@ -98,7 +98,7 @@
     <maven.eclipse.plugin.version>2.10</maven.eclipse.plugin.version>
     <maven.exec.plugin.version>3.1.0</maven.exec.plugin.version>
     <maven.versions.plugin.version>2.16.0</maven.versions.plugin.version>
-    <maven.shade.plugin.version>3.6.0</maven.shade.plugin.version>
+    <maven.shade.plugin.version>3.6.2</maven.shade.plugin.version>
     <josdk.version>5.3.4</josdk.version>
     <fabric8.version>7.7.0</fabric8.version>
     <maven.surefire.plugin.version>3.5.3</maven.surefire.plugin.version>
@@ -113,7 +113,7 @@
     <apache-directory-server.version>2.0.0-M24</apache-directory-server.version>
     <avro.version>1.12.0</avro.version>
     <bcprov-jdk18on.version>1.82</bcprov-jdk18on.version>
-    <calcite.version>1.33.0</calcite.version>
+    <calcite.version>1.42.0</calcite.version>
     <datanucleus-api-jdo.version>6.0.5</datanucleus-api-jdo.version>
     <datanucleus-core.version>6.0.11</datanucleus-core.version>
     <datanucleus-jdo.version>3.2.0-release</datanucleus-jdo.version>

ql/pom.xml

@@ -365,6 +365,14 @@
       <artifactId>hadoop-yarn-client</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents.core5</groupId>
+      <artifactId>httpcore5</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents.client5</groupId>
+      <artifactId>httpclient5</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.orc</groupId>
       <artifactId>orc-tools</artifactId>

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -28,6 +28,7 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.calcite.util.ConversionUtil;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.ObjectUtils;
@@ -569,6 +570,10 @@ private Hive(HiveConf c, boolean doRegisterAllFns) throws HiveException {
     conf = c;
     // turn off calcite rexnode normalization
     System.setProperty("calcite.enable.rexnode.digest.normalize", "false");
+    // update calcite default charset
+    // TODO is this the right place? It's not considered in cases like TestHiveRelJsonReader
+    System.setProperty("calcite.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
+    System.setProperty("calcite.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
     if (doRegisterAllFns) {
       registerAllFunctionsOnce();
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java

@@ -48,9 +48,12 @@
 import org.apache.calcite.rex.RexDynamicParam;
 import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLambda;
+import org.apache.calcite.rex.RexLambdaRef;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexNodeAndFieldIndex;
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexPatternFieldRef;
 import org.apache.calcite.rex.RexShuttle;
@@ -1139,6 +1142,21 @@ public Boolean visitSubQuery(RexSubQuery subQuery) {
     public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
       return false;
     }
+
+    @Override
+    public Boolean visitLambda(RexLambda lambda) {
+      return false;
+    }
+
+    @Override
+    public Boolean visitLambdaRef(RexLambdaRef lambdaRef) {
+      return false;
+    }
+
+    @Override
+    public Boolean visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex) {
+      return false;
+    }
   }
 
   public static Set<Integer> getInputRefs(RexNode expr) {

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelBuilder.java

@@ -89,7 +89,8 @@ public static RelBuilderFactory proto(final Context context) {
     return new RelBuilderFactory() {
       @Override
       public RelBuilder create(RelOptCluster cluster, RelOptSchema schema) {
-        Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED));
+        Context confContext = Contexts.of(Config.DEFAULT.withPruneInputOfAggregate(Bug.CALCITE_4513_FIXED)
+            .withSimplifyValues(false)); // disabled to avoid simplifications that can create non-empty HiveValues
         return new HiveRelBuilder(Contexts.chain(context, confContext), cluster, schema);
       }
     };

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelShuttleImpl.java

@@ -24,6 +24,7 @@
 import org.apache.calcite.rel.core.TableFunctionScan;
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalAsofJoin;
 import org.apache.calcite.rel.logical.LogicalCalc;
 import org.apache.calcite.rel.logical.LogicalCorrelate;
 import org.apache.calcite.rel.logical.LogicalExchange;
@@ -33,6 +34,7 @@
 import org.apache.calcite.rel.logical.LogicalMatch;
 import org.apache.calcite.rel.logical.LogicalMinus;
 import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalRepeatUnion;
 import org.apache.calcite.rel.logical.LogicalSort;
 import org.apache.calcite.rel.logical.LogicalTableModify;
 import org.apache.calcite.rel.logical.LogicalUnion;
@@ -177,6 +179,16 @@ public RelNode visit(HiveTableScan scan) {
     public RelNode visit(HiveJdbcConverter conv) {
         return visitChild(conv, 0, conv.getInput());
     }
+
+    @Override
+    public RelNode visit(LogicalAsofJoin logicalAsofJoin) {
+        return visitChildren(logicalAsofJoin);
+    }
+
+    @Override
+    public RelNode visit(LogicalRepeatUnion logicalRepeatUnion) {
+        return visitChildren(logicalRepeatUnion);
+    }
 }
 
 // End RelShuttleImpl.java

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java

@@ -19,8 +19,10 @@
 
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeFactoryImpl;
 import org.apache.calcite.rel.type.RelDataTypeSystemImpl;
 import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.type.SqlTypeUtil;
 
 import static org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum.SUM_RESULT_PRECISION_INCREASE;
 
@@ -53,7 +55,7 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl {
   public int getMaxScale(SqlTypeName typeName) {
     switch (typeName) {
     case DECIMAL:
-      return getMaxNumericScale();
+      return MAX_DECIMAL_SCALE;
     case INTERVAL_YEAR:
     case INTERVAL_MONTH:
     case INTERVAL_YEAR_MONTH:
@@ -126,7 +128,7 @@ public int getDefaultPrecision(SqlTypeName typeName) {
   public int getMaxPrecision(SqlTypeName typeName) {
     switch (typeName) {
     case DECIMAL:
-      return getMaxNumericPrecision();
+      return MAX_DECIMAL_PRECISION;
     case VARCHAR:
     case CHAR:
       return MAX_CHAR_PRECISION;
@@ -157,16 +159,6 @@ public int getMaxPrecision(SqlTypeName typeName) {
     }
   }
 
-  @Override
-  public int getMaxNumericScale() {
-    return MAX_DECIMAL_SCALE;
-  }
-
-  @Override
-  public int getMaxNumericPrecision() {
-    return MAX_DECIMAL_PRECISION;
-  }
-
   @Override
   public boolean isSchemaCaseSensitive() {
     return false;
@@ -189,4 +181,40 @@ public RelDataType deriveSumType(RelDataTypeFactory typeFactory,
     return argumentType;
   }
 
+  /**
+   * Overridden because CALCITE-6464 changed the default behavior to match MS-SQL-Server-style algorithm,
+   * which can cause a drop in the scale computation, hence a precision loss in certain cases.
+   * We override this method to keep the "old behavior" (pre-CALCITE-6464); an alternative could be not
+   * overridding it (and keep the new Calcite default MS-SQL-style decimal-divide semantics), but that
+   * would lead to "regressions" (precision loss) and would require test adjustments.
+   */
+  @Override
+  public RelDataType deriveDecimalDivideType(RelDataTypeFactory typeFactory,
+      RelDataType type1, RelDataType type2) {
+    if (SqlTypeUtil.isExactNumeric(type1) && SqlTypeUtil.isExactNumeric(type2)) {
+      if (SqlTypeUtil.isDecimal(type1) || SqlTypeUtil.isDecimal(type2)) {
+        // Java numeric will always have invalid precision/scale,
+        // use its default decimal precision/scale instead.
+        type1 = RelDataTypeFactoryImpl.isJavaType(type1) ? typeFactory.decimalOf(type1) : type1;
+        type2 = RelDataTypeFactoryImpl.isJavaType(type2) ? typeFactory.decimalOf(type2) : type2;
+        int p1 = type1.getPrecision();
+        int p2 = type2.getPrecision();
+        int s1 = type1.getScale();
+        int s2 = type2.getScale();
+
+        final int maxNumericPrecision = getMaxNumericPrecision();
+        int dout = Math.min(p1 - s1 + s2, maxNumericPrecision);
+        int scale = Math.max(6, s1 + p2 + 1);
+        scale = Math.min(scale, maxNumericPrecision - dout);
+        scale = Math.min(scale, getMaxNumericScale());
+
+        int precision = dout + scale;
+        assert precision <= maxNumericPrecision;
+        assert precision > 0;
+        return typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale);
+      }
+    }
+    return null;
+  }
+
 }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumAggFunction.java

@@ -108,7 +108,7 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
         break;
       case 2:
         node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, merges);
-        node = rexBuilder.makeAbstractCast(aggregateCall.type, node);
+        node = rexBuilder.makeAbstractCast(aggregateCall.type, node, false);
         break;
       default:
         throw new AssertionError("unexpected count " + merges);
@@ -120,6 +120,7 @@ public AggregateCall topSplit(RexBuilder rexBuilder,
           false,
           false,
           false,
+          List.of(),
           ImmutableList.of(ordinal),
           -1,
           aggregateCall.distinctKeys,

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java

@@ -75,8 +75,8 @@ private static AggregateCall replaceAggFunction(RelDataType rowType, AggregateCa
         SqlTypeUtil.projectTypes(rowType, aggCall.getArgList()),
         aggCall.getType());
     return AggregateCall.create(aggFunction, aggCall.isDistinct(), aggCall.isApproximate(), aggCall.ignoreNulls(),
-        aggCall.getArgList(), aggCall.filterArg, aggCall.distinctKeys, aggCall.getCollation(), aggCall.getType(),
-        aggCall.getName());
+        aggCall.rexList, aggCall.getArgList(), aggCall.filterArg, aggCall.distinctKeys, aggCall.getCollation(),
+        aggCall.getType(), aggCall.getName());
   }
 
   @Override

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFieldTrimmerRule.java

@@ -21,6 +21,7 @@
 import org.apache.calcite.plan.RelOptRuleCall;
 import org.apache.calcite.plan.hep.HepRelVertex;
 import org.apache.calcite.rel.RelNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
 
 
@@ -57,7 +58,7 @@ public void onMatch(RelOptRuleCall call) {
       return;
     }
     // The node is the root, release the kraken!
-    node = HiveHepExtractRelNodeRule.execute(node);
+    node = HiveCalciteUtil.stripHepVertices(node);
     call.transformTo(trim(call, node));
     triggered = true;
   }