From a148dbbc6e5242f4c746058d8fcb863f55361944 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 12 Nov 2025 08:50:16 +0800 Subject: [PATCH 01/35] simple Leader & Follower hint # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java --- .../queryengine/common/MPPQueryContext.java | 11 ++ .../plan/AbstractFragmentParallelPlanner.java | 5 + .../analyzer/StatementAnalyzer.java | 14 +++ .../sql/ast/QuerySpecification.java | 27 ++++- .../plan/relational/sql/ast/SelectHint.java | 93 +++++++++++++++ .../relational/sql/parser/AstBuilder.java | 65 ++++++++++- .../plan/relational/sql/util/QueryUtil.java | 7 +- .../relational/utils/hint/FollowerHint.java | 30 +++++ .../plan/relational/utils/hint/Hint.java | 45 ++++++++ .../relational/utils/hint/HintDefinition.java | 107 ++++++++++++++++++ .../relational/utils/hint/LeaderHint.java | 30 +++++ .../relational/grammar/sql/RelationalSql.g4 | 19 +++- 12 files changed, 444 insertions(+), 9 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index a507b98008bbd..c86f3a787c4fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; import org.apache.iotdb.db.utils.cte.CteDataStore; @@ -131,6 +132,8 @@ public enum ExplainType { private boolean debug = false; + private Map hintMap = new HashMap<>(); + private Map, Query> cteQueries = new HashMap<>(); // Stores the EXPLAIN/EXPLAIN ANALYZE results for Common Table Expressions (CTEs) @@ -185,6 +188,14 @@ public MPPQueryContext( this.initResultNodeContext(); } + public void setHintMap(Map hintMap) { + this.hintMap = hintMap; + } + + public Map getHintMap() { + return hintMap; + } + public void setReserveMemoryForSchemaTreeFunc(LongConsumer reserveMemoryForSchemaTreeFunc) { this.reserveMemoryForSchemaTreeFunc = reserveMemoryForSchemaTreeFunc; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java index 0dcc7018fc60d..d31b043734f80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java @@ -118,6 +118,11 @@ protected TDataNodeLocation selectTargetDataNode(TRegionReplicaSet regionReplica } boolean selectRandomDataNode = ReadConsistencyLevel.WEAK == this.readConsistencyLevel; + if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel + && queryContext.getHintMap().containsKey("Follower")) { + selectRandomDataNode = true; + } + // When planning fragment onto specific DataNode, the DataNode whose endPoint is in // black list won't be considered because it may have connection issue now. List availableDataNodes = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index dade85d20b1be..f99d1aceaa1c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -147,6 +147,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectHint; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetOperation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; @@ -195,6 +196,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.relational.type.CompatibleResolver; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeManager; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -1163,6 +1165,10 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional node.getWhere().ifPresent(where -> analyzeWhere(node, sourceScope, where)); List outputExpressions = analyzeSelect(node, sourceScope); + + Map hintMap = analyzeHint(node); + queryContext.setHintMap(hintMap); + Analysis.GroupingSetAnalysis groupByAnalysis = analyzeGroupBy(node, sourceScope, outputExpressions); analyzeHaving(node, sourceScope); @@ -1501,6 +1507,14 @@ private void analyzeWhere(Node node, Scope scope, Expression predicate) { analysis.setWhere(node, predicate); } + private Map analyzeHint(QuerySpecification node) { + Optional selectHint = node.getHintMap(); + if (selectHint.isPresent()) { + return selectHint.get().getHintMap(); + } + return ImmutableMap.of(); + } + private List analyzeSelect(QuerySpecification node, Scope scope) { ImmutableList.Builder outputExpressionBuilder = ImmutableList.builder(); ImmutableList.Builder selectExpressionBuilder = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java index 5de61a2dde2e2..3edf33320d37b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java @@ -45,6 +45,8 @@ public class QuerySpecification extends QueryBody { private final Optional offset; private final Optional limit; + private final Optional selectHint; + public QuerySpecification( Select select, Optional from, @@ -55,8 +57,21 @@ public QuerySpecification( List windows, Optional orderBy, Optional offset, - Optional limit) { - this(null, select, from, where, groupBy, having, fill, windows, orderBy, offset, limit); + Optional limit, + Optional selectHint) { + this( + null, + select, + from, + where, + groupBy, + having, + fill, + windows, + orderBy, + offset, + limit, + selectHint); } public QuerySpecification( @@ -70,7 +85,8 @@ public QuerySpecification( List windows, Optional orderBy, Optional offset, - Optional limit) { + Optional limit, + Optional selectHint) { super(location); this.select = requireNonNull(select, "select is null"); @@ -83,6 +99,7 @@ public QuerySpecification( this.orderBy = requireNonNull(orderBy, "orderBy is null"); this.offset = requireNonNull(offset, "offset is null"); this.limit = requireNonNull(limit, "limit is null"); + this.selectHint = requireNonNull(selectHint, "hintMap is null"); } public Select getSelect() { @@ -125,6 +142,10 @@ public Optional getLimit() { return limit; } + public Optional getHintMap() { + return selectHint; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitQuerySpecification(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java new file mode 100644 index 0000000000000..6d9ee4b6ab1f3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java @@ -0,0 +1,93 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; + +import com.google.common.collect.ImmutableList; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class SelectHint extends Node { + Map hintMap; + + public SelectHint() { + super(null); + this.hintMap = new HashMap<>(); + } + + public SelectHint(Map hintMap) { + super(null); + this.hintMap = hintMap; + } + + public Map getHintMap() { + return hintMap; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(hintMap); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SelectHint other = (SelectHint) obj; + return Objects.equals(this.hintMap, other.hintMap); + } + + @Override + public String toString() { + if (hintMap == null || hintMap.isEmpty()) { + return ""; + } + + StringBuilder sb = new StringBuilder(); + sb.append("/*+ "); + + boolean first = true; + for (Map.Entry entry : hintMap.entrySet()) { + if (!first) { + sb.append(" "); + } + sb.append(entry.getValue().toString()); + first = false; + } + + sb.append(" */"); + return sb.toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 9bd99771f53ba..002d5525cba7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -175,6 +175,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectHint; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetColumnComment; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; @@ -254,6 +255,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.QueryUtil; import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintDefinition; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -273,6 +278,8 @@ import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; @@ -2339,7 +2346,8 @@ public Node visitQueryNoWith(RelationalSqlParser.QueryNoWithContext ctx) { query.getWindows(), orderBy, offset, - limit), + limit, + query.getHintMap()), Optional.empty(), Optional.empty(), Optional.empty(), @@ -2503,6 +2511,12 @@ private Node buildQuerySpecification( NodeLocation selectLocation = selectNode != null ? getLocation(selectNode) : getLocation(parserRuleContext); + // Hint Map + Optional selectHint = + ctx.selectHint() != null + ? Optional.of((SelectHint) visitSelectHint(ctx.selectHint())) + : Optional.empty(); + return new QuerySpecification( getLocation(parserRuleContext), new Select(selectLocation, isDistinct(setQuantifier), selectItems), @@ -2514,7 +2528,8 @@ private Node buildQuerySpecification( visit(windowDefinitions, WindowDefinition.class), Optional.empty(), Optional.empty(), - Optional.empty()); + Optional.empty(), + selectHint); } @Override @@ -2543,6 +2558,52 @@ public Node visitSelectAll(RelationalSqlParser.SelectAllContext ctx) { } } + @Override + public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { + Map hintMap = new HashMap<>(); + for (RelationalSqlParser.HintItemContext hiCtx : ctx.hintItem()) { + if (hiCtx instanceof RelationalSqlParser.ParameterizedHintContext) { + // RelationalSqlParser.ParameterizedHintContext paramHint = + // (RelationalSqlParser.ParameterizedHintContext) hiCtx; + // List identifiers = paramHint.identifier(); + // String hintName = identifiers.get(0).getText(); + // List params = new ArrayList<>(); + // for (int i = 1; i < identifiers.size(); i++) { + // params.add(identifiers.get(i).getText()); + // } + // Hint hint = new LeadingHint(params); + // hintMap.put(hintName, hint); + } else if (hiCtx instanceof RelationalSqlParser.SimpleHintContext) { + RelationalSqlParser.SimpleHintContext simpleHint = + (RelationalSqlParser.SimpleHintContext) hiCtx; + String hintName = simpleHint.identifier().getText(); + addSimpleHint(hintName.toUpperCase(), hintMap); + } + } + + return new SelectHint(hintMap); + } + + private static final Map HINT_DEFINITIONS = + ImmutableMap.of( + "LEADER", + new HintDefinition(LeaderHint.hintName, LeaderHint::new, ImmutableSet.of("Follower")), + "FOLLOWER", + new HintDefinition( + FollowerHint.hintName, FollowerHint::new, ImmutableSet.of("Leader")) + // "MERGE_JOIN", new HintDefinition("MergeJoin", MergeJoinHint::new, + // ImmutableSet.of("NestLoopJoin", "HashJoin")), + // "NL_JOIN", new HintDefinition("NestLoopJoin", NestLoopJoinHint::new, + // ImmutableSet.of("MergeJoin", "HashJoin")) + ); + + private void addSimpleHint(String hintName, Map hintMap) { + HintDefinition definition = HINT_DEFINITIONS.get(hintName); + if (definition != null) { + definition.addTo(hintMap); + } + } + @Override public Node visitGroupBy(RelationalSqlParser.GroupByContext ctx) { return new GroupBy( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java index f0bd840453e2e..c04b0df5ac630 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java @@ -160,6 +160,7 @@ public static Query simpleQuery(Select select) { ImmutableList.of(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty())); } @@ -214,7 +215,8 @@ public static Query simpleQuery( ImmutableList.of(), orderBy, offset, - limit)); + limit, + Optional.empty())); } public static Query simpleQuery( @@ -239,7 +241,8 @@ public static Query simpleQuery( windows, orderBy, offset, - limit)); + limit, + Optional.empty())); } public static Query query(QueryBody body) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java new file mode 100644 index 0000000000000..125e801176a4d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -0,0 +1,30 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +public class FollowerHint extends Hint { + public static String hintName = "Follower"; + + public FollowerHint() { + super(hintName); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java new file mode 100644 index 0000000000000..8bc6903320524 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -0,0 +1,45 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import java.util.Objects; + +public abstract class Hint { + protected String hintName; + + protected Hint(String hintName) { + this.hintName = Objects.requireNonNull(hintName, "hintName can not be null"); + } + + protected void setHintName(String hintName) { + this.hintName = hintName; + } + + protected String getHintName() { + return hintName; + } + + @Override + public String toString() { + return hintName; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java new file mode 100644 index 0000000000000..ac0bd98f7584f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java @@ -0,0 +1,107 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +/** + * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, + * creation strategy, and which other hints it is mutually exclusive with. + */ +public final class HintDefinition { + + private final String key; + private final Supplier supplier; + private final Set mutuallyExclusive; + + /** + * Creates a new hint definition. + * + * @param key the key to use when storing the hint in the hint map + * @param supplier factory method to create the hint instance, receives the key as parameter + * @param mutuallyExclusive set of hint keys that are mutually exclusive with this hint + */ + public HintDefinition(String key, Supplier supplier, Set mutuallyExclusive) { + this.key = key; + this.supplier = supplier; + this.mutuallyExclusive = mutuallyExclusive; + } + + /** + * Gets the hint key used in the hint map. + * + * @return the hint key + */ + public String getKey() { + return key; + } + + /** + * Gets the supplier that creates the hint instance. + * + * @return the hint supplier + */ + public Supplier getSupplier() { + return supplier; + } + + /** + * Gets the set of hint keys that are mutually exclusive with this hint. + * + * @return the set of mutually exclusive hint keys + */ + public Set getMutuallyExclusive() { + return mutuallyExclusive; + } + + /** + * Creates the hint instance. + * + * @return the created hint + */ + public Hint createHint() { + return supplier.get(); + } + + /** + * Checks if this hint is mutually exclusive with the given hint map. + * + * @param hintMap the current hint map to check against + * @return true if this hint can be added (no conflicts), false otherwise + */ + public boolean canAddTo(Map hintMap) { + return mutuallyExclusive.stream().noneMatch(hintMap::containsKey); + } + + /** + * Adds this hint to the hint map if no mutually exclusive hints exist. + * + * @param hintMap the hint map to add to + */ + public void addTo(Map hintMap) { + if (canAddTo(hintMap)) { + hintMap.put(key, createHint()); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java new file mode 100644 index 0000000000000..b9c7f7fd82ec5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -0,0 +1,30 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +public class LeaderHint extends Hint { + public static String hintName = "Leader"; + + public LeaderHint() { + super(hintName); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index ca76646468677..18374b3ca99b9 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1026,7 +1026,7 @@ sortItem ; querySpecification - : SELECT setQuantifier? selectItem (',' selectItem)* + : SELECT selectHint? setQuantifier? selectItem (',' selectItem)* (FROM relation (',' relation)*)? (WHERE where=booleanExpression)? (GROUP BY groupBy)? @@ -1116,6 +1116,15 @@ joinCriteria | USING '(' identifier (',' identifier)* ')' ; +selectHint + : HINT_START hintItem (',' hintItem)* HINT_END + ; + +hintItem + : identifier '(' ( identifier (',' identifier)* )? ')' #parameterizedHint + | identifier #simpleHint + ; + patternRecognition : aliasedRelation ( MATCH_RECOGNIZE '(' @@ -1508,6 +1517,7 @@ nonReserved | WEEK | WHILE | WINDOW | WITHIN | WITHOUT | WORK | WRAPPER | WRITE | YEAR | ZONE + | HINT_START | HINT_END ; ABSENT: 'ABSENT'; @@ -1938,6 +1948,8 @@ CONCAT: '||'; QUESTION_MARK: '?'; SEMICOLON: ';'; +HINT_START: '/*+'; +HINT_END: '*/'; STRING : '\'' ( ~'\'' | '\'\'' )* '\'' @@ -2034,9 +2046,12 @@ SIMPLE_COMMENT ; BRACKETED_COMMENT - : '/*' .*? '*/' -> channel(HIDDEN) + : '/*' (~[+] .*?)? '*/' -> channel(HIDDEN) ; +EMPTY_HINT + : '/*+' [ \r\n\t]* '*/' -> channel(HIDDEN) + ; WS : [ \r\n\t]+ -> channel(HIDDEN) ; From f5a07fe1155f9fe40c90f524826781999a4ebf29 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 12 Nov 2025 16:29:18 +0800 Subject: [PATCH 02/35] Leader & Follower hint improvement --- .../plan/AbstractFragmentParallelPlanner.java | 8 ++-- .../relational/sql/parser/AstBuilder.java | 41 +++++++++++++------ .../relational/utils/hint/FollowerHint.java | 16 +++++++- .../plan/relational/utils/hint/Hint.java | 8 +--- .../relational/utils/hint/HintDefinition.java | 37 +++++------------ .../relational/utils/hint/LeaderHint.java | 16 +++++++- .../relational/grammar/sql/RelationalSql.g4 | 2 +- 7 files changed, 76 insertions(+), 52 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java index d31b043734f80..dfd998bf55345 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java @@ -118,10 +118,10 @@ protected TDataNodeLocation selectTargetDataNode(TRegionReplicaSet regionReplica } boolean selectRandomDataNode = ReadConsistencyLevel.WEAK == this.readConsistencyLevel; - if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel - && queryContext.getHintMap().containsKey("Follower")) { - selectRandomDataNode = true; - } + // if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel + // && queryContext.getHintMap().containsKey("Follower")) { + // selectRandomDataNode = true; + // } // When planning fragment onto specific DataNode, the DataNode whose endPoint is in // black list won't be considered because it may have connection issue now. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 002d5525cba7d..caa7cc0eef8a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2563,16 +2563,16 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { Map hintMap = new HashMap<>(); for (RelationalSqlParser.HintItemContext hiCtx : ctx.hintItem()) { if (hiCtx instanceof RelationalSqlParser.ParameterizedHintContext) { - // RelationalSqlParser.ParameterizedHintContext paramHint = - // (RelationalSqlParser.ParameterizedHintContext) hiCtx; - // List identifiers = paramHint.identifier(); - // String hintName = identifiers.get(0).getText(); - // List params = new ArrayList<>(); - // for (int i = 1; i < identifiers.size(); i++) { - // params.add(identifiers.get(i).getText()); - // } - // Hint hint = new LeadingHint(params); - // hintMap.put(hintName, hint); + RelationalSqlParser.ParameterizedHintContext paramHint = + (RelationalSqlParser.ParameterizedHintContext) hiCtx; + List identifiers = paramHint.identifier(); + String hintName = identifiers.get(0).getText(); + String[] params = + identifiers.stream() + .skip(1) + .map(RelationalSqlParser.IdentifierContext::getText) + .toArray(String[]::new); + addParamHint(hintName.toUpperCase(), params, hintMap); } else if (hiCtx instanceof RelationalSqlParser.SimpleHintContext) { RelationalSqlParser.SimpleHintContext simpleHint = (RelationalSqlParser.SimpleHintContext) hiCtx; @@ -2587,20 +2587,35 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { private static final Map HINT_DEFINITIONS = ImmutableMap.of( "LEADER", - new HintDefinition(LeaderHint.hintName, LeaderHint::new, ImmutableSet.of("Follower")), + new HintDefinition( + LeaderHint.hintName, LeaderHint::new, ImmutableSet.of(FollowerHint.hintName)), "FOLLOWER", new HintDefinition( - FollowerHint.hintName, FollowerHint::new, ImmutableSet.of("Leader")) + FollowerHint.hintName, FollowerHint::new, ImmutableSet.of(LeaderHint.hintName)) // "MERGE_JOIN", new HintDefinition("MergeJoin", MergeJoinHint::new, // ImmutableSet.of("NestLoopJoin", "HashJoin")), // "NL_JOIN", new HintDefinition("NestLoopJoin", NestLoopJoinHint::new, // ImmutableSet.of("MergeJoin", "HashJoin")) ); + private void addParamHint(String hintName, String[] params, Map hintMap) { + HintDefinition definition = HINT_DEFINITIONS.get(hintName); + if (definition != null) { + Hint hint = definition.createHint(params); + String hintKey = hint.toString(); + if (!hintMap.containsKey(hintKey)) { + hintMap.put(hintKey, hint); + } + } + } + private void addSimpleHint(String hintName, Map hintMap) { HintDefinition definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - definition.addTo(hintMap); + Hint hint = definition.createHint(); + if (definition.canAddTo(hintMap)) { + hintMap.put(hint.toString(), hint); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 125e801176a4d..3217691a5e8e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -23,8 +23,22 @@ public class FollowerHint extends Hint { public static String hintName = "Follower"; + private String targetTable = null; - public FollowerHint() { + public FollowerHint(String... tables) { super(hintName); + if (tables.length > 0) { + this.targetTable = tables[0]; + } + } + + @Override + public boolean appliesToAll() { + return targetTable == null; + } + + @Override + public String toString() { + return targetTable == null ? hintName : hintName + "-" + targetTable; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java index 8bc6903320524..5575916cc7c40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -30,12 +30,8 @@ protected Hint(String hintName) { this.hintName = Objects.requireNonNull(hintName, "hintName can not be null"); } - protected void setHintName(String hintName) { - this.hintName = hintName; - } - - protected String getHintName() { - return hintName; + public boolean appliesToAll() { + return true; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java index ac0bd98f7584f..3f8e62a363c5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.Set; -import java.util.function.Supplier; +import java.util.function.Function; /** * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, @@ -32,19 +32,20 @@ public final class HintDefinition { private final String key; - private final Supplier supplier; + private final Function hintFactory; private final Set mutuallyExclusive; /** * Creates a new hint definition. * * @param key the key to use when storing the hint in the hint map - * @param supplier factory method to create the hint instance, receives the key as parameter + * @param hintFactory factory method to create the hint instance, receives the key as parameter * @param mutuallyExclusive set of hint keys that are mutually exclusive with this hint */ - public HintDefinition(String key, Supplier supplier, Set mutuallyExclusive) { + public HintDefinition( + String key, Function hintFactory, Set mutuallyExclusive) { this.key = key; - this.supplier = supplier; + this.hintFactory = hintFactory; this.mutuallyExclusive = mutuallyExclusive; } @@ -57,15 +58,6 @@ public String getKey() { return key; } - /** - * Gets the supplier that creates the hint instance. - * - * @return the hint supplier - */ - public Supplier getSupplier() { - return supplier; - } - /** * Gets the set of hint keys that are mutually exclusive with this hint. * @@ -81,7 +73,11 @@ public Set getMutuallyExclusive() { * @return the created hint */ public Hint createHint() { - return supplier.get(); + return hintFactory.apply(new String[0]); + } + + public Hint createHint(String... parameters) { + return hintFactory.apply(parameters != null ? parameters : new String[0]); } /** @@ -93,15 +89,4 @@ public Hint createHint() { public boolean canAddTo(Map hintMap) { return mutuallyExclusive.stream().noneMatch(hintMap::containsKey); } - - /** - * Adds this hint to the hint map if no mutually exclusive hints exist. - * - * @param hintMap the hint map to add to - */ - public void addTo(Map hintMap) { - if (canAddTo(hintMap)) { - hintMap.put(key, createHint()); - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index b9c7f7fd82ec5..0a6e53da1494d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -23,8 +23,22 @@ public class LeaderHint extends Hint { public static String hintName = "Leader"; + private String targetTable = null; - public LeaderHint() { + public LeaderHint(String... tables) { super(hintName); + if (tables.length > 0) { + this.targetTable = tables[0]; + } + } + + @Override + public boolean appliesToAll() { + return targetTable == null; + } + + @Override + public String toString() { + return targetTable == null ? hintName : hintName + "-" + targetTable; } } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 18374b3ca99b9..d719b27a7815f 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1121,7 +1121,7 @@ selectHint ; hintItem - : identifier '(' ( identifier (',' identifier)* )? ')' #parameterizedHint + : identifier '(' identifier (',' identifier)* ')' #parameterizedHint | identifier #simpleHint ; From 42ff25430138fdefc13c8e3adef6deb114cdf686 Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 13 Nov 2025 11:27:49 +0800 Subject: [PATCH 03/35] follower & leader refactor # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java # Conflicts: # iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java --- .../queryengine/common/MPPQueryContext.java | 10 -- .../plan/relational/analyzer/Analysis.java | 16 ++++ .../analyzer/StatementAnalyzer.java | 93 +++++++++++++++++-- .../plan/relational/sql/ast/AstVisitor.java | 12 +++ .../sql/ast/ParameterizedHintItem.java | 80 ++++++++++++++++ .../sql/ast/QuerySpecification.java | 2 +- .../plan/relational/sql/ast/SelectHint.java | 38 ++++---- .../relational/sql/ast/SimpleHintItem.java | 73 +++++++++++++++ .../relational/sql/parser/AstBuilder.java | 82 +++++----------- .../relational/utils/hint/FollowerHint.java | 10 +- .../plan/relational/utils/hint/Hint.java | 13 +-- .../{HintDefinition.java => HintFactory.java} | 53 +++++------ .../relational/utils/hint/LeaderHint.java | 10 +- 13 files changed, 342 insertions(+), 150 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/{HintDefinition.java => HintFactory.java} (55%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index c86f3a787c4fe..50908c0ea7c51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -47,11 +47,9 @@ import java.time.ZoneId; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -188,14 +186,6 @@ public MPPQueryContext( this.initResultNodeContext(); } - public void setHintMap(Map hintMap) { - this.hintMap = hintMap; - } - - public Map getHintMap() { - return hintMap; - } - public void setReserveMemoryForSchemaTreeFunc(LongConsumer reserveMemoryForSchemaTreeFunc) { this.reserveMemoryForSchemaTreeFunc = reserveMemoryForSchemaTreeFunc; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 5149945374319..330bfc66b7691 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import com.google.common.collect.ArrayListMultimap; @@ -258,6 +259,9 @@ public class Analysis implements IAnalysis { private boolean isQuery = false; + // Hint map + private Map hintMap = new HashMap<>(); + // SqlParser is needed during query planning phase for executing uncorrelated scalar subqueries // in advance (predicate folding). The planner needs to parse and execute these subqueries // independently to utilize predicate pushdown optimization. @@ -276,6 +280,14 @@ public void updateNeedSetHighestPriority(QualifiedObjectName tableName) { needSetHighestPriority = InformationSchema.QUERIES.equals(tableName.getObjectName()); } + public void setHintMap(Map hintMap) { + this.hintMap = hintMap; + } + + public Map getHintMap() { + return hintMap; + } + public Map, Expression> getParameters() { return parameters; } @@ -858,6 +870,10 @@ public QualifiedName getRelationName(final Relation relation) { return relationNames.get(NodeRef.of(relation)); } + public List getRelationNames() { + return relationNames.values().stream().collect(toImmutableList()); + } + public void addAliased(final Relation relation) { aliasedRelations.add(NodeRef.of(relation)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index f99d1aceaa1c1..a7c9e6659593d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -133,6 +133,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; @@ -162,6 +163,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; @@ -196,7 +198,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.relational.type.CompatibleResolver; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeManager; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintFactory; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -364,6 +369,14 @@ private enum UpdateKind { MERGE, } + // Hint definition and processing methods + private static final Map HINT_DEFINITIONS = + ImmutableMap.of( + "LEADER", + new HintFactory(LeaderHint.hintName, LeaderHint::new, true), + "FOLLOWER", + new HintFactory(FollowerHint.hintName, FollowerHint::new, true)); + /** * Visitor context represents local query scope (if exists). The invariant is that the local query * scopes hierarchy should always have outer query scope (if provided) as ancestor. @@ -1165,10 +1178,6 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional node.getWhere().ifPresent(where -> analyzeWhere(node, sourceScope, where)); List outputExpressions = analyzeSelect(node, sourceScope); - - Map hintMap = analyzeHint(node); - queryContext.setHintMap(hintMap); - Analysis.GroupingSetAnalysis groupByAnalysis = analyzeGroupBy(node, sourceScope, outputExpressions); analyzeHaving(node, sourceScope); @@ -1254,6 +1263,8 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional orderByScope.orElseThrow(() -> new NoSuchElementException("No value present"))); } + // select hint + analyzeHint(node, sourceScope); return outputScope; } @@ -1507,12 +1518,76 @@ private void analyzeWhere(Node node, Scope scope, Expression predicate) { analysis.setWhere(node, predicate); } - private Map analyzeHint(QuerySpecification node) { - Optional selectHint = node.getHintMap(); - if (selectHint.isPresent()) { - return selectHint.get().getHintMap(); + private void analyzeHint(QuerySpecification node, Scope scope) { + Optional selectHint = node.getSelectHint(); + selectHint.ifPresent(hint -> process(hint, scope)); + } + + @Override + public Scope visitSelectHint(SelectHint node, final Optional context) { + Map hintMap = new HashMap<>(); + for (Node hintItem : node.getHintItems()) { + if (hintItem instanceof ParameterizedHintItem) { + ParameterizedHintItem paramHint = (ParameterizedHintItem) hintItem; + String hintName = paramHint.getHintName(); + List params = paramHint.getParameters(); + addHint(hintName, params.toArray(new String[0]), hintMap); + } else if (hintItem instanceof SimpleHintItem) { + SimpleHintItem simpleHint = (SimpleHintItem) hintItem; + String hintName = simpleHint.getHintName(); + addHint(hintName, null, hintMap); + } + } + analysis.setHintMap(hintMap); + return createAndAssignScope(node, context); + } + + private boolean invalidHintParameters(String... params) { + if (params.length == 0) { + return false; + } + + List validTables = + analysis.getRelationNames().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); + for (String tableName : params) { + if (!validTables.contains(tableName)) { + return true; + } + } + return false; + } + + private void addHint(String hintName, String[] params, Map hintMap) { + HintFactory definition = HINT_DEFINITIONS.get(hintName); + if (definition != null) { + // If this hint supports parameter expansion and has multiple parameters + if (params != null && definition.shouldExpandParameters()) { + for (String param : params) { + if (invalidHintParameters(param)) { + return; + } + + Hint hint = definition.createHint(param); + String hintKey = hint.getKey(); + if (!hintMap.containsKey(hintKey)) { + hintMap.put(hintKey, hint); + } + } + } else { + // Default behavior for single parameter or non-expanding hints + if (params != null && invalidHintParameters(params)) { + return; + } + + Hint hint = definition.createHint(params); + String hintKey = hint.getKey(); + if (!hintMap.containsKey(hintKey)) { + hintMap.put(hintKey, hint); + } + } } - return ImmutableMap.of(); } private List analyzeSelect(QuerySpecification node, Scope scope) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 93cc5cf95a4d6..fc5623cc1e52d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -137,6 +137,18 @@ protected R visitSelect(Select node, C context) { return visitNode(node, context); } + protected R visitSelectHint(SelectHint node, C context) { + return visitNode(node, context); + } + + protected R visitSimpleHintItem(SimpleHintItem node, C context) { + return visitNode(node, context); + } + + protected R visitParameterizedHintItem(ParameterizedHintItem node, C context) { + return visitNode(node, context); + } + protected R visitRelation(Relation node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java new file mode 100644 index 0000000000000..9857132ab1750 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java @@ -0,0 +1,80 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** Represents a parameterized hint, e.g., "LEADER(table1)" or "FOLLOWER(table2)". */ +public class ParameterizedHintItem extends Node { + private final String hintName; + private final List parameters; + + public ParameterizedHintItem(String hintName, List parameters) { + super(null); + this.hintName = hintName.toUpperCase(); + this.parameters = ImmutableList.copyOf(parameters); + } + + public String getHintName() { + return hintName; + } + + public List getParameters() { + return parameters; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitParameterizedHintItem(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(hintName, parameters); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ParameterizedHintItem other = (ParameterizedHintItem) obj; + return Objects.equals(this.hintName, other.hintName) + && Objects.equals(this.parameters, other.parameters); + } + + @Override + public String toString() { + return hintName + "(" + String.join(", ", parameters) + ")"; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java index 3edf33320d37b..9a4afb6c2a1f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java @@ -142,7 +142,7 @@ public Optional getLimit() { return limit; } - public Optional getHintMap() { + public Optional getSelectHint() { return selectHint; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java index 6d9ee4b6ab1f3..721504393fa32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java @@ -21,40 +21,36 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; - import com.google.common.collect.ImmutableList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; public class SelectHint extends Node { - Map hintMap; + private final List hintItems; - public SelectHint() { + public SelectHint(List hintItems) { super(null); - this.hintMap = new HashMap<>(); + this.hintItems = ImmutableList.copyOf(hintItems); } - public SelectHint(Map hintMap) { - super(null); - this.hintMap = hintMap; + public List getHintItems() { + return hintItems; } - public Map getHintMap() { - return hintMap; + @Override + public List getChildren() { + return hintItems; } @Override - public List getChildren() { - return ImmutableList.of(); + public R accept(AstVisitor visitor, C context) { + return visitor.visitSelectHint(this, context); } @Override public int hashCode() { - return Objects.hash(hintMap); + return Objects.hash(hintItems); } @Override @@ -66,25 +62,23 @@ public boolean equals(Object obj) { return false; } SelectHint other = (SelectHint) obj; - return Objects.equals(this.hintMap, other.hintMap); + return Objects.equals(this.hintItems, other.hintItems); } @Override public String toString() { - if (hintMap == null || hintMap.isEmpty()) { + if (hintItems == null || hintItems.isEmpty()) { return ""; } StringBuilder sb = new StringBuilder(); sb.append("/*+ "); - boolean first = true; - for (Map.Entry entry : hintMap.entrySet()) { - if (!first) { + for (int i = 0; i < hintItems.size(); i++) { + if (i > 0) { sb.append(" "); } - sb.append(entry.getValue().toString()); - first = false; + sb.append(hintItems.get(i).toString()); } sb.append(" */"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java new file mode 100644 index 0000000000000..cecaa1047fff0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java @@ -0,0 +1,73 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +/** Represents a simple hint without parameters, e.g., "LEADER". */ +public class SimpleHintItem extends Node { + private final String hintName; + + public SimpleHintItem(String hintName) { + super(null); + this.hintName = hintName.toUpperCase(); + } + + public String getHintName() { + return hintName; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitSimpleHintItem(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(hintName); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + SimpleHintItem other = (SimpleHintItem) obj; + return Objects.equals(this.hintName, other.hintName); + } + + @Override + public String toString() { + return hintName; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index caa7cc0eef8a9..79ad487e45415 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -145,6 +145,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OneOrMoreQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternAlternation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternConcatenation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternPermutation; @@ -214,6 +215,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVersion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; @@ -255,10 +257,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.QueryUtil; import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintDefinition; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -278,8 +276,6 @@ import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; @@ -2347,7 +2343,7 @@ public Node visitQueryNoWith(RelationalSqlParser.QueryNoWithContext ctx) { orderBy, offset, limit, - query.getHintMap()), + query.getSelectHint()), Optional.empty(), Optional.empty(), Optional.empty(), @@ -2560,63 +2556,29 @@ public Node visitSelectAll(RelationalSqlParser.SelectAllContext ctx) { @Override public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { - Map hintMap = new HashMap<>(); - for (RelationalSqlParser.HintItemContext hiCtx : ctx.hintItem()) { - if (hiCtx instanceof RelationalSqlParser.ParameterizedHintContext) { - RelationalSqlParser.ParameterizedHintContext paramHint = - (RelationalSqlParser.ParameterizedHintContext) hiCtx; - List identifiers = paramHint.identifier(); - String hintName = identifiers.get(0).getText(); - String[] params = - identifiers.stream() - .skip(1) - .map(RelationalSqlParser.IdentifierContext::getText) - .toArray(String[]::new); - addParamHint(hintName.toUpperCase(), params, hintMap); - } else if (hiCtx instanceof RelationalSqlParser.SimpleHintContext) { - RelationalSqlParser.SimpleHintContext simpleHint = - (RelationalSqlParser.SimpleHintContext) hiCtx; - String hintName = simpleHint.identifier().getText(); - addSimpleHint(hintName.toUpperCase(), hintMap); - } + List hintItems = new ArrayList<>(); + for (RelationalSqlParser.HintItemContext hintItemCtx : ctx.hintItem()) { + hintItems.add(visit(hintItemCtx)); } + return new SelectHint(hintItems); + } - return new SelectHint(hintMap); - } - - private static final Map HINT_DEFINITIONS = - ImmutableMap.of( - "LEADER", - new HintDefinition( - LeaderHint.hintName, LeaderHint::new, ImmutableSet.of(FollowerHint.hintName)), - "FOLLOWER", - new HintDefinition( - FollowerHint.hintName, FollowerHint::new, ImmutableSet.of(LeaderHint.hintName)) - // "MERGE_JOIN", new HintDefinition("MergeJoin", MergeJoinHint::new, - // ImmutableSet.of("NestLoopJoin", "HashJoin")), - // "NL_JOIN", new HintDefinition("NestLoopJoin", NestLoopJoinHint::new, - // ImmutableSet.of("MergeJoin", "HashJoin")) - ); - - private void addParamHint(String hintName, String[] params, Map hintMap) { - HintDefinition definition = HINT_DEFINITIONS.get(hintName); - if (definition != null) { - Hint hint = definition.createHint(params); - String hintKey = hint.toString(); - if (!hintMap.containsKey(hintKey)) { - hintMap.put(hintKey, hint); - } - } + @Override + public Node visitParameterizedHint(RelationalSqlParser.ParameterizedHintContext ctx) { + List identifiers = ctx.identifier(); + String hintName = identifiers.get(0).getText(); + List params = + identifiers.stream() + .skip(1) + .map(x -> x.getText().toLowerCase()) + .collect(Collectors.toList()); + return new ParameterizedHintItem(hintName, params); } - private void addSimpleHint(String hintName, Map hintMap) { - HintDefinition definition = HINT_DEFINITIONS.get(hintName); - if (definition != null) { - Hint hint = definition.createHint(); - if (definition.canAddTo(hintMap)) { - hintMap.put(hint.toString(), hint); - } - } + @Override + public Node visitSimpleHint(RelationalSqlParser.SimpleHintContext ctx) { + String hintName = ctx.identifier().getText(); + return new SimpleHintItem(hintName); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 3217691a5e8e4..25700f9e78f75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -22,19 +22,19 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; public class FollowerHint extends Hint { - public static String hintName = "Follower"; + public static String hintName = "follower"; + public static String category = "replica"; private String targetTable = null; public FollowerHint(String... tables) { - super(hintName); + super(hintName, category); if (tables.length > 0) { this.targetTable = tables[0]; } } - @Override - public boolean appliesToAll() { - return targetTable == null; + public String getKey() { + return targetTable == null ? category : category + "-" + targetTable; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java index 5575916cc7c40..ce07d51668ff8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -25,17 +25,14 @@ public abstract class Hint { protected String hintName; + protected String category; - protected Hint(String hintName) { + protected Hint(String hintName, String category) { this.hintName = Objects.requireNonNull(hintName, "hintName can not be null"); + this.category = Objects.requireNonNull(category, "category can not be null"); } - public boolean appliesToAll() { - return true; - } + public abstract String getKey(); - @Override - public String toString() { - return hintName; - } + public abstract String toString(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java similarity index 55% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java index 3f8e62a363c5d..07c920dd3b495 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintDefinition.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java @@ -21,50 +21,48 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; -import java.util.Map; -import java.util.Set; import java.util.function.Function; /** * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, - * creation strategy, and which other hints it is mutually exclusive with. + * creation strategy. */ -public final class HintDefinition { +public final class HintFactory { private final String key; - private final Function hintFactory; - private final Set mutuallyExclusive; + private final Function factory; + private final boolean expandParameters; /** * Creates a new hint definition. * * @param key the key to use when storing the hint in the hint map * @param hintFactory factory method to create the hint instance, receives the key as parameter - * @param mutuallyExclusive set of hint keys that are mutually exclusive with this hint */ - public HintDefinition( - String key, Function hintFactory, Set mutuallyExclusive) { - this.key = key; - this.hintFactory = hintFactory; - this.mutuallyExclusive = mutuallyExclusive; + public HintFactory(String key, Function hintFactory) { + this(key, hintFactory, false); } /** - * Gets the hint key used in the hint map. + * Creates a new hint definition with parameter expansion option. * - * @return the hint key + * @param key the key to use when storing the hint in the hint map + * @param hintFactory factory method to create the hint instance + * @param expandParameters whether to expand array parameters into multiple hints */ - public String getKey() { - return key; + public HintFactory(String key, Function hintFactory, boolean expandParameters) { + this.key = key; + this.factory = hintFactory; + this.expandParameters = expandParameters; } /** - * Gets the set of hint keys that are mutually exclusive with this hint. + * Gets the hint name used to create hint instance. * - * @return the set of mutually exclusive hint keys + * @return the hint key */ - public Set getMutuallyExclusive() { - return mutuallyExclusive; + public String getKey() { + return key; } /** @@ -72,21 +70,16 @@ public Set getMutuallyExclusive() { * * @return the created hint */ - public Hint createHint() { - return hintFactory.apply(new String[0]); - } - public Hint createHint(String... parameters) { - return hintFactory.apply(parameters != null ? parameters : new String[0]); + return factory.apply(parameters != null ? parameters : new String[0]); } /** - * Checks if this hint is mutually exclusive with the given hint map. + * Checks whether this hint should expand parameters into multiple hints. * - * @param hintMap the current hint map to check against - * @return true if this hint can be added (no conflicts), false otherwise + * @return true if parameters should be expanded, false otherwise */ - public boolean canAddTo(Map hintMap) { - return mutuallyExclusive.stream().noneMatch(hintMap::containsKey); + public boolean shouldExpandParameters() { + return expandParameters; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index 0a6e53da1494d..b450c0b9624dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -22,19 +22,19 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; public class LeaderHint extends Hint { - public static String hintName = "Leader"; + public static String hintName = "leader"; + public static String category = "replica"; private String targetTable = null; public LeaderHint(String... tables) { - super(hintName); + super(hintName, category); if (tables.length > 0) { this.targetTable = tables[0]; } } - @Override - public boolean appliesToAll() { - return targetTable == null; + public String getKey() { + return targetTable == null ? category : category + "-" + targetTable; } @Override From 892afc09738210944c53190a4431b0f523b36064 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 18 Nov 2025 10:52:50 +0800 Subject: [PATCH 04/35] add ReplicaHint --- .../plan/AbstractFragmentParallelPlanner.java | 5 -- .../analyzer/StatementAnalyzer.java | 45 +++++---------- .../planner/distribute/AddExchangeNodes.java | 55 ++++++++++++++++++- .../TableDistributedPlanGenerator.java | 8 +++ .../distribute/TableDistributedPlanner.java | 2 +- .../relational/utils/hint/FollowerHint.java | 32 ++++++++--- .../relational/utils/hint/HintFactory.java | 12 ++-- .../relational/utils/hint/LeaderHint.java | 33 ++++++++--- .../relational/utils/hint/ReplicaHint.java | 48 ++++++++++++++++ 9 files changed, 180 insertions(+), 60 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java index dfd998bf55345..0dcc7018fc60d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/AbstractFragmentParallelPlanner.java @@ -118,11 +118,6 @@ protected TDataNodeLocation selectTargetDataNode(TRegionReplicaSet regionReplica } boolean selectRandomDataNode = ReadConsistencyLevel.WEAK == this.readConsistencyLevel; - // if (ReadConsistencyLevel.STRONG == this.readConsistencyLevel - // && queryContext.getHintMap().containsKey("Follower")) { - // selectRandomDataNode = true; - // } - // When planning fragment onto specific DataNode, the DataNode whose endPoint is in // black list won't be considered because it may have connection issue now. List availableDataNodes = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index a7c9e6659593d..aba2eb3167ce5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1531,7 +1531,7 @@ public Scope visitSelectHint(SelectHint node, final Optional context) { ParameterizedHintItem paramHint = (ParameterizedHintItem) hintItem; String hintName = paramHint.getHintName(); List params = paramHint.getParameters(); - addHint(hintName, params.toArray(new String[0]), hintMap); + addHint(hintName, params, hintMap); } else if (hintItem instanceof SimpleHintItem) { SimpleHintItem simpleHint = (SimpleHintItem) hintItem; String hintName = simpleHint.getHintName(); @@ -1542,46 +1542,31 @@ public Scope visitSelectHint(SelectHint node, final Optional context) { return createAndAssignScope(node, context); } - private boolean invalidHintParameters(String... params) { - if (params.length == 0) { - return false; - } - - List validTables = - analysis.getRelationNames().stream() - .map(QualifiedName::getSuffix) - .collect(toImmutableList()); - for (String tableName : params) { - if (!validTables.contains(tableName)) { - return true; - } - } - return false; + private List intersect(List a, List b) { + return a.stream().filter(b::contains).collect(toImmutableList()); } - private void addHint(String hintName, String[] params, Map hintMap) { + private void addHint(String hintName, List paramTables, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - // If this hint supports parameter expansion and has multiple parameters - if (params != null && definition.shouldExpandParameters()) { - for (String param : params) { - if (invalidHintParameters(param)) { - return; - } + List existingTables = + analysis.getRelationNames().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); + + List validTables = + paramTables != null ? intersect(paramTables, existingTables) : ImmutableList.of(); - Hint hint = definition.createHint(param); + if (definition.shouldExpandParameters()) { + for (String table : validTables) { + Hint hint = definition.createHint(ImmutableList.of(table)); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { hintMap.put(hintKey, hint); } } } else { - // Default behavior for single parameter or non-expanding hints - if (params != null && invalidHintParameters(params)) { - return; - } - - Hint hint = definition.createHint(params); + Hint hint = definition.createHint(validTables); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { hintMap.put(hintKey, hint); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index c74f8bd5b4b25..38124d7bf2db1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.distribute; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistribution; @@ -36,6 +38,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.ReplicaHint; + +import java.util.List; +import java.util.Map; import static org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistributionType.DIFFERENT_FROM_ALL_CHILDREN; import static org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistributionType.NO_CHILD; @@ -96,9 +103,30 @@ public PlanNode visitPlan(PlanNode node, TableDistributedPlanGenerator.PlanConte @Override public PlanNode visitTableScan( TableScanNode node, TableDistributedPlanGenerator.PlanContext context) { + // Original region replica set + TRegionReplicaSet regionReplicaSet = node.getRegionReplicaSet(); + // Find applicable hint + ReplicaHint hint = findReplicaHint(node, context.hintMap); + + // Early return for simple cases + if (context.hintMap == null || regionReplicaSet == null || hint == null) { + context.nodeDistributionMap.put( + node.getPlanNodeId(), new NodeDistribution(SAME_WITH_ALL_CHILDREN, regionReplicaSet)); + return node; + } + + // Determine optimized locations based on hint + List optimizedLocations = + selectLocations(regionReplicaSet.getDataNodeLocations(), hint); + + // Create optimized region replica set + TRegionReplicaSet optimizedRegionReplicaSet = + new TRegionReplicaSet(regionReplicaSet.getRegionId(), optimizedLocations); + context.nodeDistributionMap.put( node.getPlanNodeId(), - new NodeDistribution(SAME_WITH_ALL_CHILDREN, node.getRegionReplicaSet())); + new NodeDistribution(SAME_WITH_ALL_CHILDREN, optimizedRegionReplicaSet)); + return node; } @@ -229,4 +257,29 @@ private PlanNode processTableDeviceSourceNode( new NodeDistribution(SAME_WITH_ALL_CHILDREN, node.getRegionReplicaSet())); return node; } + + /** + * Finds the applicable replica hint for the given table scan node. First checks for + * table-specific hint, then falls back to global hint. + */ + private ReplicaHint findReplicaHint(TableScanNode node, Map hintMap) { + if (hintMap == null || hintMap.isEmpty()) { + return null; + } + + String tableName = node.getQualifiedObjectName().getObjectName(); + String tableSpecificKey = "replica-" + tableName; + String globalKey = "replica-*"; + + return (ReplicaHint) hintMap.getOrDefault(tableSpecificKey, hintMap.get(globalKey)); + } + + /** + * Selects data node locations based on the provided hint using polymorphism. - ReplicaHint: Uses + * the hint's own selectLocations strategy - Other hints or null: Returns all original locations + */ + private List selectLocations( + List dataNodeLocations, ReplicaHint hint) { + return hint.selectLocations(dataNodeLocations); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 79ea52597bcee..e74701e75657e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -109,6 +109,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; @@ -2213,6 +2214,7 @@ public List visitUnion(UnionNode node, PlanContext context) { public static class PlanContext { final Map nodeDistributionMap; + final Map hintMap; boolean hasExchangeNode = false; boolean hasSortProperty = false; boolean pushDownGrouping = false; @@ -2222,6 +2224,12 @@ public static class PlanContext { public PlanContext() { this.nodeDistributionMap = new HashMap<>(); + this.hintMap = new HashMap<>(); + } + + public PlanContext(Map hintMap) { + this.nodeDistributionMap = new HashMap<>(); + this.hintMap = hintMap; } public NodeDistribution getNodeDistribution(PlanNodeId nodeId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java index ff7686fe86891..465a113a8f9c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java @@ -100,7 +100,7 @@ public TableDistributedPlanner( public DistributedQueryPlan plan() { TableDistributedPlanGenerator.PlanContext planContext = - new TableDistributedPlanGenerator.PlanContext(); + new TableDistributedPlanGenerator.PlanContext(analysis.getHintMap()); PlanNode outputNodeWithExchange = generateDistributedPlanWithOptimize(planContext); List planText = null; if (mppQueryContext.isExplain() && mppQueryContext.isInnerTriggeredQuery()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 25700f9e78f75..1854a1d5ef63b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -21,24 +21,38 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; -public class FollowerHint extends Hint { +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; + +import java.util.List; + +public class FollowerHint extends ReplicaHint { public static String hintName = "follower"; - public static String category = "replica"; - private String targetTable = null; + private final String targetTable; - public FollowerHint(String... tables) { - super(hintName, category); - if (tables.length > 0) { - this.targetTable = tables[0]; + public FollowerHint(List tables) { + super(hintName); + if (tables == null || tables.size() > 1) { + throw new IllegalArgumentException("FollowerHint accepts empty or exactly one table"); } + targetTable = tables.isEmpty() ? "*" : tables.get(0); } + @Override public String getKey() { - return targetTable == null ? category : category + "-" + targetTable; + return category + "-" + targetTable; } @Override public String toString() { - return targetTable == null ? hintName : hintName + "-" + targetTable; + return hintName + "-" + targetTable; + } + + @Override + public List selectLocations(List dataNodeLocations) { + if (dataNodeLocations == null || dataNodeLocations.size() <= 1) { + return dataNodeLocations; + } + // Return only followers (all locations except the first/leader) + return dataNodeLocations.subList(1, dataNodeLocations.size()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java index 07c920dd3b495..1e1c8193277aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; +import java.util.List; import java.util.function.Function; /** @@ -30,7 +31,7 @@ public final class HintFactory { private final String key; - private final Function factory; + private final Function, Hint> factory; private final boolean expandParameters; /** @@ -39,7 +40,7 @@ public final class HintFactory { * @param key the key to use when storing the hint in the hint map * @param hintFactory factory method to create the hint instance, receives the key as parameter */ - public HintFactory(String key, Function hintFactory) { + public HintFactory(String key, Function, Hint> hintFactory) { this(key, hintFactory, false); } @@ -50,7 +51,8 @@ public HintFactory(String key, Function hintFactory) { * @param hintFactory factory method to create the hint instance * @param expandParameters whether to expand array parameters into multiple hints */ - public HintFactory(String key, Function hintFactory, boolean expandParameters) { + public HintFactory( + String key, Function, Hint> hintFactory, boolean expandParameters) { this.key = key; this.factory = hintFactory; this.expandParameters = expandParameters; @@ -70,8 +72,8 @@ public String getKey() { * * @return the created hint */ - public Hint createHint(String... parameters) { - return factory.apply(parameters != null ? parameters : new String[0]); + public Hint createHint(List parameters) { + return factory.apply(parameters); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index b450c0b9624dc..6fc7a2346547f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -21,24 +21,39 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; -public class LeaderHint extends Hint { +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; + +import java.util.Collections; +import java.util.List; + +public class LeaderHint extends ReplicaHint { public static String hintName = "leader"; - public static String category = "replica"; - private String targetTable = null; + private final String targetTable; - public LeaderHint(String... tables) { - super(hintName, category); - if (tables.length > 0) { - this.targetTable = tables[0]; + public LeaderHint(List tables) { + super(hintName); + if (tables == null || tables.size() > 1) { + throw new IllegalArgumentException("LeaderHint accepts empty or exactly one table"); } + targetTable = tables.isEmpty() ? "*" : tables.get(0); } + @Override public String getKey() { - return targetTable == null ? category : category + "-" + targetTable; + return category + "-" + targetTable; } @Override public String toString() { - return targetTable == null ? hintName : hintName + "-" + targetTable; + return hintName + "-" + targetTable; + } + + @Override + public List selectLocations(List dataNodeLocations) { + if (dataNodeLocations == null || dataNodeLocations.size() <= 1) { + return dataNodeLocations; + } + // Return only the leader (first location) + return Collections.singletonList(dataNodeLocations.get(0)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java new file mode 100644 index 0000000000000..187dc27f85212 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java @@ -0,0 +1,48 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; + +import java.util.List; + +/** + * Abstract base class for replica-related hints. Provides common functionality for hints that deal + * with data node replica selection. + */ +public abstract class ReplicaHint extends Hint { + public static String category = "replica"; + + protected ReplicaHint(String hintName) { + super(hintName, category); + } + + /** + * Selects data node locations based on the replica strategy. Each replica hint implementation + * defines its own location selection logic. + * + * @param dataNodeLocations the available data node locations + * @return the selected locations based on replica hint strategy + */ + public abstract List selectLocations( + List dataNodeLocations); +} From 553d16af1450e792ed44f43a81413101d6b08bca Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 18 Nov 2025 15:44:34 +0800 Subject: [PATCH 05/35] add alias support --- ...ableModelStatementMemorySourceVisitor.java | 2 +- .../planner/plan/node/PlanGraphPrinter.java | 6 +++ .../plan/relational/analyzer/Analysis.java | 12 +++-- .../analyzer/StatementAnalyzer.java | 2 +- .../relational/planner/RelationPlanner.java | 3 +- .../planner/distribute/AddExchangeNodes.java | 5 +- .../TableDistributedPlanGenerator.java | 17 +++--- .../iterative/rule/PruneTableScanColumns.java | 3 +- .../node/AggregationTableScanNode.java | 54 +++++++++++++++++-- .../planner/node/DeviceTableScanNode.java | 49 ++++++++++++++++- .../planner/node/TableScanNode.java | 27 ++++++++++ .../UnaliasSymbolReferences.java | 3 +- .../planner/optimizations/Util.java | 3 +- .../relational/analyzer/TestPlanBuilder.java | 3 +- 14 files changed, 166 insertions(+), 23 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java index 932d941979223..2fcc8a608ada1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/memory/TableModelStatementMemorySourceVisitor.java @@ -92,7 +92,7 @@ public StatementMemorySource visitExplain( // Generate table model distributed plan final TableDistributedPlanGenerator.PlanContext planContext = - new TableDistributedPlanGenerator.PlanContext(); + new TableDistributedPlanGenerator.PlanContext(context.getAnalysis().getHintMap()); final PlanNode outputNodeWithExchange = new TableDistributedPlanner( context.getAnalysis(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 180b4b9f1be11..6de60bb867c31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -664,6 +664,9 @@ public List visitTableScan(TableScanNode node, GraphContext context) { List boxValue = new ArrayList<>(); boxValue.add(node.toString()); boxValue.add(String.format("QualifiedTableName: %s", node.getQualifiedObjectName().toString())); + if (node.getAlias() != null) { + boxValue.add(String.format("Alias: %s", node.getAlias().getValue())); + } boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); if (deviceTableScanNode != null) { @@ -751,6 +754,9 @@ public List visitAggregationTableScan( List boxValue = new ArrayList<>(); boxValue.add(node.toString()); boxValue.add(String.format("QualifiedTableName: %s", node.getQualifiedObjectName().toString())); + if (node.getAlias() != null) { + boxValue.add(String.format("Alias: %s", node.getAlias().getValue())); + } boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); int i = 0; for (org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 330bfc66b7691..6a464ffeda3a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -218,7 +218,7 @@ public class Analysis implements IAnalysis { private final Map, QualifiedName> relationNames = new LinkedHashMap<>(); - private final Set> aliasedRelations = new LinkedHashSet<>(); + private final Map, Identifier> aliasedRelations = new LinkedHashMap<>(); private final Map, TableFunctionInvocationAnalysis> tableFunctionAnalyses = new LinkedHashMap<>(); @@ -874,12 +874,16 @@ public List getRelationNames() { return relationNames.values().stream().collect(toImmutableList()); } - public void addAliased(final Relation relation) { - aliasedRelations.add(NodeRef.of(relation)); + public void addAliased(final Relation relation, Identifier alias) { + aliasedRelations.put(NodeRef.of(relation), alias); + } + + public Identifier getAliased(Relation relation) { + return aliasedRelations.get(NodeRef.of(relation)); } public boolean isAliased(Relation relation) { - return aliasedRelations.contains(NodeRef.of(relation)); + return aliasedRelations.containsKey(NodeRef.of(relation)); } public void addTableSchema( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index aba2eb3167ce5..3bdc923a476da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -3676,7 +3676,7 @@ protected Scope visitValues(Values node, Optional scope) { @Override protected Scope visitAliasedRelation(AliasedRelation relation, Optional scope) { analysis.setRelationName(relation, QualifiedName.of(ImmutableList.of(relation.getAlias()))); - analysis.addAliased(relation.getRelation()); + analysis.addAliased(relation.getRelation(), relation.getAlias()); Scope relationScope = process(relation.getRelation(), scope); RelationType relationType = relationScope.getRelationType(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 457a03ab4a012..516be38ad8eb6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -398,7 +398,8 @@ private RelationPlan processPhysicalTable(Table table, Scope scope) { qualifiedObjectName, outputSymbols, tableColumnSchema, - tagAndAttributeIndexMap); + tagAndAttributeIndexMap, + analysis.getAliased(table)); } return new RelationPlan(tableScanNode, scope, outputSymbols, outerContext); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index 38124d7bf2db1..861e4086fc280 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -267,7 +267,10 @@ private ReplicaHint findReplicaHint(TableScanNode node, Map hintMa return null; } - String tableName = node.getQualifiedObjectName().getObjectName(); + String tableName = + node.getAlias() != null + ? node.getAlias().getValue() + : node.getQualifiedObjectName().getObjectName(); String tableSpecificKey = "replica-" + tableName; String globalKey = "replica-*"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index e74701e75657e..f0b4d4751ca36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -758,7 +758,8 @@ private List constructDeviceTableScanByTags( node.getPushDownLimit(), node.getPushDownOffset(), node.isPushLimitToEachDevice(), - node.containsNonAlignedDevice()); + node.containsNonAlignedDevice(), + node.getAlias()); scanNode.setRegionReplicaSet(regionReplicaSets.get(0)); return scanNode; }); @@ -844,7 +845,8 @@ private List constructDeviceTableScanByRegionReplicaSet( node.getPushDownLimit(), node.getPushDownOffset(), node.isPushLimitToEachDevice(), - node.containsNonAlignedDevice()); + node.containsNonAlignedDevice(), + node.getAlias()); scanNode.setRegionReplicaSet(regionReplicaSet); return scanNode; }); @@ -1725,7 +1727,8 @@ private void buildRegionNodeMap( partialAggTableScanNode.getGroupingSets(), partialAggTableScanNode.getPreGroupedSymbols(), partialAggTableScanNode.getStep(), - partialAggTableScanNode.getGroupIdSymbol()); + partialAggTableScanNode.getGroupIdSymbol(), + partialAggTableScanNode.getAlias()); scanNode.setRegionReplicaSet(regionReplicaSet); return scanNode; }); @@ -2222,10 +2225,10 @@ public static class PlanContext { TRegionReplicaSet mostUsedRegion; boolean deviceCrossRegion; - public PlanContext() { - this.nodeDistributionMap = new HashMap<>(); - this.hintMap = new HashMap<>(); - } + public PlanContext() { + this.nodeDistributionMap = new HashMap<>(); + this.hintMap = new HashMap<>(); + } public PlanContext(Map hintMap) { this.nodeDistributionMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java index ffce0b6693e9d..08451ea7edb63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java @@ -126,7 +126,8 @@ public static Optional pruneColumns(TableScanNode node, Set re deviceTableScanNode.getPushDownLimit(), deviceTableScanNode.getPushDownOffset(), deviceTableScanNode.isPushLimitToEachDevice(), - deviceTableScanNode.containsNonAlignedDevice())); + deviceTableScanNode.containsNonAlignedDevice(), + deviceTableScanNode.getAlias())); } } else if (node instanceof InformationSchemaTableScanNode) { // For the convenience of process in execution stage, column-prune for diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java index 56d39f2f77dcb..1928db3c73120 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/AggregationTableScanNode.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -148,6 +149,50 @@ public AggregationTableScanNode( this.setOutputSymbols(constructOutputSymbols(groupingSets, aggregations)); } + public AggregationTableScanNode( + PlanNodeId id, + QualifiedObjectName qualifiedObjectName, + List outputSymbols, + Map assignments, + List deviceEntries, + Map tagAndAttributeIndexMap, + Ordering scanOrder, + Expression timePredicate, + Expression pushDownPredicate, + long pushDownLimit, + long pushDownOffset, + boolean pushLimitToEachDevice, + boolean containsNonAlignedDevice, + Assignments projection, + Map aggregations, + AggregationNode.GroupingSetDescriptor groupingSets, + List preGroupedSymbols, + AggregationNode.Step step, + Optional groupIdSymbol, + Identifier alias) { + this( + id, + qualifiedObjectName, + outputSymbols, + assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, + pushDownPredicate, + pushDownLimit, + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice, + projection, + aggregations, + groupingSets, + preGroupedSymbols, + step, + groupIdSymbol); + this.alias = alias; + } + protected AggregationTableScanNode() {} private static List constructOutputSymbols( @@ -278,7 +323,8 @@ public AggregationTableScanNode clone() { groupingSets, preGroupedSymbols, step, - groupIdSymbol); + groupIdSymbol, + alias); } @Override @@ -336,7 +382,8 @@ public static AggregationTableScanNode combineAggregationAndTableScan( aggregationNode.getGroupingSets(), aggregationNode.getPreGroupedSymbols(), aggregationNode.getStep(), - aggregationNode.getGroupIdSymbol()); + aggregationNode.getGroupIdSymbol(), + tableScanNode.getAlias()); } public static AggregationTableScanNode combineAggregationAndTableScan( @@ -390,7 +437,8 @@ public static AggregationTableScanNode combineAggregationAndTableScan( aggregationNode.getGroupingSets(), aggregationNode.getPreGroupedSymbols(), step, - aggregationNode.getGroupIdSymbol()); + aggregationNode.getGroupIdSymbol(), + tableScanNode.getAlias()); } public boolean mayUseLastCache() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java index b91737739351a..cead343ddcee4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.tsfile.read.filter.basic.Filter; @@ -89,6 +90,18 @@ public DeviceTableScanNode( this.tagAndAttributeIndexMap = tagAndAttributeIndexMap; } + public DeviceTableScanNode( + PlanNodeId id, + QualifiedObjectName qualifiedObjectName, + List outputSymbols, + Map assignments, + Map tagAndAttributeIndexMap, + Identifier alias) { + super(id, qualifiedObjectName, outputSymbols, assignments); + this.tagAndAttributeIndexMap = tagAndAttributeIndexMap; + this.alias = alias; + } + public DeviceTableScanNode( PlanNodeId id, QualifiedObjectName qualifiedObjectName, @@ -120,6 +133,39 @@ public DeviceTableScanNode( this.containsNonAlignedDevice = containsNonAlignedDevice; } + public DeviceTableScanNode( + PlanNodeId id, + QualifiedObjectName qualifiedObjectName, + List outputSymbols, + Map assignments, + List deviceEntries, + Map tagAndAttributeIndexMap, + Ordering scanOrder, + Expression timePredicate, + Expression pushDownPredicate, + long pushDownLimit, + long pushDownOffset, + boolean pushLimitToEachDevice, + boolean containsNonAlignedDevice, + Identifier alias) { + super( + id, + qualifiedObjectName, + outputSymbols, + assignments, + pushDownPredicate, + pushDownLimit, + pushDownOffset); + this.deviceEntries = deviceEntries; + this.tagAndAttributeIndexMap = tagAndAttributeIndexMap; + this.scanOrder = scanOrder; + this.timePredicate = timePredicate; + this.pushDownPredicate = pushDownPredicate; + this.pushLimitToEachDevice = pushLimitToEachDevice; + this.containsNonAlignedDevice = containsNonAlignedDevice; + this.alias = alias; + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitDeviceTableScan(this, context); @@ -140,7 +186,8 @@ public DeviceTableScanNode clone() { pushDownLimit, pushDownOffset, pushLimitToEachDevice, - containsNonAlignedDevice); + containsNonAlignedDevice, + alias); } protected static void serializeMemberVariables( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index 11807d9ceb499..f50f9f39b40f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableList; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -75,6 +76,9 @@ public abstract class TableScanNode extends SourceNode { // For query of schemaInfo, we only need the list of DataNodeLocation protected TRegionReplicaSet regionReplicaSet; + // alias name + protected Identifier alias; + public TableScanNode( PlanNodeId id, QualifiedObjectName qualifiedObjectName, @@ -177,6 +181,10 @@ public void open() throws Exception {} @Override public void close() throws Exception {} + public Identifier getAlias() { + return alias; + } + public QualifiedObjectName getQualifiedObjectName() { return this.qualifiedObjectName; } @@ -255,6 +263,13 @@ protected static void serializeMemberVariables( ReadWriteIOUtils.write(node.pushDownLimit, byteBuffer); ReadWriteIOUtils.write(node.pushDownOffset, byteBuffer); + + if (node.alias != null) { + ReadWriteIOUtils.write(true, byteBuffer); + Identifier.serialize(node.alias, byteBuffer); + } else { + ReadWriteIOUtils.write(false, byteBuffer); + } } protected static void serializeMemberVariables( @@ -290,6 +305,13 @@ protected static void serializeMemberVariables( ReadWriteIOUtils.write(node.pushDownLimit, stream); ReadWriteIOUtils.write(node.pushDownOffset, stream); + + if (node.alias != null) { + ReadWriteIOUtils.write(true, stream); + Identifier.serialize(node.alias, stream); + } else { + ReadWriteIOUtils.write(false, stream); + } } protected static void deserializeMemberVariables( @@ -327,6 +349,11 @@ protected static void deserializeMemberVariables( node.pushDownLimit = ReadWriteIOUtils.readLong(byteBuffer); node.pushDownOffset = ReadWriteIOUtils.readLong(byteBuffer); + + boolean hasAlias = ReadWriteIOUtils.readBool(byteBuffer); + if (hasAlias) { + node.alias = new Identifier(byteBuffer); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 24adb746df4e1..17ec744f58d8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -238,7 +238,8 @@ public PlanAndMappings visitDeviceTableScan(DeviceTableScanNode node, UnaliasCon node.getPushDownLimit(), node.getPushDownOffset(), node.isPushLimitToEachDevice(), - node.containsNonAlignedDevice()), + node.containsNonAlignedDevice(), + node.getAlias()), mapping); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java index 706b24a567b3b..27a8ea66bf67c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java @@ -188,7 +188,8 @@ public static Pair split( node.getGroupingSets(), node.getPreGroupedSymbols(), PARTIAL, - node.getGroupIdSymbol()) + node.getGroupIdSymbol(), + node.getAlias()) : new AggregationTreeDeviceViewScanNode( queryId.genPlanNodeId(), node.getQualifiedObjectName(), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java index 4b0136d892327..8422834c5cb3e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestPlanBuilder.java @@ -101,7 +101,8 @@ public TestPlanBuilder deviceTableScan( pushDownLimit, pushDownOffset, pushLimitToEachDevice, - containsNonAlignedDevice); + containsNonAlignedDevice, + null); return this; } } From a9ad438bb7d5b2f6f68e900710cd7638baf8a05f Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 18 Nov 2025 17:29:03 +0800 Subject: [PATCH 06/35] fix alias serialize --- .../plan/relational/planner/node/TableScanNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index f50f9f39b40f7..c6bda2232621c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -266,7 +266,7 @@ protected static void serializeMemberVariables( if (node.alias != null) { ReadWriteIOUtils.write(true, byteBuffer); - Identifier.serialize(node.alias, byteBuffer); + node.alias.serialize(byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } @@ -308,7 +308,7 @@ protected static void serializeMemberVariables( if (node.alias != null) { ReadWriteIOUtils.write(true, stream); - Identifier.serialize(node.alias, stream); + node.alias.serialize(stream); } else { ReadWriteIOUtils.write(false, stream); } From ee5702036037b9f7a1e827fb4bfa178889903f08 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 19 Nov 2025 14:04:31 +0800 Subject: [PATCH 07/35] Leader & Follower Hint without parameters --- .../plan/relational/analyzer/StatementAnalyzer.java | 2 +- .../relational/planner/distribute/AddExchangeNodes.java | 3 +-- .../plan/relational/utils/hint/FollowerHint.java | 6 +++--- .../queryengine/plan/relational/utils/hint/LeaderHint.java | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 3bdc923a476da..7b969454a02c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1555,7 +1555,7 @@ private void addHint(String hintName, List paramTables, Map validTables = - paramTables != null ? intersect(paramTables, existingTables) : ImmutableList.of(); + paramTables != null ? intersect(paramTables, existingTables) : existingTables; if (definition.shouldExpandParameters()) { for (String table : validTables) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index 861e4086fc280..fb72026e9792a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -272,9 +272,8 @@ private ReplicaHint findReplicaHint(TableScanNode node, Map hintMa ? node.getAlias().getValue() : node.getQualifiedObjectName().getObjectName(); String tableSpecificKey = "replica-" + tableName; - String globalKey = "replica-*"; - return (ReplicaHint) hintMap.getOrDefault(tableSpecificKey, hintMap.get(globalKey)); + return (ReplicaHint) hintMap.getOrDefault(tableSpecificKey, null); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 1854a1d5ef63b..2c1d3b3a0bafc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -31,10 +31,10 @@ public class FollowerHint extends ReplicaHint { public FollowerHint(List tables) { super(hintName); - if (tables == null || tables.size() > 1) { - throw new IllegalArgumentException("FollowerHint accepts empty or exactly one table"); + if (tables == null || tables.size() != 1) { + throw new IllegalArgumentException("FollowerHint accepts exactly one table"); } - targetTable = tables.isEmpty() ? "*" : tables.get(0); + targetTable = tables.get(0); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index 6fc7a2346547f..8c179846da986 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -32,10 +32,10 @@ public class LeaderHint extends ReplicaHint { public LeaderHint(List tables) { super(hintName); - if (tables == null || tables.size() > 1) { - throw new IllegalArgumentException("LeaderHint accepts empty or exactly one table"); + if (tables == null || tables.size() != 1) { + throw new IllegalArgumentException("LeaderHint accepts exactly one table"); } - targetTable = tables.isEmpty() ? "*" : tables.get(0); + targetTable = tables.get(0); } @Override From c9e930c5c2515cabdcd91add8ba0b7b1ec2fd801 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 19 Nov 2025 16:15:21 +0800 Subject: [PATCH 08/35] hintParameters --- .../queryengine/plan/relational/analyzer/Analysis.java | 4 ++-- .../plan/relational/analyzer/StatementAnalyzer.java | 6 ++++-- .../plan/relational/sql/parser/AstBuilder.java | 9 +++------ .../iotdb/db/relational/grammar/sql/RelationalSql.g4 | 10 ++++++++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 6a464ffeda3a6..f0f48166bcbfa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -870,8 +870,8 @@ public QualifiedName getRelationName(final Relation relation) { return relationNames.get(NodeRef.of(relation)); } - public List getRelationNames() { - return relationNames.values().stream().collect(toImmutableList()); + public Map, QualifiedName> getRelationNames() { + return relationNames; } public void addAliased(final Relation relation, Identifier alias) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 7b969454a02c1..fe580661df8a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1549,9 +1549,11 @@ private List intersect(List a, List b) { private void addHint(String hintName, List paramTables, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { + // filter the relation of aliased relation List existingTables = - analysis.getRelationNames().stream() - .map(QualifiedName::getSuffix) + analysis.getRelationNames().entrySet().stream() + .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) + .map(entry -> entry.getValue().getSuffix()) .collect(toImmutableList()); List validTables = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 79ad487e45415..96a9658a093e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2565,13 +2565,10 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { @Override public Node visitParameterizedHint(RelationalSqlParser.ParameterizedHintContext ctx) { - List identifiers = ctx.identifier(); - String hintName = identifiers.get(0).getText(); + String hintName = ctx.identifier().getText(); + List identifiers = ctx.hintParameter(); List params = - identifiers.stream() - .skip(1) - .map(x -> x.getText().toLowerCase()) - .collect(Collectors.toList()); + identifiers.stream().map(x -> x.getText().toLowerCase()).collect(Collectors.toList()); return new ParameterizedHintItem(hintName, params); } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index d719b27a7815f..e9c2313fb7ef6 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1117,14 +1117,20 @@ joinCriteria ; selectHint - : HINT_START hintItem (',' hintItem)* HINT_END + : HINT_START hintItem+ HINT_END ; hintItem - : identifier '(' identifier (',' identifier)* ')' #parameterizedHint + : identifier '(' hintParameter+ ')' #parameterizedHint | identifier #simpleHint ; +hintParameter + : identifier + | '{' + | '}' + ; + patternRecognition : aliasedRelation ( MATCH_RECOGNIZE '(' From b0630fbdb154c8e59a08f75f4ced5fede2565e64 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 23 Dec 2025 11:28:10 +0800 Subject: [PATCH 09/35] fix: rebase master --- .../queryengine/plan/relational/analyzer/StatementAnalyzer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index fe580661df8a6..0546c8ed1c7fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -133,8 +133,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; From 26d9da8291b80e2ed4a88a49f37c1ef9cff85008 Mon Sep 17 00:00:00 2001 From: shizy Date: Fri, 9 Jan 2026 15:38:51 +0800 Subject: [PATCH 10/35] leading hint --- .../queryengine/common/MPPQueryContext.java | 2 + .../analyzer/StatementAnalyzer.java | 29 +-- .../optimizations/CollectJoinConstraint.java | 73 ++++++ .../optimizations/LeadingJoinOptimizer.java | 83 +++++++ .../optimizations/LogicalOptimizeFactory.java | 4 +- .../sql/ast/ParameterizedHintItem.java | 13 ++ .../plan/relational/sql/ast/SelectHint.java | 12 + .../relational/sql/ast/SimpleHintItem.java | 12 + .../relational/utils/hint/JoinOrderHint.java | 30 +++ .../relational/utils/hint/LeadingHint.java | 219 ++++++++++++++++++ 10 files changed, 462 insertions(+), 15 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 50908c0ea7c51..e67bcd67c9d03 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -47,9 +47,11 @@ import java.time.ZoneId; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 0546c8ed1c7fa..75b56348ef1ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -202,6 +202,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintFactory; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -375,7 +376,9 @@ private enum UpdateKind { "LEADER", new HintFactory(LeaderHint.hintName, LeaderHint::new, true), "FOLLOWER", - new HintFactory(FollowerHint.hintName, FollowerHint::new, true)); + new HintFactory(FollowerHint.hintName, FollowerHint::new, true), + "LEADING", + new HintFactory(LeadingHint.hintName, LeadingHint::new, false)); /** * Visitor context represents local query scope (if exists). The invariant is that the local query @@ -1546,21 +1549,19 @@ private List intersect(List a, List b) { return a.stream().filter(b::contains).collect(toImmutableList()); } - private void addHint(String hintName, List paramTables, Map hintMap) { + private void addHint(String hintName, List parameters, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - // filter the relation of aliased relation - List existingTables = - analysis.getRelationNames().entrySet().stream() - .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) - .map(entry -> entry.getValue().getSuffix()) - .collect(toImmutableList()); - - List validTables = - paramTables != null ? intersect(paramTables, existingTables) : existingTables; - if (definition.shouldExpandParameters()) { - for (String table : validTables) { + List existingTables = + analysis.getRelationNames().entrySet().stream() + .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) + .map(entry -> entry.getValue().getSuffix()) + .collect(toImmutableList()); + for (String table : parameters) { + if (!existingTables.contains(table)) { + continue; + } Hint hint = definition.createHint(ImmutableList.of(table)); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { @@ -1568,7 +1569,7 @@ private void addHint(String hintName, List paramTables, Map { + private final Analysis analysis; + + public Rewriter(Analysis analysis) { + this.analysis = analysis; + } + + @Override + public PlanNode visitPlan(PlanNode node, Context context) { + Hint hint = analysis.getHintMap().getOrDefault(JoinOrderHint.category, null); + if (!(hint instanceof LeadingHint)) { + return node; + } + + for (PlanNode child : node.getChildren()) { + child.accept(this, context); + } + return node; + } + + @Override + public PlanNode visitJoin(JoinNode node, Context context) { + PlanNode leftNode = node.getLeftChild(); + PlanNode rightNode = node.getRightChild(); + return node; + } + + // @Override + // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { + // return visitTwoChildProcess(node, context); + // } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java new file mode 100644 index 0000000000000..14c3d1fd261f8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java @@ -0,0 +1,83 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; + +import java.util.Set; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +public class LeadingJoinOptimizer implements PlanOptimizer { + @Override + public PlanNode optimize(PlanNode plan, Context context) { + if (!context.getAnalysis().isQuery()) { + return plan; + } + + return plan.accept(new Rewriter(context.getAnalysis()), null); + } + + private static class Rewriter extends PlanVisitor { + private final Analysis analysis; + + public Rewriter(Analysis analysis) { + this.analysis = analysis; + } + + @Override + public PlanNode visitPlan(PlanNode node, Context context) { + Hint hint = analysis.getHintMap().getOrDefault(JoinOrderHint.category, null); + if (!(hint instanceof LeadingHint)) { + return node; + } + LeadingHint leadingHint = (LeadingHint) hint; + Set relationNames = + analysis.getRelationNames().values().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableSet()); + + if (!validateTableNamesMatch(relationNames, leadingHint)) { + return node; + } + + PlanNode leadingJoin = leadingHint.generateLeadingJoinPlan(); + if (leadingJoin != null) { + return leadingJoin; + } + return node; + } + + private boolean validateTableNamesMatch(Set relationNames, LeadingHint leadingHint) { + if (relationNames.size() != leadingHint.getTables().size()) { + return false; + } + return relationNames.containsAll(leadingHint.getTables()); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index afe6880394849..004a8142ee5eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -390,7 +390,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new MergeLimitWithSort(), new MergeLimitOverProjectWithSort(), new PushTopKThroughUnion())), - new ParallelizeGrouping()); + new ParallelizeGrouping(), + new CollectJoinConstraint()); + // new LeadingJoinOptimizer()); this.planOptimizers = optimizerBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java index 9857132ab1750..cc3b98d9b45b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java @@ -22,12 +22,16 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; import java.util.Objects; /** Represents a parameterized hint, e.g., "LEADER(table1)" or "FOLLOWER(table2)". */ public class ParameterizedHintItem extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ParameterizedHintItem.class); + private final String hintName; private final List parameters; @@ -77,4 +81,13 @@ public boolean equals(Object obj) { public String toString() { return hintName + "(" + String.join(", ", parameters) + ")"; } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += AstMemoryEstimationHelper.getEstimatedSizeOfStringList(parameters); + size += RamUsageEstimator.sizeOf(hintName); + return size; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java index 721504393fa32..ee583509a131e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java @@ -22,11 +22,15 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; import java.util.Objects; public class SelectHint extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SelectHint.class); + private final List hintItems; public SelectHint(List hintItems) { @@ -84,4 +88,12 @@ public String toString() { sb.append(" */"); return sb.toString(); } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeList(hintItems); + return size; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java index cecaa1047fff0..9eebb65fe9da2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java @@ -22,12 +22,16 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; import java.util.List; import java.util.Objects; /** Represents a simple hint without parameters, e.g., "LEADER". */ public class SimpleHintItem extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SimpleHintItem.class); + private final String hintName; public SimpleHintItem(String hintName) { @@ -70,4 +74,12 @@ public boolean equals(Object obj) { public String toString() { return hintName; } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += RamUsageEstimator.sizeOf(hintName); + return size; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java new file mode 100644 index 0000000000000..3bd3d2bf446d6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java @@ -0,0 +1,30 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +public abstract class JoinOrderHint extends Hint { + public static String category = "join-order"; + + protected JoinOrderHint(String hintName) { + super(hintName, category); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java new file mode 100644 index 0000000000000..7cca5eddb0d10 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -0,0 +1,219 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; + +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.Stack; + +import static org.apache.iotdb.rpc.TSStatusCode.INTERNAL_SERVER_ERROR; + +public class LeadingHint extends JoinOrderHint { + public static String hintName = "leading"; + private final List tables; + + private List addJoinParameters; + private List normalizedParameters; + + public LeadingHint(List parameters) { + super(hintName); + // /* leading(t3 {}) 会报错 + this.tables = new ArrayList<>(); + addJoinParameters = insertJoinIntoParameters(parameters); + normalizedParameters = parseIntoReversePolishNotation(addJoinParameters); + + if (tables.isEmpty()) { + throw new IllegalArgumentException("LeaderHint accepts one or more tables"); + } + if (hasDuplicateTable(tables)) { + throw new IllegalArgumentException("LeaderHint accepts no duplicate tables"); + } + } + + @Override + public String getKey() { + return category; + } + + public List getTables() { + return tables; + } + + public PlanNode generateLeadingJoinPlan() { + Stack stack = new Stack<>(); + for (String item : normalizedParameters) { + if (item.equals("join")) { + PlanNode rightChild = stack.pop(); + PlanNode leftChild = stack.pop(); + PlanNode joinPlan = makeJoinPlan(leftChild, rightChild); + if (joinPlan == null) { + return null; + } + stack.push(joinPlan); + } else { + // PlanNode logicalPlan = getLogicalPlanByName(item); + // ogicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); + // stack.push(logicalPlan); + } + } + + PlanNode finalJoin = stack.pop(); + // we want all filters been removed + // if (Utils.enableAssert && !filters.isEmpty()) { + // throw new IllegalStateException( + // "Leading hint process failed: filter should be empty, but meet: " + filters + // ); + // } + if (finalJoin == null) { + throw new IoTDBRuntimeException( + "final join plan should not be null", INTERNAL_SERVER_ERROR.getStatusCode()); + } + return finalJoin; + } + + @Override + public String toString() { + if (tables == null || tables.isEmpty()) { + return hintName; + } + return hintName + "-" + String.join("-", tables); + } + + private boolean hasDuplicateTable(List tables) { + Set tableSet = Sets.newHashSet(); + for (String table : tables) { + if (!tableSet.add(table)) { + return true; + } + } + return false; + } + + public static List insertJoinIntoParameters(List list) { + List output = new ArrayList<>(); + + for (String item : list) { + if (item.equals("{")) { + output.add(item); + continue; + } else if (item.equals("}")) { + output.remove(output.size() - 1); + output.add(item); + } else { + output.add(item); + } + output.add("join"); + } + output.remove(output.size() - 1); + return output; + } + + public List parseIntoReversePolishNotation(List list) { + Stack s1 = new Stack<>(); + List s2 = new ArrayList<>(); + + for (String item : list) { + if (!(item.equals("{") || item.equals("}") || item.equals("join"))) { + tables.add(item); + s2.add(item); + } else if (item.equals("{")) { + s1.push(item); + } else if (item.equals("}")) { + while (!s1.peek().equals("{")) { + String pop = s1.pop(); + s2.add(pop); + } + s1.pop(); + } else { + while (!s1.isEmpty() && !s1.peek().equals("{")) { + s2.add(s1.pop()); + } + s1.push(item); + } + } + while (!s1.isEmpty()) { + s2.add(s1.pop()); + } + return s2; + } + + private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { + // List conditions = getJoinConditions( + // getFilters(), leftChild, rightChild); + // Pair, List> pair = JoinUtils.extractExpressionForHashTable( + // leftChild.getOutput(), rightChild.getOutput(), conditions); + // // leading hint would set status inside if not success + // JoinType joinType = computeJoinType(getBitmap(leftChild), + // getBitmap(rightChild), conditions); + // if (joinType == null) { + // this.setStatus(HintStatus.SYNTAX_ERROR); + // this.setErrorMessage("JoinType can not be null"); + // } else if (!isConditionJoinTypeMatched(conditions, joinType)) { + // this.setStatus(HintStatus.UNUSED); + // this.setErrorMessage("condition does not matched joinType"); + // } + // if (!this.isSuccess()) { + // return null; + // } + // get joinType + // LogicalJoin logicalJoin = new LogicalJoin<>(joinType, pair.first, + // pair.second, + // distributeHint, + // Optional.empty(), + // leftChild, + // rightChild, null); + // logicalJoin.getJoinReorderContext().setLeadingJoin(true); + // logicalJoin.setBitmap(LongBitmap.or(getBitmap(leftChild), getBitmap(rightChild))); + + PlanNode joinNode = planJoin(leftChild, rightChild); + return joinNode; + } + + private PlanNode planJoin(PlanNode leftPlan, PlanNode rightPlan) { + List leftOutputSymbols = leftPlan.getOutputSymbols(); + List rightOutputSymbols = rightPlan.getOutputSymbols(); + List criteria = new ArrayList<>(); + Optional asofCriteria = Optional.empty(); + + return new JoinNode( + new PlanNodeId("join"), + JoinNode.JoinType.INNER, + leftPlan, + rightPlan, + criteria, + asofCriteria, + leftOutputSymbols, + rightOutputSymbols, + Optional.empty(), + Optional.empty()); + } +} From 90f82c35ea6b3b27e7d15afe258960d13aa63925 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 21 Jan 2026 10:16:15 +0800 Subject: [PATCH 11/35] EquiJoinClause part 1 --- .../relational/planner/RelationPlanner.java | 30 ++++++++- ...TransformFilteringSemiJoinToInnerJoin.java | 7 ++- .../relational/planner/node/JoinNode.java | 61 +++++++++++++++++-- .../PushPredicateIntoTableScan.java | 24 +++++++- .../UnaliasSymbolReferences.java | 5 +- .../relational/analyzer/AsofJoinTest.java | 14 ++--- .../plan/relational/analyzer/JoinTest.java | 14 ++++- .../analyzer/TableFunctionTest.java | 2 +- .../planner/CorrelatedSubqueryTest.java | 4 +- .../assertions/EquiJoinClauseProvider.java | 16 ++++- .../planner/assertions/JoinMatcher.java | 5 +- .../planner/assertions/PlanMatchPattern.java | 9 ++- 12 files changed, 162 insertions(+), 29 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 516be38ad8eb6..4854e9f45b1c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -142,6 +142,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -518,7 +519,9 @@ If casts are redundant (due to column type and common type being equal), rightCoercions.put(rightOutput, right.getSymbol(rightField).toSymbolReference()); rightJoinColumns.put(identifier, rightOutput); - clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput)); + Set leftTables = new HashSet<>(left.getScope().getTables()); + Set rightTables = new HashSet<>(right.getScope().getTables()); + clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput, leftTables, rightTables)); } ProjectNode leftCoercion = @@ -734,7 +737,30 @@ public RelationPlan planJoin( Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); - equiClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + // Extract tables from expressions + Set leftDependencies = + SymbolsExtractor.extractNames( + leftComparisonExpressions.get(i), analysis.getColumnReferences()); + Set rightDependencies = + SymbolsExtractor.extractNames( + rightComparisonExpressions.get(i), analysis.getColumnReferences()); + + // Convert QualifiedName to Identifier (table name is the first part) + Set leftTables = new HashSet<>(); + for (QualifiedName name : leftDependencies) { + if (name.getPrefix().isPresent()) { + leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + Set rightTables = new HashSet<>(); + for (QualifiedName name : rightDependencies) { + if (name.getPrefix().isPresent()) { + rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + + equiClauses.add( + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); } else { postInnerJoinConditions.add( new ComparisonExpression( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 1f32fdbffe157..bdc35fd8ba255 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -34,6 +34,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; @@ -130,7 +131,11 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { filteringSourceDistinct, ImmutableList.of( new JoinNode.EquiJoinClause( - semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), + semiJoin.getSourceJoinSymbol(), + semiJoin.getFilteringSourceJoinSymbol(), + // should from SemiJoin + ImmutableSet.of(), + ImmutableSet.of())), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 5e6cc5818acc1..c3c79caeb2727 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import com.google.common.collect.ImmutableList; @@ -37,6 +38,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -236,6 +238,16 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), byteBuffer); Symbol.serialize(equiJoinClause.getRight(), byteBuffer); + Set leftTables = equiJoinClause.getLeftTables(); + ReadWriteIOUtils.write(leftTables.size(), byteBuffer); + for (Identifier identifier : leftTables) { + identifier.serialize(byteBuffer); + } + Set rightTables = equiJoinClause.getRightTables(); + ReadWriteIOUtils.write(rightTables.size(), byteBuffer); + for (Identifier identifier : rightTables) { + identifier.serialize(byteBuffer); + } } if (asofCriteria.isPresent()) { @@ -268,6 +280,16 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), stream); Symbol.serialize(equiJoinClause.getRight(), stream); + Set leftTables = equiJoinClause.getLeftTables(); + ReadWriteIOUtils.write(leftTables.size(), stream); + for (Identifier identifier : leftTables) { + identifier.serialize(stream); + } + Set rightTables = equiJoinClause.getRightTables(); + ReadWriteIOUtils.write(rightTables.size(), stream); + for (Identifier identifier : rightTables) { + identifier.serialize(stream); + } } if (asofCriteria.isPresent()) { @@ -295,8 +317,19 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { int size = ReadWriteIOUtils.readInt(byteBuffer); List criteria = new ArrayList<>(size); while (size-- > 0) { - criteria.add( - new EquiJoinClause(Symbol.deserialize(byteBuffer), Symbol.deserialize(byteBuffer))); + Symbol left = Symbol.deserialize(byteBuffer); + Symbol right = Symbol.deserialize(byteBuffer); + int leftTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set leftTables = new HashSet<>(leftTablesSize); + while (leftTablesSize-- > 0) { + leftTables.add(new Identifier(byteBuffer)); + } + int rightTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set rightTables = new HashSet<>(rightTablesSize); + while (rightTablesSize-- > 0) { + rightTables.add(new Identifier(byteBuffer)); + } + criteria.add(new EquiJoinClause(left, right, leftTables, rightTables)); } Optional asofJoinClause = Optional.empty(); @@ -369,10 +402,15 @@ public String toString() { public static class EquiJoinClause { private final Symbol left; private final Symbol right; + private final Set leftTables; + private final Set rightTables; - public EquiJoinClause(Symbol left, Symbol right) { + public EquiJoinClause( + Symbol left, Symbol right, Set leftTables, Set rightTables) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTables = requireNonNull(leftTables, "leftTables is null"); + this.rightTables = requireNonNull(rightTables, "rightTables is null"); } public Symbol getLeft() { @@ -383,13 +421,21 @@ public Symbol getRight() { return right; } + public Set getLeftTables() { + return leftTables; + } + + public Set getRightTables() { + return rightTables; + } + public ComparisonExpression toExpression() { return new ComparisonExpression( ComparisonExpression.Operator.EQUAL, left.toSymbolReference(), right.toSymbolReference()); } public EquiJoinClause flip() { - return new EquiJoinClause(right, left); + return new EquiJoinClause(right, left, rightTables, leftTables); } public static List flipBatch(List input) { @@ -410,12 +456,15 @@ public boolean equals(Object obj) { EquiJoinClause other = (EquiJoinClause) obj; - return Objects.equals(this.left, other.left) && Objects.equals(this.right, other.right); + return Objects.equals(this.left, other.left) + && Objects.equals(this.right, other.right) + && Objects.equals(this.leftTables, other.leftTables) + && Objects.equals(this.rightTables, other.rightTables); } @Override public int hashCode() { - return Objects.hash(left, right); + return Objects.hash(left, right, leftTables, rightTables); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 7bd4530afd194..7de8e311489b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -68,9 +68,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.utils.TimestampPrecisionUtils; @@ -876,7 +878,27 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { rightProjections.put(rightSymbol, rightExpression); } - equiJoinClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + Set leftDependencies = + SymbolsExtractor.extractNames(equality.getLeft(), analysis.getColumnReferences()); + Set rightDependencies = + SymbolsExtractor.extractNames(equality.getRight(), analysis.getColumnReferences()); + + // Convert QualifiedName to Identifier (table name is the first part) + Set leftTables = new HashSet<>(); + for (QualifiedName name : leftDependencies) { + if (name.getPrefix().isPresent()) { + leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + Set rightTables = new HashSet<>(); + for (QualifiedName name : rightDependencies) { + if (name.getPrefix().isPresent()) { + rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); + } + } + + equiJoinClauses.add( + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 17ec744f58d8f..9a67bbe030270 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -833,7 +833,10 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { for (JoinNode.EquiJoinClause clause : node.getCriteria()) { builder.add( new JoinNode.EquiJoinClause( - mapper.map(clause.getLeft()), mapper.map(clause.getRight()))); + mapper.map(clause.getLeft()), + mapper.map(clause.getRight()), + ImmutableSet.copyOf(clause.getLeftTables()), + ImmutableSet.copyOf(clause.getRightTables()))); } List newCriteria = builder.build(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java index 62ec85d27745b..dee921c73fe9b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java @@ -154,7 +154,7 @@ public void projectInAsofCriteriaTest() { builder .asofCriteria( ComparisonExpression.Operator.GREATER_THAN, "expr", "time_0") - .equiCriteria("tag1", "tag1_1") + .equiCriteria("tag1", "tag1_1", "table1", "table2") .left( sort( ImmutableList.of( @@ -210,9 +210,9 @@ public void sortEliminateTest() { .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( - equiJoinClause("tag1", "tag1_1"), - equiJoinClause("tag2", "tag2_2"), - equiJoinClause("tag3", "tag3_3"))) + equiJoinClause("tag1", "tag1_1", "table1", "table2"), + equiJoinClause("tag2", "tag2_2", "table1", "table2"), + equiJoinClause("tag3", "tag3_3", "table1", "table2"))) .left(sort(table1)) .right(sort(table2))))); @@ -227,9 +227,9 @@ public void sortEliminateTest() { .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( - equiJoinClause("tag1", "tag1_1"), - equiJoinClause("tag2", "tag2_2"), - equiJoinClause("tag3", "tag3_3"))) + equiJoinClause("tag1", "tag1_1", "table1", "table2"), + equiJoinClause("tag2", "tag2_2", "table1", "table2"), + equiJoinClause("tag3", "tag3_3", "table1", "table2"))) .left(exchange()) .right(exchange())))); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java index 1b178b50449da..1e161eea62061 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java @@ -53,6 +53,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -187,7 +188,12 @@ private void assertInnerJoinTest1(String sql) { joinNode = (JoinNode) getChildrenNode(logicalPlanNode, 3); List joinCriteria = Collections.singletonList( - new JoinNode.EquiJoinClause(Symbol.of("time"), Symbol.of("time_0"))); + new JoinNode.EquiJoinClause( + Symbol.of("time"), + Symbol.of("time_0"), + ImmutableSet.of(new Identifier("t1")), + ImmutableSet.of(new Identifier("t2")))); + assertJoinNodeEquals( joinNode, INNER, @@ -368,7 +374,11 @@ private void assertInnerJoinTest2(String sql, boolean joinUsing) { joinNode = (JoinNode) getChildrenNode(logicalPlanNode, 4); List joinCriteria = Collections.singletonList( - new JoinNode.EquiJoinClause(Symbol.of("time"), Symbol.of("time_0"))); + new JoinNode.EquiJoinClause( + Symbol.of("time"), + Symbol.of("time_0"), + ImmutableSet.of(new Identifier("t1")), + ImmutableSet.of(new Identifier("t2")))); assertJoinNodeEquals( joinNode, INNER, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java index 344c69cfc1a71..da2edb5051dab 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java @@ -281,7 +281,7 @@ public void testLeafFunction() { builder .left(sort(tableFunctionProcessor(tableFunctionMatcher1))) .right(sort(tableFunctionProcessor(tableFunctionMatcher2))) - .equiCriteria("output", "output_0")))); + .equiCriteria("output", "output_0", "a", "b")))); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java index e7baf8078f112..c54df4742c43a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java @@ -84,7 +84,7 @@ public void testCorrelatedExistsSubquery() { JoinNode.JoinType.INNER, builder -> builder - .equiCriteria("s1", "s2_7") + .equiCriteria("s1", "s2_7", "t1", "t2") .left(sort(tableScan1)) .right( sort( @@ -133,7 +133,7 @@ public void testCorrelatedExistsSubquery() { JoinNode.JoinType.LEFT, builder -> builder - .equiCriteria("s1", "s2_3") + .equiCriteria("s1", "s2_3", "t1", "t2") .left(sort(tableScan1)) .right( sort( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java index c151b6874a6f8..b6e5280caf2ea 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java @@ -20,21 +20,33 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; + +import java.util.Set; import static java.util.Objects.requireNonNull; public class EquiJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; + private final Set leftTables; + private final Set rightTables; - public EquiJoinClauseProvider(SymbolAlias left, SymbolAlias right) { + public EquiJoinClauseProvider( + SymbolAlias left, + SymbolAlias right, + Set leftTables, + Set rightTables) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTables = requireNonNull(leftTables, "leftTables is null"); + this.rightTables = requireNonNull(rightTables, "rightTables is null"); } @Override public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) { - return new JoinNode.EquiJoinClause(left.toSymbol(aliases), right.toSymbol(aliases)); + return new JoinNode.EquiJoinClause( + left.toSymbol(aliases), right.toSymbol(aliases), leftTables, rightTables); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java index 090e4a0bc3bc9..8ea44457e331d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java @@ -166,8 +166,9 @@ public Builder equiCriteria( } @CanIgnoreReturnValue - public Builder equiCriteria(String left, String right) { - this.equiCriteria = Optional.of(ImmutableList.of(equiJoinClause(left, right))); + public Builder equiCriteria(String left, String right, String leftTable, String rightTable) { + this.equiCriteria = + Optional.of(ImmutableList.of(equiJoinClause(left, right, leftTable, rightTable))); return this; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 2ca57e5296a5b..b5ca5f227f519 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -63,6 +63,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; @@ -716,8 +717,12 @@ public static PlanMatchPattern strictProject( } public static ExpectedValueProvider equiJoinClause( - String left, String right) { - return new EquiJoinClauseProvider(new SymbolAlias(left), new SymbolAlias(right)); + String left, String right, String leftTable, String rightTable) { + return new EquiJoinClauseProvider( + new SymbolAlias(left), + new SymbolAlias(right), + ImmutableSet.of(new Identifier(leftTable)), + ImmutableSet.of(new Identifier(rightTable))); } public static AsofJoinClauseProvider asofJoinClause( From 812570e9c816fc23b81e808d42bd2318f27a420f Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 21 Jan 2026 14:36:14 +0800 Subject: [PATCH 12/35] left & right tables for JoinNode --- .../queryengine/common/MPPQueryContext.java | 10 +- .../plan/relational/analyzer/Scope.java | 17 +-- .../analyzer/StatementAnalyzer.java | 2 +- .../relational/planner/CteMaterializer.java | 3 +- .../relational/planner/RelationPlanner.java | 12 +- ...correlatedScalarSubqueryReconstructor.java | 3 +- .../iterative/rule/PruneJoinColumns.java | 4 +- ...TransformFilteringSemiJoinToInnerJoin.java | 7 +- .../relational/planner/node/JoinNode.java | 113 +++++++++++++++++- .../PushPredicateIntoTableScan.java | 16 ++- .../UnaliasSymbolReferences.java | 4 +- 11 files changed, 154 insertions(+), 37 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index e67bcd67c9d03..886da68f1c2e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -41,7 +41,7 @@ import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; import org.apache.iotdb.db.utils.cte.CteDataStore; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; @@ -152,7 +152,7 @@ public enum ExplainType { private boolean innerTriggeredQuery = false; // Tables in the subquery - private final Map, List> subQueryTables = new HashMap<>(); + private final Map, Set> subQueryTables = new HashMap<>(); @TestOnly public MPPQueryContext(QueryId queryId) { @@ -554,12 +554,12 @@ public void setCteQueries(Map, Query> cteQueries) { this.cteQueries = cteQueries; } - public void addSubQueryTables(Query query, List tables) { + public void addSubQueryTables(Query query, Set tables) { subQueryTables.put(NodeRef.of(query), tables); } - public List getTables(Query query) { - return subQueryTables.getOrDefault(NodeRef.of(query), ImmutableList.of()); + public Set getTables(Query query) { + return subQueryTables.getOrDefault(NodeRef.of(query), ImmutableSet.of()); } public void addCteExplainResult(Table table, Pair> cteExplainResult) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java index d06529c00506f..d178a5f30471e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java @@ -31,11 +31,12 @@ import com.google.common.collect.ImmutableMap; -import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; import static com.google.common.base.MoreObjects.toStringHelper; @@ -57,7 +58,7 @@ public class Scope { // Tables to access for the current relation. For CTE materialization and constant folding // subqueries, non-materialized CTEs in tables must be identified, and their definitions // attached to the subquery context. - private List tables; + private Set tables; public static Scope create() { return builder().build(); @@ -73,24 +74,24 @@ private Scope( RelationId relationId, RelationType relation, Map namedQueries, - List tables) { + Set tables) { this.parent = requireNonNull(parent, "parent is null"); this.relationId = requireNonNull(relationId, "relationId is null"); this.queryBoundary = queryBoundary; this.relation = requireNonNull(relation, "relation is null"); this.namedQueries = ImmutableMap.copyOf(requireNonNull(namedQueries, "namedQueries is null")); - this.tables = new ArrayList<>(requireNonNull(tables, "tables is null")); + this.tables = new HashSet<>(requireNonNull(tables, "tables is null")); } public void addTable(Table table) { tables.add(new Identifier(table.getName().getSuffix())); } - public void setTables(List tables) { + public void setTables(Set tables) { this.tables = tables; } - public List getTables() { + public Set getTables() { return tables; } @@ -349,7 +350,7 @@ public static final class Builder { private RelationId relationId = RelationId.anonymous(); private RelationType relationType = new RelationType(); private final Map namedQueries = new HashMap<>(); - private final List tables = new ArrayList<>(); + private final Set tables = new HashSet<>(); private Optional parent = Optional.empty(); private boolean queryBoundary; @@ -388,7 +389,7 @@ public Builder withNamedQuery(String name, WithQuery withQuery) { return this; } - public Builder withTables(List tables) { + public Builder withTables(Set tables) { this.tables.addAll(tables); return this; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 75b56348ef1ba..3ded3afd2e24e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -3730,7 +3730,7 @@ protected Scope visitJoin(Join node, Optional scope) { joinConditionCheck(criteria); // Remember original tables before processing left - List originalTables = new ArrayList<>(); + Set originalTables = new HashSet<>(); scope.ifPresent(s -> originalTables.addAll(s.getTables())); Scope left = process(node.getLeft(), scope); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java index 03eb0baa00538..7864d8428cda0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java @@ -66,6 +66,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -108,7 +109,7 @@ public CteDataStore fetchCteQueryResult( try { Query q = query; if (with != null) { - List tables = context.getTables(query); + Set tables = context.getTables(query); List withQueries = with.getQueries().stream() .filter( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 4854e9f45b1c8..fcce050da1a08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -542,7 +542,9 @@ If casts are redundant (due to column type and common type being equal), leftCoercion.getOutputSymbols(), rightCoercion.getOutputSymbols(), Optional.empty(), - Optional.empty()); + Optional.empty(), + left.getScope().getTables(), + right.getScope().getTables()); // Transform RIGHT JOIN to LEFT if (join.getJoinType() == JoinNode.JoinType.RIGHT) { join = join.flip(); @@ -782,7 +784,9 @@ public RelationPlan planJoin( leftPlanBuilder.getRoot().getOutputSymbols(), rightPlanBuilder.getRoot().getOutputSymbols(), Optional.empty(), - Optional.empty()); + Optional.empty(), + leftPlan.getScope().getTables(), + rightPlan.getScope().getTables()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } @@ -837,7 +841,9 @@ public RelationPlan planJoin( complexJoinExpressions.stream() .map(e -> coerceIfNecessary(analysis, e, translationMap.rewrite(e))) .collect(Collectors.toList()))), - Optional.empty()); + Optional.empty(), + leftPlan.getScope().getTables(), + rightPlan.getScope().getTables()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java index 49d4b7831e7d3..5f6e7fe012f83 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java @@ -55,6 +55,7 @@ import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -118,7 +119,7 @@ public Optional fetchUncorrelatedSubqueryResultForPredicate( Query query = subqueryExpression.getQuery(); Query q = query; if (with != null) { - List tables = context.getTables(query); + Set tables = context.getTables(query); List withQueries = with.getQueries().stream() .filter( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java index c09710346ed09..21f3df06ec7a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java @@ -48,6 +48,8 @@ protected Optional pushDownProjectOff( filteredCopy(joinNode.getLeftOutputSymbols(), referencedOutputs::contains), filteredCopy(joinNode.getRightOutputSymbols(), referencedOutputs::contains), joinNode.getFilter(), - joinNode.isSpillable())); + joinNode.isSpillable(), + joinNode.getLeftTables(), + joinNode.getRightTables())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index bdc35fd8ba255..1f32fdbffe157 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; @@ -131,11 +130,7 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { filteringSourceDistinct, ImmutableList.of( new JoinNode.EquiJoinClause( - semiJoin.getSourceJoinSymbol(), - semiJoin.getFilteringSourceJoinSymbol(), - // should from SemiJoin - ImmutableSet.of(), - ImmutableSet.of())), + semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index c3c79caeb2727..0f85e5990aa90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -67,6 +67,9 @@ public class JoinNode extends TwoChildProcessNode { // private final Optional distributionType; // private final Map dynamicFilters; + private final Set leftTables; + private final Set rightTables; + public JoinNode( PlanNodeId id, JoinType joinType, @@ -77,7 +80,9 @@ public JoinNode( List leftOutputSymbols, List rightOutputSymbols, Optional filter, - Optional spillable) { + Optional spillable, + Set leftTables, + Set rightTables) { super(id); requireNonNull(joinType, "type is null"); requireNonNull(leftChild, "left is null"); @@ -96,6 +101,8 @@ public JoinNode( // requireNonNull(leftHashSymbol, "leftHashSymbol is null"); // requireNonNull(rightHashSymbol, "rightHashSymbol is null"); requireNonNull(spillable, "spillable is null"); + requireNonNull(leftTables, "leftTables is null"); + requireNonNull(rightTables, "rightTables is null"); this.joinType = joinType; this.leftChild = leftChild; @@ -109,6 +116,8 @@ public JoinNode( // this.maySkipOutputDuplicates = maySkipOutputDuplicates; // this.leftHashSymbol = leftHashSymbol; // this.rightHashSymbol = rightHashSymbol; + this.leftTables = ImmutableSet.copyOf(leftTables); + this.rightTables = ImmutableSet.copyOf(rightTables); Set leftSymbols = ImmutableSet.copyOf(leftChild.getOutputSymbols()); Set rightSymbols = ImmutableSet.copyOf(rightChild.getOutputSymbols()); @@ -136,6 +145,32 @@ public JoinNode( equiJoinClause)); } + public JoinNode( + PlanNodeId id, + JoinType joinType, + PlanNode leftChild, + PlanNode rightChild, + List criteria, + Optional asofCriteria, + List leftOutputSymbols, + List rightOutputSymbols, + Optional filter, + Optional spillable) { + this( + id, + joinType, + leftChild, + rightChild, + criteria, + asofCriteria, + leftOutputSymbols, + rightOutputSymbols, + filter, + spillable, + ImmutableSet.of(), + ImmutableSet.of()); + } + // only used for deserialize public JoinNode( PlanNodeId id, @@ -143,15 +178,21 @@ public JoinNode( List criteria, Optional asofCriteria, List leftOutputSymbols, - List rightOutputSymbols) { + List rightOutputSymbols, + Set leftTables, + Set rightTables) { super(id); requireNonNull(joinType, "type is null"); requireNonNull(criteria, "criteria is null"); + requireNonNull(leftTables, "leftTables is null"); + requireNonNull(rightTables, "rightTables is null"); this.leftOutputSymbols = leftOutputSymbols; this.rightOutputSymbols = rightOutputSymbols; this.filter = Optional.empty(); this.spillable = Optional.empty(); + this.leftTables = ImmutableSet.copyOf(leftTables); + this.rightTables = ImmutableSet.copyOf(rightTables); this.joinType = joinType; this.criteria = criteria; @@ -172,7 +213,9 @@ public JoinNode flip() { rightOutputSymbols, leftOutputSymbols, filter, - spillable); + spillable, + rightTables, + leftTables); } @Override @@ -193,7 +236,9 @@ public PlanNode replaceChildren(List newChildren) { leftOutputSymbols, rightOutputSymbols, filter, - spillable); + spillable, + leftTables, + rightTables); } @Override @@ -217,7 +262,9 @@ public PlanNode clone() { leftOutputSymbols, rightOutputSymbols, filter, - spillable); + spillable, + leftTables, + rightTables); joinNode.setLeftChild(null); joinNode.setRightChild(null); return joinNode; @@ -268,6 +315,16 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (Symbol rightOutputSymbol : rightOutputSymbols) { Symbol.serialize(rightOutputSymbol, byteBuffer); } + + // Serialize JoinNode-level leftTables and rightTables + ReadWriteIOUtils.write(this.leftTables.size(), byteBuffer); + for (Identifier identifier : this.leftTables) { + identifier.serialize(byteBuffer); + } + ReadWriteIOUtils.write(this.rightTables.size(), byteBuffer); + for (Identifier identifier : this.rightTables) { + identifier.serialize(byteBuffer); + } } @Override @@ -310,6 +367,16 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (Symbol rightOutputSymbol : rightOutputSymbols) { Symbol.serialize(rightOutputSymbol, stream); } + + // Serialize JoinNode-level leftTables and rightTables + ReadWriteIOUtils.write(this.leftTables.size(), stream); + for (Identifier identifier : this.leftTables) { + identifier.serialize(stream); + } + ReadWriteIOUtils.write(this.rightTables.size(), stream); + for (Identifier identifier : this.rightTables) { + identifier.serialize(stream); + } } public static JoinNode deserialize(ByteBuffer byteBuffer) { @@ -354,9 +421,28 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { rightOutputSymbols.add(Symbol.deserialize(byteBuffer)); } + // Deserialize JoinNode-level leftTables and rightTables + int joinLeftTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set joinLeftTables = new HashSet<>(joinLeftTablesSize); + while (joinLeftTablesSize-- > 0) { + joinLeftTables.add(new Identifier(byteBuffer)); + } + int joinRightTablesSize = ReadWriteIOUtils.readInt(byteBuffer); + Set joinRightTables = new HashSet<>(joinRightTablesSize); + while (joinRightTablesSize-- > 0) { + joinRightTables.add(new Identifier(byteBuffer)); + } + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); return new JoinNode( - planNodeId, joinType, criteria, asofJoinClause, leftOutputSymbols, rightOutputSymbols); + planNodeId, + joinType, + criteria, + asofJoinClause, + leftOutputSymbols, + rightOutputSymbols, + joinLeftTables, + joinRightTables); } public JoinType getJoinType() { @@ -387,6 +473,14 @@ public Optional isSpillable() { return spillable; } + public Set getLeftTables() { + return leftTables; + } + + public Set getRightTables() { + return rightTables; + } + public boolean isCrossJoin() { return !asofCriteria.isPresent() && criteria.isEmpty() @@ -405,6 +499,13 @@ public static class EquiJoinClause { private final Set leftTables; private final Set rightTables; + public EquiJoinClause(Symbol left, Symbol right) { + this.left = requireNonNull(left, "left is null"); + this.right = requireNonNull(right, "right is null"); + this.leftTables = ImmutableSet.of(); + this.rightTables = ImmutableSet.of(); + } + public EquiJoinClause( Symbol left, Symbol right, Set leftTables, Set rightTables) { this.left = requireNonNull(left, "left is null"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 7de8e311489b2..12d9bed387b70 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -981,7 +981,9 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { leftSource.getOutputSymbols(), rightSource.getOutputSymbols(), newJoinFilter, - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } JoinNode outputJoinNode = (JoinNode) output; @@ -1384,7 +1386,9 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } if (canConvertToLeftJoin) { return new JoinNode( @@ -1397,7 +1401,9 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } else { // temp fix because right join is not supported for now. return node; @@ -1432,7 +1438,9 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable()); + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()); } private boolean canConvertOuterToInner( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 9a67bbe030270..0b1a434d3cbc4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -878,7 +878,9 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { newLeftOutputSymbols, newRightOutputSymbols, newFilter, - node.isSpillable()), + node.isSpillable(), + node.getLeftTables(), + node.getRightTables()), outputMapping); } From e0380a52934043bcffc8a9caec16e8c02fbb3809 Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 21 Jan 2026 15:58:16 +0800 Subject: [PATCH 13/35] Update PushPredicateIntoTableScan --- .../PushPredicateIntoTableScan.java | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 12d9bed387b70..5d34f62173f99 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -72,7 +72,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.utils.TimestampPrecisionUtils; @@ -855,6 +854,15 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { node.getRightChild().getOutputSymbols().stream() .collect(toImmutableMap(key -> key, Symbol::toSymbolReference))); + // Build a map from (left, right) symbols to their (leftTables, rightTables) + // This preserves table info when rebuilding EquiJoinClause + Map, Pair, Set>> symbolToTablesMap = + node.getCriteria().stream() + .collect( + toImmutableMap( + clause -> new Pair<>(clause.getLeft(), clause.getRight()), + clause -> new Pair<>(clause.getLeftTables(), clause.getRightTables()))); + // Create new projections for the new join clauses List equiJoinClauses = new ArrayList<>(); ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); @@ -878,27 +886,14 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { rightProjections.put(rightSymbol, rightExpression); } - Set leftDependencies = - SymbolsExtractor.extractNames(equality.getLeft(), analysis.getColumnReferences()); - Set rightDependencies = - SymbolsExtractor.extractNames(equality.getRight(), analysis.getColumnReferences()); - - // Convert QualifiedName to Identifier (table name is the first part) - Set leftTables = new HashSet<>(); - for (QualifiedName name : leftDependencies) { - if (name.getPrefix().isPresent()) { - leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } - } - Set rightTables = new HashSet<>(); - for (QualifiedName name : rightDependencies) { - if (name.getPrefix().isPresent()) { - rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } - } + // Retrieve leftTables and rightTables from original criteria + Pair, Set> tables = + symbolToTablesMap.getOrDefault( + new Pair<>(leftSymbol, rightSymbol), + new Pair<>(ImmutableSet.of(), ImmutableSet.of())); equiJoinClauses.add( - new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, tables.left, tables.right)); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; From 97f06c9b1cc232e8a118126527a001d20046ec6e Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 22 Jan 2026 16:17:02 +0800 Subject: [PATCH 14/35] simple inner join reorder --- .../relational/planner/node/JoinNode.java | 8 + .../optimizations/CollectJoinConstraint.java | 62 ++++++- .../optimizations/LeadingJoinOptimizer.java | 40 ++-- .../optimizations/LogicalOptimizeFactory.java | 6 +- .../relational/utils/hint/LeadingHint.java | 171 +++++++++++++----- 5 files changed, 225 insertions(+), 62 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 0f85e5990aa90..aeaf4d0dbb7c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -32,6 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -498,12 +499,14 @@ public static class EquiJoinClause { private final Symbol right; private final Set leftTables; private final Set rightTables; + private final Set tables; public EquiJoinClause(Symbol left, Symbol right) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); this.leftTables = ImmutableSet.of(); this.rightTables = ImmutableSet.of(); + this.tables = ImmutableSet.of(); } public EquiJoinClause( @@ -512,6 +515,7 @@ public EquiJoinClause( this.right = requireNonNull(right, "right is null"); this.leftTables = requireNonNull(leftTables, "leftTables is null"); this.rightTables = requireNonNull(rightTables, "rightTables is null"); + this.tables = Sets.union(leftTables, rightTables); } public Symbol getLeft() { @@ -530,6 +534,10 @@ public Set getRightTables() { return rightTables; } + public Set getTables() { + return tables; + } + public ComparisonExpression toExpression() { return new ComparisonExpression( ComparisonExpression.Operator.EQUAL, left.toSymbolReference(), right.toSymbolReference()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 5922d7dc3bcae..5450c9b60f8fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -24,11 +24,21 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.tsfile.utils.Pair; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + public class CollectJoinConstraint implements PlanOptimizer { @Override public PlanNode optimize(PlanNode plan, Context context) { @@ -58,10 +68,41 @@ public PlanNode visitPlan(PlanNode node, Context context) { return node; } + @Override + public PlanNode visitDeviceTableScan(DeviceTableScanNode node, Context context) { + String tableName = node.getQualifiedObjectName().getObjectName(); + LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + leading.getRelationToScanMap().put(tableName, node); + return node; + } + @Override public PlanNode visitJoin(JoinNode node, Context context) { - PlanNode leftNode = node.getLeftChild(); - PlanNode rightNode = node.getRightChild(); + for (PlanNode child : node.getChildren()) { + child.accept(this, context); + } + + LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + Set leadingTables = + leading.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); + + Set leftTables = node.getLeftTables(); + Set rightTables = node.getRightTables(); + Set totalJoinTables = ImmutableSet.of(); + + // join conjunctions + List criteria = node.getCriteria(); + for (JoinNode.EquiJoinClause equiJoin : criteria) { + Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); + totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); + if (node.getJoinType() == JoinNode.JoinType.LEFT) { + // do something + } + leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); + // leading.putConditionJoinType(expression, join.getJoinType()); + } + collectJoinConstraintList(leading, leftTables, rightTables, node, totalJoinTables); + return node; } @@ -69,5 +110,22 @@ public PlanNode visitJoin(JoinNode node, Context context) { // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { // return visitTwoChildProcess(node, context); // } + + private void collectJoinConstraintList( + LeadingHint leading, + Set leftHand, + Set rightHand, + JoinNode join, + Set joinTables) { + Set totalTables = Sets.union(leftHand, rightHand); + if (join.getJoinType() == JoinNode.JoinType.INNER) { + leading.setInnerJoinTables(Sets.union(leading.getInnerJoinTables(), totalTables)); + return; + } + if (join.getJoinType() == JoinNode.JoinType.FULL) { + // full join + return; + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java index 14c3d1fd261f8..3bd3ebcdbd021 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java @@ -24,14 +24,16 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; -import java.util.Set; +import com.google.common.collect.Sets; -import static com.google.common.collect.ImmutableSet.toImmutableSet; +import java.util.Set; +import java.util.stream.Collectors; public class LeadingJoinOptimizer implements PlanOptimizer { @Override @@ -56,13 +58,22 @@ public PlanNode visitPlan(PlanNode node, Context context) { if (!(hint instanceof LeadingHint)) { return node; } - LeadingHint leadingHint = (LeadingHint) hint; - Set relationNames = - analysis.getRelationNames().values().stream() - .map(QualifiedName::getSuffix) - .collect(toImmutableSet()); - if (!validateTableNamesMatch(relationNames, leadingHint)) { + PlanNode newNode = node.clone(); + for (PlanNode child : node.getChildren()) { + newNode.addChild(child.accept(this, context)); + } + return newNode; + } + + @Override + public PlanNode visitJoin(JoinNode node, Context context) { + LeadingHint leadingHint = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + Set currentTables = Sets.union(node.getLeftTables(), node.getRightTables()); + Set leadingTables = + leadingHint.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); + + if (!currentTables.equals(leadingTables)) { return node; } @@ -73,11 +84,10 @@ public PlanNode visitPlan(PlanNode node, Context context) { return node; } - private boolean validateTableNamesMatch(Set relationNames, LeadingHint leadingHint) { - if (relationNames.size() != leadingHint.getTables().size()) { - return false; - } - return relationNames.containsAll(leadingHint.getTables()); - } + // @Override + // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { + // return visitTwoChildProcess(node, context); + // } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 004a8142ee5eb..b98d5dfade10e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -286,6 +286,8 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new RemoveRedundantIdentityProjections())) .build()), simplifyOptimizer, + new CollectJoinConstraint(), + new LeadingJoinOptimizer(), new UnaliasSymbolReferences(plannerContext.getMetadata()), new IterativeOptimizer( plannerContext, @@ -390,9 +392,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new MergeLimitWithSort(), new MergeLimitOverProjectWithSort(), new PushTopKThroughUnion())), - new ParallelizeGrouping(), - new CollectJoinConstraint()); - // new LeadingJoinOptimizer()); + new ParallelizeGrouping()); this.planOptimizers = optimizerBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 7cca5eddb0d10..856efcec77f01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -25,12 +25,22 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import org.apache.tsfile.utils.Pair; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.Stack; @@ -44,6 +54,11 @@ public class LeadingHint extends JoinOrderHint { private List addJoinParameters; private List normalizedParameters; + private final List, Expression>> filters = new ArrayList<>(); + private final Map relationToScanMap = new HashMap<>(); + + private Set innerJoinTables = ImmutableSet.of(); + public LeadingHint(List parameters) { super(hintName); // /* leading(t3 {}) 会报错 @@ -68,6 +83,22 @@ public List getTables() { return tables; } + public List, Expression>> getFilters() { + return filters; + } + + public Map getRelationToScanMap() { + return relationToScanMap; + } + + public Set getInnerJoinTables() { + return innerJoinTables; + } + + public void setInnerJoinTables(Set innerJoinTables) { + this.innerJoinTables = innerJoinTables; + } + public PlanNode generateLeadingJoinPlan() { Stack stack = new Stack<>(); for (String item : normalizedParameters) { @@ -80,9 +111,12 @@ public PlanNode generateLeadingJoinPlan() { } stack.push(joinPlan); } else { - // PlanNode logicalPlan = getLogicalPlanByName(item); - // ogicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); - // stack.push(logicalPlan); + PlanNode logicalPlan = getPlanByName(item); + if (logicalPlan == null) { + return null; + } + logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); + stack.push(logicalPlan); } } @@ -166,54 +200,107 @@ public List parseIntoReversePolishNotation(List list) { return s2; } + public PlanNode getPlanByName(String name) { + if (!relationToScanMap.containsKey(name)) { + return null; + } + return relationToScanMap.get(name); + } + private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { - // List conditions = getJoinConditions( - // getFilters(), leftChild, rightChild); - // Pair, List> pair = JoinUtils.extractExpressionForHashTable( - // leftChild.getOutput(), rightChild.getOutput(), conditions); - // // leading hint would set status inside if not success - // JoinType joinType = computeJoinType(getBitmap(leftChild), - // getBitmap(rightChild), conditions); - // if (joinType == null) { - // this.setStatus(HintStatus.SYNTAX_ERROR); - // this.setErrorMessage("JoinType can not be null"); - // } else if (!isConditionJoinTypeMatched(conditions, joinType)) { - // this.setStatus(HintStatus.UNUSED); - // this.setErrorMessage("condition does not matched joinType"); - // } - // if (!this.isSuccess()) { - // return null; - // } - // get joinType - // LogicalJoin logicalJoin = new LogicalJoin<>(joinType, pair.first, - // pair.second, - // distributeHint, - // Optional.empty(), - // leftChild, - // rightChild, null); - // logicalJoin.getJoinReorderContext().setLeadingJoin(true); - // logicalJoin.setBitmap(LongBitmap.or(getBitmap(leftChild), getBitmap(rightChild))); - - PlanNode joinNode = planJoin(leftChild, rightChild); - return joinNode; - } - - private PlanNode planJoin(PlanNode leftPlan, PlanNode rightPlan) { - List leftOutputSymbols = leftPlan.getOutputSymbols(); - List rightOutputSymbols = rightPlan.getOutputSymbols(); - List criteria = new ArrayList<>(); + List conditions = getJoinConditions(getFilters(), leftChild, rightChild); + JoinNode.JoinType joinType = computeJoinType(); + if (joinType == null) { + return null; + } else if (!isConditionJoinTypeMatched()) { + return null; + } + + List leftOutputSymbols = leftChild.getOutputSymbols(); + List rightOutputSymbols = rightChild.getOutputSymbols(); Optional asofCriteria = Optional.empty(); + List criteria = new ArrayList<>(); + + for (Expression conjunct : conditions) { + ComparisonExpression equality = (ComparisonExpression) conjunct; + Symbol leftSymbol = Symbol.from(equality.getLeft()); + Symbol rightSymbol = Symbol.from(equality.getRight()); + + if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { + criteria.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + } else if (leftOutputSymbols.contains(rightSymbol) + && rightOutputSymbols.contains(leftSymbol)) { + criteria.add(new JoinNode.EquiJoinClause(rightSymbol, leftSymbol)); + } else { + throw new IllegalArgumentException("Invalid join condition"); + } + } return new JoinNode( new PlanNodeId("join"), - JoinNode.JoinType.INNER, - leftPlan, - rightPlan, + joinType, + leftChild, + rightChild, criteria, asofCriteria, leftOutputSymbols, rightOutputSymbols, Optional.empty(), - Optional.empty()); + Optional.empty(), + getTables(leftChild), + getTables(rightChild)); + } + + private List getJoinConditions( + List, Expression>> filters, PlanNode left, PlanNode right) { + List joinConditions = new ArrayList<>(); + for (int i = filters.size() - 1; i >= 0; i--) { + Pair, Expression> filterPair = filters.get(i); + Set tables = Sets.union(getTables(left), getTables(right)); + // left one is smaller set + if (tables.containsAll(filterPair.left)) { + joinConditions.add(filterPair.right); + filters.remove(i); + } + } + return joinConditions; + } + + private PlanNode makeFilterPlanIfExist( + List, Expression>> filters, PlanNode plan) { + if (filters.isEmpty()) { + return plan; + } + for (int i = filters.size() - 1; i >= 0; i--) { + Pair, Expression> filterPair = filters.get(i); + if (getTables(plan).containsAll(filterPair.left)) { + plan = new FilterNode(plan.getPlanNodeId(), plan, filterPair.right); + filters.remove(i); + } + } + return plan; + } + + private Set getTables(PlanNode root) { + if (root instanceof JoinNode) { + return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); + } else if (root instanceof DeviceTableScanNode) { + return ImmutableSet.of( + new Identifier(((DeviceTableScanNode) root).getQualifiedObjectName().getObjectName())); + } else if (root instanceof FilterNode) { + return getTables(((FilterNode) root).getChild()); + } else if (root instanceof ProjectNode) { + return getTables(((ProjectNode) root).getChild()); + } else { + return null; + } + } + + public JoinNode.JoinType computeJoinType() { + return JoinNode.JoinType.INNER; + } + + public boolean isConditionJoinTypeMatched() { + return true; } } From 5728188669b3775ac4030f752b918dc24cf185f9 Mon Sep 17 00:00:00 2001 From: shizy Date: Mon, 9 Feb 2026 15:04:14 +0800 Subject: [PATCH 15/35] left & right outer join --- .../plan/planner/plan/node/PlanNode.java | 12 ++ .../relational/planner/RelationPlanner.java | 12 +- .../iterative/rule/PruneJoinColumns.java | 4 +- .../relational/planner/node/JoinNode.java | 4 +- .../planner/node/TableScanNode.java | 9 + .../optimizations/CollectJoinConstraint.java | 86 +++++++- .../PushPredicateIntoTableScan.java | 16 +- .../UnaliasSymbolReferences.java | 4 +- .../relational/utils/hint/JoinConstraint.java | 80 ++++++++ .../relational/utils/hint/LeadingHint.java | 192 +++++++++++++++--- 10 files changed, 350 insertions(+), 69 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java index 4e3196c8d42c4..9746d06d2a10a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java @@ -23,6 +23,7 @@ import org.apache.iotdb.consensus.common.request.IConsensusRequest; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -32,8 +33,10 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import static java.util.Objects.requireNonNull; @@ -78,6 +81,15 @@ public void markAsGeneratedByPipe() { public abstract void addChild(PlanNode child); + public Set getInputTables() { + Set tables = new HashSet<>(); + for (PlanNode child : getChildren()) { + tables.addAll(child.getInputTables()); + } + ; + return tables; + } + /** * If this plan node has to be serialized or deserialized, override this method. If this method is * overridden, the serialization and deserialization methods must be implemented. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index fcce050da1a08..4854e9f45b1c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -542,9 +542,7 @@ If casts are redundant (due to column type and common type being equal), leftCoercion.getOutputSymbols(), rightCoercion.getOutputSymbols(), Optional.empty(), - Optional.empty(), - left.getScope().getTables(), - right.getScope().getTables()); + Optional.empty()); // Transform RIGHT JOIN to LEFT if (join.getJoinType() == JoinNode.JoinType.RIGHT) { join = join.flip(); @@ -784,9 +782,7 @@ public RelationPlan planJoin( leftPlanBuilder.getRoot().getOutputSymbols(), rightPlanBuilder.getRoot().getOutputSymbols(), Optional.empty(), - Optional.empty(), - leftPlan.getScope().getTables(), - rightPlan.getScope().getTables()); + Optional.empty()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } @@ -841,9 +837,7 @@ public RelationPlan planJoin( complexJoinExpressions.stream() .map(e -> coerceIfNecessary(analysis, e, translationMap.rewrite(e))) .collect(Collectors.toList()))), - Optional.empty(), - leftPlan.getScope().getTables(), - rightPlan.getScope().getTables()); + Optional.empty()); if (type == RIGHT && asofCriteria == null) { root = ((JoinNode) root).flip(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java index 21f3df06ec7a8..c09710346ed09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneJoinColumns.java @@ -48,8 +48,6 @@ protected Optional pushDownProjectOff( filteredCopy(joinNode.getLeftOutputSymbols(), referencedOutputs::contains), filteredCopy(joinNode.getRightOutputSymbols(), referencedOutputs::contains), joinNode.getFilter(), - joinNode.isSpillable(), - joinNode.getLeftTables(), - joinNode.getRightTables())); + joinNode.isSpillable())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index aeaf4d0dbb7c9..09994aaeb7704 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -168,8 +168,8 @@ public JoinNode( rightOutputSymbols, filter, spillable, - ImmutableSet.of(), - ImmutableSet.of()); + leftChild.getInputTables(), + rightChild.getInputTables()); } // only used for deserialize diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index c6bda2232621c..dff5887961994 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -43,10 +43,12 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -114,6 +116,13 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visitTableScan(this, context); } + @Override + public Set getInputTables() { + Set tables = new HashSet<>(); + tables.add(alias != null ? alias : new Identifier(qualifiedObjectName.getObjectName())); + return tables; + } + @Override public List getChildren() { return ImmutableList.of(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 5450c9b60f8fc..91b52a450e128 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinConstraint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; @@ -86,8 +87,8 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set leadingTables = leading.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); - Set leftTables = node.getLeftTables(); - Set rightTables = node.getRightTables(); + Set leftHand = node.getLeftTables(); + Set rightHand = node.getRightTables(); Set totalJoinTables = ImmutableSet.of(); // join conjunctions @@ -96,21 +97,25 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { - // do something + /* + SELECT * + FROM A + LEFT JOIN B ON A.id = B.id + LEFT JOIN C ON A.id = C.id; + + join conjunction A.id = C.id（A ⋈ B ⋈ C）, and join table set is {A, B, C}. + */ + equiJoinTables = Sets.union(equiJoinTables, leftHand); } leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); - // leading.putConditionJoinType(expression, join.getJoinType()); + leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); } - collectJoinConstraintList(leading, leftTables, rightTables, node, totalJoinTables); + // join constraint + collectJoinConstraintList(leading, leftHand, rightHand, node, totalJoinTables); return node; } - // @Override - // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { - // return visitTwoChildProcess(node, context); - // } - private void collectJoinConstraintList( LeadingHint leading, Set leftHand, @@ -123,9 +128,68 @@ private void collectJoinConstraintList( return; } if (join.getJoinType() == JoinNode.JoinType.FULL) { - // full join + JoinConstraint joinConstraint = + new JoinConstraint(leftHand, rightHand, leftHand, rightHand, JoinNode.JoinType.FULL); + leading.getJoinConstraintList().add(joinConstraint); return; } + Set minLeftHand = Sets.intersection(joinTables, leftHand); + Set innerJoinTables = + Sets.intersection(totalTables, leading.getInnerJoinTables()); + Set filterAndInnerBelow = Sets.union(joinTables, innerJoinTables); + Set minRightHand = Sets.intersection(filterAndInnerBelow, rightHand); + + for (JoinConstraint other : leading.getJoinConstraintList()) { + if (other.getJoinType() == JoinNode.JoinType.FULL) { + if (isOverlap(leftHand, other.getLeftHand()) + || isOverlap(leftHand, other.getRightHand())) { + minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); + minLeftHand = Sets.union(minLeftHand, other.getRightHand()); + } + + if (isOverlap(rightHand, other.getLeftHand()) + || isOverlap(rightHand, other.getRightHand())) { + minRightHand = Sets.union(minRightHand, other.getLeftHand()); + minRightHand = Sets.union(minRightHand, other.getRightHand()); + } + /* Needn't do anything else with the full join */ + continue; + } + + // 当前join的左表包含之前某个join的右表 & join条件包含之前某个join的右表 + if (isOverlap(leftHand, other.getRightHand()) + && isOverlap(joinTables, other.getRightHand())) { + minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); + minLeftHand = Sets.union(minLeftHand, other.getRightHand()); + } + + // 当前join的右表包含之前某个join的右表 + if (isOverlap(rightHand, other.getRightHand())) { + if (isOverlap(joinTables, other.getRightHand()) + || !isOverlap(joinTables, other.getMinLeftHand())) { + minRightHand = Sets.union(minRightHand, other.getLeftHand()); + minRightHand = Sets.union(minRightHand, other.getRightHand()); + } + } + } + if (minLeftHand.isEmpty()) { + minLeftHand = leftHand; + } + if (minRightHand.isEmpty()) { + minRightHand = rightHand; + } + + JoinConstraint joinConstraint = + new JoinConstraint( + minLeftHand, minRightHand, minLeftHand, minRightHand, join.getJoinType()); + leading.getJoinConstraintList().add(joinConstraint); + } + + private boolean isOverlap(Set set1, Set set2) { + if (set1 == null || set2 == null || set1.isEmpty() || set2.isEmpty()) { + return false; + } + return !Sets.intersection(set1, set2).isEmpty(); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 5d34f62173f99..7b296a60a4d23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -976,9 +976,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { leftSource.getOutputSymbols(), rightSource.getOutputSymbols(), newJoinFilter, - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } JoinNode outputJoinNode = (JoinNode) output; @@ -1381,9 +1379,7 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } if (canConvertToLeftJoin) { return new JoinNode( @@ -1396,9 +1392,7 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } else { // temp fix because right join is not supported for now. return node; @@ -1433,9 +1427,7 @@ private JoinNode tryNormalizeToOuterToInnerJoin(JoinNode node, Expression inheri node.getLeftOutputSymbols(), node.getRightOutputSymbols(), node.getFilter(), - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()); + node.isSpillable()); } private boolean canConvertOuterToInner( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 0b1a434d3cbc4..9a67bbe030270 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -878,9 +878,7 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { newLeftOutputSymbols, newRightOutputSymbols, newFilter, - node.isSpillable(), - node.getLeftTables(), - node.getRightTables()), + node.isSpillable()), outputMapping); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java new file mode 100644 index 0000000000000..547929479dc20 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java @@ -0,0 +1,80 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; + +import java.util.Set; + +public class JoinConstraint { + private final Set minLeftHand; + private final Set minRightHand; + + private final Set leftHand; + private final Set rightHand; + + private final JoinNode.JoinType joinType; + + private boolean isReversed; + + public JoinConstraint( + Set minLeftHand, + Set minRightHand, + Set leftHand, + Set rightHand, + JoinNode.JoinType joinType) { + this.minLeftHand = minLeftHand; + this.minRightHand = minRightHand; + this.leftHand = leftHand; + this.rightHand = rightHand; + this.joinType = joinType; + } + + public JoinNode.JoinType getJoinType() { + return joinType; + } + + public Set getLeftHand() { + return leftHand; + } + + public Set getRightHand() { + return rightHand; + } + + public Set getMinLeftHand() { + return minLeftHand; + } + + public Set getMinRightHand() { + return minRightHand; + } + + public void setReversed(boolean reversed) { + isReversed = reversed; + } + + public boolean isReversed() { + return isReversed; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 856efcec77f01..5e65fb3943826 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -25,15 +25,14 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.tsfile.utils.Pair; @@ -59,6 +58,10 @@ public class LeadingHint extends JoinOrderHint { private Set innerJoinTables = ImmutableSet.of(); + private final List joinConstraintList = new ArrayList<>(); + + private final Map conditionJoinType = Maps.newLinkedHashMap(); + public LeadingHint(List parameters) { super(hintName); // /* leading(t3 {}) 会报错 @@ -83,10 +86,18 @@ public List getTables() { return tables; } + public List getJoinConstraintList() { + return joinConstraintList; + } + public List, Expression>> getFilters() { return filters; } + public void putConditionJoinType(Expression filter, JoinNode.JoinType joinType) { + conditionJoinType.put(filter, joinType); + } + public Map getRelationToScanMap() { return relationToScanMap; } @@ -122,11 +133,10 @@ public PlanNode generateLeadingJoinPlan() { PlanNode finalJoin = stack.pop(); // we want all filters been removed - // if (Utils.enableAssert && !filters.isEmpty()) { - // throw new IllegalStateException( - // "Leading hint process failed: filter should be empty, but meet: " + filters - // ); - // } + if (!filters.isEmpty()) { + throw new IllegalStateException( + "Leading hint process failed: filter should be empty, but meet: " + filters); + } if (finalJoin == null) { throw new IoTDBRuntimeException( "final join plan should not be null", INTERNAL_SERVER_ERROR.getStatusCode()); @@ -209,10 +219,11 @@ public PlanNode getPlanByName(String name) { private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { List conditions = getJoinConditions(getFilters(), leftChild, rightChild); - JoinNode.JoinType joinType = computeJoinType(); + JoinNode.JoinType joinType = + computeJoinType(leftChild.getInputTables(), rightChild.getInputTables(), conditions); if (joinType == null) { return null; - } else if (!isConditionJoinTypeMatched()) { + } else if (!isConditionJoinTypeMatched(conditions, joinType)) { return null; } @@ -246,9 +257,7 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { leftOutputSymbols, rightOutputSymbols, Optional.empty(), - Optional.empty(), - getTables(leftChild), - getTables(rightChild)); + Optional.empty()); } private List getJoinConditions( @@ -256,7 +265,7 @@ private List getJoinConditions( List joinConditions = new ArrayList<>(); for (int i = filters.size() - 1; i >= 0; i--) { Pair, Expression> filterPair = filters.get(i); - Set tables = Sets.union(getTables(left), getTables(right)); + Set tables = Sets.union(left.getInputTables(), right.getInputTables()); // left one is smaller set if (tables.containsAll(filterPair.left)) { joinConditions.add(filterPair.right); @@ -273,7 +282,7 @@ private PlanNode makeFilterPlanIfExist( } for (int i = filters.size() - 1; i >= 0; i--) { Pair, Expression> filterPair = filters.get(i); - if (getTables(plan).containsAll(filterPair.left)) { + if (plan.getInputTables().containsAll(filterPair.left)) { plan = new FilterNode(plan.getPlanNodeId(), plan, filterPair.right); filters.remove(i); } @@ -281,26 +290,151 @@ private PlanNode makeFilterPlanIfExist( return plan; } - private Set getTables(PlanNode root) { - if (root instanceof JoinNode) { - return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); - } else if (root instanceof DeviceTableScanNode) { - return ImmutableSet.of( - new Identifier(((DeviceTableScanNode) root).getQualifiedObjectName().getObjectName())); - } else if (root instanceof FilterNode) { - return getTables(((FilterNode) root).getChild()); - } else if (root instanceof ProjectNode) { - return getTables(((ProjectNode) root).getChild()); - } else { + // private Set getTables(PlanNode root) { + // if (root instanceof JoinNode) { + // return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); + // } else if (root instanceof DeviceTableScanNode) { + // return ImmutableSet.of( + // new Identifier(((DeviceTableScanNode) + // root).getQualifiedObjectName().getObjectName())); + // } else if (root instanceof FilterNode) { + // return getTables(((FilterNode) root).getChild()); + // } else if (root instanceof ProjectNode) { + // return getTables(((ProjectNode) root).getChild()); + // } else { + // return null; + // } + // } + + public JoinNode.JoinType computeJoinType( + Set left, Set right, List conditions) { + Pair joinConstraintBooleanPair = + getJoinConstraint(Sets.union(left, right), left, right); + if (!joinConstraintBooleanPair.right) { return null; + } else if (joinConstraintBooleanPair.left == null) { + return JoinNode.JoinType.INNER; + } else { + JoinConstraint joinConstraint = joinConstraintBooleanPair.left; + if (joinConstraint.isReversed()) { + JoinNode.JoinType joinType = joinConstraint.getJoinType(); + if (joinType == JoinNode.JoinType.LEFT) { + return JoinNode.JoinType.RIGHT; + } else if (joinType == JoinNode.JoinType.RIGHT) { + return JoinNode.JoinType.LEFT; + } else { + return joinType; + } + } } - } - - public JoinNode.JoinType computeJoinType() { return JoinNode.JoinType.INNER; } - public boolean isConditionJoinTypeMatched() { + public boolean isConditionJoinTypeMatched( + List conditions, JoinNode.JoinType joinType) { + for (Expression condition : conditions) { + JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); + if (originalJoinType == joinType + || (originalJoinType == JoinNode.JoinType.LEFT && joinType == JoinNode.JoinType.RIGHT) + || (originalJoinType == JoinNode.JoinType.RIGHT && joinType == JoinNode.JoinType.LEFT)) { + continue; + } + return false; + } return true; } + + public Pair getJoinConstraint( + Set joinTables, Set leftHand, Set rightHand) { + boolean reversed = false; + boolean mustBeLeftJoin = false; + + JoinConstraint matchedJoinConstraint = null; + for (JoinConstraint joinConstraint : joinConstraintList) { + if (joinConstraint.getJoinType() == JoinNode.JoinType.FULL) { + if ((isEqual(joinConstraint.getLeftHand(), leftHand) + && isEqual(joinConstraint.getRightHand(), rightHand)) + || (isEqual(joinConstraint.getLeftHand(), rightHand) + && isEqual(joinConstraint.getRightHand(), leftHand))) { + if (matchedJoinConstraint != null) { + return new Pair<>(null, false); + } + matchedJoinConstraint = joinConstraint; + reversed = false; + break; + } else { + continue; + } + } + + if (!isOverlap(joinConstraint.getRightHand(), joinTables)) { + continue; + } + + if (joinTables.containsAll(joinConstraint.getMinRightHand())) { + continue; + } + + if ((joinConstraint.getMinLeftHand().containsAll(leftHand)) + && joinConstraint.getMinRightHand().containsAll(leftHand)) { + continue; + } + + if (joinConstraint.getMinLeftHand().containsAll(rightHand) + && joinConstraint.getMinRightHand().containsAll(rightHand)) { + continue; + } + + if (joinConstraint.getMinLeftHand().containsAll(leftHand) + && joinConstraint.getMinRightHand().containsAll(rightHand)) { + if (matchedJoinConstraint != null) { + return new Pair<>(null, false); + } + matchedJoinConstraint = joinConstraint; + reversed = false; + } else if (joinConstraint.getMinLeftHand().containsAll(rightHand) + && joinConstraint.getMinRightHand().containsAll(leftHand)) { + if (matchedJoinConstraint != null) { + return new Pair<>(null, false); + } + matchedJoinConstraint = joinConstraint; + reversed = true; + } else { + if (isOverlap(leftHand, joinConstraint.getMinRightHand()) + && isOverlap(rightHand, joinConstraint.getRightHand())) { + continue; + } + if (joinConstraint.getJoinType() == JoinNode.JoinType.LEFT + || isOverlap(joinTables, joinConstraint.getMinLeftHand())) { + return new Pair<>(null, false); + } + mustBeLeftJoin = true; + } + } + if (mustBeLeftJoin + && (matchedJoinConstraint == null + || matchedJoinConstraint.getJoinType() != JoinNode.JoinType.LEFT)) { + return new Pair<>(null, false); + } + // inner join + if (matchedJoinConstraint == null) { + return new Pair<>(null, true); + } + matchedJoinConstraint.setReversed(reversed); + return new Pair<>(matchedJoinConstraint, true); + } + + private boolean isEqual(Set set1, Set set2) { + if (set1 == null || set2 == null) { + return set1 == set2; + } + return set1.size() == set2.size() && set1.containsAll(set2); + } + + private boolean isOverlap(Set set1, Set set2) { + if (set1 == null || set2 == null || set1.isEmpty() || set2.isEmpty()) { + return false; + } + return !Sets.intersection(set1, set2).isEmpty(); + } } From 136e04dcc8dabf95cc54505b8e652904d64890d2 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 10 Feb 2026 09:28:28 +0800 Subject: [PATCH 16/35] fix leading hint --- ...TransformFilteringSemiJoinToInnerJoin.java | 5 +- .../relational/planner/node/JoinNode.java | 8 -- .../optimizations/CollectJoinConstraint.java | 22 ++-- .../relational/utils/hint/JoinConstraint.java | 10 -- .../relational/utils/hint/LeadingHint.java | 105 ++++++++---------- 5 files changed, 60 insertions(+), 90 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 1f32fdbffe157..10f2eb51dd0ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -130,7 +130,10 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { filteringSourceDistinct, ImmutableList.of( new JoinNode.EquiJoinClause( - semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), + semiJoin.getSourceJoinSymbol(), + semiJoin.getFilteringSourceJoinSymbol(), + semiJoin.getSource().getInputTables(), + semiJoin.getFilteringSource().getInputTables())), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 09994aaeb7704..a0521bb7e5373 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -501,14 +501,6 @@ public static class EquiJoinClause { private final Set rightTables; private final Set tables; - public EquiJoinClause(Symbol left, Symbol right) { - this.left = requireNonNull(left, "left is null"); - this.right = requireNonNull(right, "right is null"); - this.leftTables = ImmutableSet.of(); - this.rightTables = ImmutableSet.of(); - this.tables = ImmutableSet.of(); - } - public EquiJoinClause( Symbol left, Symbol right, Set leftTables, Set rightTables) { this.left = requireNonNull(left, "left is null"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 91b52a450e128..d1768e4eeca5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -97,15 +97,14 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { - /* - SELECT * - FROM A - LEFT JOIN B ON A.id = B.id - LEFT JOIN C ON A.id = C.id; - - join conjunction A.id = C.id（A ⋈ B ⋈ C）, and join table set is {A, B, C}. - */ - equiJoinTables = Sets.union(equiJoinTables, leftHand); + // leading.getFilters() is used to determine which join the join condition should apply + // to. + // For LEFT join, we need to add rightHand tables as well. In the getJoinConditions + // function, + // we ensure that the tables joined by the Join include both the tables in the join + // condition + // and the right tables of the outer join, so that the join condition can be applied. + equiJoinTables = Sets.union(equiJoinTables, rightHand); } leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); @@ -156,14 +155,15 @@ private void collectJoinConstraintList( continue; } - // 当前join的左表包含之前某个join的右表 & join条件包含之前某个join的右表 + // Current join's left table contains the right table of a previous join & join condition + // contains the right table of a previous join if (isOverlap(leftHand, other.getRightHand()) && isOverlap(joinTables, other.getRightHand())) { minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); minLeftHand = Sets.union(minLeftHand, other.getRightHand()); } - // 当前join的右表包含之前某个join的右表 + // Current join's right table contains the right table of a previous join if (isOverlap(rightHand, other.getRightHand())) { if (isOverlap(joinTables, other.getRightHand()) || !isOverlap(joinTables, other.getMinLeftHand())) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java index 547929479dc20..243308cb6dd67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java @@ -35,8 +35,6 @@ public class JoinConstraint { private final JoinNode.JoinType joinType; - private boolean isReversed; - public JoinConstraint( Set minLeftHand, Set minRightHand, @@ -69,12 +67,4 @@ public Set getMinLeftHand() { public Set getMinRightHand() { return minRightHand; } - - public void setReversed(boolean reversed) { - isReversed = reversed; - } - - public boolean isReversed() { - return isReversed; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 5e65fb3943826..5b4566b2451ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -220,7 +220,7 @@ public PlanNode getPlanByName(String name) { private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { List conditions = getJoinConditions(getFilters(), leftChild, rightChild); JoinNode.JoinType joinType = - computeJoinType(leftChild.getInputTables(), rightChild.getInputTables(), conditions); + computeJoinType(leftChild.getInputTables(), rightChild.getInputTables()); if (joinType == null) { return null; } else if (!isConditionJoinTypeMatched(conditions, joinType)) { @@ -238,10 +238,14 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { Symbol rightSymbol = Symbol.from(equality.getRight()); if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { - criteria.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); + criteria.add( + new JoinNode.EquiJoinClause( + leftSymbol, rightSymbol, leftChild.getInputTables(), rightChild.getInputTables())); } else if (leftOutputSymbols.contains(rightSymbol) && rightOutputSymbols.contains(leftSymbol)) { - criteria.add(new JoinNode.EquiJoinClause(rightSymbol, leftSymbol)); + criteria.add( + new JoinNode.EquiJoinClause( + rightSymbol, leftSymbol, leftChild.getInputTables(), rightChild.getInputTables())); } else { throw new IllegalArgumentException("Invalid join condition"); } @@ -266,7 +270,7 @@ private List getJoinConditions( for (int i = filters.size() - 1; i >= 0; i--) { Pair, Expression> filterPair = filters.get(i); Set tables = Sets.union(left.getInputTables(), right.getInputTables()); - // left one is smaller set + // it should contain all tables in join conjunctions & right tables if it's left join if (tables.containsAll(filterPair.left)) { joinConditions.add(filterPair.right); filters.remove(i); @@ -290,53 +294,25 @@ private PlanNode makeFilterPlanIfExist( return plan; } - // private Set getTables(PlanNode root) { - // if (root instanceof JoinNode) { - // return Sets.union(((JoinNode) root).getLeftTables(), ((JoinNode) root).getRightTables()); - // } else if (root instanceof DeviceTableScanNode) { - // return ImmutableSet.of( - // new Identifier(((DeviceTableScanNode) - // root).getQualifiedObjectName().getObjectName())); - // } else if (root instanceof FilterNode) { - // return getTables(((FilterNode) root).getChild()); - // } else if (root instanceof ProjectNode) { - // return getTables(((ProjectNode) root).getChild()); - // } else { - // return null; - // } - // } - - public JoinNode.JoinType computeJoinType( - Set left, Set right, List conditions) { + public JoinNode.JoinType computeJoinType(Set left, Set right) { Pair joinConstraintBooleanPair = getJoinConstraint(Sets.union(left, right), left, right); if (!joinConstraintBooleanPair.right) { return null; - } else if (joinConstraintBooleanPair.left == null) { + } + if (joinConstraintBooleanPair.left == null) { return JoinNode.JoinType.INNER; - } else { - JoinConstraint joinConstraint = joinConstraintBooleanPair.left; - if (joinConstraint.isReversed()) { - JoinNode.JoinType joinType = joinConstraint.getJoinType(); - if (joinType == JoinNode.JoinType.LEFT) { - return JoinNode.JoinType.RIGHT; - } else if (joinType == JoinNode.JoinType.RIGHT) { - return JoinNode.JoinType.LEFT; - } else { - return joinType; - } - } } - return JoinNode.JoinType.INNER; + + JoinConstraint joinConstraint = joinConstraintBooleanPair.left; + return joinConstraint.getJoinType(); } public boolean isConditionJoinTypeMatched( List conditions, JoinNode.JoinType joinType) { for (Expression condition : conditions) { JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); - if (originalJoinType == joinType - || (originalJoinType == JoinNode.JoinType.LEFT && joinType == JoinNode.JoinType.RIGHT) - || (originalJoinType == JoinNode.JoinType.RIGHT && joinType == JoinNode.JoinType.LEFT)) { + if (originalJoinType == joinType) { continue; } return false; @@ -344,9 +320,14 @@ public boolean isConditionJoinTypeMatched( return true; } + /** + * try to get join constraint. If it can not be found, it means join is inner join + * + * @return boolean value used for judging whether the join is legal, and should this join need to + * reverse + */ public Pair getJoinConstraint( Set joinTables, Set leftHand, Set rightHand) { - boolean reversed = false; boolean mustBeLeftJoin = false; JoinConstraint matchedJoinConstraint = null; @@ -360,54 +341,59 @@ && isEqual(joinConstraint.getRightHand(), leftHand))) { return new Pair<>(null, false); } matchedJoinConstraint = joinConstraint; - reversed = false; break; } else { continue; } } - if (!isOverlap(joinConstraint.getRightHand(), joinTables)) { + // join操作完全不涉及最小右表约束，跳过 + if (!isOverlap(joinConstraint.getMinRightHand(), joinTables)) { continue; } - if (joinTables.containsAll(joinConstraint.getMinRightHand())) { + // 如果当前join的所有表都在约束的"右边界"内，说明这个约束当前还没法应用，跳过 + if (joinConstraint.getMinRightHand().containsAll(joinTables)) { continue; } - if ((joinConstraint.getMinLeftHand().containsAll(leftHand)) - && joinConstraint.getMinRightHand().containsAll(leftHand)) { + // 当前join的左表包含最小左右约束，跳过 + if (leftHand.containsAll(joinConstraint.getMinLeftHand()) + && leftHand.containsAll(joinConstraint.getMinRightHand())) { continue; } - if (joinConstraint.getMinLeftHand().containsAll(rightHand) - && joinConstraint.getMinRightHand().containsAll(rightHand)) { + // 当前join的右表包含最小左右约束，跳过 + if (rightHand.containsAll(joinConstraint.getMinLeftHand()) + && rightHand.containsAll(joinConstraint.getMinRightHand())) { continue; } - if (joinConstraint.getMinLeftHand().containsAll(leftHand) - && joinConstraint.getMinRightHand().containsAll(rightHand)) { - if (matchedJoinConstraint != null) { - return new Pair<>(null, false); - } - matchedJoinConstraint = joinConstraint; - reversed = false; - } else if (joinConstraint.getMinLeftHand().containsAll(rightHand) - && joinConstraint.getMinRightHand().containsAll(leftHand)) { + if (leftHand.containsAll(joinConstraint.getMinLeftHand()) + && rightHand.containsAll(joinConstraint.getMinRightHand())) { + // 当前join的左表包含最小左约束，右表包含最小右约束 if (matchedJoinConstraint != null) { return new Pair<>(null, false); } matchedJoinConstraint = joinConstraint; - reversed = true; + } else if (rightHand.containsAll(joinConstraint.getMinLeftHand()) + && leftHand.containsAll(joinConstraint.getMinRightHand())) { + // 不支持left join转换为right join + return new Pair<>(null, false); } else { + // 当前join的左右表和最小右约束均有交集，跳过 + // minRightHand中的表被分散在了join的两边 if (isOverlap(leftHand, joinConstraint.getMinRightHand()) - && isOverlap(rightHand, joinConstraint.getRightHand())) { + && isOverlap(rightHand, joinConstraint.getMinRightHand())) { continue; } - if (joinConstraint.getJoinType() == JoinNode.JoinType.LEFT + // LEFT JOIN且joinTables 与 minLeftHand 有交集。如果当前JOIN执行完毕，再和minRightHand连接， + // 无法再满足 "minLeftHand作为左边界"的要求 + if (joinConstraint.getJoinType() != JoinNode.JoinType.LEFT || isOverlap(joinTables, joinConstraint.getMinLeftHand())) { return new Pair<>(null, false); } + // LEFT JOIN 且joinTables 与 minLeftHand 无交集 mustBeLeftJoin = true; } } @@ -420,7 +406,6 @@ && isOverlap(rightHand, joinConstraint.getRightHand())) { if (matchedJoinConstraint == null) { return new Pair<>(null, true); } - matchedJoinConstraint.setReversed(reversed); return new Pair<>(matchedJoinConstraint, true); } From 1f635750a322d08cc8a906a1f9187c6b8e5d31d1 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 24 Feb 2026 11:41:59 +0800 Subject: [PATCH 17/35] fix leader/follower bug --- .../analyzer/StatementAnalyzer.java | 21 ++++++++++--------- .../planner/distribute/AddExchangeNodes.java | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 3ded3afd2e24e..5bbed730ad4f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1558,16 +1558,17 @@ private void addHint(String hintName, List parameters, Map .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) .map(entry -> entry.getValue().getSuffix()) .collect(toImmutableList()); - for (String table : parameters) { - if (!existingTables.contains(table)) { - continue; - } - Hint hint = definition.createHint(ImmutableList.of(table)); - String hintKey = hint.getKey(); - if (!hintMap.containsKey(hintKey)) { - hintMap.put(hintKey, hint); - } - } + + List tablesToProcess = parameters == null ? existingTables : parameters; + + tablesToProcess.stream() + .filter(table -> parameters == null || existingTables.contains(table)) + .forEach( + table -> { + Hint hint = definition.createHint(ImmutableList.of(table)); + String hintKey = hint.getKey(); + hintMap.putIfAbsent(hintKey, hint); + }); } else { Hint hint = definition.createHint(parameters); String hintKey = hint.getKey(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index fb72026e9792a..dc7834cc21924 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -109,7 +109,7 @@ public PlanNode visitTableScan( ReplicaHint hint = findReplicaHint(node, context.hintMap); // Early return for simple cases - if (context.hintMap == null || regionReplicaSet == null || hint == null) { + if (regionReplicaSet == null || hint == null) { context.nodeDistributionMap.put( node.getPlanNodeId(), new NodeDistribution(SAME_WITH_ALL_CHILDREN, regionReplicaSet)); return node; From 573f8b473dfabb989781e4b2bf552e8f1d5d122d Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 25 Feb 2026 16:17:51 +0800 Subject: [PATCH 18/35] handle filter & sort & mergesort node for leading hint --- .../analyzer/StatementAnalyzer.java | 21 ++++++---- .../optimizations/CollectJoinConstraint.java | 41 +++++++++++++++---- .../optimizations/LogicalOptimizeFactory.java | 6 +-- .../PushPredicateIntoTableScan.java | 19 +-------- 4 files changed, 53 insertions(+), 34 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 5bbed730ad4f7..d749c5925a725 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1552,15 +1552,13 @@ private List intersect(List a, List b) { private void addHint(String hintName, List parameters, Map hintMap) { HintFactory definition = HINT_DEFINITIONS.get(hintName); if (definition != null) { - if (definition.shouldExpandParameters()) { - List existingTables = - analysis.getRelationNames().entrySet().stream() - .filter(entry -> !analysis.isAliased(entry.getKey().getNode())) - .map(entry -> entry.getValue().getSuffix()) - .collect(toImmutableList()); + List existingTables = + analysis.getRelationNames().values().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); + if (definition.shouldExpandParameters()) { List tablesToProcess = parameters == null ? existingTables : parameters; - tablesToProcess.stream() .filter(table -> parameters == null || existingTables.contains(table)) .forEach( @@ -1570,6 +1568,15 @@ private void addHint(String hintName, List parameters, Map hintMap.putIfAbsent(hintKey, hint); }); } else { + // Skip if parameters contain tables that don't exist in the query + if (parameters != null) { + boolean hasInvalidTable = + parameters.stream().anyMatch(table -> !existingTables.contains(table)); + if (hasInvalidTable) { + return; + } + } + Hint hint = definition.createHint(parameters); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index d1768e4eeca5a..9ced7f8aa22ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -25,7 +25,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinConstraint; @@ -71,12 +74,38 @@ public PlanNode visitPlan(PlanNode node, Context context) { @Override public PlanNode visitDeviceTableScan(DeviceTableScanNode node, Context context) { - String tableName = node.getQualifiedObjectName().getObjectName(); + Identifier alias = node.getAlias(); + String tableName = + alias != null ? alias.getValue() : node.getQualifiedObjectName().getObjectName(); LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); leading.getRelationToScanMap().put(tableName, node); return node; } + @Override + public PlanNode visitMergeSort(MergeSortNode node, Context context) { + return visitSingleTableNode(node, context); + } + + public PlanNode visitFilter(FilterNode node, Context context) { + return visitSingleTableNode(node, context); + } + + public PlanNode visitSort(SortNode node, Context context) { + return visitSingleTableNode(node, context); + } + + private PlanNode visitSingleTableNode(PlanNode node, Context context) { + visitPlan(node, context); + Set tables = node.getInputTables(); + if (tables != null && tables.size() == 1) { + Identifier tableName = tables.iterator().next(); + LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); + leading.getRelationToScanMap().put(tableName.getValue(), node); + } + return node; + } + @Override public PlanNode visitJoin(JoinNode node, Context context) { for (PlanNode child : node.getChildren()) { @@ -98,12 +127,10 @@ public PlanNode visitJoin(JoinNode node, Context context) { totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { // leading.getFilters() is used to determine which join the join condition should apply - // to. - // For LEFT join, we need to add rightHand tables as well. In the getJoinConditions - // function, - // we ensure that the tables joined by the Join include both the tables in the join - // condition - // and the right tables of the outer join, so that the join condition can be applied. + // to. For LEFT join, we need to add rightHand tables as well. In the getJoinConditions + // function, we ensure that the tables joined by the Join include both the tables in the + // join condition and the right tables of the outer join, so that the join condition + // can be applied. equiJoinTables = Sets.union(equiJoinTables, rightHand); } leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index b98d5dfade10e..4386a9255f85f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -286,8 +286,6 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new RemoveRedundantIdentityProjections())) .build()), simplifyOptimizer, - new CollectJoinConstraint(), - new LeadingJoinOptimizer(), new UnaliasSymbolReferences(plannerContext.getMetadata()), new IterativeOptimizer( plannerContext, @@ -392,7 +390,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new MergeLimitWithSort(), new MergeLimitOverProjectWithSort(), new PushTopKThroughUnion())), - new ParallelizeGrouping()); + new ParallelizeGrouping(), + new CollectJoinConstraint(), + new LeadingJoinOptimizer()); this.planOptimizers = optimizerBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 7b296a60a4d23..82ceed3618351 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -68,7 +68,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; @@ -854,15 +853,6 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { node.getRightChild().getOutputSymbols().stream() .collect(toImmutableMap(key -> key, Symbol::toSymbolReference))); - // Build a map from (left, right) symbols to their (leftTables, rightTables) - // This preserves table info when rebuilding EquiJoinClause - Map, Pair, Set>> symbolToTablesMap = - node.getCriteria().stream() - .collect( - toImmutableMap( - clause -> new Pair<>(clause.getLeft(), clause.getRight()), - clause -> new Pair<>(clause.getLeftTables(), clause.getRightTables()))); - // Create new projections for the new join clauses List equiJoinClauses = new ArrayList<>(); ImmutableList.Builder joinFilterBuilder = ImmutableList.builder(); @@ -886,14 +876,9 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { rightProjections.put(rightSymbol, rightExpression); } - // Retrieve leftTables and rightTables from original criteria - Pair, Set> tables = - symbolToTablesMap.getOrDefault( - new Pair<>(leftSymbol, rightSymbol), - new Pair<>(ImmutableSet.of(), ImmutableSet.of())); - equiJoinClauses.add( - new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, tables.left, tables.right)); + new JoinNode.EquiJoinClause( + leftSymbol, rightSymbol, node.getLeftTables(), node.getRightTables())); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; From 78c4726c525e26ed5993c88ddc1111017631b57a Mon Sep 17 00:00:00 2001 From: shizy Date: Wed, 25 Feb 2026 19:56:47 +0800 Subject: [PATCH 19/35] EquiJoinClause left & right table --- .../relational/planner/RelationPlanner.java | 83 +++++++++++++++---- ...TransformFilteringSemiJoinToInnerJoin.java | 18 +++- .../relational/planner/node/JoinNode.java | 68 +++++---------- .../planner/optimizations/JoinUtils.java | 38 +++++++++ .../PushPredicateIntoTableScan.java | 17 +++- .../UnaliasSymbolReferences.java | 4 +- .../relational/utils/hint/LeadingHint.java | 35 +++++++- 7 files changed, 192 insertions(+), 71 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 4854e9f45b1c8..bfae738667aed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -142,7 +142,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -519,9 +518,28 @@ If casts are redundant (due to column type and common type being equal), rightCoercions.put(rightOutput, right.getSymbol(rightField).toSymbolReference()); rightJoinColumns.put(identifier, rightOutput); - Set leftTables = new HashSet<>(left.getScope().getTables()); - Set rightTables = new HashSet<>(right.getScope().getTables()); - clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput, leftTables, rightTables)); + // Extract tables from the actual fields used in the join condition + Field leftFieldObj = left.getScope().getRelationType().getFieldByIndex(leftField); + Identifier leftTable = + extractTableName(leftFieldObj) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for field %s in JOIN USING clause", + leftFieldObj.getName()))); + + Field rightFieldObj = right.getScope().getRelationType().getFieldByIndex(rightField); + Identifier rightTable = + extractTableName(rightFieldObj) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for field %s in JOIN USING clause", + rightFieldObj.getName()))); + + clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput, leftTable, rightTable)); } ProjectNode leftCoercion = @@ -745,22 +763,37 @@ public RelationPlan planJoin( SymbolsExtractor.extractNames( rightComparisonExpressions.get(i), analysis.getColumnReferences()); - // Convert QualifiedName to Identifier (table name is the first part) - Set leftTables = new HashSet<>(); - for (QualifiedName name : leftDependencies) { - if (name.getPrefix().isPresent()) { - leftTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } - } - Set rightTables = new HashSet<>(); - for (QualifiedName name : rightDependencies) { - if (name.getPrefix().isPresent()) { - rightTables.add(new Identifier(name.getPrefix().get().getSuffix())); - } + if (leftDependencies.size() != 1 || rightDependencies.size() != 1) { + throw new IllegalStateException("Cannot find source table for symbol " + leftSymbol); } + QualifiedName leftQualifiedName = leftDependencies.iterator().next(); + QualifiedName rightQualifiedName = rightDependencies.iterator().next(); + + Identifier leftTable = + leftQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause", + leftSymbol))); + + Identifier rightTable = + rightQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause", + rightSymbol))); + equiClauses.add( - new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTables, rightTables)); + new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTable, rightTable)); } else { postInnerJoinConditions.add( new ComparisonExpression( @@ -1673,4 +1706,20 @@ public Map getVariableDefinitions() { return variableDefinitions; } } + + /** + * Extracts the table name from a field. Priority: 1) relation alias (if exists), 2) origin table + * name. + * + * @param field the field to extract the table name from + * @return the table identifier, or empty if not found + */ + private static Optional extractTableName(Field field) { + Optional fromAlias = + field.getRelationAlias().map(alias -> new Identifier(alias.getSuffix())); + if (fromAlias.isPresent()) { + return fromAlias; + } + return field.getOriginTable().map(originTable -> new Identifier(originTable.getObjectName())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 10f2eb51dd0ef..910fe9e9ce8d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; @@ -132,8 +133,21 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { new JoinNode.EquiJoinClause( semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol(), - semiJoin.getSource().getInputTables(), - semiJoin.getFilteringSource().getInputTables())), + JoinUtils.findSourceTable(semiJoin.getSource(), semiJoin.getSourceJoinSymbol()) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in semi-join source", + semiJoin.getSourceJoinSymbol()))), + JoinUtils.findSourceTable( + semiJoin.getFilteringSource(), semiJoin.getFilteringSourceJoinSymbol()) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in semi-join filtering source", + semiJoin.getFilteringSourceJoinSymbol()))))), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index a0521bb7e5373..a7049702a2fce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -286,16 +285,8 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), byteBuffer); Symbol.serialize(equiJoinClause.getRight(), byteBuffer); - Set leftTables = equiJoinClause.getLeftTables(); - ReadWriteIOUtils.write(leftTables.size(), byteBuffer); - for (Identifier identifier : leftTables) { - identifier.serialize(byteBuffer); - } - Set rightTables = equiJoinClause.getRightTables(); - ReadWriteIOUtils.write(rightTables.size(), byteBuffer); - for (Identifier identifier : rightTables) { - identifier.serialize(byteBuffer); - } + equiJoinClause.getLeftTable().serialize(byteBuffer); + equiJoinClause.getRightTable().serialize(byteBuffer); } if (asofCriteria.isPresent()) { @@ -338,16 +329,8 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), stream); Symbol.serialize(equiJoinClause.getRight(), stream); - Set leftTables = equiJoinClause.getLeftTables(); - ReadWriteIOUtils.write(leftTables.size(), stream); - for (Identifier identifier : leftTables) { - identifier.serialize(stream); - } - Set rightTables = equiJoinClause.getRightTables(); - ReadWriteIOUtils.write(rightTables.size(), stream); - for (Identifier identifier : rightTables) { - identifier.serialize(stream); - } + equiJoinClause.getLeftTable().serialize(stream); + equiJoinClause.getRightTable().serialize(stream); } if (asofCriteria.isPresent()) { @@ -387,17 +370,9 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { while (size-- > 0) { Symbol left = Symbol.deserialize(byteBuffer); Symbol right = Symbol.deserialize(byteBuffer); - int leftTablesSize = ReadWriteIOUtils.readInt(byteBuffer); - Set leftTables = new HashSet<>(leftTablesSize); - while (leftTablesSize-- > 0) { - leftTables.add(new Identifier(byteBuffer)); - } - int rightTablesSize = ReadWriteIOUtils.readInt(byteBuffer); - Set rightTables = new HashSet<>(rightTablesSize); - while (rightTablesSize-- > 0) { - rightTables.add(new Identifier(byteBuffer)); - } - criteria.add(new EquiJoinClause(left, right, leftTables, rightTables)); + Identifier leftTable = new Identifier(byteBuffer); + Identifier rightTable = new Identifier(byteBuffer); + criteria.add(new EquiJoinClause(left, right, leftTable, rightTable)); } Optional asofJoinClause = Optional.empty(); @@ -497,17 +472,16 @@ public String toString() { public static class EquiJoinClause { private final Symbol left; private final Symbol right; - private final Set leftTables; - private final Set rightTables; + private final Identifier leftTable; + private final Identifier rightTable; private final Set tables; - public EquiJoinClause( - Symbol left, Symbol right, Set leftTables, Set rightTables) { + public EquiJoinClause(Symbol left, Symbol right, Identifier leftTable, Identifier rightTable) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTables = requireNonNull(leftTables, "leftTables is null"); - this.rightTables = requireNonNull(rightTables, "rightTables is null"); - this.tables = Sets.union(leftTables, rightTables); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); + this.tables = ImmutableSet.of(leftTable, rightTable); } public Symbol getLeft() { @@ -518,12 +492,12 @@ public Symbol getRight() { return right; } - public Set getLeftTables() { - return leftTables; + public Identifier getLeftTable() { + return leftTable; } - public Set getRightTables() { - return rightTables; + public Identifier getRightTable() { + return rightTable; } public Set getTables() { @@ -536,7 +510,7 @@ public ComparisonExpression toExpression() { } public EquiJoinClause flip() { - return new EquiJoinClause(right, left, rightTables, leftTables); + return new EquiJoinClause(right, left, rightTable, leftTable); } public static List flipBatch(List input) { @@ -559,13 +533,13 @@ public boolean equals(Object obj) { return Objects.equals(this.left, other.left) && Objects.equals(this.right, other.right) - && Objects.equals(this.leftTables, other.leftTables) - && Objects.equals(this.rightTables, other.rightTables); + && Objects.equals(this.leftTable, other.leftTable) + && Objects.equals(this.rightTable, other.rightTable); } @Override public int hashCode() { - return Objects.hash(left, right, leftTables, rightTables); + return Objects.hash(left, right, leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java index 9e751d2dd4d31..6542b46df93bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java @@ -19,18 +19,22 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.planner.EqualityInference; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.Collection; import java.util.Objects; +import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -404,4 +408,38 @@ public Expression getPostJoinPredicate() { return postJoinPredicate; } } + + /** + * Finds the source table for a given symbol by traversing the plan tree. Returns Optional.empty() + * if the symbol cannot be traced to a specific table. + * + * @param node the plan node to search + * @param symbol the symbol to find the source table for + * @return the table identifier if found, Optional.empty() otherwise + */ + public static Optional findSourceTable(PlanNode node, Symbol symbol) { + if (node instanceof DeviceTableScanNode) { + DeviceTableScanNode scanNode = (DeviceTableScanNode) node; + if (scanNode.getOutputSymbols().contains(symbol)) { + Identifier alias = scanNode.getAlias(); + if (alias != null) { + return Optional.of(alias); + } + return Optional.of(new Identifier(scanNode.getQualifiedObjectName().getObjectName())); + } + return Optional.empty(); + } + + // For other node types, recursively check children + for (PlanNode child : node.getChildren()) { + if (child.getOutputSymbols().contains(symbol)) { + Optional result = findSourceTable(child, symbol); + if (result.isPresent()) { + return result; + } + } + } + + return Optional.empty(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 82ceed3618351..f4ced2b977ef4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -878,7 +878,22 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { equiJoinClauses.add( new JoinNode.EquiJoinClause( - leftSymbol, rightSymbol, node.getLeftTables(), node.getRightTables())); + leftSymbol, + rightSymbol, + JoinUtils.findSourceTable(node.getLeftChild(), leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in join left child", + leftSymbol))), + JoinUtils.findSourceTable(node.getRightChild(), rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in join right child", + rightSymbol))))); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 9a67bbe030270..2fe01ce4c9bd0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -835,8 +835,8 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { new JoinNode.EquiJoinClause( mapper.map(clause.getLeft()), mapper.map(clause.getRight()), - ImmutableSet.copyOf(clause.getLeftTables()), - ImmutableSet.copyOf(clause.getRightTables()))); + clause.getLeftTable(), + clause.getRightTable())); } List newCriteria = builder.build(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 5b4566b2451ca..31d0f35026379 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; @@ -240,12 +241,42 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { criteria.add( new JoinNode.EquiJoinClause( - leftSymbol, rightSymbol, leftChild.getInputTables(), rightChild.getInputTables())); + leftSymbol, + rightSymbol, + JoinUtils.findSourceTable(leftChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + leftSymbol))), + JoinUtils.findSourceTable(rightChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + rightSymbol))))); } else if (leftOutputSymbols.contains(rightSymbol) && rightOutputSymbols.contains(leftSymbol)) { criteria.add( new JoinNode.EquiJoinClause( - rightSymbol, leftSymbol, leftChild.getInputTables(), rightChild.getInputTables())); + rightSymbol, + leftSymbol, + JoinUtils.findSourceTable(leftChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + rightSymbol))), + JoinUtils.findSourceTable(rightChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + leftSymbol))))); } else { throw new IllegalArgumentException("Invalid join condition"); } From b63c2d1f0485cbff5b55824463ed31b9ac75cf24 Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 26 Feb 2026 22:23:00 +0800 Subject: [PATCH 20/35] leading hint for asof join --- .../relational/planner/RelationPlanner.java | 78 +++++++------ .../relational/planner/node/JoinNode.java | 41 ++++++- .../optimizations/CollectJoinConstraint.java | 21 +++- .../relational/utils/hint/LeadingHint.java | 108 ++++++++++++++++-- .../relational/analyzer/AsofJoinTest.java | 40 ++++++- .../assertions/AsofJoinClauseProvider.java | 14 ++- .../assertions/EquiJoinClauseProvider.java | 17 +-- .../planner/assertions/JoinMatcher.java | 10 +- .../planner/assertions/PlanMatchPattern.java | 17 ++- 9 files changed, 263 insertions(+), 83 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index bfae738667aed..7b9354297e23c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -740,6 +740,41 @@ public RelationPlan planJoin( rightPlanBuilder = rightCoercions.getSubPlan(); for (int i = 0; i < leftComparisonExpressions.size(); i++) { + // Extract tables from expressions + Set leftDependencies = + SymbolsExtractor.extractNames( + leftComparisonExpressions.get(i), analysis.getColumnReferences()); + Set rightDependencies = + SymbolsExtractor.extractNames( + rightComparisonExpressions.get(i), analysis.getColumnReferences()); + + if (leftDependencies.size() != 1 || rightDependencies.size() != 1) { + throw new IllegalStateException("Cannot find source table for symbol"); + } + + QualifiedName leftQualifiedName = leftDependencies.iterator().next(); + QualifiedName rightQualifiedName = rightDependencies.iterator().next(); + + Identifier leftTable = + leftQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause"))); + + Identifier rightTable = + rightQualifiedName + .getPrefix() + .map(prefix -> new Identifier(prefix.getSuffix())) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in JOIN ON clause"))); + if (asofCriteria != null && i == 0) { Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); @@ -747,7 +782,11 @@ public RelationPlan planJoin( asofJoinClause = Optional.of( new JoinNode.AsofJoinClause( - joinConditionComparisonOperators.get(i), leftSymbol, rightSymbol)); + joinConditionComparisonOperators.get(i), + leftSymbol, + rightSymbol, + leftTable, + rightTable)); continue; } @@ -755,43 +794,6 @@ public RelationPlan planJoin( Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); - // Extract tables from expressions - Set leftDependencies = - SymbolsExtractor.extractNames( - leftComparisonExpressions.get(i), analysis.getColumnReferences()); - Set rightDependencies = - SymbolsExtractor.extractNames( - rightComparisonExpressions.get(i), analysis.getColumnReferences()); - - if (leftDependencies.size() != 1 || rightDependencies.size() != 1) { - throw new IllegalStateException("Cannot find source table for symbol " + leftSymbol); - } - - QualifiedName leftQualifiedName = leftDependencies.iterator().next(); - QualifiedName rightQualifiedName = rightDependencies.iterator().next(); - - Identifier leftTable = - leftQualifiedName - .getPrefix() - .map(prefix -> new Identifier(prefix.getSuffix())) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in JOIN ON clause", - leftSymbol))); - - Identifier rightTable = - rightQualifiedName - .getPrefix() - .map(prefix -> new Identifier(prefix.getSuffix())) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in JOIN ON clause", - rightSymbol))); - equiClauses.add( new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTable, rightTable)); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index a7049702a2fce..9334dde0611ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -295,6 +295,8 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(asofJoinClause.getOperator().ordinal(), byteBuffer); Symbol.serialize(asofJoinClause.getLeft(), byteBuffer); Symbol.serialize(asofJoinClause.getRight(), byteBuffer); + asofJoinClause.getLeftTable().serialize(byteBuffer); + asofJoinClause.getRightTable().serialize(byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } @@ -339,6 +341,8 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(asofJoinClause.getOperator().ordinal(), stream); Symbol.serialize(asofJoinClause.getLeft(), stream); Symbol.serialize(asofJoinClause.getRight(), stream); + asofJoinClause.getLeftTable().serialize(stream); + asofJoinClause.getRightTable().serialize(stream); } else { ReadWriteIOUtils.write(false, stream); } @@ -382,7 +386,9 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { new AsofJoinClause( ComparisonExpression.Operator.values()[ReadWriteIOUtils.readInt(byteBuffer)], Symbol.deserialize(byteBuffer), - Symbol.deserialize(byteBuffer))); + Symbol.deserialize(byteBuffer), + new Identifier(byteBuffer), + new Identifier(byteBuffer))); } size = ReadWriteIOUtils.readInt(byteBuffer); @@ -551,12 +557,23 @@ public String toString() { public static class AsofJoinClause { private final Symbol left; private final Symbol right; + private final Identifier leftTable; + private final Identifier rightTable; + private final Set tables; private final ComparisonExpression.Operator operator; - public AsofJoinClause(ComparisonExpression.Operator operator, Symbol left, Symbol right) { + public AsofJoinClause( + ComparisonExpression.Operator operator, + Symbol left, + Symbol right, + Identifier leftTable, + Identifier rightTable) { this.operator = operator; this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); + this.tables = ImmutableSet.of(leftTable, rightTable); } public Symbol getLeft() { @@ -567,6 +584,18 @@ public Symbol getRight() { return right; } + public Identifier getLeftTable() { + return leftTable; + } + + public Identifier getRightTable() { + return rightTable; + } + + public Set getTables() { + return tables; + } + public ComparisonExpression.Operator getOperator() { return operator; } @@ -577,7 +606,7 @@ public ComparisonExpression toExpression() { } public AsofJoinClause flip() { - return new AsofJoinClause(operator.flip(), right, left); + return new AsofJoinClause(operator.flip(), right, left, rightTable, leftTable); } public boolean isOperatorContainsGreater() { @@ -607,12 +636,14 @@ public boolean equals(Object obj) { return Objects.equals(this.operator, other.operator) && Objects.equals(this.left, other.left) - && Objects.equals(this.right, other.right); + && Objects.equals(this.right, other.right) + && Objects.equals(this.leftTable, other.leftTable) + && Objects.equals(this.rightTable, other.rightTable); } @Override public int hashCode() { - return Objects.hash(operator, left, right); + return Objects.hash(operator, left, right, leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 9ced7f8aa22ea..cad983b631c5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -39,7 +39,7 @@ import com.google.common.collect.Sets; import org.apache.tsfile.utils.Pair; -import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -121,8 +121,7 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set totalJoinTables = ImmutableSet.of(); // join conjunctions - List criteria = node.getCriteria(); - for (JoinNode.EquiJoinClause equiJoin : criteria) { + for (JoinNode.EquiJoinClause equiJoin : node.getCriteria()) { Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); if (node.getJoinType() == JoinNode.JoinType.LEFT) { @@ -133,9 +132,23 @@ public PlanNode visitJoin(JoinNode node, Context context) { // can be applied. equiJoinTables = Sets.union(equiJoinTables, rightHand); } - leading.getFilters().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); + leading.getEquiJoins().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); } + + Optional asofJoin = node.getAsofCriteria(); + if (asofJoin.isPresent()) { + JoinNode.AsofJoinClause asofJoinClause = asofJoin.get(); + Set asofJoinTables = + Sets.intersection(leadingTables, asofJoinClause.getTables()); + totalJoinTables = Sets.union(totalJoinTables, asofJoinTables); + if (node.getJoinType() == JoinNode.JoinType.LEFT) { + asofJoinTables = Sets.union(asofJoinTables, rightHand); + } + leading.setAsofJoin(new Pair<>(asofJoinTables, asofJoinClause.toExpression())); + leading.putConditionJoinType(asofJoinClause.toExpression(), node.getJoinType()); + } + // join constraint collectJoinConstraintList(leading, leftHand, rightHand, node, totalJoinTables); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index 31d0f35026379..d7710d451dd38 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -54,7 +54,8 @@ public class LeadingHint extends JoinOrderHint { private List addJoinParameters; private List normalizedParameters; - private final List, Expression>> filters = new ArrayList<>(); + private final List, Expression>> equiJoins = new ArrayList<>(); + private Pair, Expression> asofJoin; private final Map relationToScanMap = new HashMap<>(); private Set innerJoinTables = ImmutableSet.of(); @@ -91,8 +92,16 @@ public List getJoinConstraintList() { return joinConstraintList; } - public List, Expression>> getFilters() { - return filters; + public List, Expression>> getEquiJoins() { + return equiJoins; + } + + public Pair, Expression> getAsofJoin() { + return asofJoin; + } + + public void setAsofJoin(Pair, Expression> asofJoin) { + this.asofJoin = asofJoin; } public void putConditionJoinType(Expression filter, JoinNode.JoinType joinType) { @@ -127,16 +136,16 @@ public PlanNode generateLeadingJoinPlan() { if (logicalPlan == null) { return null; } - logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); + logicalPlan = makeFilterPlanIfExist(getEquiJoins(), logicalPlan); stack.push(logicalPlan); } } PlanNode finalJoin = stack.pop(); // we want all filters been removed - if (!filters.isEmpty()) { + if (!equiJoins.isEmpty()) { throw new IllegalStateException( - "Leading hint process failed: filter should be empty, but meet: " + filters); + "Leading hint process failed: filter should be empty, but meet: " + equiJoins); } if (finalJoin == null) { throw new IoTDBRuntimeException( @@ -219,12 +228,14 @@ public PlanNode getPlanByName(String name) { } private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { - List conditions = getJoinConditions(getFilters(), leftChild, rightChild); JoinNode.JoinType joinType = computeJoinType(leftChild.getInputTables(), rightChild.getInputTables()); if (joinType == null) { return null; - } else if (!isConditionJoinTypeMatched(conditions, joinType)) { + } + + List equiJoins = getEquiJoinConditions(getEquiJoins(), leftChild, rightChild); + if (!isConditionJoinTypeMatched(equiJoins, joinType)) { return null; } @@ -233,8 +244,8 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { Optional asofCriteria = Optional.empty(); List criteria = new ArrayList<>(); - for (Expression conjunct : conditions) { - ComparisonExpression equality = (ComparisonExpression) conjunct; + for (Expression equiJoin : equiJoins) { + ComparisonExpression equality = (ComparisonExpression) equiJoin; Symbol leftSymbol = Symbol.from(equality.getLeft()); Symbol rightSymbol = Symbol.from(equality.getRight()); @@ -282,6 +293,62 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { } } + Expression asofJoin = getAsofJoinCondition(getAsofJoin(), leftChild, rightChild); + if (asofJoin != null) { + if (!isConditionJoinTypeMatched(asofJoin, joinType)) { + return null; + } + + ComparisonExpression asofJoinExpr = (ComparisonExpression) asofJoin; + Symbol leftSymbol = Symbol.from(asofJoinExpr.getLeft()); + Symbol rightSymbol = Symbol.from(asofJoinExpr.getRight()); + + if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { + asofCriteria = + Optional.of( + new JoinNode.AsofJoinClause( + asofJoinExpr.getOperator(), + leftSymbol, + rightSymbol, + JoinUtils.findSourceTable(leftChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + leftSymbol))), + JoinUtils.findSourceTable(rightChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + rightSymbol))))); + } else if (leftOutputSymbols.contains(rightSymbol) + && rightOutputSymbols.contains(leftSymbol)) { + asofCriteria = + Optional.of( + new JoinNode.AsofJoinClause( + asofJoinExpr.getOperator().flip(), + rightSymbol, + leftSymbol, + JoinUtils.findSourceTable(leftChild, rightSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join left child", + rightSymbol))), + JoinUtils.findSourceTable(rightChild, leftSymbol) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Cannot find source table for symbol %s in leading hint join right child", + leftSymbol))))); + } + } + return new JoinNode( new PlanNodeId("join"), joinType, @@ -295,7 +362,7 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { Optional.empty()); } - private List getJoinConditions( + private List getEquiJoinConditions( List, Expression>> filters, PlanNode left, PlanNode right) { List joinConditions = new ArrayList<>(); for (int i = filters.size() - 1; i >= 0; i--) { @@ -310,6 +377,20 @@ private List getJoinConditions( return joinConditions; } + private Expression getAsofJoinCondition( + Pair, Expression> filterPair, PlanNode left, PlanNode right) { + if (filterPair == null) { + return null; + } + + Set tables = Sets.union(left.getInputTables(), right.getInputTables()); + // it should contain all tables in join conjunctions & right tables if it's left join + if (tables.containsAll(filterPair.left)) { + return filterPair.right; + } + return null; + } + private PlanNode makeFilterPlanIfExist( List, Expression>> filters, PlanNode plan) { if (filters.isEmpty()) { @@ -351,6 +432,11 @@ public boolean isConditionJoinTypeMatched( return true; } + public boolean isConditionJoinTypeMatched(Expression condition, JoinNode.JoinType joinType) { + JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); + return originalJoinType == joinType; + } + /** * try to get join constraint. If it can not be found, it means join is inner join * diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java index dee921c73fe9b..5648ce16a88ae 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java @@ -61,7 +61,12 @@ public void simpleTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.LESS_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.LESS_THAN, + "time", + "time_0", + "table1", + "table2") .left( sort( ImmutableList.of(sort("time", ASCENDING, LAST)), @@ -85,7 +90,12 @@ public void simpleTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .left( sort( ImmutableList.of(sort("time", DESCENDING, LAST)), @@ -118,7 +128,11 @@ public void toleranceTest() { builder -> builder .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .left( sort( ImmutableList.of(sort("time", DESCENDING, LAST)), @@ -153,7 +167,11 @@ public void projectInAsofCriteriaTest() { builder -> builder .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, "expr", "time_0") + ComparisonExpression.Operator.GREATER_THAN, + "expr", + "time_0", + "table1", + "table2") .equiCriteria("tag1", "tag1_1", "table1", "table2") .left( sort( @@ -207,7 +225,12 @@ public void sortEliminateTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .equiCriteria( ImmutableList.of( equiJoinClause("tag1", "tag1_1", "table1", "table2"), @@ -224,7 +247,12 @@ public void sortEliminateTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") + .asofCriteria( + ComparisonExpression.Operator.GREATER_THAN, + "time", + "time_0", + "table1", + "table2") .equiCriteria( ImmutableList.of( equiJoinClause("tag1", "tag1_1", "table1", "table2"), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java index c02f41bc8ae06..2ac69c2bf818e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import static java.lang.String.format; @@ -29,13 +30,21 @@ public class AsofJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; + private final Identifier leftTable; + private final Identifier rightTable; private final ComparisonExpression.Operator operator; public AsofJoinClauseProvider( - ComparisonExpression.Operator operator, SymbolAlias left, SymbolAlias right) { + ComparisonExpression.Operator operator, + SymbolAlias left, + SymbolAlias right, + Identifier leftTable, + Identifier rightTable) { this.operator = requireNonNull(operator, "operator is null"); this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); } public ComparisonExpression toExpression() { @@ -45,7 +54,8 @@ public ComparisonExpression toExpression() { @Override public JoinNode.AsofJoinClause getExpectedValue(SymbolAliases aliases) { - return new JoinNode.AsofJoinClause(operator, left.toSymbol(aliases), right.toSymbol(aliases)); + return new JoinNode.AsofJoinClause( + operator, left.toSymbol(aliases), right.toSymbol(aliases), leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java index b6e5280caf2ea..99ca1fc8af2df 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java @@ -22,31 +22,26 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; -import java.util.Set; - import static java.util.Objects.requireNonNull; public class EquiJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; - private final Set leftTables; - private final Set rightTables; + private final Identifier leftTable; + private final Identifier rightTable; public EquiJoinClauseProvider( - SymbolAlias left, - SymbolAlias right, - Set leftTables, - Set rightTables) { + SymbolAlias left, SymbolAlias right, Identifier leftTable, Identifier rightTable) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTables = requireNonNull(leftTables, "leftTables is null"); - this.rightTables = requireNonNull(rightTables, "rightTables is null"); + this.leftTable = requireNonNull(leftTable, "leftTable is null"); + this.rightTable = requireNonNull(rightTable, "rightTable is null"); } @Override public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) { return new JoinNode.EquiJoinClause( - left.toSymbol(aliases), right.toSymbol(aliases), leftTables, rightTables); + left.toSymbol(aliases), right.toSymbol(aliases), leftTable, rightTable); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java index 8ea44457e331d..29b8314f5b50f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java @@ -174,8 +174,14 @@ public Builder equiCriteria(String left, String right, String leftTable, String } @CanIgnoreReturnValue - public Builder asofCriteria(ComparisonExpression.Operator operator, String left, String right) { - this.asofJoinCriteria = Optional.of(asofJoinClause(operator, left, right)); + public Builder asofCriteria( + ComparisonExpression.Operator operator, + String left, + String right, + String leftTable, + String rightTable) { + this.asofJoinCriteria = + Optional.of(asofJoinClause(operator, left, right, leftTable, rightTable)); return this; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index b5ca5f227f519..317c5860e7d70 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -721,13 +721,22 @@ public static ExpectedValueProvider equiJoinClause( return new EquiJoinClauseProvider( new SymbolAlias(left), new SymbolAlias(right), - ImmutableSet.of(new Identifier(leftTable)), - ImmutableSet.of(new Identifier(rightTable))); + new Identifier(leftTable), + new Identifier(rightTable)); } public static AsofJoinClauseProvider asofJoinClause( - ComparisonExpression.Operator operator, String left, String right) { - return new AsofJoinClauseProvider(operator, new SymbolAlias(left), new SymbolAlias(right)); + ComparisonExpression.Operator operator, + String left, + String right, + String leftTable, + String rightTable) { + return new AsofJoinClauseProvider( + operator, + new SymbolAlias(left), + new SymbolAlias(right), + new Identifier(leftTable), + new Identifier(rightTable)); } public static SymbolAlias symbol(String alias) { From 295606f2808fc7e5c34fb1c2663d4a59c97ff97e Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 5 Mar 2026 10:40:32 +0800 Subject: [PATCH 21/35] fix some issues --- .../db/queryengine/plan/planner/plan/node/PlanNode.java | 1 - .../plan/relational/analyzer/StatementAnalyzer.java | 4 +++- .../planner/optimizations/CollectJoinConstraint.java | 3 +-- .../planner/optimizations/LeadingJoinOptimizer.java | 6 ------ .../plan/relational/utils/hint/FollowerHint.java | 8 ++++---- .../plan/relational/utils/hint/LeaderHint.java | 8 ++++---- 6 files changed, 12 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java index 9746d06d2a10a..73bb65e486617 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java @@ -86,7 +86,6 @@ public Set getInputTables() { for (PlanNode child : getChildren()) { tables.addAll(child.getInputTables()); } - ; return tables; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index d749c5925a725..3c7bb4773fbea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1571,7 +1571,9 @@ private void addHint(String hintName, List parameters, Map // Skip if parameters contain tables that don't exist in the query if (parameters != null) { boolean hasInvalidTable = - parameters.stream().anyMatch(table -> !existingTables.contains(table)); + parameters.stream() + .filter(table -> !table.equals("{") && !table.equals("}")) + .anyMatch(table -> !existingTables.contains(table)); if (hasInvalidTable) { return; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index cad983b631c5d..4fbe9ddf912b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -220,8 +220,7 @@ && isOverlap(joinTables, other.getRightHand())) { } JoinConstraint joinConstraint = - new JoinConstraint( - minLeftHand, minRightHand, minLeftHand, minRightHand, join.getJoinType()); + new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand, join.getJoinType()); leading.getJoinConstraintList().add(joinConstraint); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java index 3bd3ebcdbd021..8d1afefb8f8a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java @@ -83,11 +83,5 @@ public PlanNode visitJoin(JoinNode node, Context context) { } return node; } - - // @Override - // public PlanNode visitSemiJoin(SemiJoinNode node, Context context) { - // return visitTwoChildProcess(node, context); - // } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 2c1d3b3a0bafc..496c38d5e060a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -27,24 +27,24 @@ public class FollowerHint extends ReplicaHint { public static String hintName = "follower"; - private final String targetTable; + private final String table; public FollowerHint(List tables) { super(hintName); if (tables == null || tables.size() != 1) { throw new IllegalArgumentException("FollowerHint accepts exactly one table"); } - targetTable = tables.get(0); + table = tables.get(0); } @Override public String getKey() { - return category + "-" + targetTable; + return category + "-" + table; } @Override public String toString() { - return hintName + "-" + targetTable; + return hintName + "-" + table; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index 8c179846da986..c752b35bb59c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -28,24 +28,24 @@ public class LeaderHint extends ReplicaHint { public static String hintName = "leader"; - private final String targetTable; + private final String table; public LeaderHint(List tables) { super(hintName); if (tables == null || tables.size() != 1) { throw new IllegalArgumentException("LeaderHint accepts exactly one table"); } - targetTable = tables.get(0); + table = tables.get(0); } @Override public String getKey() { - return category + "-" + targetTable; + return category + "-" + table; } @Override public String toString() { - return hintName + "-" + targetTable; + return hintName + "-" + table; } @Override From 54c1acc8856bc79f382a05304e6e90643f4cbcac Mon Sep 17 00:00:00 2001 From: shizy Date: Fri, 6 Mar 2026 16:43:48 +0800 Subject: [PATCH 22/35] code change --- .../analyzer/StatementAnalyzer.java | 4 +- .../TableDistributedPlanGenerator.java | 3 +- .../relational/planner/node/JoinNode.java | 107 +----------------- .../optimizations/CollectJoinConstraint.java | 6 +- .../planner/optimizations/JoinUtils.java | 35 ++++++ .../optimizations/LeadingJoinOptimizer.java | 4 +- .../relational/utils/hint/FollowerHint.java | 3 +- .../relational/utils/hint/HintFactory.java | 24 ++-- .../relational/utils/hint/LeaderHint.java | 3 +- .../relational/utils/hint/LeadingHint.java | 68 ++--------- 10 files changed, 74 insertions(+), 183 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 3c7bb4773fbea..a28e5e26fa4ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1563,7 +1563,7 @@ private void addHint(String hintName, List parameters, Map .filter(table -> parameters == null || existingTables.contains(table)) .forEach( table -> { - Hint hint = definition.createHint(ImmutableList.of(table)); + Hint hint = definition.createHint(ImmutableList.of(table), queryContext); String hintKey = hint.getKey(); hintMap.putIfAbsent(hintKey, hint); }); @@ -1579,7 +1579,7 @@ private void addHint(String hintName, List parameters, Map } } - Hint hint = definition.createHint(parameters); + Hint hint = definition.createHint(parameters, queryContext); String hintKey = hint.getKey(); if (!hintMap.containsKey(hintKey)) { hintMap.put(hintKey, hint); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index f0b4d4751ca36..6978d15ac8cf9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -635,7 +635,8 @@ public List visitJoin(JoinNode node, PlanContext context) { node.setRightChild(mergeChildrenViaCollectOrMergeSort(rightChildOrdering, rightChildrenNodes)); // Now the join implement but CROSS is MergeSortJoin, so it can keep order - if (!node.isCrossJoin() && !node.getAsofCriteria().isPresent()) { + // if (!node.isCrossJoin() && !node.getAsofCriteria().isPresent()) { // cross join？ + if (!node.getAsofCriteria().isPresent()) { switch (node.getJoinType()) { case FULL: // If join type is FULL Join, we will process SortProperties in ProjectNode above this diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 9334dde0611ee..73535f0c6e660 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -67,9 +66,6 @@ public class JoinNode extends TwoChildProcessNode { // private final Optional distributionType; // private final Map dynamicFilters; - private final Set leftTables; - private final Set rightTables; - public JoinNode( PlanNodeId id, JoinType joinType, @@ -80,9 +76,7 @@ public JoinNode( List leftOutputSymbols, List rightOutputSymbols, Optional filter, - Optional spillable, - Set leftTables, - Set rightTables) { + Optional spillable) { super(id); requireNonNull(joinType, "type is null"); requireNonNull(leftChild, "left is null"); @@ -101,8 +95,6 @@ public JoinNode( // requireNonNull(leftHashSymbol, "leftHashSymbol is null"); // requireNonNull(rightHashSymbol, "rightHashSymbol is null"); requireNonNull(spillable, "spillable is null"); - requireNonNull(leftTables, "leftTables is null"); - requireNonNull(rightTables, "rightTables is null"); this.joinType = joinType; this.leftChild = leftChild; @@ -116,8 +108,6 @@ public JoinNode( // this.maySkipOutputDuplicates = maySkipOutputDuplicates; // this.leftHashSymbol = leftHashSymbol; // this.rightHashSymbol = rightHashSymbol; - this.leftTables = ImmutableSet.copyOf(leftTables); - this.rightTables = ImmutableSet.copyOf(rightTables); Set leftSymbols = ImmutableSet.copyOf(leftChild.getOutputSymbols()); Set rightSymbols = ImmutableSet.copyOf(rightChild.getOutputSymbols()); @@ -145,32 +135,6 @@ public JoinNode( equiJoinClause)); } - public JoinNode( - PlanNodeId id, - JoinType joinType, - PlanNode leftChild, - PlanNode rightChild, - List criteria, - Optional asofCriteria, - List leftOutputSymbols, - List rightOutputSymbols, - Optional filter, - Optional spillable) { - this( - id, - joinType, - leftChild, - rightChild, - criteria, - asofCriteria, - leftOutputSymbols, - rightOutputSymbols, - filter, - spillable, - leftChild.getInputTables(), - rightChild.getInputTables()); - } - // only used for deserialize public JoinNode( PlanNodeId id, @@ -178,21 +142,15 @@ public JoinNode( List criteria, Optional asofCriteria, List leftOutputSymbols, - List rightOutputSymbols, - Set leftTables, - Set rightTables) { + List rightOutputSymbols) { super(id); requireNonNull(joinType, "type is null"); requireNonNull(criteria, "criteria is null"); - requireNonNull(leftTables, "leftTables is null"); - requireNonNull(rightTables, "rightTables is null"); this.leftOutputSymbols = leftOutputSymbols; this.rightOutputSymbols = rightOutputSymbols; this.filter = Optional.empty(); this.spillable = Optional.empty(); - this.leftTables = ImmutableSet.copyOf(leftTables); - this.rightTables = ImmutableSet.copyOf(rightTables); this.joinType = joinType; this.criteria = criteria; @@ -213,9 +171,7 @@ public JoinNode flip() { rightOutputSymbols, leftOutputSymbols, filter, - spillable, - rightTables, - leftTables); + spillable); } @Override @@ -236,9 +192,7 @@ public PlanNode replaceChildren(List newChildren) { leftOutputSymbols, rightOutputSymbols, filter, - spillable, - leftTables, - rightTables); + spillable); } @Override @@ -262,9 +216,7 @@ public PlanNode clone() { leftOutputSymbols, rightOutputSymbols, filter, - spillable, - leftTables, - rightTables); + spillable); joinNode.setLeftChild(null); joinNode.setRightChild(null); return joinNode; @@ -309,16 +261,6 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (Symbol rightOutputSymbol : rightOutputSymbols) { Symbol.serialize(rightOutputSymbol, byteBuffer); } - - // Serialize JoinNode-level leftTables and rightTables - ReadWriteIOUtils.write(this.leftTables.size(), byteBuffer); - for (Identifier identifier : this.leftTables) { - identifier.serialize(byteBuffer); - } - ReadWriteIOUtils.write(this.rightTables.size(), byteBuffer); - for (Identifier identifier : this.rightTables) { - identifier.serialize(byteBuffer); - } } @Override @@ -355,16 +297,6 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (Symbol rightOutputSymbol : rightOutputSymbols) { Symbol.serialize(rightOutputSymbol, stream); } - - // Serialize JoinNode-level leftTables and rightTables - ReadWriteIOUtils.write(this.leftTables.size(), stream); - for (Identifier identifier : this.leftTables) { - identifier.serialize(stream); - } - ReadWriteIOUtils.write(this.rightTables.size(), stream); - for (Identifier identifier : this.rightTables) { - identifier.serialize(stream); - } } public static JoinNode deserialize(ByteBuffer byteBuffer) { @@ -403,28 +335,9 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { rightOutputSymbols.add(Symbol.deserialize(byteBuffer)); } - // Deserialize JoinNode-level leftTables and rightTables - int joinLeftTablesSize = ReadWriteIOUtils.readInt(byteBuffer); - Set joinLeftTables = new HashSet<>(joinLeftTablesSize); - while (joinLeftTablesSize-- > 0) { - joinLeftTables.add(new Identifier(byteBuffer)); - } - int joinRightTablesSize = ReadWriteIOUtils.readInt(byteBuffer); - Set joinRightTables = new HashSet<>(joinRightTablesSize); - while (joinRightTablesSize-- > 0) { - joinRightTables.add(new Identifier(byteBuffer)); - } - PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); return new JoinNode( - planNodeId, - joinType, - criteria, - asofJoinClause, - leftOutputSymbols, - rightOutputSymbols, - joinLeftTables, - joinRightTables); + planNodeId, joinType, criteria, asofJoinClause, leftOutputSymbols, rightOutputSymbols); } public JoinType getJoinType() { @@ -455,14 +368,6 @@ public Optional isSpillable() { return spillable; } - public Set getLeftTables() { - return leftTables; - } - - public Set getRightTables() { - return rightTables; - } - public boolean isCrossJoin() { return !asofCriteria.isPresent() && criteria.isEmpty() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 4fbe9ddf912b3..6d7e28cc803f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -116,8 +116,8 @@ public PlanNode visitJoin(JoinNode node, Context context) { Set leadingTables = leading.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); - Set leftHand = node.getLeftTables(); - Set rightHand = node.getRightTables(); + Set leftHand = JoinUtils.collectAllTables(node.getLeftChild()); + Set rightHand = JoinUtils.collectAllTables(node.getRightChild()); Set totalJoinTables = ImmutableSet.of(); // join conjunctions @@ -133,7 +133,6 @@ public PlanNode visitJoin(JoinNode node, Context context) { equiJoinTables = Sets.union(equiJoinTables, rightHand); } leading.getEquiJoins().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); - leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); } Optional asofJoin = node.getAsofCriteria(); @@ -146,7 +145,6 @@ public PlanNode visitJoin(JoinNode node, Context context) { asofJoinTables = Sets.union(asofJoinTables, rightHand); } leading.setAsofJoin(new Pair<>(asofJoinTables, asofJoinClause.toExpression())); - leading.putConditionJoinType(asofJoinClause.toExpression(), node.getJoinType()); } // join constraint diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java index 6542b46df93bd..fff297d18a20a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java @@ -442,4 +442,39 @@ public static Optional findSourceTable(PlanNode node, Symbol symbol) return Optional.empty(); } + + /** + * Collects all table names or aliases from a PlanNode by traversing the plan tree. + * + * @param node the plan node to traverse + * @return a set of all table identifiers (names or aliases) found in the plan + */ + public static Set collectAllTables(PlanNode node) { + Set tables = new java.util.HashSet<>(); + collectAllTables(node, tables); + return tables; + } + + /** + * Recursively collects all table names or aliases from a PlanNode and its children. + * + * @param node the plan node to traverse + * @param tables the set to accumulate table identifiers + */ + private static void collectAllTables(PlanNode node, Set tables) { + if (node instanceof DeviceTableScanNode) { + DeviceTableScanNode scanNode = (DeviceTableScanNode) node; + Identifier alias = scanNode.getAlias(); + if (alias != null) { + tables.add(alias); + } else { + tables.add(new Identifier(scanNode.getQualifiedObjectName().getObjectName())); + } + } + + // Recursively process all children + for (PlanNode child : node.getChildren()) { + collectAllTables(child, tables); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java index 8d1afefb8f8a9..df7b4534736c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java @@ -69,7 +69,9 @@ public PlanNode visitPlan(PlanNode node, Context context) { @Override public PlanNode visitJoin(JoinNode node, Context context) { LeadingHint leadingHint = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); - Set currentTables = Sets.union(node.getLeftTables(), node.getRightTables()); + Set leftTables = JoinUtils.collectAllTables(node.getLeftChild()); + Set rightTables = JoinUtils.collectAllTables(node.getRightChild()); + Set currentTables = Sets.union(leftTables, rightTables); Set leadingTables = leadingHint.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 496c38d5e060a..5e412f1a650cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import java.util.List; @@ -29,7 +30,7 @@ public class FollowerHint extends ReplicaHint { public static String hintName = "follower"; private final String table; - public FollowerHint(List tables) { + public FollowerHint(List tables, MPPQueryContext queryContext) { super(hintName); if (tables == null || tables.size() != 1) { throw new IllegalArgumentException("FollowerHint accepts exactly one table"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java index 1e1c8193277aa..ea7dea56516f6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java @@ -21,8 +21,10 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; + import java.util.List; -import java.util.function.Function; +import java.util.function.BiFunction; /** * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, @@ -31,19 +33,9 @@ public final class HintFactory { private final String key; - private final Function, Hint> factory; + private final BiFunction, MPPQueryContext, Hint> factory; private final boolean expandParameters; - /** - * Creates a new hint definition. - * - * @param key the key to use when storing the hint in the hint map - * @param hintFactory factory method to create the hint instance, receives the key as parameter - */ - public HintFactory(String key, Function, Hint> hintFactory) { - this(key, hintFactory, false); - } - /** * Creates a new hint definition with parameter expansion option. * @@ -52,7 +44,9 @@ public HintFactory(String key, Function, Hint> hintFactory) { * @param expandParameters whether to expand array parameters into multiple hints */ public HintFactory( - String key, Function, Hint> hintFactory, boolean expandParameters) { + String key, + BiFunction, MPPQueryContext, Hint> hintFactory, + boolean expandParameters) { this.key = key; this.factory = hintFactory; this.expandParameters = expandParameters; @@ -72,8 +66,8 @@ public String getKey() { * * @return the created hint */ - public Hint createHint(List parameters) { - return factory.apply(parameters); + public Hint createHint(List parameters, MPPQueryContext context) { + return factory.apply(parameters, context); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index c752b35bb59c6..1118270f8a0b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import java.util.Collections; import java.util.List; @@ -30,7 +31,7 @@ public class LeaderHint extends ReplicaHint { public static String hintName = "leader"; private final String table; - public LeaderHint(List tables) { + public LeaderHint(List tables, MPPQueryContext queryContext) { super(hintName); if (tables == null || tables.size() != 1) { throw new IllegalArgumentException("LeaderHint accepts exactly one table"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index d7710d451dd38..b1161b3920e0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -22,10 +22,10 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; @@ -33,7 +33,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.tsfile.utils.Pair; @@ -51,6 +50,8 @@ public class LeadingHint extends JoinOrderHint { public static String hintName = "leading"; private final List tables; + private QueryId idAllocator; + private List addJoinParameters; private List normalizedParameters; @@ -62,14 +63,13 @@ public class LeadingHint extends JoinOrderHint { private final List joinConstraintList = new ArrayList<>(); - private final Map conditionJoinType = Maps.newLinkedHashMap(); - - public LeadingHint(List parameters) { + public LeadingHint(List parameters, MPPQueryContext queryContext) { super(hintName); // /* leading(t3 {}) 会报错 this.tables = new ArrayList<>(); - addJoinParameters = insertJoinIntoParameters(parameters); - normalizedParameters = parseIntoReversePolishNotation(addJoinParameters); + this.idAllocator = queryContext.getQueryId(); + this.addJoinParameters = insertJoinIntoParameters(parameters); + this.normalizedParameters = parseIntoReversePolishNotation(addJoinParameters); if (tables.isEmpty()) { throw new IllegalArgumentException("LeaderHint accepts one or more tables"); @@ -104,10 +104,6 @@ public void setAsofJoin(Pair, Expression> asofJoin) { this.asofJoin = asofJoin; } - public void putConditionJoinType(Expression filter, JoinNode.JoinType joinType) { - conditionJoinType.put(filter, joinType); - } - public Map getRelationToScanMap() { return relationToScanMap; } @@ -136,7 +132,6 @@ public PlanNode generateLeadingJoinPlan() { if (logicalPlan == null) { return null; } - logicalPlan = makeFilterPlanIfExist(getEquiJoins(), logicalPlan); stack.push(logicalPlan); } } @@ -235,10 +230,6 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { } List equiJoins = getEquiJoinConditions(getEquiJoins(), leftChild, rightChild); - if (!isConditionJoinTypeMatched(equiJoins, joinType)) { - return null; - } - List leftOutputSymbols = leftChild.getOutputSymbols(); List rightOutputSymbols = rightChild.getOutputSymbols(); Optional asofCriteria = Optional.empty(); @@ -289,16 +280,12 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { "Cannot find source table for symbol %s in leading hint join right child", leftSymbol))))); } else { - throw new IllegalArgumentException("Invalid join condition"); + return null; } } Expression asofJoin = getAsofJoinCondition(getAsofJoin(), leftChild, rightChild); if (asofJoin != null) { - if (!isConditionJoinTypeMatched(asofJoin, joinType)) { - return null; - } - ComparisonExpression asofJoinExpr = (ComparisonExpression) asofJoin; Symbol leftSymbol = Symbol.from(asofJoinExpr.getLeft()); Symbol rightSymbol = Symbol.from(asofJoinExpr.getRight()); @@ -350,7 +337,7 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { } return new JoinNode( - new PlanNodeId("join"), + idAllocator.genPlanNodeId(), joinType, leftChild, rightChild, @@ -391,21 +378,6 @@ private Expression getAsofJoinCondition( return null; } - private PlanNode makeFilterPlanIfExist( - List, Expression>> filters, PlanNode plan) { - if (filters.isEmpty()) { - return plan; - } - for (int i = filters.size() - 1; i >= 0; i--) { - Pair, Expression> filterPair = filters.get(i); - if (plan.getInputTables().containsAll(filterPair.left)) { - plan = new FilterNode(plan.getPlanNodeId(), plan, filterPair.right); - filters.remove(i); - } - } - return plan; - } - public JoinNode.JoinType computeJoinType(Set left, Set right) { Pair joinConstraintBooleanPair = getJoinConstraint(Sets.union(left, right), left, right); @@ -420,28 +392,10 @@ public JoinNode.JoinType computeJoinType(Set left, Set r return joinConstraint.getJoinType(); } - public boolean isConditionJoinTypeMatched( - List conditions, JoinNode.JoinType joinType) { - for (Expression condition : conditions) { - JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); - if (originalJoinType == joinType) { - continue; - } - return false; - } - return true; - } - - public boolean isConditionJoinTypeMatched(Expression condition, JoinNode.JoinType joinType) { - JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); - return originalJoinType == joinType; - } - /** * try to get join constraint. If it can not be found, it means join is inner join * - * @return boolean value used for judging whether the join is legal, and should this join need to - * reverse + * @return boolean value used for judging whether the join is legal */ public Pair getJoinConstraint( Set joinTables, Set leftHand, Set rightHand) { From 8cef1d4a509f68b7179c5f18066edf30d6648921 Mon Sep 17 00:00:00 2001 From: shizy Date: Fri, 6 Mar 2026 17:08:13 +0800 Subject: [PATCH 23/35] revert conditionJoinType --- .../optimizations/CollectJoinConstraint.java | 9 +++--- .../relational/utils/hint/LeadingHint.java | 30 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java index 6d7e28cc803f7..59e70e0550edf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java @@ -133,6 +133,7 @@ public PlanNode visitJoin(JoinNode node, Context context) { equiJoinTables = Sets.union(equiJoinTables, rightHand); } leading.getEquiJoins().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); + leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); } Optional asofJoin = node.getAsofCriteria(); @@ -145,6 +146,7 @@ public PlanNode visitJoin(JoinNode node, Context context) { asofJoinTables = Sets.union(asofJoinTables, rightHand); } leading.setAsofJoin(new Pair<>(asofJoinTables, asofJoinClause.toExpression())); + leading.putConditionJoinType(asofJoinClause.toExpression(), node.getJoinType()); } // join constraint @@ -203,11 +205,8 @@ && isOverlap(joinTables, other.getRightHand())) { // Current join's right table contains the right table of a previous join if (isOverlap(rightHand, other.getRightHand())) { - if (isOverlap(joinTables, other.getRightHand()) - || !isOverlap(joinTables, other.getMinLeftHand())) { - minRightHand = Sets.union(minRightHand, other.getLeftHand()); - minRightHand = Sets.union(minRightHand, other.getRightHand()); - } + minRightHand = Sets.union(minRightHand, other.getLeftHand()); + minRightHand = Sets.union(minRightHand, other.getRightHand()); } } if (minLeftHand.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java index b1161b3920e0a..2d89b93eb0c7a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.tsfile.utils.Pair; @@ -63,6 +64,8 @@ public class LeadingHint extends JoinOrderHint { private final List joinConstraintList = new ArrayList<>(); + private final Map conditionJoinType = Maps.newLinkedHashMap(); + public LeadingHint(List parameters, MPPQueryContext queryContext) { super(hintName); // /* leading(t3 {}) 会报错 @@ -108,6 +111,10 @@ public Map getRelationToScanMap() { return relationToScanMap; } + public void putConditionJoinType(Expression filter, JoinNode.JoinType joinType) { + conditionJoinType.put(filter, joinType); + } + public Set getInnerJoinTables() { return innerJoinTables; } @@ -230,6 +237,9 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { } List equiJoins = getEquiJoinConditions(getEquiJoins(), leftChild, rightChild); + if (!isConditionJoinTypeMatched(equiJoins, joinType)) { + return null; + } List leftOutputSymbols = leftChild.getOutputSymbols(); List rightOutputSymbols = rightChild.getOutputSymbols(); Optional asofCriteria = Optional.empty(); @@ -286,6 +296,9 @@ private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { Expression asofJoin = getAsofJoinCondition(getAsofJoin(), leftChild, rightChild); if (asofJoin != null) { + if (!isConditionJoinTypeMatched(asofJoin, joinType)) { + return null; + } ComparisonExpression asofJoinExpr = (ComparisonExpression) asofJoin; Symbol leftSymbol = Symbol.from(asofJoinExpr.getLeft()); Symbol rightSymbol = Symbol.from(asofJoinExpr.getRight()); @@ -392,6 +405,23 @@ public JoinNode.JoinType computeJoinType(Set left, Set r return joinConstraint.getJoinType(); } + public boolean isConditionJoinTypeMatched( + List conditions, JoinNode.JoinType joinType) { + for (Expression condition : conditions) { + JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); + if (originalJoinType == joinType) { + continue; + } + return false; + } + return true; + } + + public boolean isConditionJoinTypeMatched(Expression condition, JoinNode.JoinType joinType) { + JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); + return originalJoinType == joinType; + } + /** * try to get join constraint. If it can not be found, it means join is inner join * From bb032199e6be1ef07d3b4b712d6587d5f5c25df0 Mon Sep 17 00:00:00 2001 From: shizy Date: Sat, 14 Mar 2026 21:16:49 +0800 Subject: [PATCH 24/35] remove leading hint --- .../analyzer/StatementAnalyzer.java | 5 +- .../relational/planner/RelationPlanner.java | 11 +- ...TransformFilteringSemiJoinToInnerJoin.java | 19 +- .../relational/planner/node/JoinNode.java | 79 +-- .../optimizations/CollectJoinConstraint.java | 231 -------- .../optimizations/LeadingJoinOptimizer.java | 89 --- .../optimizations/LogicalOptimizeFactory.java | 4 +- .../PushPredicateIntoTableScan.java | 19 +- .../UnaliasSymbolReferences.java | 5 +- .../relational/utils/hint/JoinConstraint.java | 70 --- .../relational/utils/hint/JoinOrderHint.java | 30 - .../relational/utils/hint/LeadingHint.java | 526 ------------------ .../relational/analyzer/AsofJoinTest.java | 40 +- .../plan/relational/analyzer/JoinTest.java | 6 +- .../assertions/AsofJoinClauseProvider.java | 14 +- .../planner/assertions/JoinMatcher.java | 10 +- .../planner/assertions/PlanMatchPattern.java | 13 +- 17 files changed, 31 insertions(+), 1140 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index a28e5e26fa4ac..8a0a286fd953c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -202,7 +202,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintFactory; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -376,9 +375,7 @@ private enum UpdateKind { "LEADER", new HintFactory(LeaderHint.hintName, LeaderHint::new, true), "FOLLOWER", - new HintFactory(FollowerHint.hintName, FollowerHint::new, true), - "LEADING", - new HintFactory(LeadingHint.hintName, LeadingHint::new, false)); + new HintFactory(FollowerHint.hintName, FollowerHint::new, true)); /** * Visitor context represents local query scope (if exists). The invariant is that the local query diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 7b9354297e23c..deecb649a5556 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -539,7 +539,7 @@ If casts are redundant (due to column type and common type being equal), "Cannot find source table for field %s in JOIN USING clause", rightFieldObj.getName()))); - clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput, leftTable, rightTable)); + clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput)); } ProjectNode leftCoercion = @@ -782,11 +782,7 @@ public RelationPlan planJoin( asofJoinClause = Optional.of( new JoinNode.AsofJoinClause( - joinConditionComparisonOperators.get(i), - leftSymbol, - rightSymbol, - leftTable, - rightTable)); + joinConditionComparisonOperators.get(i), leftSymbol, rightSymbol)); continue; } @@ -794,8 +790,7 @@ public RelationPlan planJoin( Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); - equiClauses.add( - new JoinNode.EquiJoinClause(leftSymbol, rightSymbol, leftTable, rightTable)); + equiClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); } else { postInnerJoinConditions.add( new ComparisonExpression( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java index 910fe9e9ce8d8..1f32fdbffe157 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -27,7 +27,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture; import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; @@ -131,23 +130,7 @@ public Result apply(FilterNode filterNode, Captures captures, Context context) { filteringSourceDistinct, ImmutableList.of( new JoinNode.EquiJoinClause( - semiJoin.getSourceJoinSymbol(), - semiJoin.getFilteringSourceJoinSymbol(), - JoinUtils.findSourceTable(semiJoin.getSource(), semiJoin.getSourceJoinSymbol()) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in semi-join source", - semiJoin.getSourceJoinSymbol()))), - JoinUtils.findSourceTable( - semiJoin.getFilteringSource(), semiJoin.getFilteringSourceJoinSymbol()) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in semi-join filtering source", - semiJoin.getFilteringSourceJoinSymbol()))))), + semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), Optional.empty(), semiJoin.getSource().getOutputSymbols(), ImmutableList.of(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index 73535f0c6e660..ce82dbe73e926 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -27,7 +27,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import com.google.common.collect.ImmutableList; @@ -237,8 +236,6 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), byteBuffer); Symbol.serialize(equiJoinClause.getRight(), byteBuffer); - equiJoinClause.getLeftTable().serialize(byteBuffer); - equiJoinClause.getRightTable().serialize(byteBuffer); } if (asofCriteria.isPresent()) { @@ -247,8 +244,6 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(asofJoinClause.getOperator().ordinal(), byteBuffer); Symbol.serialize(asofJoinClause.getLeft(), byteBuffer); Symbol.serialize(asofJoinClause.getRight(), byteBuffer); - asofJoinClause.getLeftTable().serialize(byteBuffer); - asofJoinClause.getRightTable().serialize(byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } @@ -273,8 +268,6 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { for (EquiJoinClause equiJoinClause : criteria) { Symbol.serialize(equiJoinClause.getLeft(), stream); Symbol.serialize(equiJoinClause.getRight(), stream); - equiJoinClause.getLeftTable().serialize(stream); - equiJoinClause.getRightTable().serialize(stream); } if (asofCriteria.isPresent()) { @@ -283,8 +276,6 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(asofJoinClause.getOperator().ordinal(), stream); Symbol.serialize(asofJoinClause.getLeft(), stream); Symbol.serialize(asofJoinClause.getRight(), stream); - asofJoinClause.getLeftTable().serialize(stream); - asofJoinClause.getRightTable().serialize(stream); } else { ReadWriteIOUtils.write(false, stream); } @@ -306,9 +297,7 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { while (size-- > 0) { Symbol left = Symbol.deserialize(byteBuffer); Symbol right = Symbol.deserialize(byteBuffer); - Identifier leftTable = new Identifier(byteBuffer); - Identifier rightTable = new Identifier(byteBuffer); - criteria.add(new EquiJoinClause(left, right, leftTable, rightTable)); + criteria.add(new EquiJoinClause(left, right)); } Optional asofJoinClause = Optional.empty(); @@ -318,9 +307,7 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { new AsofJoinClause( ComparisonExpression.Operator.values()[ReadWriteIOUtils.readInt(byteBuffer)], Symbol.deserialize(byteBuffer), - Symbol.deserialize(byteBuffer), - new Identifier(byteBuffer), - new Identifier(byteBuffer))); + Symbol.deserialize(byteBuffer))); } size = ReadWriteIOUtils.readInt(byteBuffer); @@ -383,16 +370,10 @@ public String toString() { public static class EquiJoinClause { private final Symbol left; private final Symbol right; - private final Identifier leftTable; - private final Identifier rightTable; - private final Set tables; - public EquiJoinClause(Symbol left, Symbol right, Identifier leftTable, Identifier rightTable) { + public EquiJoinClause(Symbol left, Symbol right) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTable = requireNonNull(leftTable, "leftTable is null"); - this.rightTable = requireNonNull(rightTable, "rightTable is null"); - this.tables = ImmutableSet.of(leftTable, rightTable); } public Symbol getLeft() { @@ -403,25 +384,13 @@ public Symbol getRight() { return right; } - public Identifier getLeftTable() { - return leftTable; - } - - public Identifier getRightTable() { - return rightTable; - } - - public Set getTables() { - return tables; - } - public ComparisonExpression toExpression() { return new ComparisonExpression( ComparisonExpression.Operator.EQUAL, left.toSymbolReference(), right.toSymbolReference()); } public EquiJoinClause flip() { - return new EquiJoinClause(right, left, rightTable, leftTable); + return new EquiJoinClause(right, left); } public static List flipBatch(List input) { @@ -442,15 +411,12 @@ public boolean equals(Object obj) { EquiJoinClause other = (EquiJoinClause) obj; - return Objects.equals(this.left, other.left) - && Objects.equals(this.right, other.right) - && Objects.equals(this.leftTable, other.leftTable) - && Objects.equals(this.rightTable, other.rightTable); + return Objects.equals(this.left, other.left) && Objects.equals(this.right, other.right); } @Override public int hashCode() { - return Objects.hash(left, right, leftTable, rightTable); + return Objects.hash(left, right); } @Override @@ -462,23 +428,12 @@ public String toString() { public static class AsofJoinClause { private final Symbol left; private final Symbol right; - private final Identifier leftTable; - private final Identifier rightTable; - private final Set tables; private final ComparisonExpression.Operator operator; - public AsofJoinClause( - ComparisonExpression.Operator operator, - Symbol left, - Symbol right, - Identifier leftTable, - Identifier rightTable) { + public AsofJoinClause(ComparisonExpression.Operator operator, Symbol left, Symbol right) { this.operator = operator; this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTable = requireNonNull(leftTable, "leftTable is null"); - this.rightTable = requireNonNull(rightTable, "rightTable is null"); - this.tables = ImmutableSet.of(leftTable, rightTable); } public Symbol getLeft() { @@ -489,18 +444,6 @@ public Symbol getRight() { return right; } - public Identifier getLeftTable() { - return leftTable; - } - - public Identifier getRightTable() { - return rightTable; - } - - public Set getTables() { - return tables; - } - public ComparisonExpression.Operator getOperator() { return operator; } @@ -511,7 +454,7 @@ public ComparisonExpression toExpression() { } public AsofJoinClause flip() { - return new AsofJoinClause(operator.flip(), right, left, rightTable, leftTable); + return new AsofJoinClause(operator.flip(), right, left); } public boolean isOperatorContainsGreater() { @@ -541,14 +484,12 @@ public boolean equals(Object obj) { return Objects.equals(this.operator, other.operator) && Objects.equals(this.left, other.left) - && Objects.equals(this.right, other.right) - && Objects.equals(this.leftTable, other.leftTable) - && Objects.equals(this.rightTable, other.rightTable); + && Objects.equals(this.right, other.right); } @Override public int hashCode() { - return Objects.hash(operator, left, right, leftTable, rightTable); + return Objects.hash(operator, left, right); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java deleted file mode 100644 index 59e70e0550edf..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/CollectJoinConstraint.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; - -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinConstraint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; - -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import org.apache.tsfile.utils.Pair; - -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -public class CollectJoinConstraint implements PlanOptimizer { - @Override - public PlanNode optimize(PlanNode plan, Context context) { - if (!context.getAnalysis().isQuery()) { - return plan; - } - return plan.accept(new Rewriter(context.getAnalysis()), null); - } - - private static class Rewriter extends PlanVisitor { - private final Analysis analysis; - - public Rewriter(Analysis analysis) { - this.analysis = analysis; - } - - @Override - public PlanNode visitPlan(PlanNode node, Context context) { - Hint hint = analysis.getHintMap().getOrDefault(JoinOrderHint.category, null); - if (!(hint instanceof LeadingHint)) { - return node; - } - - for (PlanNode child : node.getChildren()) { - child.accept(this, context); - } - return node; - } - - @Override - public PlanNode visitDeviceTableScan(DeviceTableScanNode node, Context context) { - Identifier alias = node.getAlias(); - String tableName = - alias != null ? alias.getValue() : node.getQualifiedObjectName().getObjectName(); - LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); - leading.getRelationToScanMap().put(tableName, node); - return node; - } - - @Override - public PlanNode visitMergeSort(MergeSortNode node, Context context) { - return visitSingleTableNode(node, context); - } - - public PlanNode visitFilter(FilterNode node, Context context) { - return visitSingleTableNode(node, context); - } - - public PlanNode visitSort(SortNode node, Context context) { - return visitSingleTableNode(node, context); - } - - private PlanNode visitSingleTableNode(PlanNode node, Context context) { - visitPlan(node, context); - Set tables = node.getInputTables(); - if (tables != null && tables.size() == 1) { - Identifier tableName = tables.iterator().next(); - LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); - leading.getRelationToScanMap().put(tableName.getValue(), node); - } - return node; - } - - @Override - public PlanNode visitJoin(JoinNode node, Context context) { - for (PlanNode child : node.getChildren()) { - child.accept(this, context); - } - - LeadingHint leading = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); - Set leadingTables = - leading.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); - - Set leftHand = JoinUtils.collectAllTables(node.getLeftChild()); - Set rightHand = JoinUtils.collectAllTables(node.getRightChild()); - Set totalJoinTables = ImmutableSet.of(); - - // join conjunctions - for (JoinNode.EquiJoinClause equiJoin : node.getCriteria()) { - Set equiJoinTables = Sets.intersection(leadingTables, equiJoin.getTables()); - totalJoinTables = Sets.union(totalJoinTables, equiJoinTables); - if (node.getJoinType() == JoinNode.JoinType.LEFT) { - // leading.getFilters() is used to determine which join the join condition should apply - // to. For LEFT join, we need to add rightHand tables as well. In the getJoinConditions - // function, we ensure that the tables joined by the Join include both the tables in the - // join condition and the right tables of the outer join, so that the join condition - // can be applied. - equiJoinTables = Sets.union(equiJoinTables, rightHand); - } - leading.getEquiJoins().add(new Pair<>(equiJoinTables, equiJoin.toExpression())); - leading.putConditionJoinType(equiJoin.toExpression(), node.getJoinType()); - } - - Optional asofJoin = node.getAsofCriteria(); - if (asofJoin.isPresent()) { - JoinNode.AsofJoinClause asofJoinClause = asofJoin.get(); - Set asofJoinTables = - Sets.intersection(leadingTables, asofJoinClause.getTables()); - totalJoinTables = Sets.union(totalJoinTables, asofJoinTables); - if (node.getJoinType() == JoinNode.JoinType.LEFT) { - asofJoinTables = Sets.union(asofJoinTables, rightHand); - } - leading.setAsofJoin(new Pair<>(asofJoinTables, asofJoinClause.toExpression())); - leading.putConditionJoinType(asofJoinClause.toExpression(), node.getJoinType()); - } - - // join constraint - collectJoinConstraintList(leading, leftHand, rightHand, node, totalJoinTables); - - return node; - } - - private void collectJoinConstraintList( - LeadingHint leading, - Set leftHand, - Set rightHand, - JoinNode join, - Set joinTables) { - Set totalTables = Sets.union(leftHand, rightHand); - if (join.getJoinType() == JoinNode.JoinType.INNER) { - leading.setInnerJoinTables(Sets.union(leading.getInnerJoinTables(), totalTables)); - return; - } - if (join.getJoinType() == JoinNode.JoinType.FULL) { - JoinConstraint joinConstraint = - new JoinConstraint(leftHand, rightHand, leftHand, rightHand, JoinNode.JoinType.FULL); - leading.getJoinConstraintList().add(joinConstraint); - return; - } - Set minLeftHand = Sets.intersection(joinTables, leftHand); - Set innerJoinTables = - Sets.intersection(totalTables, leading.getInnerJoinTables()); - Set filterAndInnerBelow = Sets.union(joinTables, innerJoinTables); - Set minRightHand = Sets.intersection(filterAndInnerBelow, rightHand); - - for (JoinConstraint other : leading.getJoinConstraintList()) { - if (other.getJoinType() == JoinNode.JoinType.FULL) { - if (isOverlap(leftHand, other.getLeftHand()) - || isOverlap(leftHand, other.getRightHand())) { - minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); - minLeftHand = Sets.union(minLeftHand, other.getRightHand()); - } - - if (isOverlap(rightHand, other.getLeftHand()) - || isOverlap(rightHand, other.getRightHand())) { - minRightHand = Sets.union(minRightHand, other.getLeftHand()); - minRightHand = Sets.union(minRightHand, other.getRightHand()); - } - /* Needn't do anything else with the full join */ - continue; - } - - // Current join's left table contains the right table of a previous join & join condition - // contains the right table of a previous join - if (isOverlap(leftHand, other.getRightHand()) - && isOverlap(joinTables, other.getRightHand())) { - minLeftHand = Sets.union(minLeftHand, other.getLeftHand()); - minLeftHand = Sets.union(minLeftHand, other.getRightHand()); - } - - // Current join's right table contains the right table of a previous join - if (isOverlap(rightHand, other.getRightHand())) { - minRightHand = Sets.union(minRightHand, other.getLeftHand()); - minRightHand = Sets.union(minRightHand, other.getRightHand()); - } - } - if (minLeftHand.isEmpty()) { - minLeftHand = leftHand; - } - if (minRightHand.isEmpty()) { - minRightHand = rightHand; - } - - JoinConstraint joinConstraint = - new JoinConstraint(minLeftHand, minRightHand, leftHand, rightHand, join.getJoinType()); - leading.getJoinConstraintList().add(joinConstraint); - } - - private boolean isOverlap(Set set1, Set set2) { - if (set1 == null || set2 == null || set1.isEmpty() || set2.isEmpty()) { - return false; - } - return !Sets.intersection(set1, set2).isEmpty(); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java deleted file mode 100644 index df7b4534736c0..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LeadingJoinOptimizer.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; - -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.JoinOrderHint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeadingHint; - -import com.google.common.collect.Sets; - -import java.util.Set; -import java.util.stream.Collectors; - -public class LeadingJoinOptimizer implements PlanOptimizer { - @Override - public PlanNode optimize(PlanNode plan, Context context) { - if (!context.getAnalysis().isQuery()) { - return plan; - } - - return plan.accept(new Rewriter(context.getAnalysis()), null); - } - - private static class Rewriter extends PlanVisitor { - private final Analysis analysis; - - public Rewriter(Analysis analysis) { - this.analysis = analysis; - } - - @Override - public PlanNode visitPlan(PlanNode node, Context context) { - Hint hint = analysis.getHintMap().getOrDefault(JoinOrderHint.category, null); - if (!(hint instanceof LeadingHint)) { - return node; - } - - PlanNode newNode = node.clone(); - for (PlanNode child : node.getChildren()) { - newNode.addChild(child.accept(this, context)); - } - return newNode; - } - - @Override - public PlanNode visitJoin(JoinNode node, Context context) { - LeadingHint leadingHint = (LeadingHint) analysis.getHintMap().get(JoinOrderHint.category); - Set leftTables = JoinUtils.collectAllTables(node.getLeftChild()); - Set rightTables = JoinUtils.collectAllTables(node.getRightChild()); - Set currentTables = Sets.union(leftTables, rightTables); - Set leadingTables = - leadingHint.getTables().stream().map(Identifier::new).collect(Collectors.toSet()); - - if (!currentTables.equals(leadingTables)) { - return node; - } - - PlanNode leadingJoin = leadingHint.generateLeadingJoinPlan(); - if (leadingJoin != null) { - return leadingJoin; - } - return node; - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 4386a9255f85f..afe6880394849 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -390,9 +390,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new MergeLimitWithSort(), new MergeLimitOverProjectWithSort(), new PushTopKThroughUnion())), - new ParallelizeGrouping(), - new CollectJoinConstraint(), - new LeadingJoinOptimizer()); + new ParallelizeGrouping()); this.planOptimizers = optimizerBuilder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index f4ced2b977ef4..7bd4530afd194 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -876,24 +876,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { rightProjections.put(rightSymbol, rightExpression); } - equiJoinClauses.add( - new JoinNode.EquiJoinClause( - leftSymbol, - rightSymbol, - JoinUtils.findSourceTable(node.getLeftChild(), leftSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in join left child", - leftSymbol))), - JoinUtils.findSourceTable(node.getRightChild(), rightSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in join right child", - rightSymbol))))); + equiJoinClauses.add(new JoinNode.EquiJoinClause(leftSymbol, rightSymbol)); } else { if (conjunct.equals(TRUE_LITERAL) && node.getAsofCriteria().isPresent()) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 2fe01ce4c9bd0..17ec744f58d8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -833,10 +833,7 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { for (JoinNode.EquiJoinClause clause : node.getCriteria()) { builder.add( new JoinNode.EquiJoinClause( - mapper.map(clause.getLeft()), - mapper.map(clause.getRight()), - clause.getLeftTable(), - clause.getRightTable())); + mapper.map(clause.getLeft()), mapper.map(clause.getRight()))); } List newCriteria = builder.build(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java deleted file mode 100644 index 243308cb6dd67..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinConstraint.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; - -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; - -import java.util.Set; - -public class JoinConstraint { - private final Set minLeftHand; - private final Set minRightHand; - - private final Set leftHand; - private final Set rightHand; - - private final JoinNode.JoinType joinType; - - public JoinConstraint( - Set minLeftHand, - Set minRightHand, - Set leftHand, - Set rightHand, - JoinNode.JoinType joinType) { - this.minLeftHand = minLeftHand; - this.minRightHand = minRightHand; - this.leftHand = leftHand; - this.rightHand = rightHand; - this.joinType = joinType; - } - - public JoinNode.JoinType getJoinType() { - return joinType; - } - - public Set getLeftHand() { - return leftHand; - } - - public Set getRightHand() { - return rightHand; - } - - public Set getMinLeftHand() { - return minLeftHand; - } - - public Set getMinRightHand() { - return minRightHand; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java deleted file mode 100644 index 3bd3d2bf446d6..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/JoinOrderHint.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; - -public abstract class JoinOrderHint extends Hint { - public static String category = "join-order"; - - protected JoinOrderHint(String hintName) { - super(hintName, category); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java deleted file mode 100644 index 2d89b93eb0c7a..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeadingHint.java +++ /dev/null @@ -1,526 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; - -import org.apache.iotdb.commons.exception.IoTDBRuntimeException; -import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.common.QueryId; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; - -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.tsfile.utils.Pair; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.Stack; - -import static org.apache.iotdb.rpc.TSStatusCode.INTERNAL_SERVER_ERROR; - -public class LeadingHint extends JoinOrderHint { - public static String hintName = "leading"; - private final List tables; - - private QueryId idAllocator; - - private List addJoinParameters; - private List normalizedParameters; - - private final List, Expression>> equiJoins = new ArrayList<>(); - private Pair, Expression> asofJoin; - private final Map relationToScanMap = new HashMap<>(); - - private Set innerJoinTables = ImmutableSet.of(); - - private final List joinConstraintList = new ArrayList<>(); - - private final Map conditionJoinType = Maps.newLinkedHashMap(); - - public LeadingHint(List parameters, MPPQueryContext queryContext) { - super(hintName); - // /* leading(t3 {}) 会报错 - this.tables = new ArrayList<>(); - this.idAllocator = queryContext.getQueryId(); - this.addJoinParameters = insertJoinIntoParameters(parameters); - this.normalizedParameters = parseIntoReversePolishNotation(addJoinParameters); - - if (tables.isEmpty()) { - throw new IllegalArgumentException("LeaderHint accepts one or more tables"); - } - if (hasDuplicateTable(tables)) { - throw new IllegalArgumentException("LeaderHint accepts no duplicate tables"); - } - } - - @Override - public String getKey() { - return category; - } - - public List getTables() { - return tables; - } - - public List getJoinConstraintList() { - return joinConstraintList; - } - - public List, Expression>> getEquiJoins() { - return equiJoins; - } - - public Pair, Expression> getAsofJoin() { - return asofJoin; - } - - public void setAsofJoin(Pair, Expression> asofJoin) { - this.asofJoin = asofJoin; - } - - public Map getRelationToScanMap() { - return relationToScanMap; - } - - public void putConditionJoinType(Expression filter, JoinNode.JoinType joinType) { - conditionJoinType.put(filter, joinType); - } - - public Set getInnerJoinTables() { - return innerJoinTables; - } - - public void setInnerJoinTables(Set innerJoinTables) { - this.innerJoinTables = innerJoinTables; - } - - public PlanNode generateLeadingJoinPlan() { - Stack stack = new Stack<>(); - for (String item : normalizedParameters) { - if (item.equals("join")) { - PlanNode rightChild = stack.pop(); - PlanNode leftChild = stack.pop(); - PlanNode joinPlan = makeJoinPlan(leftChild, rightChild); - if (joinPlan == null) { - return null; - } - stack.push(joinPlan); - } else { - PlanNode logicalPlan = getPlanByName(item); - if (logicalPlan == null) { - return null; - } - stack.push(logicalPlan); - } - } - - PlanNode finalJoin = stack.pop(); - // we want all filters been removed - if (!equiJoins.isEmpty()) { - throw new IllegalStateException( - "Leading hint process failed: filter should be empty, but meet: " + equiJoins); - } - if (finalJoin == null) { - throw new IoTDBRuntimeException( - "final join plan should not be null", INTERNAL_SERVER_ERROR.getStatusCode()); - } - return finalJoin; - } - - @Override - public String toString() { - if (tables == null || tables.isEmpty()) { - return hintName; - } - return hintName + "-" + String.join("-", tables); - } - - private boolean hasDuplicateTable(List tables) { - Set tableSet = Sets.newHashSet(); - for (String table : tables) { - if (!tableSet.add(table)) { - return true; - } - } - return false; - } - - public static List insertJoinIntoParameters(List list) { - List output = new ArrayList<>(); - - for (String item : list) { - if (item.equals("{")) { - output.add(item); - continue; - } else if (item.equals("}")) { - output.remove(output.size() - 1); - output.add(item); - } else { - output.add(item); - } - output.add("join"); - } - output.remove(output.size() - 1); - return output; - } - - public List parseIntoReversePolishNotation(List list) { - Stack s1 = new Stack<>(); - List s2 = new ArrayList<>(); - - for (String item : list) { - if (!(item.equals("{") || item.equals("}") || item.equals("join"))) { - tables.add(item); - s2.add(item); - } else if (item.equals("{")) { - s1.push(item); - } else if (item.equals("}")) { - while (!s1.peek().equals("{")) { - String pop = s1.pop(); - s2.add(pop); - } - s1.pop(); - } else { - while (!s1.isEmpty() && !s1.peek().equals("{")) { - s2.add(s1.pop()); - } - s1.push(item); - } - } - while (!s1.isEmpty()) { - s2.add(s1.pop()); - } - return s2; - } - - public PlanNode getPlanByName(String name) { - if (!relationToScanMap.containsKey(name)) { - return null; - } - return relationToScanMap.get(name); - } - - private PlanNode makeJoinPlan(PlanNode leftChild, PlanNode rightChild) { - JoinNode.JoinType joinType = - computeJoinType(leftChild.getInputTables(), rightChild.getInputTables()); - if (joinType == null) { - return null; - } - - List equiJoins = getEquiJoinConditions(getEquiJoins(), leftChild, rightChild); - if (!isConditionJoinTypeMatched(equiJoins, joinType)) { - return null; - } - List leftOutputSymbols = leftChild.getOutputSymbols(); - List rightOutputSymbols = rightChild.getOutputSymbols(); - Optional asofCriteria = Optional.empty(); - List criteria = new ArrayList<>(); - - for (Expression equiJoin : equiJoins) { - ComparisonExpression equality = (ComparisonExpression) equiJoin; - Symbol leftSymbol = Symbol.from(equality.getLeft()); - Symbol rightSymbol = Symbol.from(equality.getRight()); - - if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { - criteria.add( - new JoinNode.EquiJoinClause( - leftSymbol, - rightSymbol, - JoinUtils.findSourceTable(leftChild, leftSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join left child", - leftSymbol))), - JoinUtils.findSourceTable(rightChild, rightSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join right child", - rightSymbol))))); - } else if (leftOutputSymbols.contains(rightSymbol) - && rightOutputSymbols.contains(leftSymbol)) { - criteria.add( - new JoinNode.EquiJoinClause( - rightSymbol, - leftSymbol, - JoinUtils.findSourceTable(leftChild, rightSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join left child", - rightSymbol))), - JoinUtils.findSourceTable(rightChild, leftSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join right child", - leftSymbol))))); - } else { - return null; - } - } - - Expression asofJoin = getAsofJoinCondition(getAsofJoin(), leftChild, rightChild); - if (asofJoin != null) { - if (!isConditionJoinTypeMatched(asofJoin, joinType)) { - return null; - } - ComparisonExpression asofJoinExpr = (ComparisonExpression) asofJoin; - Symbol leftSymbol = Symbol.from(asofJoinExpr.getLeft()); - Symbol rightSymbol = Symbol.from(asofJoinExpr.getRight()); - - if (leftOutputSymbols.contains(leftSymbol) && rightOutputSymbols.contains(rightSymbol)) { - asofCriteria = - Optional.of( - new JoinNode.AsofJoinClause( - asofJoinExpr.getOperator(), - leftSymbol, - rightSymbol, - JoinUtils.findSourceTable(leftChild, leftSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join left child", - leftSymbol))), - JoinUtils.findSourceTable(rightChild, rightSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join right child", - rightSymbol))))); - } else if (leftOutputSymbols.contains(rightSymbol) - && rightOutputSymbols.contains(leftSymbol)) { - asofCriteria = - Optional.of( - new JoinNode.AsofJoinClause( - asofJoinExpr.getOperator().flip(), - rightSymbol, - leftSymbol, - JoinUtils.findSourceTable(leftChild, rightSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join left child", - rightSymbol))), - JoinUtils.findSourceTable(rightChild, leftSymbol) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in leading hint join right child", - leftSymbol))))); - } - } - - return new JoinNode( - idAllocator.genPlanNodeId(), - joinType, - leftChild, - rightChild, - criteria, - asofCriteria, - leftOutputSymbols, - rightOutputSymbols, - Optional.empty(), - Optional.empty()); - } - - private List getEquiJoinConditions( - List, Expression>> filters, PlanNode left, PlanNode right) { - List joinConditions = new ArrayList<>(); - for (int i = filters.size() - 1; i >= 0; i--) { - Pair, Expression> filterPair = filters.get(i); - Set tables = Sets.union(left.getInputTables(), right.getInputTables()); - // it should contain all tables in join conjunctions & right tables if it's left join - if (tables.containsAll(filterPair.left)) { - joinConditions.add(filterPair.right); - filters.remove(i); - } - } - return joinConditions; - } - - private Expression getAsofJoinCondition( - Pair, Expression> filterPair, PlanNode left, PlanNode right) { - if (filterPair == null) { - return null; - } - - Set tables = Sets.union(left.getInputTables(), right.getInputTables()); - // it should contain all tables in join conjunctions & right tables if it's left join - if (tables.containsAll(filterPair.left)) { - return filterPair.right; - } - return null; - } - - public JoinNode.JoinType computeJoinType(Set left, Set right) { - Pair joinConstraintBooleanPair = - getJoinConstraint(Sets.union(left, right), left, right); - if (!joinConstraintBooleanPair.right) { - return null; - } - if (joinConstraintBooleanPair.left == null) { - return JoinNode.JoinType.INNER; - } - - JoinConstraint joinConstraint = joinConstraintBooleanPair.left; - return joinConstraint.getJoinType(); - } - - public boolean isConditionJoinTypeMatched( - List conditions, JoinNode.JoinType joinType) { - for (Expression condition : conditions) { - JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); - if (originalJoinType == joinType) { - continue; - } - return false; - } - return true; - } - - public boolean isConditionJoinTypeMatched(Expression condition, JoinNode.JoinType joinType) { - JoinNode.JoinType originalJoinType = conditionJoinType.get(condition); - return originalJoinType == joinType; - } - - /** - * try to get join constraint. If it can not be found, it means join is inner join - * - * @return boolean value used for judging whether the join is legal - */ - public Pair getJoinConstraint( - Set joinTables, Set leftHand, Set rightHand) { - boolean mustBeLeftJoin = false; - - JoinConstraint matchedJoinConstraint = null; - for (JoinConstraint joinConstraint : joinConstraintList) { - if (joinConstraint.getJoinType() == JoinNode.JoinType.FULL) { - if ((isEqual(joinConstraint.getLeftHand(), leftHand) - && isEqual(joinConstraint.getRightHand(), rightHand)) - || (isEqual(joinConstraint.getLeftHand(), rightHand) - && isEqual(joinConstraint.getRightHand(), leftHand))) { - if (matchedJoinConstraint != null) { - return new Pair<>(null, false); - } - matchedJoinConstraint = joinConstraint; - break; - } else { - continue; - } - } - - // join操作完全不涉及最小右表约束，跳过 - if (!isOverlap(joinConstraint.getMinRightHand(), joinTables)) { - continue; - } - - // 如果当前join的所有表都在约束的"右边界"内，说明这个约束当前还没法应用，跳过 - if (joinConstraint.getMinRightHand().containsAll(joinTables)) { - continue; - } - - // 当前join的左表包含最小左右约束，跳过 - if (leftHand.containsAll(joinConstraint.getMinLeftHand()) - && leftHand.containsAll(joinConstraint.getMinRightHand())) { - continue; - } - - // 当前join的右表包含最小左右约束，跳过 - if (rightHand.containsAll(joinConstraint.getMinLeftHand()) - && rightHand.containsAll(joinConstraint.getMinRightHand())) { - continue; - } - - if (leftHand.containsAll(joinConstraint.getMinLeftHand()) - && rightHand.containsAll(joinConstraint.getMinRightHand())) { - // 当前join的左表包含最小左约束，右表包含最小右约束 - if (matchedJoinConstraint != null) { - return new Pair<>(null, false); - } - matchedJoinConstraint = joinConstraint; - } else if (rightHand.containsAll(joinConstraint.getMinLeftHand()) - && leftHand.containsAll(joinConstraint.getMinRightHand())) { - // 不支持left join转换为right join - return new Pair<>(null, false); - } else { - // 当前join的左右表和最小右约束均有交集，跳过 - // minRightHand中的表被分散在了join的两边 - if (isOverlap(leftHand, joinConstraint.getMinRightHand()) - && isOverlap(rightHand, joinConstraint.getMinRightHand())) { - continue; - } - // LEFT JOIN且joinTables 与 minLeftHand 有交集。如果当前JOIN执行完毕，再和minRightHand连接， - // 无法再满足 "minLeftHand作为左边界"的要求 - if (joinConstraint.getJoinType() != JoinNode.JoinType.LEFT - || isOverlap(joinTables, joinConstraint.getMinLeftHand())) { - return new Pair<>(null, false); - } - // LEFT JOIN 且joinTables 与 minLeftHand 无交集 - mustBeLeftJoin = true; - } - } - if (mustBeLeftJoin - && (matchedJoinConstraint == null - || matchedJoinConstraint.getJoinType() != JoinNode.JoinType.LEFT)) { - return new Pair<>(null, false); - } - // inner join - if (matchedJoinConstraint == null) { - return new Pair<>(null, true); - } - return new Pair<>(matchedJoinConstraint, true); - } - - private boolean isEqual(Set set1, Set set2) { - if (set1 == null || set2 == null) { - return set1 == set2; - } - return set1.size() == set2.size() && set1.containsAll(set2); - } - - private boolean isOverlap(Set set1, Set set2) { - if (set1 == null || set2 == null || set1.isEmpty() || set2.isEmpty()) { - return false; - } - return !Sets.intersection(set1, set2).isEmpty(); - } -} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java index 5648ce16a88ae..dee921c73fe9b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java @@ -61,12 +61,7 @@ public void simpleTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria( - ComparisonExpression.Operator.LESS_THAN, - "time", - "time_0", - "table1", - "table2") + .asofCriteria(ComparisonExpression.Operator.LESS_THAN, "time", "time_0") .left( sort( ImmutableList.of(sort("time", ASCENDING, LAST)), @@ -90,12 +85,7 @@ public void simpleTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, - "time", - "time_0", - "table1", - "table2") + .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .left( sort( ImmutableList.of(sort("time", DESCENDING, LAST)), @@ -128,11 +118,7 @@ public void toleranceTest() { builder -> builder .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, - "time", - "time_0", - "table1", - "table2") + ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .left( sort( ImmutableList.of(sort("time", DESCENDING, LAST)), @@ -167,11 +153,7 @@ public void projectInAsofCriteriaTest() { builder -> builder .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, - "expr", - "time_0", - "table1", - "table2") + ComparisonExpression.Operator.GREATER_THAN, "expr", "time_0") .equiCriteria("tag1", "tag1_1", "table1", "table2") .left( sort( @@ -225,12 +207,7 @@ public void sortEliminateTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, - "time", - "time_0", - "table1", - "table2") + .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( equiJoinClause("tag1", "tag1_1", "table1", "table2"), @@ -247,12 +224,7 @@ public void sortEliminateTest() { JoinNode.JoinType.INNER, builder -> builder - .asofCriteria( - ComparisonExpression.Operator.GREATER_THAN, - "time", - "time_0", - "table1", - "table2") + .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( equiJoinClause("tag1", "tag1_1", "table1", "table2"), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java index 1e161eea62061..9fbdd88e1cf38 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java @@ -188,11 +188,7 @@ private void assertInnerJoinTest1(String sql) { joinNode = (JoinNode) getChildrenNode(logicalPlanNode, 3); List joinCriteria = Collections.singletonList( - new JoinNode.EquiJoinClause( - Symbol.of("time"), - Symbol.of("time_0"), - ImmutableSet.of(new Identifier("t1")), - ImmutableSet.of(new Identifier("t2")))); + new JoinNode.EquiJoinClause(Symbol.of("time"), Symbol.of("time_0"))); assertJoinNodeEquals( joinNode, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java index 2ac69c2bf818e..c02f41bc8ae06 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AsofJoinClauseProvider.java @@ -21,7 +21,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import static java.lang.String.format; @@ -30,21 +29,13 @@ public class AsofJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; - private final Identifier leftTable; - private final Identifier rightTable; private final ComparisonExpression.Operator operator; public AsofJoinClauseProvider( - ComparisonExpression.Operator operator, - SymbolAlias left, - SymbolAlias right, - Identifier leftTable, - Identifier rightTable) { + ComparisonExpression.Operator operator, SymbolAlias left, SymbolAlias right) { this.operator = requireNonNull(operator, "operator is null"); this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTable = requireNonNull(leftTable, "leftTable is null"); - this.rightTable = requireNonNull(rightTable, "rightTable is null"); } public ComparisonExpression toExpression() { @@ -54,8 +45,7 @@ public ComparisonExpression toExpression() { @Override public JoinNode.AsofJoinClause getExpectedValue(SymbolAliases aliases) { - return new JoinNode.AsofJoinClause( - operator, left.toSymbol(aliases), right.toSymbol(aliases), leftTable, rightTable); + return new JoinNode.AsofJoinClause(operator, left.toSymbol(aliases), right.toSymbol(aliases)); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java index 29b8314f5b50f..8ea44457e331d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java @@ -174,14 +174,8 @@ public Builder equiCriteria(String left, String right, String leftTable, String } @CanIgnoreReturnValue - public Builder asofCriteria( - ComparisonExpression.Operator operator, - String left, - String right, - String leftTable, - String rightTable) { - this.asofJoinCriteria = - Optional.of(asofJoinClause(operator, left, right, leftTable, rightTable)); + public Builder asofCriteria(ComparisonExpression.Operator operator, String left, String right) { + this.asofJoinCriteria = Optional.of(asofJoinClause(operator, left, right)); return this; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 317c5860e7d70..0b021135c65e6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -726,17 +726,8 @@ public static ExpectedValueProvider equiJoinClause( } public static AsofJoinClauseProvider asofJoinClause( - ComparisonExpression.Operator operator, - String left, - String right, - String leftTable, - String rightTable) { - return new AsofJoinClauseProvider( - operator, - new SymbolAlias(left), - new SymbolAlias(right), - new Identifier(leftTable), - new Identifier(rightTable)); + ComparisonExpression.Operator operator, String left, String right) { + return new AsofJoinClauseProvider(operator, new SymbolAlias(left), new SymbolAlias(right)); } public static SymbolAlias symbol(String alias) { From cec41ca830b1efdbbc79370c1f4f49a5bca80c28 Mon Sep 17 00:00:00 2001 From: shizy Date: Sat, 14 Mar 2026 21:46:12 +0800 Subject: [PATCH 25/35] remove leading hint 2 --- .../plan/planner/plan/node/PlanNode.java | 11 --- .../relational/planner/RelationPlanner.java | 72 ------------------ .../TableDistributedPlanGenerator.java | 3 +- .../planner/node/DeviceTableScanNode.java | 17 ++--- .../relational/planner/node/JoinNode.java | 5 +- .../planner/node/TableScanNode.java | 9 --- .../planner/optimizations/JoinUtils.java | 73 ------------------- .../relational/analyzer/AsofJoinTest.java | 14 ++-- .../analyzer/TableFunctionTest.java | 2 +- .../planner/CorrelatedSubqueryTest.java | 4 +- .../assertions/EquiJoinClauseProvider.java | 11 +-- .../planner/assertions/JoinMatcher.java | 5 +- .../planner/assertions/PlanMatchPattern.java | 9 +-- 13 files changed, 27 insertions(+), 208 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java index 73bb65e486617..4e3196c8d42c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNode.java @@ -23,7 +23,6 @@ import org.apache.iotdb.consensus.common.request.IConsensusRequest; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -33,10 +32,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.Set; import static java.util.Objects.requireNonNull; @@ -81,14 +78,6 @@ public void markAsGeneratedByPipe() { public abstract void addChild(PlanNode child); - public Set getInputTables() { - Set tables = new HashSet<>(); - for (PlanNode child : getChildren()) { - tables.addAll(child.getInputTables()); - } - return tables; - } - /** * If this plan node has to be serialized or deserialized, override this method. If this method is * overridden, the serialization and deserialization methods must be implemented. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index deecb649a5556..516be38ad8eb6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -518,27 +518,6 @@ If casts are redundant (due to column type and common type being equal), rightCoercions.put(rightOutput, right.getSymbol(rightField).toSymbolReference()); rightJoinColumns.put(identifier, rightOutput); - // Extract tables from the actual fields used in the join condition - Field leftFieldObj = left.getScope().getRelationType().getFieldByIndex(leftField); - Identifier leftTable = - extractTableName(leftFieldObj) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for field %s in JOIN USING clause", - leftFieldObj.getName()))); - - Field rightFieldObj = right.getScope().getRelationType().getFieldByIndex(rightField); - Identifier rightTable = - extractTableName(rightFieldObj) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for field %s in JOIN USING clause", - rightFieldObj.getName()))); - clauses.add(new JoinNode.EquiJoinClause(leftOutput, rightOutput)); } @@ -740,41 +719,6 @@ public RelationPlan planJoin( rightPlanBuilder = rightCoercions.getSubPlan(); for (int i = 0; i < leftComparisonExpressions.size(); i++) { - // Extract tables from expressions - Set leftDependencies = - SymbolsExtractor.extractNames( - leftComparisonExpressions.get(i), analysis.getColumnReferences()); - Set rightDependencies = - SymbolsExtractor.extractNames( - rightComparisonExpressions.get(i), analysis.getColumnReferences()); - - if (leftDependencies.size() != 1 || rightDependencies.size() != 1) { - throw new IllegalStateException("Cannot find source table for symbol"); - } - - QualifiedName leftQualifiedName = leftDependencies.iterator().next(); - QualifiedName rightQualifiedName = rightDependencies.iterator().next(); - - Identifier leftTable = - leftQualifiedName - .getPrefix() - .map(prefix -> new Identifier(prefix.getSuffix())) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in JOIN ON clause"))); - - Identifier rightTable = - rightQualifiedName - .getPrefix() - .map(prefix -> new Identifier(prefix.getSuffix())) - .orElseThrow( - () -> - new IllegalStateException( - String.format( - "Cannot find source table for symbol %s in JOIN ON clause"))); - if (asofCriteria != null && i == 0) { Symbol leftSymbol = leftCoercions.get(leftComparisonExpressions.get(i)); Symbol rightSymbol = rightCoercions.get(rightComparisonExpressions.get(i)); @@ -1703,20 +1647,4 @@ public Map getVariableDefinitions() { return variableDefinitions; } } - - /** - * Extracts the table name from a field. Priority: 1) relation alias (if exists), 2) origin table - * name. - * - * @param field the field to extract the table name from - * @return the table identifier, or empty if not found - */ - private static Optional extractTableName(Field field) { - Optional fromAlias = - field.getRelationAlias().map(alias -> new Identifier(alias.getSuffix())); - if (fromAlias.isPresent()) { - return fromAlias; - } - return field.getOriginTable().map(originTable -> new Identifier(originTable.getObjectName())); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 6978d15ac8cf9..f0b4d4751ca36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -635,8 +635,7 @@ public List visitJoin(JoinNode node, PlanContext context) { node.setRightChild(mergeChildrenViaCollectOrMergeSort(rightChildOrdering, rightChildrenNodes)); // Now the join implement but CROSS is MergeSortJoin, so it can keep order - // if (!node.isCrossJoin() && !node.getAsofCriteria().isPresent()) { // cross join？ - if (!node.getAsofCriteria().isPresent()) { + if (!node.isCrossJoin() && !node.getAsofCriteria().isPresent()) { switch (node.getJoinType()) { case FULL: // If join type is FULL Join, we will process SortProperties in ProjectNode above this diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java index cead343ddcee4..44eef92451f95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/DeviceTableScanNode.java @@ -148,21 +148,20 @@ public DeviceTableScanNode( boolean pushLimitToEachDevice, boolean containsNonAlignedDevice, Identifier alias) { - super( + this( id, qualifiedObjectName, outputSymbols, assignments, + deviceEntries, + tagAndAttributeIndexMap, + scanOrder, + timePredicate, pushDownPredicate, pushDownLimit, - pushDownOffset); - this.deviceEntries = deviceEntries; - this.tagAndAttributeIndexMap = tagAndAttributeIndexMap; - this.scanOrder = scanOrder; - this.timePredicate = timePredicate; - this.pushDownPredicate = pushDownPredicate; - this.pushLimitToEachDevice = pushLimitToEachDevice; - this.containsNonAlignedDevice = containsNonAlignedDevice; + pushDownOffset, + pushLimitToEachDevice, + containsNonAlignedDevice); this.alias = alias; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java index ce82dbe73e926..5e6cc5818acc1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/JoinNode.java @@ -295,9 +295,8 @@ public static JoinNode deserialize(ByteBuffer byteBuffer) { int size = ReadWriteIOUtils.readInt(byteBuffer); List criteria = new ArrayList<>(size); while (size-- > 0) { - Symbol left = Symbol.deserialize(byteBuffer); - Symbol right = Symbol.deserialize(byteBuffer); - criteria.add(new EquiJoinClause(left, right)); + criteria.add( + new EquiJoinClause(Symbol.deserialize(byteBuffer), Symbol.deserialize(byteBuffer))); } Optional asofJoinClause = Optional.empty(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index dff5887961994..c6bda2232621c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -43,12 +43,10 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -116,13 +114,6 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visitTableScan(this, context); } - @Override - public Set getInputTables() { - Set tables = new HashSet<>(); - tables.add(alias != null ? alias : new Identifier(qualifiedObjectName.getObjectName())); - return tables; - } - @Override public List getChildren() { return ImmutableList.of(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java index fff297d18a20a..9e751d2dd4d31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/JoinUtils.java @@ -19,22 +19,18 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.planner.EqualityInference; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.Collection; import java.util.Objects; -import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; @@ -408,73 +404,4 @@ public Expression getPostJoinPredicate() { return postJoinPredicate; } } - - /** - * Finds the source table for a given symbol by traversing the plan tree. Returns Optional.empty() - * if the symbol cannot be traced to a specific table. - * - * @param node the plan node to search - * @param symbol the symbol to find the source table for - * @return the table identifier if found, Optional.empty() otherwise - */ - public static Optional findSourceTable(PlanNode node, Symbol symbol) { - if (node instanceof DeviceTableScanNode) { - DeviceTableScanNode scanNode = (DeviceTableScanNode) node; - if (scanNode.getOutputSymbols().contains(symbol)) { - Identifier alias = scanNode.getAlias(); - if (alias != null) { - return Optional.of(alias); - } - return Optional.of(new Identifier(scanNode.getQualifiedObjectName().getObjectName())); - } - return Optional.empty(); - } - - // For other node types, recursively check children - for (PlanNode child : node.getChildren()) { - if (child.getOutputSymbols().contains(symbol)) { - Optional result = findSourceTable(child, symbol); - if (result.isPresent()) { - return result; - } - } - } - - return Optional.empty(); - } - - /** - * Collects all table names or aliases from a PlanNode by traversing the plan tree. - * - * @param node the plan node to traverse - * @return a set of all table identifiers (names or aliases) found in the plan - */ - public static Set collectAllTables(PlanNode node) { - Set tables = new java.util.HashSet<>(); - collectAllTables(node, tables); - return tables; - } - - /** - * Recursively collects all table names or aliases from a PlanNode and its children. - * - * @param node the plan node to traverse - * @param tables the set to accumulate table identifiers - */ - private static void collectAllTables(PlanNode node, Set tables) { - if (node instanceof DeviceTableScanNode) { - DeviceTableScanNode scanNode = (DeviceTableScanNode) node; - Identifier alias = scanNode.getAlias(); - if (alias != null) { - tables.add(alias); - } else { - tables.add(new Identifier(scanNode.getQualifiedObjectName().getObjectName())); - } - } - - // Recursively process all children - for (PlanNode child : node.getChildren()) { - collectAllTables(child, tables); - } - } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java index dee921c73fe9b..62ec85d27745b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AsofJoinTest.java @@ -154,7 +154,7 @@ public void projectInAsofCriteriaTest() { builder .asofCriteria( ComparisonExpression.Operator.GREATER_THAN, "expr", "time_0") - .equiCriteria("tag1", "tag1_1", "table1", "table2") + .equiCriteria("tag1", "tag1_1") .left( sort( ImmutableList.of( @@ -210,9 +210,9 @@ public void sortEliminateTest() { .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( - equiJoinClause("tag1", "tag1_1", "table1", "table2"), - equiJoinClause("tag2", "tag2_2", "table1", "table2"), - equiJoinClause("tag3", "tag3_3", "table1", "table2"))) + equiJoinClause("tag1", "tag1_1"), + equiJoinClause("tag2", "tag2_2"), + equiJoinClause("tag3", "tag3_3"))) .left(sort(table1)) .right(sort(table2))))); @@ -227,9 +227,9 @@ public void sortEliminateTest() { .asofCriteria(ComparisonExpression.Operator.GREATER_THAN, "time", "time_0") .equiCriteria( ImmutableList.of( - equiJoinClause("tag1", "tag1_1", "table1", "table2"), - equiJoinClause("tag2", "tag2_2", "table1", "table2"), - equiJoinClause("tag3", "tag3_3", "table1", "table2"))) + equiJoinClause("tag1", "tag1_1"), + equiJoinClause("tag2", "tag2_2"), + equiJoinClause("tag3", "tag3_3"))) .left(exchange()) .right(exchange())))); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java index da2edb5051dab..344c69cfc1a71 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TableFunctionTest.java @@ -281,7 +281,7 @@ public void testLeafFunction() { builder .left(sort(tableFunctionProcessor(tableFunctionMatcher1))) .right(sort(tableFunctionProcessor(tableFunctionMatcher2))) - .equiCriteria("output", "output_0", "a", "b")))); + .equiCriteria("output", "output_0")))); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java index c54df4742c43a..e7baf8078f112 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java @@ -84,7 +84,7 @@ public void testCorrelatedExistsSubquery() { JoinNode.JoinType.INNER, builder -> builder - .equiCriteria("s1", "s2_7", "t1", "t2") + .equiCriteria("s1", "s2_7") .left(sort(tableScan1)) .right( sort( @@ -133,7 +133,7 @@ public void testCorrelatedExistsSubquery() { JoinNode.JoinType.LEFT, builder -> builder - .equiCriteria("s1", "s2_3", "t1", "t2") + .equiCriteria("s1", "s2_3") .left(sort(tableScan1)) .right( sort( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java index 99ca1fc8af2df..c151b6874a6f8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/EquiJoinClauseProvider.java @@ -20,28 +20,21 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import static java.util.Objects.requireNonNull; public class EquiJoinClauseProvider implements ExpectedValueProvider { private final SymbolAlias left; private final SymbolAlias right; - private final Identifier leftTable; - private final Identifier rightTable; - public EquiJoinClauseProvider( - SymbolAlias left, SymbolAlias right, Identifier leftTable, Identifier rightTable) { + public EquiJoinClauseProvider(SymbolAlias left, SymbolAlias right) { this.left = requireNonNull(left, "left is null"); this.right = requireNonNull(right, "right is null"); - this.leftTable = requireNonNull(leftTable, "leftTable is null"); - this.rightTable = requireNonNull(rightTable, "rightTable is null"); } @Override public JoinNode.EquiJoinClause getExpectedValue(SymbolAliases aliases) { - return new JoinNode.EquiJoinClause( - left.toSymbol(aliases), right.toSymbol(aliases), leftTable, rightTable); + return new JoinNode.EquiJoinClause(left.toSymbol(aliases), right.toSymbol(aliases)); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java index 8ea44457e331d..090e4a0bc3bc9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/JoinMatcher.java @@ -166,9 +166,8 @@ public Builder equiCriteria( } @CanIgnoreReturnValue - public Builder equiCriteria(String left, String right, String leftTable, String rightTable) { - this.equiCriteria = - Optional.of(ImmutableList.of(equiJoinClause(left, right, leftTable, rightTable))); + public Builder equiCriteria(String left, String right) { + this.equiCriteria = Optional.of(ImmutableList.of(equiJoinClause(left, right))); return this; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 0b021135c65e6..2ca57e5296a5b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -63,7 +63,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; @@ -717,12 +716,8 @@ public static PlanMatchPattern strictProject( } public static ExpectedValueProvider equiJoinClause( - String left, String right, String leftTable, String rightTable) { - return new EquiJoinClauseProvider( - new SymbolAlias(left), - new SymbolAlias(right), - new Identifier(leftTable), - new Identifier(rightTable)); + String left, String right) { + return new EquiJoinClauseProvider(new SymbolAlias(left), new SymbolAlias(right)); } public static AsofJoinClauseProvider asofJoinClause( From 8c1d37cc3fb20bc2619b9aa372727bcc36f91ae2 Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 19 Mar 2026 08:38:35 +0800 Subject: [PATCH 26/35] refactor leader/follower hint --- .../analyzer/StatementAnalyzer.java | 113 +++++++++--------- .../plan/relational/sql/ast/AstVisitor.java | 8 +- .../relational/sql/ast/FollowerHintItem.java | 110 +++++++++++++++++ ...rizedHintItem.java => LeaderHintItem.java} | 40 +++---- ...pleHintItem.java => ParallelHintItem.java} | 33 +++-- .../relational/sql/parser/AstBuilder.java | 36 ++++-- .../relational/utils/hint/FollowerHint.java | 38 ++++-- .../relational/utils/hint/HintFactory.java | 81 ------------- .../relational/utils/hint/LeaderHint.java | 8 +- .../relational/utils/hint/ParallelHint.java | 44 +++++++ .../plan/relational/analyzer/JoinTest.java | 7 +- .../relational/grammar/sql/RelationalSql.g4 | 27 ++++- 12 files changed, 331 insertions(+), 214 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{ParameterizedHintItem.java => LeaderHintItem.java} (64%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{SimpleHintItem.java => ParallelHintItem.java} (74%) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 8a0a286fd953c..505d5a356df74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -100,6 +100,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FollowerHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupBy; @@ -120,6 +121,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinCriteria; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LeaderHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; @@ -133,8 +135,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParallelHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; @@ -163,7 +165,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; @@ -200,8 +201,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.type.TypeManager; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.HintFactory; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.ParallelHint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -369,14 +370,6 @@ private enum UpdateKind { MERGE, } - // Hint definition and processing methods - private static final Map HINT_DEFINITIONS = - ImmutableMap.of( - "LEADER", - new HintFactory(LeaderHint.hintName, LeaderHint::new, true), - "FOLLOWER", - new HintFactory(FollowerHint.hintName, FollowerHint::new, true)); - /** * Visitor context represents local query scope (if exists). The invariant is that the local query * scopes hierarchy should always have outer query scope (if provided) as ancestor. @@ -1527,60 +1520,70 @@ private void analyzeHint(QuerySpecification node, Scope scope) { public Scope visitSelectHint(SelectHint node, final Optional context) { Map hintMap = new HashMap<>(); for (Node hintItem : node.getHintItems()) { - if (hintItem instanceof ParameterizedHintItem) { - ParameterizedHintItem paramHint = (ParameterizedHintItem) hintItem; - String hintName = paramHint.getHintName(); - List params = paramHint.getParameters(); - addHint(hintName, params, hintMap); - } else if (hintItem instanceof SimpleHintItem) { - SimpleHintItem simpleHint = (SimpleHintItem) hintItem; - String hintName = simpleHint.getHintName(); - addHint(hintName, null, hintMap); + if (hintItem instanceof LeaderHintItem) { + LeaderHintItem leaderHintItem = (LeaderHintItem) hintItem; + List tables = leaderHintItem.getTables(); + addLeaderHints(tables, hintMap); + } else if (hintItem instanceof FollowerHintItem) { + FollowerHintItem followerHintItem = (FollowerHintItem) hintItem; + List tables = followerHintItem.getTables(); + List> nodeIds = followerHintItem.getNodeIds(); + addFollowerHints(tables, nodeIds, hintMap); + } else if (hintItem instanceof ParallelHintItem) { + ParallelHintItem parallelHintItem = (ParallelHintItem) hintItem; + int parallelism = parallelHintItem.getParallelism(); + Hint hint = new ParallelHint(parallelism); + hintMap.putIfAbsent(hint.getKey(), hint); } } analysis.setHintMap(hintMap); return createAndAssignScope(node, context); } - private List intersect(List a, List b) { - return a.stream().filter(b::contains).collect(toImmutableList()); - } - - private void addHint(String hintName, List parameters, Map hintMap) { - HintFactory definition = HINT_DEFINITIONS.get(hintName); - if (definition != null) { - List existingTables = - analysis.getRelationNames().values().stream() - .map(QualifiedName::getSuffix) - .collect(toImmutableList()); + private void addLeaderHints(List tables, Map hintMap) { + List existingTables = + analysis.getRelationNames().values().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); - if (definition.shouldExpandParameters()) { - List tablesToProcess = parameters == null ? existingTables : parameters; - tablesToProcess.stream() - .filter(table -> parameters == null || existingTables.contains(table)) - .forEach( - table -> { - Hint hint = definition.createHint(ImmutableList.of(table), queryContext); - String hintKey = hint.getKey(); - hintMap.putIfAbsent(hintKey, hint); - }); - } else { - // Skip if parameters contain tables that don't exist in the query - if (parameters != null) { - boolean hasInvalidTable = - parameters.stream() - .filter(table -> !table.equals("{") && !table.equals("}")) - .anyMatch(table -> !existingTables.contains(table)); - if (hasInvalidTable) { - return; - } + if (tables == null || tables.isEmpty()) { + existingTables.forEach( + table -> { + Hint hint = new LeaderHint(table); + hintMap.putIfAbsent(hint.getKey(), hint); + }); + } else { + for (String table : tables) { + if (!existingTables.contains(table)) { + continue; } + Hint hint = new LeaderHint(table); + hintMap.putIfAbsent(hint.getKey(), hint); + } + } + } - Hint hint = definition.createHint(parameters, queryContext); - String hintKey = hint.getKey(); - if (!hintMap.containsKey(hintKey)) { - hintMap.put(hintKey, hint); + private void addFollowerHints( + List tables, List> nodeIds, Map hintMap) { + List existingTables = + analysis.getRelationNames().values().stream() + .map(QualifiedName::getSuffix) + .collect(toImmutableList()); + + if (tables == null || tables.isEmpty()) { + existingTables.forEach( + table -> { + Hint hint = new FollowerHint(table, ImmutableList.of()); + hintMap.putIfAbsent(hint.getKey(), hint); + }); + } else { + for (int i = 0; i < tables.size(); i++) { + String table = tables.get(i); + if (!existingTables.contains(table)) { + continue; } + Hint hint = new FollowerHint(table, nodeIds.get(i)); + hintMap.putIfAbsent(hint.getKey(), hint); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index fc5623cc1e52d..310f5d16c13e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -141,11 +141,15 @@ protected R visitSelectHint(SelectHint node, C context) { return visitNode(node, context); } - protected R visitSimpleHintItem(SimpleHintItem node, C context) { + protected R visitLeaderHintItem(LeaderHintItem node, C context) { return visitNode(node, context); } - protected R visitParameterizedHintItem(ParameterizedHintItem node, C context) { + protected R visitFollowerHintItem(FollowerHintItem node, C context) { + return visitNode(node, context); + } + + protected R visitParallelHintItem(ParallelHintItem node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java new file mode 100644 index 0000000000000..e091b8d39ae8e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java @@ -0,0 +1,110 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class FollowerHintItem extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(FollowerHintItem.class); + + private static final String hintName = "FOLLOWER"; + private final List tables; + private final List> nodeIds; + + public FollowerHintItem(List tables, List> nodeIds) { + super(null); + this.tables = ImmutableList.copyOf(tables); + this.nodeIds = ImmutableList.copyOf(nodeIds); + } + + public List getTables() { + return tables; + } + + public List> getNodeIds() { + return nodeIds; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitFollowerHintItem(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(tables); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + FollowerHintItem other = (FollowerHintItem) obj; + return Objects.equals(this.tables, other.tables) && Objects.equals(this.nodeIds, other.nodeIds); + } + + @Override + public String toString() { + return hintName + + "(" + + IntStream.range(0, tables.size()) + .mapToObj( + i -> { + String table = tables.get(i); + List ids = nodeIds.get(i); + if (ids == null || ids.isEmpty()) { + return table; + } + return table + + "(" + + ids.stream().map(String::valueOf).collect(Collectors.joining(",")) + + ")"; + }) + .collect(Collectors.joining(",")) + + ")"; + } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += AstMemoryEstimationHelper.getEstimatedSizeOfStringList(tables); + size += RamUsageEstimator.sizeOfArrayList(nodeIds); + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java similarity index 64% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java index cc3b98d9b45b8..2ad2ae4c2131b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParameterizedHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java @@ -27,26 +27,25 @@ import java.util.List; import java.util.Objects; -/** Represents a parameterized hint, e.g., "LEADER(table1)" or "FOLLOWER(table2)". */ -public class ParameterizedHintItem extends Node { +public class LeaderHintItem extends Node { private static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(ParameterizedHintItem.class); + RamUsageEstimator.shallowSizeOfInstance(LeaderHintItem.class); - private final String hintName; - private final List parameters; + private static final String hintName = "LEADER"; + private final List tables; - public ParameterizedHintItem(String hintName, List parameters) { + public LeaderHintItem(List tables) { super(null); - this.hintName = hintName.toUpperCase(); - this.parameters = ImmutableList.copyOf(parameters); + this.tables = ImmutableList.copyOf(tables); } - public String getHintName() { - return hintName; + public List getTables() { + return tables; } - public List getParameters() { - return parameters; + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitLeaderHintItem(this, context); } @Override @@ -54,14 +53,9 @@ public List getChildren() { return ImmutableList.of(); } - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitParameterizedHintItem(this, context); - } - @Override public int hashCode() { - return Objects.hash(hintName, parameters); + return Objects.hash(tables); } @Override @@ -72,22 +66,20 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { return false; } - ParameterizedHintItem other = (ParameterizedHintItem) obj; - return Objects.equals(this.hintName, other.hintName) - && Objects.equals(this.parameters, other.parameters); + LeaderHintItem other = (LeaderHintItem) obj; + return Objects.equals(this.tables, other.tables); } @Override public String toString() { - return hintName + "(" + String.join(", ", parameters) + ")"; + return hintName + "(" + String.join(", ", tables) + ")"; } @Override public long ramBytesUsed() { long size = INSTANCE_SIZE; size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); - size += AstMemoryEstimationHelper.getEstimatedSizeOfStringList(parameters); - size += RamUsageEstimator.sizeOf(hintName); + size += AstMemoryEstimationHelper.getEstimatedSizeOfStringList(tables); return size; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java similarity index 74% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java index 9eebb65fe9da2..bc464abe7422c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SimpleHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java @@ -27,35 +27,35 @@ import java.util.List; import java.util.Objects; -/** Represents a simple hint without parameters, e.g., "LEADER". */ -public class SimpleHintItem extends Node { +public class ParallelHintItem extends Node { private static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(SimpleHintItem.class); + RamUsageEstimator.shallowSizeOfInstance(ParallelHintItem.class); - private final String hintName; + private static final String hintName = "PARALLEL"; + private final int parallelism; - public SimpleHintItem(String hintName) { + public ParallelHintItem(int parallelism) { super(null); - this.hintName = hintName.toUpperCase(); + this.parallelism = parallelism; } - public String getHintName() { - return hintName; + public int getParallelism() { + return parallelism; } @Override - public List getChildren() { - return ImmutableList.of(); + public R accept(AstVisitor visitor, C context) { + return visitor.visitParallelHintItem(this, context); } @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitSimpleHintItem(this, context); + public List getChildren() { + return ImmutableList.of(); } @Override public int hashCode() { - return Objects.hash(hintName); + return Objects.hash(parallelism); } @Override @@ -66,20 +66,19 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { return false; } - SimpleHintItem other = (SimpleHintItem) obj; - return Objects.equals(this.hintName, other.hintName); + ParallelHintItem other = (ParallelHintItem) obj; + return this.parallelism == other.parallelism; } @Override public String toString() { - return hintName; + return hintName + "(" + parallelism + ")"; } @Override public long ramBytesUsed() { long size = INSTANCE_SIZE; size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); - size += RamUsageEstimator.sizeOf(hintName); return size; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 96a9658a093e4..26f2f43a22382 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -104,6 +104,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FollowerHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; @@ -124,6 +125,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.KillQuery; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LeaderHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; @@ -144,8 +146,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OneOrMoreQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParallelHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ParameterizedHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternAlternation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternConcatenation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternPermutation; @@ -215,7 +217,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVersion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; @@ -2564,18 +2565,33 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { } @Override - public Node visitParameterizedHint(RelationalSqlParser.ParameterizedHintContext ctx) { - String hintName = ctx.identifier().getText(); - List identifiers = ctx.hintParameter(); - List params = + public Node visitLeaderHint(RelationalSqlParser.LeaderHintContext ctx) { + List identifiers = ctx.identifier(); + List tables = identifiers.stream().map(x -> x.getText().toLowerCase()).collect(Collectors.toList()); - return new ParameterizedHintItem(hintName, params); + return new LeaderHintItem(tables); } @Override - public Node visitSimpleHint(RelationalSqlParser.SimpleHintContext ctx) { - String hintName = ctx.identifier().getText(); - return new SimpleHintItem(hintName); + public Node visitFollowerHint(RelationalSqlParser.FollowerHintContext ctx) { + List followerParameters = ctx.followerParameter(); + List tables = + followerParameters.stream().map(x -> x.identifier().getText()).collect(Collectors.toList()); + List> nodeIds = + followerParameters.stream() + .map( + x -> + x.number().stream() + .map(y -> Integer.parseInt(y.getText())) + .collect(Collectors.toList())) + .collect(Collectors.toList()); + return new FollowerHintItem(tables, nodeIds); + } + + @Override + public Node visitParallelHint(RelationalSqlParser.ParallelHintContext ctx) { + int parallelism = Integer.parseInt(ctx.number().getText()); + return new ParallelHintItem(parallelism); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java index 5e412f1a650cc..b9ed579d504d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java @@ -22,20 +22,21 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class FollowerHint extends ReplicaHint { public static String hintName = "follower"; private final String table; + private final List nodeIds; - public FollowerHint(List tables, MPPQueryContext queryContext) { + public FollowerHint(String table, List nodeIds) { super(hintName); - if (tables == null || tables.size() != 1) { - throw new IllegalArgumentException("FollowerHint accepts exactly one table"); - } - table = tables.get(0); + this.table = table; + this.nodeIds = nodeIds; } @Override @@ -53,7 +54,28 @@ public List selectLocations(List dataNodeL if (dataNodeLocations == null || dataNodeLocations.size() <= 1) { return dataNodeLocations; } - // Return only followers (all locations except the first/leader) - return dataNodeLocations.subList(1, dataNodeLocations.size()); + + List followerLocations = + dataNodeLocations.subList(1, dataNodeLocations.size()); + if (nodeIds == null || nodeIds.isEmpty()) { + return followerLocations; + } + + // nodeId -> Location for O(1) lookup + Map followerLocationMap = new HashMap<>(); + for (TDataNodeLocation location : followerLocations) { + followerLocationMap.put(location.getDataNodeId(), location); + } + + // Find the first nodeId that exists in followerLocations and return its location + for (Integer nodeId : nodeIds) { + TDataNodeLocation location = followerLocationMap.get(nodeId); + if (location != null) { + return Collections.singletonList(location); + } + } + + // If no matching nodeId found, return all follower locations + return followerLocations; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java deleted file mode 100644 index ea7dea56516f6..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/HintFactory.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; - -import org.apache.iotdb.db.queryengine.common.MPPQueryContext; - -import java.util.List; -import java.util.function.BiFunction; - -/** - * Defines the properties and creation logic for a SQL hint. This class encapsulates the hint's key, - * creation strategy. - */ -public final class HintFactory { - - private final String key; - private final BiFunction, MPPQueryContext, Hint> factory; - private final boolean expandParameters; - - /** - * Creates a new hint definition with parameter expansion option. - * - * @param key the key to use when storing the hint in the hint map - * @param hintFactory factory method to create the hint instance - * @param expandParameters whether to expand array parameters into multiple hints - */ - public HintFactory( - String key, - BiFunction, MPPQueryContext, Hint> hintFactory, - boolean expandParameters) { - this.key = key; - this.factory = hintFactory; - this.expandParameters = expandParameters; - } - - /** - * Gets the hint name used to create hint instance. - * - * @return the hint key - */ - public String getKey() { - return key; - } - - /** - * Creates the hint instance. - * - * @return the created hint - */ - public Hint createHint(List parameters, MPPQueryContext context) { - return factory.apply(parameters, context); - } - - /** - * Checks whether this hint should expand parameters into multiple hints. - * - * @return true if parameters should be expanded, false otherwise - */ - public boolean shouldExpandParameters() { - return expandParameters; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java index 1118270f8a0b8..352a713e94a6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java @@ -22,7 +22,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import java.util.Collections; import java.util.List; @@ -31,12 +30,9 @@ public class LeaderHint extends ReplicaHint { public static String hintName = "leader"; private final String table; - public LeaderHint(List tables, MPPQueryContext queryContext) { + public LeaderHint(String table) { super(hintName); - if (tables == null || tables.size() != 1) { - throw new IllegalArgumentException("LeaderHint accepts exactly one table"); - } - table = tables.get(0); + this.table = table; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java new file mode 100644 index 0000000000000..70ca8da9dfbf9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java @@ -0,0 +1,44 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + */ + +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +public class ParallelHint extends Hint { + public static String category = "parallel"; + public static String hintName = "parallel"; + + private final int parallelism; + + public ParallelHint(int parallelism) { + super(hintName, category); + this.parallelism = parallelism; + } + + @Override + public String getKey() { + return category; + } + + @Override + public String toString() { + return hintName + "(" + parallelism + ")"; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java index 9fbdd88e1cf38..d3e90cdc061a6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java @@ -53,7 +53,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -370,11 +369,7 @@ private void assertInnerJoinTest2(String sql, boolean joinUsing) { joinNode = (JoinNode) getChildrenNode(logicalPlanNode, 4); List joinCriteria = Collections.singletonList( - new JoinNode.EquiJoinClause( - Symbol.of("time"), - Symbol.of("time_0"), - ImmutableSet.of(new Identifier("t1")), - ImmutableSet.of(new Identifier("t2")))); + new JoinNode.EquiJoinClause(Symbol.of("time"), Symbol.of("time_0"))); assertJoinNodeEquals( joinNode, INNER, diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index e9c2313fb7ef6..3f95434e83120 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1121,14 +1121,28 @@ selectHint ; hintItem - : identifier '(' hintParameter+ ')' #parameterizedHint - | identifier #simpleHint + : leaderHint + | followerHint + | parallelHint ; -hintParameter +leaderHint + : LEADER + | LEADER '(' identifier (',' identifier)* ')' + ; + +followerHint + : FOLLOWER + | FOLLOWER '(' followerParameter (',' followerParameter)* ')' + ; + +followerParameter : identifier - | '{' - | '}' + | identifier '(' number (',' number)* ')' + ; + +parallelHint + : PARALLEL '(' number ')' ; patternRecognition @@ -1936,6 +1950,9 @@ WRITE: 'WRITE'; YEAR: 'YEAR' | 'Y'; ZONE: 'ZONE'; AUDIT: 'AUDIT'; +LEADER: 'LEADER'; +FOLLOWER: 'FOLLOWER'; +PARALLEL: 'PARALLEL'; AT_SIGN: '@'; EQ: '='; From 035d0101dd2e3b652c638b28978ddbf9640a7f51 Mon Sep 17 00:00:00 2001 From: shizy Date: Thu, 19 Mar 2026 09:22:46 +0800 Subject: [PATCH 27/35] parallel hint implementation --- .../db/queryengine/common/MPPQueryContext.java | 12 +++++++++++- .../fragment/FragmentInstanceContext.java | 11 +++++++++++ .../fragment/FragmentInstanceManager.java | 3 +++ .../plan/planner/plan/FragmentInstance.java | 16 ++++++++++++++++ .../relational/analyzer/StatementAnalyzer.java | 6 ++++-- .../distribute/TableDistributedPlanner.java | 9 ++++++++- .../TableModelQueryFragmentPlanner.java | 3 +++ .../relational/sql/ast/FollowerHintItem.java | 4 ++-- .../plan/relational/sql/ast/LeaderHintItem.java | 4 ++-- .../relational/sql/ast/ParallelHintItem.java | 4 ++-- .../plan/relational/utils/hint/ParallelHint.java | 4 ++++ 11 files changed, 66 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 886da68f1c2e3..b9266ba6e90f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -37,7 +37,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; import org.apache.iotdb.db.utils.cte.CteDataStore; @@ -154,6 +153,9 @@ public enum ExplainType { // Tables in the subquery private final Map, Set> subQueryTables = new HashMap<>(); + // parallel hint + private int parallelism = 0; + @TestOnly public MPPQueryContext(QueryId queryId) { this.queryId = queryId; @@ -677,5 +679,13 @@ public IAuditEntity setSqlString(String sqlString) { return this; } + public int getParallelism() { + return parallelism; + } + + public void setParallelism(int parallelism) { + this.parallelism = parallelism; + } + // ================= Authentication Interfaces ========================= } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index df9ac2d2f936f..314cbb59aebe4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -162,6 +162,9 @@ public class FragmentInstanceContext extends QueryContext { private long closedSeqFileNum = 0; private long closedUnseqFileNum = 0; + // parallel hint + private int parallelism = 0; + public static FragmentInstanceContext createFragmentInstanceContext( FragmentInstanceId id, FragmentInstanceStateMachine stateMachine, @@ -1193,4 +1196,12 @@ public boolean ignoreNotExistsDevice() { public boolean isSingleSourcePath() { return singleSourcePath; } + + public int getParallelism() { + return parallelism; + } + + public void setParallelism(int parallelism) { + this.parallelism = parallelism; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java index 1898cbfe53ccb..3a4241cab3a08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java @@ -164,6 +164,9 @@ public FragmentInstanceInfo execDataQueryFragmentInstance( instance.isDebug(), instance.isVerbose())); + // set parallelism from fragment instance + context.setParallelism(instance.getParallelism()); + try { List driverFactories = planner.plan( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java index d53e29e16924f..e592f18f7cf80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java @@ -72,6 +72,9 @@ public class FragmentInstance implements IConsensusRequest { private boolean isHighestPriority; + // parallel hint + private int parallelism = 0; + // indicate which index we are retrying private transient int nextRetryIndex = 0; @@ -101,6 +104,7 @@ public FragmentInstance( this.timeOut = timeOut > 0 ? timeOut : CONFIG.getQueryTimeoutThreshold(); this.isRoot = false; this.sessionInfo = sessionInfo; + this.parallelism = 0; this.debug = debug; this.verbose = verbose; } @@ -213,6 +217,14 @@ public void setDataNodeFINum(int dataNodeFINum) { this.dataNodeFINum = dataNodeFINum; } + public int getParallelism() { + return parallelism; + } + + public void setParallelism(int parallelism) { + this.parallelism = parallelism; + } + public boolean isDebug() { return debug; } @@ -250,6 +262,7 @@ public static FragmentInstance deserializeFrom(ByteBuffer buffer) { TimePredicate globalTimePredicate = hasTimePredicate ? TimePredicate.deserialize(buffer) : null; QueryType queryType = QueryType.values()[ReadWriteIOUtils.readInt(buffer)]; int dataNodeFINum = ReadWriteIOUtils.readInt(buffer); + int parallelism = ReadWriteIOUtils.readInt(buffer); boolean debug = ReadWriteIOUtils.readBool(buffer); boolean verbose = ReadWriteIOUtils.readBool(buffer); FragmentInstance fragmentInstance = @@ -263,6 +276,8 @@ public static FragmentInstance deserializeFrom(ByteBuffer buffer) { dataNodeFINum, debug, verbose); + planFragment, id, globalTimePredicate, queryType, timeOut, sessionInfo, dataNodeFINum); + fragmentInstance.setParallelism(parallelism); boolean hasHostDataNode = ReadWriteIOUtils.readBool(buffer); fragmentInstance.hostDataNode = hasHostDataNode ? ThriftCommonsSerDeUtils.deserializeTDataNodeLocation(buffer) : null; @@ -286,6 +301,7 @@ public ByteBuffer serializeToByteBuffer() { } ReadWriteIOUtils.write(type.ordinal(), outputStream); ReadWriteIOUtils.write(dataNodeFINum, outputStream); + ReadWriteIOUtils.write(parallelism, outputStream); ReadWriteIOUtils.write(debug, outputStream); ReadWriteIOUtils.write(verbose, outputStream); ReadWriteIOUtils.write(hostDataNode != null, outputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 505d5a356df74..352e1db100d42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1532,8 +1532,10 @@ public Scope visitSelectHint(SelectHint node, final Optional context) { } else if (hintItem instanceof ParallelHintItem) { ParallelHintItem parallelHintItem = (ParallelHintItem) hintItem; int parallelism = parallelHintItem.getParallelism(); - Hint hint = new ParallelHint(parallelism); - hintMap.putIfAbsent(hint.getKey(), hint); + if (parallelism > 0) { + Hint hint = new ParallelHint(parallelism); + hintMap.putIfAbsent(hint.getKey(), hint); + } } } analysis.setHintMap(hintMap); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java index 465a113a8f9c4..2a12af6ac6cb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.DistributedOptimizeFactory; import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PlanOptimizer; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.ParallelHint; import java.util.Collections; import java.util.HashMap; @@ -130,8 +131,14 @@ public DistributedQueryPlan plan() { public PlanNode generateDistributedPlanWithOptimize( TableDistributedPlanGenerator.PlanContext planContext) { - // generate table model distributed plan + // set parallel hint + ParallelHint parallelHint = + (ParallelHint) planContext.hintMap.getOrDefault(ParallelHint.category, null); + if (parallelHint != null) { + mppQueryContext.setParallelism(parallelHint.getParallelism()); + } + // generate table model distributed plan List distributedPlanResult = new TableDistributedPlanGenerator( mppQueryContext, analysis, symbolAllocator, dataNodeLocationSupplier) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelQueryFragmentPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelQueryFragmentPlanner.java index 6ad998a5a5335..7de271b4a669e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelQueryFragmentPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelQueryFragmentPlanner.java @@ -187,6 +187,9 @@ private void produceFragmentInstance( fragment.isRoot(), queryContext.isVerbose()); + // set parallelism from MppQueryContext + fragmentInstance.setParallelism(queryContext.getParallelism()); + selectExecutorAndHost( fragment, fragmentInstance, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java index e091b8d39ae8e..3c7e093f82542 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java @@ -33,7 +33,7 @@ public class FollowerHintItem extends Node { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(FollowerHintItem.class); - private static final String hintName = "FOLLOWER"; + private static final String hintItemName = "follower"; private final List tables; private final List> nodeIds; @@ -80,7 +80,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return hintName + return hintItemName + "(" + IntStream.range(0, tables.size()) .mapToObj( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java index 2ad2ae4c2131b..3ed3d6197ab41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java @@ -31,7 +31,7 @@ public class LeaderHintItem extends Node { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(LeaderHintItem.class); - private static final String hintName = "LEADER"; + private static final String hintItemName = "leader"; private final List tables; public LeaderHintItem(List tables) { @@ -72,7 +72,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return hintName + "(" + String.join(", ", tables) + ")"; + return hintItemName + "(" + String.join(", ", tables) + ")"; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java index bc464abe7422c..87a54b37171ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java @@ -31,7 +31,7 @@ public class ParallelHintItem extends Node { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ParallelHintItem.class); - private static final String hintName = "PARALLEL"; + private static final String hintItemName = "parallel"; private final int parallelism; public ParallelHintItem(int parallelism) { @@ -72,7 +72,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return hintName + "(" + parallelism + ")"; + return hintItemName + "(" + parallelism + ")"; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java index 70ca8da9dfbf9..9c1baaff3a65c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java @@ -32,6 +32,10 @@ public ParallelHint(int parallelism) { this.parallelism = parallelism; } + public int getParallelism() { + return parallelism; + } + @Override public String getKey() { return category; From 06db278d8eedeb73c90758d28979462dd3be0c40 Mon Sep 17 00:00:00 2001 From: shizy Date: Fri, 17 Apr 2026 11:22:03 +0800 Subject: [PATCH 28/35] rebase master --- .../db/queryengine/common/MPPQueryContext.java | 2 -- .../plan/planner/plan/FragmentInstance.java | 1 - .../plan/relational/planner/RelationPlanner.java | 2 +- .../distribute/TableDistributedPlanGenerator.java | 10 +++++----- .../plan/relational/sql/parser/AstBuilder.java | 13 ++++++++----- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index b9266ba6e90f1..da13ac5b0e636 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -131,8 +131,6 @@ public enum ExplainType { private boolean debug = false; - private Map hintMap = new HashMap<>(); - private Map, Query> cteQueries = new HashMap<>(); // Stores the EXPLAIN/EXPLAIN ANALYZE results for Common Table Expressions (CTEs) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java index e592f18f7cf80..3b78c8c34f49b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java @@ -276,7 +276,6 @@ public static FragmentInstance deserializeFrom(ByteBuffer buffer) { dataNodeFINum, debug, verbose); - planFragment, id, globalTimePredicate, queryType, timeOut, sessionInfo, dataNodeFINum); fragmentInstance.setParallelism(parallelism); boolean hasHostDataNode = ReadWriteIOUtils.readBool(buffer); fragmentInstance.hostDataNode = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 516be38ad8eb6..03859d4499da7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -399,7 +399,7 @@ private RelationPlan processPhysicalTable(Table table, Scope scope) { outputSymbols, tableColumnSchema, tagAndAttributeIndexMap, - analysis.getAliased(table)); + analysis.getAliased(table)); } return new RelationPlan(tableScanNode, scope, outputSymbols, outerContext); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index f0b4d4751ca36..d66938a85ea5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -1728,7 +1728,7 @@ private void buildRegionNodeMap( partialAggTableScanNode.getPreGroupedSymbols(), partialAggTableScanNode.getStep(), partialAggTableScanNode.getGroupIdSymbol(), - partialAggTableScanNode.getAlias()); + partialAggTableScanNode.getAlias()); scanNode.setRegionReplicaSet(regionReplicaSet); return scanNode; }); @@ -2225,10 +2225,10 @@ public static class PlanContext { TRegionReplicaSet mostUsedRegion; boolean deviceCrossRegion; - public PlanContext() { - this.nodeDistributionMap = new HashMap<>(); - this.hintMap = new HashMap<>(); - } + public PlanContext() { + this.nodeDistributionMap = new HashMap<>(); + this.hintMap = new HashMap<>(); + } public PlanContext(Map hintMap) { this.nodeDistributionMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 26f2f43a22382..e8bdfec65885a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2453,7 +2453,8 @@ public Node visitQuerySpecification(RelationalSqlParser.QuerySpecificationContex ctx.where, ctx.groupBy(), ctx.having, - ctx.windowDefinition()); + ctx.windowDefinition(), + ctx.selectHint()); } @Override @@ -2468,7 +2469,8 @@ public Node visitFromFirstQuerySpecification( ctx.where, ctx.groupBy(), ctx.having, - ctx.windowDefinition()); + ctx.windowDefinition(), + null); } private Node buildQuerySpecification( @@ -2480,7 +2482,8 @@ private Node buildQuerySpecification( RelationalSqlParser.BooleanExpressionContext where, RelationalSqlParser.GroupByContext groupBy, RelationalSqlParser.BooleanExpressionContext having, - List windowDefinitions) { + List windowDefinitions, + RelationalSqlParser.SelectHintContext selectHintContext) { Optional from = Optional.empty(); List selectItems = visit(selectItemContexts, SelectItem.class); @@ -2510,8 +2513,8 @@ private Node buildQuerySpecification( // Hint Map Optional selectHint = - ctx.selectHint() != null - ? Optional.of((SelectHint) visitSelectHint(ctx.selectHint())) + selectHintContext != null + ? Optional.of((SelectHint) visitSelectHint(selectHintContext)) : Optional.empty(); return new QuerySpecification( From 9a2a66589737f1acabd731a9b5e6f9d247ddec4b Mon Sep 17 00:00:00 2001 From: shizy Date: Mon, 20 Apr 2026 14:39:40 +0800 Subject: [PATCH 29/35] change to replica & region_route hint --- .../analyzer/StatementAnalyzer.java | 92 ++++++--------- .../planner/distribute/AddExchangeNodes.java | 67 +++++++---- .../distribute/TableDistributedPlanner.java | 2 +- .../plan/relational/sql/ast/AstVisitor.java | 4 +- .../relational/sql/ast/FollowerHintItem.java | 110 ------------------ .../relational/sql/ast/LeaderHintItem.java | 85 -------------- .../relational/sql/ast/ParallelHintItem.java | 8 +- .../sql/ast/RegionRouteHintItem.java | 70 +++++++++++ .../relational/sql/ast/ReplicaHintItem.java | 67 +++++++++++ .../relational/sql/parser/AstBuilder.java | 47 ++++---- .../relational/utils/hint/FollowerHint.java | 81 ------------- .../plan/relational/utils/hint/Hint.java | 4 +- .../relational/utils/hint/LeaderHint.java | 56 --------- .../relational/utils/hint/ParallelHint.java | 9 +- .../utils/hint/RegionRouteHint.java | 58 +++++++++ .../relational/utils/hint/ReplicaHint.java | 41 +++++-- .../relational/grammar/sql/RelationalSql.g4 | 25 ++-- 17 files changed, 350 insertions(+), 476 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 352e1db100d42..1e6efd560e2b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -100,7 +100,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FollowerHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupBy; @@ -121,7 +120,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinCriteria; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LeaderHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; @@ -144,9 +142,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RegionRouteHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReplicaHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; @@ -199,10 +199,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.relational.type.CompatibleResolver; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeManager; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.FollowerHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; -import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.LeaderHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.ParallelHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.RegionRouteHint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.ReplicaHint; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -1520,74 +1520,50 @@ private void analyzeHint(QuerySpecification node, Scope scope) { public Scope visitSelectHint(SelectHint node, final Optional context) { Map hintMap = new HashMap<>(); for (Node hintItem : node.getHintItems()) { - if (hintItem instanceof LeaderHintItem) { - LeaderHintItem leaderHintItem = (LeaderHintItem) hintItem; - List tables = leaderHintItem.getTables(); - addLeaderHints(tables, hintMap); - } else if (hintItem instanceof FollowerHintItem) { - FollowerHintItem followerHintItem = (FollowerHintItem) hintItem; - List tables = followerHintItem.getTables(); - List> nodeIds = followerHintItem.getNodeIds(); - addFollowerHints(tables, nodeIds, hintMap); + Hint hint = null; + if (hintItem instanceof ReplicaHintItem) { + ReplicaHintItem replicaHintItem = (ReplicaHintItem) hintItem; + QualifiedName table = getValidTable(replicaHintItem.getTable()); + hint = new ReplicaHint(table, replicaHintItem.getReplicaIndex()); + } else if (hintItem instanceof RegionRouteHintItem) { + RegionRouteHintItem regionRouteHintItem = (RegionRouteHintItem) hintItem; + QualifiedName table = getValidTable(regionRouteHintItem.getTable()); + hint = new RegionRouteHint(table, regionRouteHintItem.getRegionDatanodeMap()); } else if (hintItem instanceof ParallelHintItem) { ParallelHintItem parallelHintItem = (ParallelHintItem) hintItem; int parallelism = parallelHintItem.getParallelism(); - if (parallelism > 0) { - Hint hint = new ParallelHint(parallelism); - hintMap.putIfAbsent(hint.getKey(), hint); - } + hint = new ParallelHint(parallelism); + } + if (hint != null) { + hintMap.putIfAbsent(hint.getKey(), hint); } } analysis.setHintMap(hintMap); return createAndAssignScope(node, context); } - private void addLeaderHints(List tables, Map hintMap) { - List existingTables = - analysis.getRelationNames().values().stream() - .map(QualifiedName::getSuffix) - .collect(toImmutableList()); + private QualifiedName getValidTable(QualifiedName table) { + if (table == null) { + return null; + } - if (tables == null || tables.isEmpty()) { - existingTables.forEach( - table -> { - Hint hint = new LeaderHint(table); - hintMap.putIfAbsent(hint.getKey(), hint); - }); - } else { - for (String table : tables) { - if (!existingTables.contains(table)) { - continue; - } - Hint hint = new LeaderHint(table); - hintMap.putIfAbsent(hint.getKey(), hint); - } + List existingTables = + analysis.getRelationNames().values().stream().collect(toImmutableList()); + + // Alias + if (existingTables.contains(table)) { + return table; } - } - private void addFollowerHints( - List tables, List> nodeIds, Map hintMap) { - List existingTables = - analysis.getRelationNames().values().stream() - .map(QualifiedName::getSuffix) - .collect(toImmutableList()); + // Table + QualifiedObjectName tableObjectName = createQualifiedObjectName(sessionContext, table); + QualifiedName tableName = + QualifiedName.of(tableObjectName.getDatabaseName(), tableObjectName.getObjectName()); - if (tables == null || tables.isEmpty()) { - existingTables.forEach( - table -> { - Hint hint = new FollowerHint(table, ImmutableList.of()); - hintMap.putIfAbsent(hint.getKey(), hint); - }); - } else { - for (int i = 0; i < tables.size(); i++) { - String table = tables.get(i); - if (!existingTables.contains(table)) { - continue; - } - Hint hint = new FollowerHint(table, nodeIds.get(i)); - hintMap.putIfAbsent(hint.getKey(), hint); - } + if (!existingTables.contains(tableName)) { + throw new SemanticException(String.format("Invalid table : %s", tableName)); } + return tableName; } private List analyzeSelect(QuerySpecification node, Scope scope) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index dc7834cc21924..9c408792543e6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CopyToNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CteScanNode; @@ -39,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.Hint; +import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.RegionRouteHint; import org.apache.iotdb.db.queryengine.plan.relational.utils.hint.ReplicaHint; import java.util.List; @@ -105,27 +107,40 @@ public PlanNode visitTableScan( TableScanNode node, TableDistributedPlanGenerator.PlanContext context) { // Original region replica set TRegionReplicaSet regionReplicaSet = node.getRegionReplicaSet(); - // Find applicable hint - ReplicaHint hint = findReplicaHint(node, context.hintMap); - // Early return for simple cases - if (regionReplicaSet == null || hint == null) { - context.nodeDistributionMap.put( - node.getPlanNodeId(), new NodeDistribution(SAME_WITH_ALL_CHILDREN, regionReplicaSet)); - return node; + // Determine optimized locations based on hint + List optimizedLocations = null; + + // Find region_route hint + RegionRouteHint regionRouteHint = + (RegionRouteHint) findHint(RegionRouteHint.HINT_NAME, node, context.hintMap); + if (regionRouteHint != null) { + optimizedLocations = + regionRouteHint.selectLocations( + regionReplicaSet.getDataNodeLocations(), regionReplicaSet.getRegionId().getId()); } - // Determine optimized locations based on hint - List optimizedLocations = - selectLocations(regionReplicaSet.getDataNodeLocations(), hint); + // Find replica hint + if (optimizedLocations == null) { + ReplicaHint replicaHint = + (ReplicaHint) findHint(ReplicaHint.HINT_NAME, node, context.hintMap); + if (replicaHint != null) { + optimizedLocations = replicaHint.selectLocations(regionReplicaSet.getDataNodeLocations()); + } + } - // Create optimized region replica set - TRegionReplicaSet optimizedRegionReplicaSet = - new TRegionReplicaSet(regionReplicaSet.getRegionId(), optimizedLocations); + if (optimizedLocations == null) { + context.nodeDistributionMap.put( + node.getPlanNodeId(), new NodeDistribution(SAME_WITH_ALL_CHILDREN, regionReplicaSet)); + } else { + // Create optimized region replica set + TRegionReplicaSet optimizedRegionReplicaSet = + new TRegionReplicaSet(regionReplicaSet.getRegionId(), optimizedLocations); - context.nodeDistributionMap.put( - node.getPlanNodeId(), - new NodeDistribution(SAME_WITH_ALL_CHILDREN, optimizedRegionReplicaSet)); + context.nodeDistributionMap.put( + node.getPlanNodeId(), + new NodeDistribution(SAME_WITH_ALL_CHILDREN, optimizedRegionReplicaSet)); + } return node; } @@ -262,18 +277,24 @@ private PlanNode processTableDeviceSourceNode( * Finds the applicable replica hint for the given table scan node. First checks for * table-specific hint, then falls back to global hint. */ - private ReplicaHint findReplicaHint(TableScanNode node, Map hintMap) { + private Hint findHint(String hintName, TableScanNode node, Map hintMap) { if (hintMap == null || hintMap.isEmpty()) { return null; } - String tableName = - node.getAlias() != null - ? node.getAlias().getValue() - : node.getQualifiedObjectName().getObjectName(); - String tableSpecificKey = "replica-" + tableName; + String tableName = getTableName(node); + String tableSpecificKey = hintName + "-" + tableName; + + Hint hint = hintMap.get(tableSpecificKey); + return hint != null ? hint : hintMap.get(hintName); + } - return (ReplicaHint) hintMap.getOrDefault(tableSpecificKey, null); + private String getTableName(TableScanNode node) { + if (node.getAlias() != null) { + return node.getAlias().getValue(); + } + QualifiedObjectName fullName = node.getQualifiedObjectName(); + return fullName.getDatabaseName() + "." + fullName.getObjectName(); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java index 2a12af6ac6cb4..7ac985cbfbb6c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java @@ -133,7 +133,7 @@ public PlanNode generateDistributedPlanWithOptimize( TableDistributedPlanGenerator.PlanContext planContext) { // set parallel hint ParallelHint parallelHint = - (ParallelHint) planContext.hintMap.getOrDefault(ParallelHint.category, null); + (ParallelHint) planContext.hintMap.getOrDefault(ParallelHint.HINT_NAME, null); if (parallelHint != null) { mppQueryContext.setParallelism(parallelHint.getParallelism()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 310f5d16c13e1..50c18668cd032 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -141,11 +141,11 @@ protected R visitSelectHint(SelectHint node, C context) { return visitNode(node, context); } - protected R visitLeaderHintItem(LeaderHintItem node, C context) { + protected R visitReplicaHintItem(ReplicaHintItem node, C context) { return visitNode(node, context); } - protected R visitFollowerHintItem(FollowerHintItem node, C context) { + protected R visitRegionRouteHintItem(RegionRouteHintItem node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java deleted file mode 100644 index 3c7e093f82542..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FollowerHintItem.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import com.google.common.collect.ImmutableList; -import org.apache.tsfile.utils.RamUsageEstimator; - -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class FollowerHintItem extends Node { - private static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(FollowerHintItem.class); - - private static final String hintItemName = "follower"; - private final List tables; - private final List> nodeIds; - - public FollowerHintItem(List tables, List> nodeIds) { - super(null); - this.tables = ImmutableList.copyOf(tables); - this.nodeIds = ImmutableList.copyOf(nodeIds); - } - - public List getTables() { - return tables; - } - - public List> getNodeIds() { - return nodeIds; - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitFollowerHintItem(this, context); - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public int hashCode() { - return Objects.hash(tables); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - FollowerHintItem other = (FollowerHintItem) obj; - return Objects.equals(this.tables, other.tables) && Objects.equals(this.nodeIds, other.nodeIds); - } - - @Override - public String toString() { - return hintItemName - + "(" - + IntStream.range(0, tables.size()) - .mapToObj( - i -> { - String table = tables.get(i); - List ids = nodeIds.get(i); - if (ids == null || ids.isEmpty()) { - return table; - } - return table - + "(" - + ids.stream().map(String::valueOf).collect(Collectors.joining(",")) - + ")"; - }) - .collect(Collectors.joining(",")) - + ")"; - } - - @Override - public long ramBytesUsed() { - long size = INSTANCE_SIZE; - size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); - size += AstMemoryEstimationHelper.getEstimatedSizeOfStringList(tables); - size += RamUsageEstimator.sizeOfArrayList(nodeIds); - return size; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java deleted file mode 100644 index 3ed3d6197ab41..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LeaderHintItem.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import com.google.common.collect.ImmutableList; -import org.apache.tsfile.utils.RamUsageEstimator; - -import java.util.List; -import java.util.Objects; - -public class LeaderHintItem extends Node { - private static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(LeaderHintItem.class); - - private static final String hintItemName = "leader"; - private final List tables; - - public LeaderHintItem(List tables) { - super(null); - this.tables = ImmutableList.copyOf(tables); - } - - public List getTables() { - return tables; - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitLeaderHintItem(this, context); - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public int hashCode() { - return Objects.hash(tables); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - LeaderHintItem other = (LeaderHintItem) obj; - return Objects.equals(this.tables, other.tables); - } - - @Override - public String toString() { - return hintItemName + "(" + String.join(", ", tables) + ")"; - } - - @Override - public long ramBytesUsed() { - long size = INSTANCE_SIZE; - size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); - size += AstMemoryEstimationHelper.getEstimatedSizeOfStringList(tables); - return size; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java index 87a54b37171ed..94e7feaea5a50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java @@ -31,7 +31,7 @@ public class ParallelHintItem extends Node { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ParallelHintItem.class); - private static final String hintItemName = "parallel"; + private static final String HINT_NAME_ITEM = "parallel"; private final int parallelism; public ParallelHintItem(int parallelism) { @@ -72,13 +72,11 @@ public boolean equals(Object obj) { @Override public String toString() { - return hintItemName + "(" + parallelism + ")"; + return HINT_NAME_ITEM + "(" + parallelism + ")"; } @Override public long ramBytesUsed() { - long size = INSTANCE_SIZE; - size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); - return size; + return INSTANCE_SIZE; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java new file mode 100644 index 0000000000000..419d688292e92 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java @@ -0,0 +1,70 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class RegionRouteHintItem extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(RegionRouteHintItem.class); + private static final String HINT_NAME_ITEM = "region_route"; + + private final QualifiedName table; + private final Map regionDatanodeMap; + + public RegionRouteHintItem(QualifiedName table, Map regionDatanodeMap) { + super(null); + this.table = table; + this.regionDatanodeMap = regionDatanodeMap; + } + + public QualifiedName getTable() { + return table; + } + + public Map getRegionDatanodeMap() { + return regionDatanodeMap; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitRegionRouteHintItem(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(table, regionDatanodeMap); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RegionRouteHintItem other = (RegionRouteHintItem) obj; + return this.table.equals(other.table) && regionDatanodeMap.equals(other.regionDatanodeMap); + } + + @Override + public String toString() { + return HINT_NAME_ITEM + (table == null ? "" : "-" + table) + "(" + regionDatanodeMap + ")"; + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + AstMemoryEstimationHelper.getEstimatedSizeOfAccountableObject(table) + + RamUsageEstimator.sizeOfMap(regionDatanodeMap); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java new file mode 100644 index 0000000000000..149f72fdc5b51 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java @@ -0,0 +1,67 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.Objects; + +public class ReplicaHintItem extends Node { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ReplicaHintItem.class); + private static final String HINT_NAME_ITEM = "replica"; + + private final QualifiedName table; + private final int replicaIndex; + + public ReplicaHintItem(QualifiedName table, int replicaIndex) { + super(null); + this.table = table; + this.replicaIndex = replicaIndex; + } + + public QualifiedName getTable() { + return table; + } + + public int getReplicaIndex() { + return replicaIndex; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitReplicaHintItem(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(table, replicaIndex); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ReplicaHintItem other = (ReplicaHintItem) obj; + return this.table.equals(other.table) && this.replicaIndex == other.replicaIndex; + } + + @Override + public String toString() { + return HINT_NAME_ITEM + (table == null ? "" : "-" + table) + "(" + replicaIndex + ")"; + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + AstMemoryEstimationHelper.getEstimatedSizeOfAccountableObject(table); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index e8bdfec65885a..10430cc024256 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -104,7 +104,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FollowerHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; @@ -125,7 +124,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.KillQuery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LeaderHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; @@ -166,6 +164,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RegionRouteHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveAINode; @@ -174,6 +173,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReplicaHintItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; @@ -2568,32 +2568,37 @@ public Node visitSelectHint(RelationalSqlParser.SelectHintContext ctx) { } @Override - public Node visitLeaderHint(RelationalSqlParser.LeaderHintContext ctx) { - List identifiers = ctx.identifier(); - List tables = - identifiers.stream().map(x -> x.getText().toLowerCase()).collect(Collectors.toList()); - return new LeaderHintItem(tables); + public Node visitReplicaHint(RelationalSqlParser.ReplicaHintContext ctx) { + QualifiedName table = ctx.tableName == null ? null : getQualifiedName(ctx.tableName); + int replicaIndex = Integer.parseInt(ctx.INTEGER_VALUE().getText()); + return new ReplicaHintItem(table, replicaIndex); } @Override - public Node visitFollowerHint(RelationalSqlParser.FollowerHintContext ctx) { - List followerParameters = ctx.followerParameter(); - List tables = - followerParameters.stream().map(x -> x.identifier().getText()).collect(Collectors.toList()); - List> nodeIds = - followerParameters.stream() - .map( - x -> - x.number().stream() - .map(y -> Integer.parseInt(y.getText())) - .collect(Collectors.toList())) - .collect(Collectors.toList()); - return new FollowerHintItem(tables, nodeIds); + public Node visitRegionRouteHint(RelationalSqlParser.RegionRouteHintContext ctx) { + QualifiedName table = ctx.tableName == null ? null : getQualifiedName(ctx.tableName); + Map regionDatanodeMap = new HashMap<>(); + try { + for (RelationalSqlParser.RegionRouteItemContext itemCtx : ctx.regionRoutes) { + int regionId = Integer.parseInt(itemCtx.regionId.getText()); + if (regionId < -1) { + throw parseError("Region ID must not be less than -1", ctx); + } + int datanodeId = Integer.parseInt(itemCtx.datanodeId.getText()); + if (datanodeId < 0) { + throw parseError("DataNode ID must not be less than 0", ctx); + } + regionDatanodeMap.putIfAbsent(regionId, datanodeId); + } + } catch (NumberFormatException e) { + throw parseError("Region ID and DataNode ID must be integers", ctx); + } + return new RegionRouteHintItem(table, regionDatanodeMap); } @Override public Node visitParallelHint(RelationalSqlParser.ParallelHintContext ctx) { - int parallelism = Integer.parseInt(ctx.number().getText()); + int parallelism = Integer.parseInt(ctx.INTEGER_VALUE().getText()); return new ParallelHintItem(parallelism); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java deleted file mode 100644 index b9ed579d504d6..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/FollowerHint.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; - -import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class FollowerHint extends ReplicaHint { - public static String hintName = "follower"; - private final String table; - private final List nodeIds; - - public FollowerHint(String table, List nodeIds) { - super(hintName); - this.table = table; - this.nodeIds = nodeIds; - } - - @Override - public String getKey() { - return category + "-" + table; - } - - @Override - public String toString() { - return hintName + "-" + table; - } - - @Override - public List selectLocations(List dataNodeLocations) { - if (dataNodeLocations == null || dataNodeLocations.size() <= 1) { - return dataNodeLocations; - } - - List followerLocations = - dataNodeLocations.subList(1, dataNodeLocations.size()); - if (nodeIds == null || nodeIds.isEmpty()) { - return followerLocations; - } - - // nodeId -> Location for O(1) lookup - Map followerLocationMap = new HashMap<>(); - for (TDataNodeLocation location : followerLocations) { - followerLocationMap.put(location.getDataNodeId(), location); - } - - // Find the first nodeId that exists in followerLocations and return its location - for (Integer nodeId : nodeIds) { - TDataNodeLocation location = followerLocationMap.get(nodeId); - if (location != null) { - return Collections.singletonList(location); - } - } - - // If no matching nodeId found, return all follower locations - return followerLocations; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java index ce07d51668ff8..628322aa581f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -25,11 +25,9 @@ public abstract class Hint { protected String hintName; - protected String category; - protected Hint(String hintName, String category) { + protected Hint(String hintName) { this.hintName = Objects.requireNonNull(hintName, "hintName can not be null"); - this.category = Objects.requireNonNull(category, "category can not be null"); } public abstract String getKey(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java deleted file mode 100644 index 352a713e94a6f..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/LeaderHint.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. - * - */ - -package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; - -import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; - -import java.util.Collections; -import java.util.List; - -public class LeaderHint extends ReplicaHint { - public static String hintName = "leader"; - private final String table; - - public LeaderHint(String table) { - super(hintName); - this.table = table; - } - - @Override - public String getKey() { - return category + "-" + table; - } - - @Override - public String toString() { - return hintName + "-" + table; - } - - @Override - public List selectLocations(List dataNodeLocations) { - if (dataNodeLocations == null || dataNodeLocations.size() <= 1) { - return dataNodeLocations; - } - // Return only the leader (first location) - return Collections.singletonList(dataNodeLocations.get(0)); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java index 9c1baaff3a65c..2883d093c3c73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java @@ -22,13 +22,12 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; public class ParallelHint extends Hint { - public static String category = "parallel"; - public static String hintName = "parallel"; + public static String HINT_NAME = "parallel"; private final int parallelism; public ParallelHint(int parallelism) { - super(hintName, category); + super(HINT_NAME); this.parallelism = parallelism; } @@ -38,11 +37,11 @@ public int getParallelism() { @Override public String getKey() { - return category; + return HINT_NAME; } @Override public String toString() { - return hintName + "(" + parallelism + ")"; + return HINT_NAME + "(" + parallelism + ")"; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java new file mode 100644 index 0000000000000..1f727b3fc08f7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java @@ -0,0 +1,58 @@ +package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Map; + +public class RegionRouteHint extends Hint { + public static String HINT_NAME = "region_route"; + public static int ANY_TABLE = -1; + + private final QualifiedName table; + private final Map regionDatanodeMap; + + public RegionRouteHint(QualifiedName table, Map regionDatanodeMa) { + super(HINT_NAME); + this.table = table; + this.regionDatanodeMap = regionDatanodeMa; + } + + @Override + public String getKey() { + return HINT_NAME + (table == null ? "" : "-" + table); + } + + @Override + public String toString() { + return HINT_NAME + (table == null ? "" : "-" + table) + "(" + regionDatanodeMap + ")"; + } + + /** + * Selects data node locations based on the region_route strategy. + * + * @param dataNodeLocations the available data node locations + * @param regionId the region ID to route + * @return the selected locations based on region_route hint strategy, or null if no match + */ + public List selectLocations( + List dataNodeLocations, int regionId) { + if (dataNodeLocations == null || dataNodeLocations.isEmpty()) { + return null; + } + + Integer datanodeId = regionDatanodeMap.getOrDefault(regionId, regionDatanodeMap.get(ANY_TABLE)); + if (datanodeId == null) { + return null; + } + + return dataNodeLocations.stream() + .filter(location -> location.getDataNodeId() == datanodeId) + .findFirst() + .map(ImmutableList::of) + .orElse(null); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java index 187dc27f85212..bab241a70192d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java @@ -22,27 +22,44 @@ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; + +import com.google.common.collect.ImmutableList; import java.util.List; -/** - * Abstract base class for replica-related hints. Provides common functionality for hints that deal - * with data node replica selection. - */ -public abstract class ReplicaHint extends Hint { - public static String category = "replica"; +public class ReplicaHint extends Hint { + public static String HINT_NAME = "replica"; + + private final QualifiedName table; + private final int replicaIndex; + + public ReplicaHint(QualifiedName table, int replicaIndex) { + super(HINT_NAME); + this.table = table; + this.replicaIndex = replicaIndex; + } + + @Override + public String getKey() { + return HINT_NAME + (table == null ? "" : "-" + table); + } - protected ReplicaHint(String hintName) { - super(hintName, category); + @Override + public String toString() { + return HINT_NAME + (table == null ? "" : "-" + table) + "(" + replicaIndex + ")"; } /** - * Selects data node locations based on the replica strategy. Each replica hint implementation - * defines its own location selection logic. + * Selects data node locations based on the replica strategy. * * @param dataNodeLocations the available data node locations * @return the selected locations based on replica hint strategy */ - public abstract List selectLocations( - List dataNodeLocations); + public List selectLocations(List dataNodeLocations) { + if (dataNodeLocations == null || dataNodeLocations.isEmpty()) { + return null; + } + return ImmutableList.of(dataNodeLocations.get(replicaIndex % dataNodeLocations.size())); + } } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 3f95434e83120..0bb86f8485cd3 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -1121,28 +1121,25 @@ selectHint ; hintItem - : leaderHint - | followerHint + : replicaHint + | regionRouteHint | parallelHint ; -leaderHint - : LEADER - | LEADER '(' identifier (',' identifier)* ')' +replicaHint + : REPLICA '(' (tableName=qualifiedName ',')? INTEGER_VALUE ')' ; -followerHint - : FOLLOWER - | FOLLOWER '(' followerParameter (',' followerParameter)* ')' +regionRouteHint + : REGION_ROUTE '(' (tableName=qualifiedName ',')? regionRoutes+=regionRouteItem (',' regionRoutes+=regionRouteItem)* ')' ; -followerParameter - : identifier - | identifier '(' number (',' number)* ')' +regionRouteItem + : '(' regionId=number ',' datanodeId=number ')' ; parallelHint - : PARALLEL '(' number ')' + : PARALLEL '(' INTEGER_VALUE ')' ; patternRecognition @@ -1950,8 +1947,8 @@ WRITE: 'WRITE'; YEAR: 'YEAR' | 'Y'; ZONE: 'ZONE'; AUDIT: 'AUDIT'; -LEADER: 'LEADER'; -FOLLOWER: 'FOLLOWER'; +REPLICA: 'REPLICA'; +REGION_ROUTE: 'REGION_ROUTE'; PARALLEL: 'PARALLEL'; AT_SIGN: '@'; From fd8dc6649df84b166412a36a89e5c06240a8b811 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 21 Apr 2026 09:10:37 +0800 Subject: [PATCH 30/35] add test cases --- .../it/query/recent/IoTDBHintIT.java | 175 ++++++++++++++ .../plan/relational/planner/PlanTester.java | 11 + .../planner/ReplicaHintPlannerTest.java | 221 ++++++++++++++++++ 3 files changed, 407 insertions(+) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java new file mode 100644 index 0000000000000..531f1c200412d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Locale; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBHintIT { + private static final String DATABASE_NAME = "testdb"; + + private static final String[] creationSqls = + new String[] { + "CREATE DATABASE IF NOT EXISTS testdb", + "USE testdb", + "CREATE TABLE IF NOT EXISTS testtb(voltage FLOAT FIELD, manufacturer STRING FIELD, deviceid STRING TAG)", + "INSERT INTO testtb VALUES(1000, 100.0, 'a', 'd1')", + "INSERT INTO testtb VALUES(2000, 200.0, 'b', 'd1')", + "INSERT INTO testtb VALUES(1000, 300.0, 'c', 'd2')", + }; + + private static final String dropDbSqls = "DROP DATABASE IF EXISTS testdb"; + + @BeforeClass + public static void setUpClass() { + Locale.setDefault(Locale.ENGLISH); + + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setPartitionInterval(1000) + .setMemtableSizeThreshold(10000); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDownClass() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Before + public void setUp() { + prepareData(); + } + + @After + public void tearDown() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute(dropDbSqls); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testReplicaHintWithInvalidTable() { + String sql = "SELECT /*+ REPLICA(t1, 2) */ * FROM testtb"; + String expectedErrMsg = "Invalid table : testdb.t1"; + tableAssertTestFail(sql, expectedErrMsg, DATABASE_NAME); + } + + @Test + public void testReplicaHintWithSystemTableQuery() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + String sql = "SELECT /*+ REPLICA(0) */ * FROM information_schema.tables"; + statement.executeQuery(sql); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testReplicaHintWithCTE() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + // REPLICA hint in CTE definition + String sql1 = + "WITH cte1 AS (SELECT /*+ REPLICA(testtb,0) */ * FROM testtb) SELECT * FROM cte1"; + statement.executeQuery(sql1); + // REPLICA hint in materialized CTE definition + String sql2 = + "WITH cte1 AS materialized (SELECT /*+ REPLICA(testtb,0) */ * FROM testtb) SELECT * FROM cte1"; + statement.executeQuery(sql2); + // REPLICA hint in main query + String sql3 = + "WITH cte1 AS (SELECT * FROM testtb) SELECT /*+ REPLICA(testtb, 0) */ * FROM cte1"; + statement.executeQuery(sql3); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testReplicaHintWithExplain() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + String sql = "EXPLAIN SELECT /*+ REPLICA(0) */ * FROM testtb"; + statement.executeQuery(sql); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + @Test + public void testReplicaHintWithJoin() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + String sql = + "SELECT /*+ REPLICA(a, 0) REPLICA(b, 1) */ * FROM testtb as a INNER JOIN testtb as b ON a.voltage = b.voltage"; + ResultSet rs = statement.executeQuery(sql); + int count = 0; + while (rs.next()) { + count++; + } + assertEquals("Expected 3 rows from the join query", 3, count); + } catch (Exception e) { + fail(e.getMessage()); + } + } + + private static void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : creationSqls) { + statement.execute(sql); + } + } catch (Exception e) { + fail(e.getMessage()); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanTester.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanTester.java index 968b0d28de9fa..296085bae0046 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanTester.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanTester.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector; import org.apache.iotdb.db.queryengine.plan.planner.plan.DistributedQueryPlan; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; @@ -238,4 +239,14 @@ public PlanNode getFragmentPlan(int index) { } return distributedQueryPlan.getFragments().get(index).getPlanNodeTree().getChildren().get(0); } + + public FragmentInstance getFragmentInstance(int index) { + if (distributedQueryPlan == null) { + distributedQueryPlan = + new TableDistributedPlanner( + analysis, symbolAllocator, plan, metadata, dataNodeLocationSupplier) + .plan(); + } + return distributedQueryPlan.getInstances().get(index); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java new file mode 100644 index 0000000000000..f825db771a372 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingException; + +import com.google.common.collect.ImmutableMap; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class ReplicaHintPlannerTest extends PlanTester { + + @Test + public void testReplicaHint() { + String sql = "SELECT /*+ REPLICA(0) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("table1", 0)); + } + } + + @Test + public void testReplicaHintWithNegativeIndex() { + try { + String sql = "SELECT /*+ REPLICA(-1) */ * FROM table1"; + createPlan(sql); + } catch (ParsingException ex) { + assertTrue( + ex.getMessage().contains("mismatched input '-'. Expecting: , ")); + } + } + + @Test + public void testReplicaHintWithTable() { + String sql = "SELECT /*+ REPLICA(table1, 1) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("table1", 1)); + } + } + + @Test + public void testReplicaHintWithDatabasePrefix() { + String sql = "SELECT /*+ REPLICA(testdb.table1, 0) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("table1", 0)); + } + } + + @Test + public void testReplicaHintWithAlias() { + String sql = "SELECT /*+ REPLICA(t, 1) */ * FROM table1 as t"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("t", 1)); + } + } + + @Test + public void testReplicaHintWithConflict() { + String sql = "SELECT /*+ REPLICA(1) REPLICA(0) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("table1", 1)); + } + } + + @Test + public void testRegionRouteHintWithNegativeRegion() { + String sql = "SELECT /*+ REGION_ROUTE((-1, 1)) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyRegionRoute(fragment, ImmutableMap.of(10, 1, 12, 1)); + } + } + + @Test + public void testRegionRouteHintWithConflict() { + String sql = "SELECT /*+ REGION_ROUTE(table1, (10, 1), (10, 2), (11, 3)) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyRegionRoute(fragment, ImmutableMap.of(10, 1, 11, 3)); + } + } + + @Test + public void testHintPriority1() { + String sql = + "SELECT /*+ REPLICA(2) REPLICA(table1, 2) REGION_ROUTE((-1, 2), (10, 2)) REGION_ROUTE(table1, (-1, 2), (10, 1)) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyRegionRoute(fragment, ImmutableMap.of(10, 1)); + } + } + + @Test + public void testHintPriority2() { + String sql = + "SELECT /*+ REPLICA(2) REPLICA(table1, 2) REGION_ROUTE((-1, 2), (10, 2)) REGION_ROUTE(table1, (-1, 1)) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyRegionRoute(fragment, ImmutableMap.of(10, 1)); + } + } + + @Test + public void testHintPriority3() { + String sql = + "SELECT /*+ REPLICA(2) REPLICA(table1, 2) REGION_ROUTE((-1, 2), (10, 1)) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyRegionRoute(fragment, ImmutableMap.of(10, 1)); + } + } + + @Test + public void testHintPriority4() { + String sql = "SELECT /*+ REPLICA(2) REPLICA(table1, 2) REGION_ROUTE((-1, 1)) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyRegionRoute(fragment, ImmutableMap.of(10, 1)); + } + } + + @Test + public void testHintPriority5() { + String sql = "SELECT /*+ REPLICA(2) REPLICA(table1, 1) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("table1", 1)); + } + } + + @Test + public void testHintPriority6() { + String sql = "SELECT /*+ REPLICA(1) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("table1", 1)); + } + } + + @Test + public void testParallelHint() { + String sql = "SELECT /*+ PARALLEL(5) */ * FROM table1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + FragmentInstance fragmentInstance = getFragmentInstance(fragment); + assertEquals(5, fragmentInstance.getParallelism()); + } + } + + private void verifyReplica(int fragment, Map tableToReplica) { + FragmentInstance fragmentInstance = getFragmentInstance(fragment); + TableScanNode fragmentPlan = (TableScanNode) getFragmentPlan(fragment); + assertEquals( + fragmentPlan.getRegionReplicaSet().getRegionId().getId(), + fragmentInstance.getRegionReplicaSet().getRegionId().getId()); + + String tableName = + fragmentPlan.getAlias() != null + ? fragmentPlan.getAlias().getValue() + : fragmentPlan.getQualifiedObjectName().getObjectName(); + + List replicaNodes = + fragmentPlan.getRegionReplicaSet().getDataNodeLocations(); + TDataNodeLocation queryNode = fragmentInstance.getHostDataNode(); + + assertEquals(2, replicaNodes.size()); + assertEquals( + queryNode.getDataNodeId(), replicaNodes.get(tableToReplica.get(tableName)).getDataNodeId()); + } + + private void verifyRegionRoute(int fragment, Map regionToDatanode) { + FragmentInstance fragmentInstance = getFragmentInstance(fragment); + TableScanNode fragmentPlan = (TableScanNode) getFragmentPlan(fragment); + + // Verify region IDs match + int regionId = fragmentPlan.getRegionReplicaSet().getRegionId().getId(); + assertEquals(regionId, fragmentInstance.getRegionReplicaSet().getRegionId().getId()); + + List replicaNodes = + fragmentPlan.getRegionReplicaSet().getDataNodeLocations(); + TDataNodeLocation queryNode = fragmentInstance.getHostDataNode(); + + assertEquals(2, replicaNodes.size()); + + if (regionToDatanode.containsKey(regionId)) { + int expectedDatanodeId = regionToDatanode.get(regionId); + assertEquals(expectedDatanodeId, queryNode.getDataNodeId()); + boolean datanodeExists = + replicaNodes.stream().anyMatch(dn -> dn.getDataNodeId() == expectedDatanodeId); + assertTrue(datanodeExists); + } + } +} From bd1dc03fd4ac948da9c8bd3a7408af194772f873 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 21 Apr 2026 10:19:47 +0800 Subject: [PATCH 31/35] invalid table does not throw SemanticException --- .../it/query/recent/IoTDBHintIT.java | 12 +++-- .../analyzer/StatementAnalyzer.java | 47 ++++++++++++------- .../planner/ReplicaHintPlannerTest.java | 3 +- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java index 531f1c200412d..b5c83fc16c630 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBHintIT.java @@ -38,7 +38,6 @@ import java.sql.Statement; import java.util.Locale; -import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -93,9 +92,14 @@ public void tearDown() { @Test public void testReplicaHintWithInvalidTable() { - String sql = "SELECT /*+ REPLICA(t1, 2) */ * FROM testtb"; - String expectedErrMsg = "Invalid table : testdb.t1"; - tableAssertTestFail(sql, expectedErrMsg, DATABASE_NAME); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + String sql = "SELECT /*+ REPLICA(t1, 2) */ * FROM testtb"; + statement.executeQuery(sql); + } catch (Exception e) { + fail(e.getMessage()); + } } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 1e6efd560e2b1..905a173812648 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1520,20 +1520,7 @@ private void analyzeHint(QuerySpecification node, Scope scope) { public Scope visitSelectHint(SelectHint node, final Optional context) { Map hintMap = new HashMap<>(); for (Node hintItem : node.getHintItems()) { - Hint hint = null; - if (hintItem instanceof ReplicaHintItem) { - ReplicaHintItem replicaHintItem = (ReplicaHintItem) hintItem; - QualifiedName table = getValidTable(replicaHintItem.getTable()); - hint = new ReplicaHint(table, replicaHintItem.getReplicaIndex()); - } else if (hintItem instanceof RegionRouteHintItem) { - RegionRouteHintItem regionRouteHintItem = (RegionRouteHintItem) hintItem; - QualifiedName table = getValidTable(regionRouteHintItem.getTable()); - hint = new RegionRouteHint(table, regionRouteHintItem.getRegionDatanodeMap()); - } else if (hintItem instanceof ParallelHintItem) { - ParallelHintItem parallelHintItem = (ParallelHintItem) hintItem; - int parallelism = parallelHintItem.getParallelism(); - hint = new ParallelHint(parallelism); - } + Hint hint = processHintItem(hintItem); if (hint != null) { hintMap.putIfAbsent(hint.getKey(), hint); } @@ -1542,11 +1529,35 @@ public Scope visitSelectHint(SelectHint node, final Optional context) { return createAndAssignScope(node, context); } - private QualifiedName getValidTable(QualifiedName table) { - if (table == null) { - return null; + private Hint processHintItem(Node hintItem) { + if (hintItem instanceof ReplicaHintItem) { + ReplicaHintItem item = (ReplicaHintItem) hintItem; + QualifiedName table = item.getTable(); + if (table == null) { + return new ReplicaHint(null, item.getReplicaIndex()); + } + QualifiedName resolvedTable = resolveTable(table); + if (resolvedTable != null) { + return new ReplicaHint(resolvedTable, item.getReplicaIndex()); + } + } else if (hintItem instanceof RegionRouteHintItem) { + RegionRouteHintItem item = (RegionRouteHintItem) hintItem; + QualifiedName table = item.getTable(); + if (table == null) { + return new RegionRouteHint(null, item.getRegionDatanodeMap()); + } + QualifiedName resolvedTable = resolveTable(table); + if (resolvedTable != null) { + return new RegionRouteHint(resolvedTable, item.getRegionDatanodeMap()); + } + } else if (hintItem instanceof ParallelHintItem) { + ParallelHintItem item = (ParallelHintItem) hintItem; + return new ParallelHint(item.getParallelism()); } + return null; + } + private QualifiedName resolveTable(QualifiedName table) { List existingTables = analysis.getRelationNames().values().stream().collect(toImmutableList()); @@ -1561,7 +1572,7 @@ private QualifiedName getValidTable(QualifiedName table) { QualifiedName.of(tableObjectName.getDatabaseName(), tableObjectName.getObjectName()); if (!existingTables.contains(tableName)) { - throw new SemanticException(String.format("Invalid table : %s", tableName)); + return null; } return tableName; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java index f825db771a372..3dd96a6158f5a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java @@ -30,7 +30,8 @@ import java.util.List; import java.util.Map; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class ReplicaHintPlannerTest extends PlanTester { From 55f9384bae0205871f865ae669ccc8ba16e1fa12 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 21 Apr 2026 10:41:45 +0800 Subject: [PATCH 32/35] add nullable annotation --- .../plan/relational/sql/ast/RegionRouteHintItem.java | 9 ++++++--- .../plan/relational/sql/ast/ReplicaHintItem.java | 8 +++++--- .../plan/relational/utils/hint/RegionRouteHint.java | 6 ++++-- .../plan/relational/utils/hint/ReplicaHint.java | 6 ++++-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java index 419d688292e92..8b37423d55151 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java @@ -3,6 +3,8 @@ import com.google.common.collect.ImmutableList; import org.apache.tsfile.utils.RamUsageEstimator; +import javax.annotation.Nullable; + import java.util.List; import java.util.Map; import java.util.Objects; @@ -12,16 +14,17 @@ public class RegionRouteHintItem extends Node { RamUsageEstimator.shallowSizeOfInstance(RegionRouteHintItem.class); private static final String HINT_NAME_ITEM = "region_route"; - private final QualifiedName table; + private @Nullable final QualifiedName table; private final Map regionDatanodeMap; - public RegionRouteHintItem(QualifiedName table, Map regionDatanodeMap) { + public RegionRouteHintItem( + @Nullable QualifiedName table, Map regionDatanodeMap) { super(null); this.table = table; this.regionDatanodeMap = regionDatanodeMap; } - public QualifiedName getTable() { + public @Nullable QualifiedName getTable() { return table; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java index 149f72fdc5b51..861011a969aff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java @@ -3,6 +3,8 @@ import com.google.common.collect.ImmutableList; import org.apache.tsfile.utils.RamUsageEstimator; +import javax.annotation.Nullable; + import java.util.List; import java.util.Objects; @@ -11,16 +13,16 @@ public class ReplicaHintItem extends Node { RamUsageEstimator.shallowSizeOfInstance(ReplicaHintItem.class); private static final String HINT_NAME_ITEM = "replica"; - private final QualifiedName table; + private @Nullable final QualifiedName table; private final int replicaIndex; - public ReplicaHintItem(QualifiedName table, int replicaIndex) { + public ReplicaHintItem(@Nullable QualifiedName table, int replicaIndex) { super(null); this.table = table; this.replicaIndex = replicaIndex; } - public QualifiedName getTable() { + public @Nullable QualifiedName getTable() { return table; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java index 1f727b3fc08f7..2adfd8272eb3d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java @@ -5,6 +5,8 @@ import com.google.common.collect.ImmutableList; +import javax.annotation.Nullable; + import java.util.List; import java.util.Map; @@ -12,10 +14,10 @@ public class RegionRouteHint extends Hint { public static String HINT_NAME = "region_route"; public static int ANY_TABLE = -1; - private final QualifiedName table; + private @Nullable final QualifiedName table; private final Map regionDatanodeMap; - public RegionRouteHint(QualifiedName table, Map regionDatanodeMa) { + public RegionRouteHint(@Nullable QualifiedName table, Map regionDatanodeMa) { super(HINT_NAME); this.table = table; this.regionDatanodeMap = regionDatanodeMa; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java index bab241a70192d..a4e3ae6e5594e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java @@ -26,15 +26,17 @@ import com.google.common.collect.ImmutableList; +import javax.annotation.Nullable; + import java.util.List; public class ReplicaHint extends Hint { public static String HINT_NAME = "replica"; - private final QualifiedName table; + private @Nullable final QualifiedName table; private final int replicaIndex; - public ReplicaHint(QualifiedName table, int replicaIndex) { + public ReplicaHint(@Nullable QualifiedName table, int replicaIndex) { super(HINT_NAME); this.table = table; this.replicaIndex = replicaIndex; From 2bccb24dad669260eee91a83eeb502ea9c51affc Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 21 Apr 2026 13:42:21 +0800 Subject: [PATCH 33/35] add aggregation test --- .../planner/ReplicaHintPlannerTest.java | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java index 3dd96a6158f5a..82261198c1c98 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ReplicaHintPlannerTest.java @@ -21,6 +21,8 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingException; @@ -32,6 +34,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class ReplicaHintPlannerTest extends PlanTester { @@ -91,6 +94,15 @@ public void testReplicaHintWithConflict() { } } + @Test + public void testReplicaHintWithAgg() { + String sql = "SELECT /*+ REPLICA(0) */ tag1, avg(s1) FROM table1 GROUP BY tag1"; + createPlan(sql); + for (int fragment = 1; fragment <= 3; fragment++) { + verifyReplica(fragment, ImmutableMap.of("table1", 0)); + } + } + @Test public void testRegionRouteHintWithNegativeRegion() { String sql = "SELECT /*+ REGION_ROUTE((-1, 1)) */ * FROM table1"; @@ -178,18 +190,29 @@ public void testParallelHint() { private void verifyReplica(int fragment, Map tableToReplica) { FragmentInstance fragmentInstance = getFragmentInstance(fragment); - TableScanNode fragmentPlan = (TableScanNode) getFragmentPlan(fragment); + PlanNode fragmentPlan = getFragmentPlan(fragment); + + TableScanNode tableScanNode = null; + if (fragmentPlan instanceof AggregationNode) { + tableScanNode = (TableScanNode) fragmentPlan.getChildren().get(0); + } else if (fragmentPlan instanceof TableScanNode) { + tableScanNode = (TableScanNode) fragmentPlan; + } + if (tableScanNode == null) { + fail("tableScanNode must not be null"); + } + assertEquals( - fragmentPlan.getRegionReplicaSet().getRegionId().getId(), + tableScanNode.getRegionReplicaSet().getRegionId().getId(), fragmentInstance.getRegionReplicaSet().getRegionId().getId()); String tableName = - fragmentPlan.getAlias() != null - ? fragmentPlan.getAlias().getValue() - : fragmentPlan.getQualifiedObjectName().getObjectName(); + tableScanNode.getAlias() != null + ? tableScanNode.getAlias().getValue() + : tableScanNode.getQualifiedObjectName().getObjectName(); List replicaNodes = - fragmentPlan.getRegionReplicaSet().getDataNodeLocations(); + tableScanNode.getRegionReplicaSet().getDataNodeLocations(); TDataNodeLocation queryNode = fragmentInstance.getHostDataNode(); assertEquals(2, replicaNodes.size()); From 4bc8764273f2026001240794f43f1e42baf6015f Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 21 Apr 2026 14:15:16 +0800 Subject: [PATCH 34/35] revert some code --- .../db/queryengine/common/MPPQueryContext.java | 10 +++++----- .../plan/relational/analyzer/Scope.java | 17 ++++++++--------- .../relational/analyzer/StatementAnalyzer.java | 2 +- .../relational/planner/CteMaterializer.java | 3 +-- .../planner/distribute/AddExchangeNodes.java | 9 --------- ...UncorrelatedScalarSubqueryReconstructor.java | 3 +-- 6 files changed, 16 insertions(+), 28 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index da13ac5b0e636..681233fe97779 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -40,7 +40,7 @@ import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; import org.apache.iotdb.db.utils.cte.CteDataStore; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableList; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; @@ -149,7 +149,7 @@ public enum ExplainType { private boolean innerTriggeredQuery = false; // Tables in the subquery - private final Map, Set> subQueryTables = new HashMap<>(); + private final Map, List> subQueryTables = new HashMap<>(); // parallel hint private int parallelism = 0; @@ -554,12 +554,12 @@ public void setCteQueries(Map, Query> cteQueries) { this.cteQueries = cteQueries; } - public void addSubQueryTables(Query query, Set tables) { + public void addSubQueryTables(Query query, List tables) { subQueryTables.put(NodeRef.of(query), tables); } - public Set getTables(Query query) { - return subQueryTables.getOrDefault(NodeRef.of(query), ImmutableSet.of()); + public List getTables(Query query) { + return subQueryTables.getOrDefault(NodeRef.of(query), ImmutableList.of()); } public void addCteExplainResult(Table table, Pair> cteExplainResult) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java index d178a5f30471e..d06529c00506f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java @@ -31,12 +31,11 @@ import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.function.Predicate; import static com.google.common.base.MoreObjects.toStringHelper; @@ -58,7 +57,7 @@ public class Scope { // Tables to access for the current relation. For CTE materialization and constant folding // subqueries, non-materialized CTEs in tables must be identified, and their definitions // attached to the subquery context. - private Set tables; + private List tables; public static Scope create() { return builder().build(); @@ -74,24 +73,24 @@ private Scope( RelationId relationId, RelationType relation, Map namedQueries, - Set tables) { + List tables) { this.parent = requireNonNull(parent, "parent is null"); this.relationId = requireNonNull(relationId, "relationId is null"); this.queryBoundary = queryBoundary; this.relation = requireNonNull(relation, "relation is null"); this.namedQueries = ImmutableMap.copyOf(requireNonNull(namedQueries, "namedQueries is null")); - this.tables = new HashSet<>(requireNonNull(tables, "tables is null")); + this.tables = new ArrayList<>(requireNonNull(tables, "tables is null")); } public void addTable(Table table) { tables.add(new Identifier(table.getName().getSuffix())); } - public void setTables(Set tables) { + public void setTables(List tables) { this.tables = tables; } - public Set getTables() { + public List getTables() { return tables; } @@ -350,7 +349,7 @@ public static final class Builder { private RelationId relationId = RelationId.anonymous(); private RelationType relationType = new RelationType(); private final Map namedQueries = new HashMap<>(); - private final Set tables = new HashSet<>(); + private final List tables = new ArrayList<>(); private Optional parent = Optional.empty(); private boolean queryBoundary; @@ -389,7 +388,7 @@ public Builder withNamedQuery(String name, WithQuery withQuery) { return this; } - public Builder withTables(Set tables) { + public Builder withTables(List tables) { this.tables.addAll(tables); return this; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 905a173812648..8ee48c066b7c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -3729,7 +3729,7 @@ protected Scope visitJoin(Join node, Optional scope) { joinConditionCheck(criteria); // Remember original tables before processing left - Set originalTables = new HashSet<>(); + List originalTables = new ArrayList<>(); scope.ifPresent(s -> originalTables.addAll(s.getTables())); Scope left = process(node.getLeft(), scope); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java index 7864d8428cda0..03eb0baa00538 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CteMaterializer.java @@ -66,7 +66,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -109,7 +108,7 @@ public CteDataStore fetchCteQueryResult( try { Query q = query; if (with != null) { - Set tables = context.getTables(query); + List tables = context.getTables(query); List withQueries = with.getQueries().stream() .filter( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index 9c408792543e6..af5b0bf6181b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -296,13 +296,4 @@ private String getTableName(TableScanNode node) { QualifiedObjectName fullName = node.getQualifiedObjectName(); return fullName.getDatabaseName() + "." + fullName.getObjectName(); } - - /** - * Selects data node locations based on the provided hint using polymorphism. - ReplicaHint: Uses - * the hint's own selectLocations strategy - Other hints or null: Returns all original locations - */ - private List selectLocations( - List dataNodeLocations, ReplicaHint hint) { - return hint.selectLocations(dataNodeLocations); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java index 5f6e7fe012f83..49d4b7831e7d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/PredicateWithUncorrelatedScalarSubqueryReconstructor.java @@ -55,7 +55,6 @@ import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; @@ -119,7 +118,7 @@ public Optional fetchUncorrelatedSubqueryResultForPredicate( Query query = subqueryExpression.getQuery(); Query q = query; if (with != null) { - Set tables = context.getTables(query); + List tables = context.getTables(query); List withQueries = with.getQueries().stream() .filter( From c7c76a7ca1b35784d378ed413fa49acc8b923f16 Mon Sep 17 00:00:00 2001 From: shizy Date: Tue, 21 Apr 2026 14:43:33 +0800 Subject: [PATCH 35/35] code format issue --- .../relational/sql/ast/ParallelHintItem.java | 30 +++++++++---------- .../sql/ast/RegionRouteHintItem.java | 19 ++++++++++++ .../relational/sql/ast/ReplicaHintItem.java | 19 ++++++++++++ .../plan/relational/sql/ast/SelectHint.java | 30 +++++++++---------- .../plan/relational/utils/hint/Hint.java | 30 +++++++++---------- .../relational/utils/hint/ParallelHint.java | 30 +++++++++---------- .../utils/hint/RegionRouteHint.java | 19 ++++++++++++ .../relational/utils/hint/ReplicaHint.java | 30 +++++++++---------- .../plan/relational/analyzer/JoinTest.java | 1 - 9 files changed, 127 insertions(+), 81 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java index 94e7feaea5a50..3c37c84c1c309 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ParallelHintItem.java @@ -1,22 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java index 8b37423d55151..c4c6838c46139 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RegionRouteHintItem.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java index 861011a969aff..87ff1666c8a81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReplicaHintItem.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java index ee583509a131e..03dcc1e979a6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SelectHint.java @@ -1,22 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java index 628322aa581f0..e64940510e2c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/Hint.java @@ -1,22 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java index 2883d093c3c73..b478d887ba3c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ParallelHint.java @@ -1,22 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java index 2adfd8272eb3d..37cbae4a6a33e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/RegionRouteHint.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java index a4e3ae6e5594e..b2b9dfeedaea1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/hint/ReplicaHint.java @@ -1,22 +1,20 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, - * * software distributed under the License is distributed on an - * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * * KIND, either express or implied. See the License for the - * * specific language governing permissions and limitations - * * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.iotdb.db.queryengine.plan.relational.utils.hint; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java index d3e90cdc061a6..1b178b50449da 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java @@ -188,7 +188,6 @@ private void assertInnerJoinTest1(String sql) { List joinCriteria = Collections.singletonList( new JoinNode.EquiJoinClause(Symbol.of("time"), Symbol.of("time_0"))); - assertJoinNodeEquals( joinNode, INNER,