This is an automated email from the ASF dual-hosted git repository.
924060929 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 060c7dbe817 [fix](local shuffle) Populate distribution keys for select
nodes (#67941)
060c7dbe817 is described below
commit 060c7dbe81758fb0d673567fb361234d06b9a1b1
Author: 924060929 <[email protected]>
AuthorDate: Wed Sep 16 11:13:17 2026 +0800
[fix](local shuffle) Populate distribution keys for select nodes (#67941)
### What problem does this PR solve?
Related PR: #67177
Problem Summary:
The physical plan translator created Select nodes without their output
and child distribution expressions. When a Select node forwarded a
parent hash requirement, local exchange planning could therefore build
an empty-key hash exchange below it. A shuffle join could then
incorrectly treat mismatched task placement as aligned and return
incomplete results.
This PR centralizes all Select node construction in the physical plan
translator and populates both the Select output distribution expressions
and its child distribution expressions. Local exchange planning can then
use the actual join keys. It does not change SelectNode,
AddLocalExchange, or BE behavior.
### Release note
Fix incorrect query results for Window output filtered before a shuffle
join when FE local shuffle planning is enabled.
---
.../glue/translator/PhysicalPlanTranslator.java | 23 ++++----
.../test_select_analytic_shuffle_join.out | 6 +++
.../test_select_analytic_shuffle_join.groovy | 62 ++++++++++++++++++++++
3 files changed, 82 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index a6c35b4216d..9bfe285a13e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -109,6 +109,7 @@ import
org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.Relation;
import org.apache.doris.nereids.trees.plans.commands.merge.MergeOperation;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
+import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
import
org.apache.doris.nereids.trees.plans.physical.PhysicalBaseExternalTableSink;
@@ -1449,9 +1450,7 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
PlanNode planNode = inputFragment.getPlanRoot();
// the three nodes don't support conjuncts, need create a SelectNode
to filter data
if (planNode instanceof ExchangeNode || planNode instanceof SortNode
|| planNode instanceof UnionNode) {
- SelectNode selectNode = new SelectNode(context.nextPlanNodeId(),
planNode);
- selectNode.setNereidsId(filter.getId());
- context.getNereidsIdToPlanNodeIdMap().put(filter.getId(),
selectNode.getId());
+ SelectNode selectNode = createSelectNode(filter, planNode,
context);
addConjunctsToPlanNode(filter, selectNode, context);
addPlanRoot(inputFragment, selectNode, filter);
} else {
@@ -1462,12 +1461,10 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
||
CollectionUtils.isNotEmpty(planNode.getProjectList())
// already have limit on this node, filter need
execute after limit, so need a new node
|| planNode.hasLimit()) {
- planNode = new SelectNode(context.nextPlanNodeId(),
planNode);
- planNode.setNereidsId(filter.getId());
+ planNode = createSelectNode(filter, planNode, context);
// NOTE: can't collect planNode.getId() on filter's child,
such as scan node
// since if the filter is embedded into scan, the id
mapping relation is not correct
// i.e, the physical filter's nereids's id will be mapped
to final plan's scan node
- context.getNereidsIdToPlanNodeIdMap().put(filter.getId(),
planNode.getId());
addPlanRoot(inputFragment, planNode, filter);
}
addConjunctsToPlanNode(filter, planNode, context);
@@ -1481,6 +1478,16 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
return inputFragment;
}
+ private SelectNode createSelectNode(AbstractPhysicalPlan physicalPlan,
PlanNode child,
+ PlanTranslatorContext context) {
+ SelectNode selectNode = new SelectNode(context.nextPlanNodeId(),
child);
+ selectNode.setNereidsId(physicalPlan.getId());
+ context.getNereidsIdToPlanNodeIdMap().put(physicalPlan.getId(),
selectNode.getId());
+ selectNode.setDistributeExprLists(getDistributeExpr(physicalPlan));
+
selectNode.setChildrenDistributeExprLists(getDistributeExprs(physicalPlan.child(0)));
+ return selectNode;
+ }
+
@Override
public PlanFragment visitPhysicalGenerate(PhysicalGenerate<? extends Plan>
generate,
PlanTranslatorContext context) {
@@ -2114,9 +2121,7 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
PlanNode inputPlanNode = inputFragment.getPlanRoot();
// this means already have project on this node, filter need execute
after project, so need a new node
if (CollectionUtils.isNotEmpty(inputPlanNode.getProjectList())) {
- SelectNode selectNode = new SelectNode(context.nextPlanNodeId(),
inputPlanNode);
- selectNode.setNereidsId(project.getId());
- context.getNereidsIdToPlanNodeIdMap().put(project.getId(),
selectNode.getId());
+ SelectNode selectNode = createSelectNode(project, inputPlanNode,
context);
addPlanRoot(inputFragment, selectNode, project);
inputPlanNode = selectNode;
}
diff --git
a/regression-test/data/nereids_p0/local_shuffle/test_select_analytic_shuffle_join.out
b/regression-test/data/nereids_p0/local_shuffle/test_select_analytic_shuffle_join.out
new file mode 100644
index 00000000000..55c70cdf195
--- /dev/null
+++
b/regression-test/data/nereids_p0/local_shuffle/test_select_analytic_shuffle_join.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_over_analytic_shuffle_join --
+10 45 55 1045
+
+-- !select_over_aligned_analytic_shuffle_join --
+10 45 55 1045
diff --git
a/regression-test/suites/nereids_p0/local_shuffle/test_select_analytic_shuffle_join.groovy
b/regression-test/suites/nereids_p0/local_shuffle/test_select_analytic_shuffle_join.groovy
new file mode 100644
index 00000000000..d295968cd1a
--- /dev/null
+++
b/regression-test/suites/nereids_p0/local_shuffle/test_select_analytic_shuffle_join.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_select_analytic_shuffle_join") {
+ sql "DROP TABLE IF EXISTS select_analytic_join_left"
+ sql "DROP TABLE IF EXISTS select_analytic_join_right"
+
+ sql """CREATE TABLE select_analytic_join_left (
+ k INT, v INT
+ ) ENGINE=OLAP DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 10
+ PROPERTIES ("replication_num"="1")"""
+ sql """CREATE TABLE select_analytic_join_right (
+ k INT, w INT
+ ) ENGINE=OLAP DUPLICATE KEY(k)
+ DISTRIBUTED BY HASH(k) BUCKETS 10
+ PROPERTIES ("replication_num"="1")"""
+
+ sql """INSERT INTO select_analytic_join_left VALUES
+ (0,1),(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)"""
+ sql """INSERT INTO select_analytic_join_right VALUES
+ (0,100),(1,101),(2,102),(3,103),(4,104),
+ (5,105),(6,106),(7,107),(8,108),(9,109)"""
+
+ def variables =
"enable_local_shuffle_planner=true,enable_local_shuffle=true," +
+
"enable_bucket_shuffle_join=false,ignore_storage_data_distribution=true," +
+ "parallel_pipeline_task_num=3,enable_sql_cache=false"
+
+ order_qt_select_over_analytic_shuffle_join """SELECT
/*+SET_VAR(${variables})*/
+ COUNT(*), SUM(s.k), SUM(s.running_v), SUM(d.w)
+ FROM (
+ SELECT k, SUM(v) OVER (PARTITION BY v ORDER BY v
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS
running_v,
+ random(0) AS r
+ FROM select_analytic_join_left
+ ) s JOIN [shuffle] select_analytic_join_right d ON s.k=d.k
+ WHERE s.r < 2.0"""
+
+ order_qt_select_over_aligned_analytic_shuffle_join """SELECT
/*+SET_VAR(${variables})*/
+ COUNT(*), SUM(s.k), SUM(s.running_v), SUM(d.w)
+ FROM (
+ SELECT k, SUM(v) OVER (PARTITION BY k ORDER BY k
+ ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS
running_v,
+ random(0) AS r
+ FROM select_analytic_join_left
+ ) s JOIN [shuffle] select_analytic_join_right d ON s.k=d.k
+ WHERE s.r < 2.0"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
