This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b4945e5be7a [fix](mark join) Fix invalid semi-join transpose when the
bottom join is a mark join (#66574)
b4945e5be7a is described below
commit b4945e5be7a86845363bd2d0649f47223b159f3d
Author: starocean999 <[email protected]>
AuthorDate: Mon Aug 10 19:43:07 2026 +0800
[fix](mark join) Fix invalid semi-join transpose when the bottom join is a
mark join (#66574)
### What problem does this PR solve?
Problem Summary:
A query with a subquery inside the ON condition of an anti join failed at
physical planning with "A expression contains slot not from children", e.g.:
```sql
SELECT t1.* FROM t1 LEFT ANTI JOIN t2
ON t1.k2 = t2.k3 AND t1.k1 NOT IN (SELECT t3.k1 FROM t3 WHERE t1.k2 =
t3.k2);
```
The subquery is unnested into a mark join `(t1 LEFT ANTI JOIN t3)` that
produces
a mark slot, and the outer anti join references the mark slot in its
conjuncts.
Root cause: the exploration rule `SemiJoinSemiJoinTransposeProject`
transposes
two nested left semi/anti joins `(A ⟕̸ B) ⟕̸ C` into `(A ⟕̸ C) ⟕̸ B`.
When the bottom join `(A ⟕̸ B)` is a mark join, the transposed plan builds
`newBottomSemi = (A ⟕̸ C)` from the top join, inheriting the top join's
conjuncts
that reference the bottom mark slot, while the mark slot is now produced
above by
the new top mark join. The mark slot is therefore referenced by a join whose
children don't output it, and physical planning fails with "slot not from
children".
The fix rejects the transpose when the bottom semi join is a mark join and
the
top semi join references the bottom mark slot in its conjuncts, so the mark
join
is always kept below the join that consumes the mark slot. After the fix the
query above executes correctly and returns the expected result.
---
.../join/SemiJoinSemiJoinTransposeProject.java | 30 +++++++++++-
.../join/SemiJoinSemiJoinTransposeProjectTest.java | 55 ++++++++++++++++++++++
2 files changed, 84 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java
index 359d6e13552..8d1d67af5c0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java
@@ -68,7 +68,15 @@ public class SemiJoinSemiJoinTransposeProject extends
OneExplorationRuleFactory
.when(this::typeChecker)
.when(topSemi -> InnerJoinLAsscomProject.checkReorder(topSemi,
topSemi.left().child(), false))
.whenNot(join -> join.hasDistributeHint() ||
join.left().child().hasDistributeHint())
- .when(join -> join.left().isAllSlots()))
+ .when(join -> join.left().isAllSlots())
+ // the transpose swaps the bottom semi join to the top, so the
mark slot
+ // produced by the bottom mark join would be produced by the
new top semi
+ // join. if the top semi join references the mark slot in its
conjuncts,
+ // those conjuncts would be moved to the new bottom semi join
whose children
+ // don't output the mark slot, which makes the mark slot
dangling and fails
+ // physical planning with "slot not from children", so the
transpose must be
+ // rejected in this case
+ .whenNot(this::isMarkSlotReferencedByTopJoin))
.then(topProject -> {
LogicalJoin<LogicalProject<LogicalJoin<GroupPlan,
GroupPlan>>, GroupPlan> topSemi
= topProject.child();
@@ -119,4 +127,24 @@ public class SemiJoinSemiJoinTransposeProject extends
OneExplorationRuleFactory
public boolean
typeChecker(LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>,
GroupPlan> topJoin) {
return VALID_TYPE_PAIR_SET.contains(Pair.of(topJoin.getJoinType(),
topJoin.left().child().getJoinType()));
}
+
+ /**
+ * check whether the top semi join references the mark slot produced by
the bottom mark
+ * join in its conjuncts. in the transposed plan the mark slot is produced
by the new
+ * top semi join (built from the bottom semi join), while the top semi
join becomes the
+ * new bottom semi join whose children are A and C, which don't output the
mark slot.
+ * so if the top semi join's conjuncts reference the mark slot, the
transpose would make
+ * the mark slot dangling and must be rejected.
+ */
+ private boolean isMarkSlotReferencedByTopJoin(
+ LogicalJoin<LogicalProject<LogicalJoin<GroupPlan, GroupPlan>>,
GroupPlan> topSemi) {
+ LogicalJoin<GroupPlan, GroupPlan> bottomSemi = topSemi.left().child();
+ if (!bottomSemi.isMarkJoin()) {
+ return false;
+ }
+ ExprId markSlotExprId =
bottomSemi.getMarkJoinSlotReference().get().getExprId();
+ return topSemi.getExpressions().stream()
+ .flatMap(expr -> expr.getInputSlotExprIds().stream())
+ .anyMatch(markSlotExprId::equals);
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java
index d37be0a1a13..99a0c452138 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.exploration.join;
import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -95,4 +96,58 @@ public class SemiJoinSemiJoinTransposeProjectTest implements
MemoPatternMatchSup
)
);
}
+
+ @Test
+ public void
testSemiProjectSemiCommuteRejectedWhenTopJoinReferencesBottomMarkSlot() {
+ /*
+ * the transpose must be rejected when the bottom semi join is a mark
join and the
+ * top semi join references the mark slot in its conjuncts. otherwise
the transposed
+ * plan would move the conjuncts that reference the mark slot to a
join whose
+ * children don't output the mark slot, which fails physical planning
with
+ * "slot not from children".
+ *
+ * topJoin(references mark) the transpose is rejected,
the plan
+ * / \ keeps the original order:
+ * abProject t3 topJoin
+ * | / \
+ * bottomMarkJoin(t1 anti t2) abProject t3
+ * / \ |
+ * t1 t2 bottomMarkJoin
+ * / \
+ * t1 t2
+ */
+ // bottom mark join: t1 left anti t2, markJoinConjuncts = (t1#0 =
t2#0),
+ // output = [t1#0, t1#1, markSlot]
+ LogicalPlan bottomMarkJoin = new LogicalPlanBuilder(scan1)
+ .markJoinWithMarkConjuncts(scan2, JoinType.LEFT_ANTI_JOIN,
Pair.of(0, 0))
+ .build();
+ // project exposes [t1#0, markSlot]
+ LogicalPlan abProject = new LogicalPlanBuilder(bottomMarkJoin)
+ .project(ImmutableList.of(0, 2))
+ .build();
+ // top anti join on t3 whose other conjunct references the mark slot
of the bottom
+ // mark join, this is exactly the plan shape that used to trigger the
bug
+ Slot markSlot = abProject.getOutput().get(1);
+ LogicalPlan topJoin = new LogicalPlanBuilder(abProject)
+ .join(scan3, JoinType.LEFT_ANTI_JOIN, ImmutableList.of(),
ImmutableList.of(markSlot))
+ .projectAll()
+ .build();
+ // the transpose is rejected, so the plan keeps the original order and
the
+ // mark join still produces the mark slot below the top anti join
+ PlanChecker.from(MemoTestUtils.createConnectContext(), topJoin)
+
.applyExploration(SemiJoinSemiJoinTransposeProject.INSTANCE.build())
+ .matches(
+ logicalProject(
+ logicalJoin(
+ logicalProject(
+ logicalJoin(
+
logicalOlapScan().when(s -> s.getTable().getName().equals("t1")),
+
logicalOlapScan().when(s -> s.getTable().getName().equals("t2"))
+ ).when(join ->
join.getJoinType() == JoinType.LEFT_ANTI_JOIN)
+ ),
+ logicalOlapScan().when(s ->
s.getTable().getName().equals("t3"))
+ ).when(join -> join.getJoinType() ==
JoinType.LEFT_ANTI_JOIN)
+ )
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
