morrySnow commented on code in PR #58228:
URL: https://github.com/apache/doris/pull/58228#discussion_r2644703755
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##########
@@ -4331,13 +4384,30 @@ private LogicalPlan withJoinRelations(LogicalPlan
input, RelationContext ctx) {
}
}
if (ids == null) {
- last = new LogicalJoin<>(joinType,
ExpressionUtils.EMPTY_CONDITION,
- condition.map(ExpressionUtils::extractConjunction)
- .orElse(ExpressionUtils.EMPTY_CONDITION),
- distributeHint,
- Optional.empty(),
- last,
- plan(join.relationPrimary()), null);
+ LogicalPlan right = plan(join.relationPrimary());
+ if (right instanceof LogicalGenerate
+ && right.child(0) instanceof LogicalOneRowRelation
Review Comment:
why must `LogicalOneRowRelation`?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalGenerate.java:
##########
@@ -74,6 +66,7 @@ public PhysicalGenerate(List<Function> generators, List<Slot>
generatorOutput,
this.generators =
ImmutableList.copyOf(Objects.requireNonNull(generators, "predicates can not be
null"));
this.generatorOutput =
ImmutableList.copyOf(Objects.requireNonNull(generatorOutput,
"generatorOutput can not be null"));
+ this.conjuncts = ImmutableList.copyOf(conjuncts);
Review Comment:
Objects.requireNonNull
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##########
@@ -2692,6 +2693,58 @@ public LogicalPlan
visitTableValuedFunction(TableValuedFunctionContext ctx) {
});
}
+ @Override
+ public LogicalPlan visitUnnestFunction(DorisParser.UnnestFunctionContext
ctx) {
+ return withUnnest(ctx.unnest());
+ }
+
+ private LogicalPlan withUnnest(DorisParser.UnnestContext ctx) {
+ String defaultNestedColumnName = "unnest";
+ String defaultOrdinalityColumnName = "ordinality";
+ List<Expression> arguments = ctx.expression().stream()
+ .<Expression>map(this::typedVisit)
+ .collect(ImmutableList.toImmutableList());
+ boolean needOrdinality = ctx.ORDINALITY() != null;
+ int size = arguments.size();
+
+ String generateName = ctx.tableName != null ? ctx.tableName.getText()
: defaultNestedColumnName;
+ // do same thing as later view explode map type, we need to add a
project to convert map to struct
+ int argumentsSize = size + (needOrdinality ? 1 : 0);
+ List<String> nestedColumnNames = new ArrayList<>(argumentsSize);
+ int columnNamesSize = ctx.columnNames.size();
+ if (!ctx.columnNames.isEmpty()) {
+ for (int i = 0; i < columnNamesSize; ++i) {
+ nestedColumnNames.add(ctx.columnNames.get(i).getText());
+ }
+ for (int i = 0; i < size - columnNamesSize; ++i) {
+ nestedColumnNames.add(defaultNestedColumnName);
+ }
+ if (needOrdinality && columnNamesSize < argumentsSize) {
+ nestedColumnNames.add(defaultOrdinalityColumnName);
+ }
Review Comment:
so, here we do not check the name length? i think we should do more check
here to process nagtive case
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##########
@@ -4331,13 +4384,30 @@ private LogicalPlan withJoinRelations(LogicalPlan
input, RelationContext ctx) {
}
}
if (ids == null) {
- last = new LogicalJoin<>(joinType,
ExpressionUtils.EMPTY_CONDITION,
- condition.map(ExpressionUtils::extractConjunction)
- .orElse(ExpressionUtils.EMPTY_CONDITION),
- distributeHint,
- Optional.empty(),
- last,
- plan(join.relationPrimary()), null);
+ LogicalPlan right = plan(join.relationPrimary());
+ if (right instanceof LogicalGenerate
+ && right.child(0) instanceof LogicalOneRowRelation
+ && ((LogicalGenerate<?>) right).getGenerators().get(0)
instanceof Unnest) {
Review Comment:
add comment for this if
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java:
##########
@@ -672,10 +682,9 @@ public static boolean
canInferNotNullForMarkSlot(Expression predicate, Expressio
* the mark slot can be non-nullable boolean
* and in semi join, we can safely change the mark conjunct to hash
conjunct
*/
- ImmutableList<Literal> literals =
- ImmutableList.of(NullLiteral.BOOLEAN_INSTANCE,
BooleanLiteral.FALSE);
- List<MarkJoinSlotReference> markJoinSlotReferenceList =
- new
ArrayList<>((predicate.collect(MarkJoinSlotReference.class::isInstance)));
+ ImmutableList<Literal> literals =
ImmutableList.of(NullLiteral.BOOLEAN_INSTANCE, BooleanLiteral.FALSE);
+ List<MarkJoinSlotReference> markJoinSlotReferenceList = new
ArrayList<>(
+ (predicate.collect(MarkJoinSlotReference.class::isInstance)));
Review Comment:
why reformat this code block? do not reformat unrelated code
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java:
##########
@@ -160,7 +161,7 @@ private void checkUnexpectedExpressionTypes(Plan plan,
Class<? extends Expressio
}
expr.foreachUp(e -> {
for (Class<? extends Expression> type :
unexpectedExpressionTypes) {
- if (type.isInstance(e)) {
+ if (type.isInstance(e) && !(e instanceof Unnest)) {
Review Comment:
why exclude Unnest? add comment
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Unnest.java:
##########
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.generator;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
+import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision;
+import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
+import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.BitmapType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * unnest([1, 2, 3]), generate three rows include 1, 2 and 3.
+ * unnest([1, 2, 3], [4, 5, 6]) generates two columns and three rows
+ * where the first column contains 1, 2, 3, and the second column contains 4,
5, 6.
+ */
+public class Unnest extends TableGeneratingFunction implements
CustomSignature, ComputePrecision, AlwaysNullable {
+
+ private boolean isOuter;
+ private boolean needOrdinality;
+
+ /**
+ * constructor with one argument.
+ */
+ public Unnest(Expression argument) {
+ this(false, false, ImmutableList.of(argument));
+ }
+
+ /**
+ * constructor with more argument.
+ */
+ public Unnest(boolean isOuter, boolean needOrdinality, List<Expression>
arguments) {
+ super("unnest", arguments);
+ this.isOuter = isOuter;
+ this.needOrdinality = needOrdinality;
+ }
+
+ /** constructor for withChildren and reuse signature */
+ private Unnest(boolean isOuter, boolean needOrdinality,
GeneratorFunctionParams functionParams) {
Review Comment:
same as upper ctor
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##########
@@ -2692,6 +2693,58 @@ public LogicalPlan
visitTableValuedFunction(TableValuedFunctionContext ctx) {
});
}
+ @Override
+ public LogicalPlan visitUnnestFunction(DorisParser.UnnestFunctionContext
ctx) {
+ return withUnnest(ctx.unnest());
+ }
+
+ private LogicalPlan withUnnest(DorisParser.UnnestContext ctx) {
+ String defaultNestedColumnName = "unnest";
+ String defaultOrdinalityColumnName = "ordinality";
Review Comment:
maybe they should be static varaible in `Unnest`
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Unnest.java:
##########
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.generator;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
+import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision;
+import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
+import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.BitmapType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * unnest([1, 2, 3]), generate three rows include 1, 2 and 3.
+ * unnest([1, 2, 3], [4, 5, 6]) generates two columns and three rows
+ * where the first column contains 1, 2, 3, and the second column contains 4,
5, 6.
+ */
+public class Unnest extends TableGeneratingFunction implements
CustomSignature, ComputePrecision, AlwaysNullable {
+
+ private boolean isOuter;
+ private boolean needOrdinality;
+
+ /**
+ * constructor with one argument.
+ */
+ public Unnest(Expression argument) {
+ this(false, false, ImmutableList.of(argument));
+ }
+
+ /**
+ * constructor with more argument.
+ */
+ public Unnest(boolean isOuter, boolean needOrdinality, List<Expression>
arguments) {
Review Comment:
```suggestion
public Unnest(List<Expression> arguments, boolean needOrdinality,
boolean isOuter) {
```
##########
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4:
##########
@@ -1348,6 +1348,17 @@ lateralView
tableName=identifier AS columnNames+=identifier (COMMA
columnNames+=identifier)*
;
+unnest:
+ LATERAL? UNNEST LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN (
Review Comment:
add neriedsParser ut
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java:
##########
@@ -312,18 +312,61 @@ boundSlot, new StringLiteral(fields.get(idx).getName())),
}
}
}
- LogicalGenerate<Plan> ret = new LogicalGenerate<>(
- boundGenerators.build(), outputSlots.build(),
generate.child());
+ /*
+ * SELECT
+ * id,
+ * tags
+ * FROM
+ * items
+ * LEFT JOIN lateral unnest(tags) AS t(tag) ON t.tag = name;
+ *
+ * t.tag is unnest's output, so the conjunct t.tag = name may
reference slot from child and its own output
+ *
+ */
+ int conjunctSize = generate.getConjuncts().size();
+ List<Expression> newConjuncts = new ArrayList<>(conjunctSize);
+ if (conjunctSize > 0) {
+ List<Slot> childOutputs = generate.child().getOutput();
+ List<Slot> conjunctsScopeSlots = new
ArrayList<>(expandAlias.size() + childOutputs.size());
+ for (Alias alias : expandAlias) {
+ conjunctsScopeSlots.add(alias.toSlot());
+ }
+
+ conjunctsScopeSlots.addAll(childOutputs);
+ Scope conjunctsScope = toScope(cascadesContext,
conjunctsScopeSlots);
+ ExpressionAnalyzer conjunctsAnalyzer = new ExpressionAnalyzer(
+ generate, conjunctsScope, cascadesContext, true, false);
+ Map<Slot, Expression> replaceMap =
ExpressionUtils.generateReplaceMap(expandAlias);
+ for (Expression expression : generate.getConjuncts()) {
+ expression = conjunctsAnalyzer.analyze(expression);
+ Expression newExpression = expression.rewriteDownShortCircuit(
+ e -> replaceMap.getOrDefault(e, e));
+ newConjuncts.add(newExpression);
+ }
+ }
+
+ LogicalGenerate<Plan> logicalGenerate = new LogicalGenerate<>(
+ boundGenerators.build(), outputSlots.build(),
ImmutableList.of(), newConjuncts, generate.child());
if (!expandAlias.isEmpty()) {
+ // project should contain: generator.child slot + expandAlias
+ List<NamedExpression> allProjectSlots = new
ArrayList<>(generate.child().getOutput().size()
+ + expandAlias.size());
+ if (!(generate.child() instanceof LogicalOneRowRelation)) {
+ // project should contain: generator.child slot + expandAlias
except:
+ allProjectSlots.addAll(generate.child().getOutput().stream()
+ .map(NamedExpression.class::cast)
+ .collect(Collectors.toList()));
Review Comment:
use `(List)` to avoid cast
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeAggregate.java:
##########
@@ -178,7 +179,8 @@ private LogicalPlan normalizeAgg(LogicalAggregate<Plan>
aggregate, Optional<Logi
if (arg instanceof Literal) {
continue;
}
- if (arg.containsType(SubqueryExpr.class,
WindowExpression.class, PreferPushDownProject.class)) {
+ if (arg.containsType(SubqueryExpr.class,
WindowExpression.class, Unnest.class,
Review Comment:
need a new ut for it
##########
fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java:
##########
@@ -56,12 +57,17 @@ public TableFunctionNode(PlanNodeId id, PlanNode inputNode,
TupleId lateralViewT
this.fnCallExprList = fnCallExprList;
this.outputSlotIds = outputSlotIds;
this.children.add(inputNode);
+ this.expandConjuncts.addAll(expandConjuncts);
Review Comment:
why use `addAll` here?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##########
@@ -2692,6 +2693,58 @@ public LogicalPlan
visitTableValuedFunction(TableValuedFunctionContext ctx) {
});
}
+ @Override
+ public LogicalPlan visitUnnestFunction(DorisParser.UnnestFunctionContext
ctx) {
+ return withUnnest(ctx.unnest());
+ }
+
+ private LogicalPlan withUnnest(DorisParser.UnnestContext ctx) {
+ String defaultNestedColumnName = "unnest";
+ String defaultOrdinalityColumnName = "ordinality";
+ List<Expression> arguments = ctx.expression().stream()
+ .<Expression>map(this::typedVisit)
+ .collect(ImmutableList.toImmutableList());
+ boolean needOrdinality = ctx.ORDINALITY() != null;
+ int size = arguments.size();
+
+ String generateName = ctx.tableName != null ? ctx.tableName.getText()
: defaultNestedColumnName;
+ // do same thing as later view explode map type, we need to add a
project to convert map to struct
+ int argumentsSize = size + (needOrdinality ? 1 : 0);
+ List<String> nestedColumnNames = new ArrayList<>(argumentsSize);
+ int columnNamesSize = ctx.columnNames.size();
+ if (!ctx.columnNames.isEmpty()) {
+ for (int i = 0; i < columnNamesSize; ++i) {
+ nestedColumnNames.add(ctx.columnNames.get(i).getText());
+ }
+ for (int i = 0; i < size - columnNamesSize; ++i) {
+ nestedColumnNames.add(defaultNestedColumnName);
+ }
+ if (needOrdinality && columnNamesSize < argumentsSize) {
+ nestedColumnNames.add(defaultOrdinalityColumnName);
+ }
+ } else {
+ if (size == 1) {
+ nestedColumnNames.add(generateName);
+ } else {
+ for (int i = 0; i < size; ++i) {
+ nestedColumnNames.add(defaultNestedColumnName);
+ }
Review Comment:
all columns have same name?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java:
##########
@@ -4559,15 +4629,19 @@ private LogicalPlan withRelations(LogicalPlan
inputPlan, List<RelationContext> r
for (RelationContext relation : relations) {
// build left deep join tree
LogicalPlan right = withJoinRelations(visitRelation(relation),
relation);
- left = (left == null) ? right :
- new LogicalJoin<>(
- JoinType.CROSS_JOIN,
- ExpressionUtils.EMPTY_CONDITION,
- ExpressionUtils.EMPTY_CONDITION,
- new DistributeHint(DistributeType.NONE),
- Optional.empty(),
- left,
- right, null);
+ // check if it's unnest
+ boolean shouldBeParent = right instanceof LogicalGenerate
Review Comment:
what's the `shouldBeParent` mean? add more comment
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java:
##########
@@ -94,7 +111,15 @@ public <R, C> R accept(PlanVisitor<R, C> visitor, C
context) {
@Override
public List<? extends Expression> getExpressions() {
- return generators;
+ return new ImmutableList.Builder<Expression>()
+ .addAll(generators)
+ .addAll(conjuncts)
+ .build();
+ }
+
+ @Override
+ public Set<Slot> getInputSlots() {
+ return PlanUtils.fastGetInputSlots(generators);
Review Comment:
why only return generators input slots?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownUnnestInProject.java:
##########
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Before:
+ * project(unnest(x) as a, unnest(y) as b)
+ *
+ * After:
+ *
+ * project(struct_element($c$1, col1) as a, struct_element($c$1, co2)
as b)
+ * │
+ * ▼
+ * generate(unnest(x, y) as $c$1)
+ */
+public class PushDownUnnestInProject extends OneRewriteRuleFactory {
Review Comment:
this rule need a fe ut
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownUnnestInProject.java:
##########
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Before:
+ * project(unnest(x) as a, unnest(y) as b)
+ *
+ * After:
+ *
+ * project(struct_element($c$1, col1) as a, struct_element($c$1, co2)
as b)
+ * │
+ * ▼
+ * generate(unnest(x, y) as $c$1)
+ */
+public class PushDownUnnestInProject extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalProject().when(project ->
containUnnestFunction(project.getProjects())).then(project -> {
+ List<NamedExpression> outputs = project.getProjects();
+ Set<Alias> existedAlias = ExpressionUtils.collect(outputs,
Alias.class::isInstance);
+ List<Unnest> toBePushedDown =
ExpressionUtils.collectToList(outputs, Unnest.class::isInstance);
+ Function newFunction =
ExpressionUtils.convertUnnest(validateAndMergeUnnest(toBePushedDown));
+ NormalizeToSlot.NormalizeToSlotContext context =
+
NormalizeToSlot.NormalizeToSlotContext.buildContext(existedAlias,
toBePushedDown);
+ String columnName = ConnectContext.get() != null
+ ?
ConnectContext.get().getStatementContext().generateColumnName() : "expand_cols";
+ SlotReference outputSlot = new SlotReference("unnest_temp_table",
newFunction.getDataType(),
Review Comment:
these names(`expand_cols`, `unnest_temp_table`) are same as explode_xxx
functions?
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeGenerates.java:
##########
@@ -35,18 +35,19 @@
public class MergeGenerates extends OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalGenerate(logicalGenerate()).then(top -> {
- LogicalGenerate<Plan> bottom = top.child();
- Set<Slot> topGeneratorSlots = top.getInputSlots();
- if
(bottom.getGeneratorOutput().stream().anyMatch(topGeneratorSlots::contains)) {
- // top generators use bottom's generator's output, cannot
merge.
- return top;
- }
- List<Function> generators =
Lists.newArrayList(bottom.getGenerators());
- generators.addAll(top.getGenerators());
- List<Slot> generatorsOutput =
Lists.newArrayList(bottom.getGeneratorOutput());
- generatorsOutput.addAll(top.getGeneratorOutput());
- return new LogicalGenerate<>(generators, generatorsOutput,
bottom.child());
- }).toRule(RuleType.MERGE_GENERATES);
+ return logicalGenerate(logicalGenerate().when(generate ->
generate.getConjuncts().isEmpty()))
Review Comment:
need a ut
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java:
##########
@@ -312,18 +312,61 @@ boundSlot, new StringLiteral(fields.get(idx).getName())),
}
}
}
- LogicalGenerate<Plan> ret = new LogicalGenerate<>(
- boundGenerators.build(), outputSlots.build(),
generate.child());
+ /*
+ * SELECT
+ * id,
+ * tags
+ * FROM
+ * items
+ * LEFT JOIN lateral unnest(tags) AS t(tag) ON t.tag = name;
+ *
+ * t.tag is unnest's output, so the conjunct t.tag = name may
reference slot from child and its own output
+ *
+ */
+ int conjunctSize = generate.getConjuncts().size();
+ List<Expression> newConjuncts = new ArrayList<>(conjunctSize);
+ if (conjunctSize > 0) {
+ List<Slot> childOutputs = generate.child().getOutput();
+ List<Slot> conjunctsScopeSlots = new
ArrayList<>(expandAlias.size() + childOutputs.size());
+ for (Alias alias : expandAlias) {
+ conjunctsScopeSlots.add(alias.toSlot());
+ }
+
+ conjunctsScopeSlots.addAll(childOutputs);
+ Scope conjunctsScope = toScope(cascadesContext,
conjunctsScopeSlots);
+ ExpressionAnalyzer conjunctsAnalyzer = new ExpressionAnalyzer(
+ generate, conjunctsScope, cascadesContext, true, false);
+ Map<Slot, Expression> replaceMap =
ExpressionUtils.generateReplaceMap(expandAlias);
+ for (Expression expression : generate.getConjuncts()) {
+ expression = conjunctsAnalyzer.analyze(expression);
+ Expression newExpression = expression.rewriteDownShortCircuit(
+ e -> replaceMap.getOrDefault(e, e));
+ newConjuncts.add(newExpression);
+ }
+ }
+
+ LogicalGenerate<Plan> logicalGenerate = new LogicalGenerate<>(
+ boundGenerators.build(), outputSlots.build(),
ImmutableList.of(), newConjuncts, generate.child());
if (!expandAlias.isEmpty()) {
+ // project should contain: generator.child slot + expandAlias
+ List<NamedExpression> allProjectSlots = new
ArrayList<>(generate.child().getOutput().size()
Review Comment:
use ImmutableListBuilder
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownUnnestInProject.java:
##########
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Before:
+ * project(unnest(x) as a, unnest(y) as b)
+ *
+ * After:
+ *
+ * project(struct_element($c$1, col1) as a, struct_element($c$1, co2)
as b)
+ * │
+ * ▼
+ * generate(unnest(x, y) as $c$1)
+ */
+public class PushDownUnnestInProject extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalProject().when(project ->
containUnnestFunction(project.getProjects())).then(project -> {
+ List<NamedExpression> outputs = project.getProjects();
+ Set<Alias> existedAlias = ExpressionUtils.collect(outputs,
Alias.class::isInstance);
+ List<Unnest> toBePushedDown =
ExpressionUtils.collectToList(outputs, Unnest.class::isInstance);
+ Function newFunction =
ExpressionUtils.convertUnnest(validateAndMergeUnnest(toBePushedDown));
+ NormalizeToSlot.NormalizeToSlotContext context =
+
NormalizeToSlot.NormalizeToSlotContext.buildContext(existedAlias,
toBePushedDown);
+ String columnName = ConnectContext.get() != null
+ ?
ConnectContext.get().getStatementContext().generateColumnName() : "expand_cols";
+ SlotReference outputSlot = new SlotReference("unnest_temp_table",
newFunction.getDataType(),
+ newFunction.nullable(), ImmutableList.of(columnName));
+ List<Alias> newProjects = new ArrayList<>(toBePushedDown.size());
+ Map<Expression, NormalizeToSlot.NormalizeToSlotTriplet>
slotTripletMap = context.getNormalizeToSlotMap();
+ if (toBePushedDown.size() > 1) {
+ // struct_element(#expand_col#k, #k) as #k
+ // struct_element(#expand_col#v, #v) as #v
+ List<StructField> fields = ((StructType)
outputSlot.getDataType()).getFields();
+ Preconditions.checkState(fields.size() ==
toBePushedDown.size(), String.format("push down"
+ + "unnest function has error, function count
is %d, pushed down count is %d",
+ toBePushedDown.size(), fields.size()));
+ for (int i = 0; i < fields.size(); ++i) {
+ Slot remainExpr =
slotTripletMap.get(toBePushedDown.get(i)).remainExpr;
+ newProjects.add(new Alias(remainExpr.getExprId(), new
StructElement(
+ outputSlot, new
StringLiteral(fields.get(i).getName())), remainExpr.getName()));
+ }
+ } else {
+ Slot remainExpr =
slotTripletMap.get(toBePushedDown.get(0)).remainExpr;
+ newProjects.add(new Alias(remainExpr.getExprId(), outputSlot,
remainExpr.getName()));
+ }
+ Map<Slot, Expression> replaceMap =
ExpressionUtils.generateReplaceMap(newProjects);
+ List<NamedExpression> newOutputs =
context.normalizeToUseSlotRef(outputs);
+ return
project.withProjectsAndChild(ExpressionUtils.replaceNamedExpressions(newOutputs,
replaceMap),
+ new LogicalGenerate(ImmutableList.of(newFunction),
ImmutableList.of(outputSlot), project.child()));
+ }).toRule(RuleType.PUSH_DOWN_UNNEST_IN_PROJECT);
+ }
+
+ private boolean containUnnestFunction(List<NamedExpression> expressions) {
+ for (NamedExpression expr : expressions) {
+ if (expr.containsType(Unnest.class)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private Unnest validateAndMergeUnnest(List<Unnest> functions) {
+ // TODO: PG only support ARRAY type, and explode_map and
explode_bitmap only support 1 argument in doris
+ // so we only allow array for now, too
+ int typeCounter = 0;
+ int size = functions.size();
+ List<Expression> expressions = new ArrayList<>(size);
+ for (Unnest unnest : functions) {
+ if (unnest.children().size() > 1) {
+ throw new AnalysisException("UNNEST function can have more
than one arguments only in from clause");
+ }
+ Expression expr = unnest.child(0);
+ DataType dataType = expr.getDataType();
+ if (dataType.isArrayType()) {
+ typeCounter += 1;
+ } else if (dataType.isMapType()) {
+ typeCounter += 2;
+ } else if (dataType.isBitmapType()) {
+ typeCounter += 3;
+ }
+ expressions.add(expr);
+ }
+ // TODO: remove this after doris support multiple arguments for
explode_map and explode_bitmap
+ if (expressions.size() > 1 && typeCounter != 1 * size) {
+ throw new AnalysisException("multiple UNNEST functions in same
place must have ARRAY argument type");
+ }
+ if (typeCounter == 1 * size || typeCounter == 2 * size || typeCounter
== 3 * size) {
Review Comment:
what's this if mean? add comment. if u want check all argument is same type,
this impl is not very intuitive
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java:
##########
@@ -1300,4 +1306,37 @@ public static boolean
containUniqueFunctionExistMultiple(Collection<? extends Ex
}
return false;
}
+
+ /**
+ * convert unnest to explode_* functions
+ */
+ public static
org.apache.doris.nereids.trees.expressions.functions.Function convertUnnest(
Review Comment:
why not import
`org.apache.doris.nereids.trees.expressions.functions.Function`
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java:
##########
@@ -1300,4 +1306,37 @@ public static boolean
containUniqueFunctionExistMultiple(Collection<? extends Ex
}
return false;
}
+
+ /**
+ * convert unnest to explode_* functions
+ */
+ public static
org.apache.doris.nereids.trees.expressions.functions.Function convertUnnest(
Review Comment:
need a ut for it
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownUnnestInProject.java:
##########
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.functions.generator.Unnest;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Before:
+ * project(unnest(x) as a, unnest(y) as b)
+ *
+ * After:
+ *
+ * project(struct_element($c$1, col1) as a, struct_element($c$1, co2)
as b)
+ * │
+ * ▼
+ * generate(unnest(x, y) as $c$1)
+ */
+public class PushDownUnnestInProject extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalProject().when(project ->
containUnnestFunction(project.getProjects())).then(project -> {
Review Comment:
add a new test case for struct sub column push down and struct sub column
prunning
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java:
##########
@@ -312,18 +312,61 @@ boundSlot, new StringLiteral(fields.get(idx).getName())),
}
}
}
- LogicalGenerate<Plan> ret = new LogicalGenerate<>(
- boundGenerators.build(), outputSlots.build(),
generate.child());
+ /*
+ * SELECT
+ * id,
+ * tags
+ * FROM
+ * items
+ * LEFT JOIN lateral unnest(tags) AS t(tag) ON t.tag = name;
+ *
+ * t.tag is unnest's output, so the conjunct t.tag = name may
reference slot from child and its own output
+ *
+ */
+ int conjunctSize = generate.getConjuncts().size();
+ List<Expression> newConjuncts = new ArrayList<>(conjunctSize);
Review Comment:
use ImmutableListBuilder
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
