This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/nested_column_prune by this
push:
new 22c43e0b18f fix access path case-insensitive if the type is struct or
root column
22c43e0b18f is described below
commit 22c43e0b18f6c160455933db6a43b517ed4d73c9
Author: 924060929 <[email protected]>
AuthorDate: Tue Oct 28 12:31:43 2025 +0800
fix access path case-insensitive if the type is struct or root column
---
.../rewrite/AccessPathExpressionCollector.java | 4 +-
.../nereids/rules/rewrite/NestedColumnPruning.java | 51 +++++++++++++---------
.../apache/doris/nereids/types/StructField.java | 3 +-
.../rules/rewrite/PruneNestedColumnTest.java | 9 ++++
4 files changed, 42 insertions(+), 25 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index 8bbb1a8c25e..4fe9e9e2f74 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -104,7 +104,7 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
public Void visitSlotReference(SlotReference slotReference,
CollectorContext context) {
DataType dataType = slotReference.getDataType();
if (dataType instanceof NestedColumnPrunable) {
- context.accessPathBuilder.addPrefix(slotReference.getName());
+
context.accessPathBuilder.addPrefix(slotReference.getName().toLowerCase());
ImmutableList<String> path =
Utils.fastToImmutableList(context.accessPathBuilder.accessPath);
int slotId = slotReference.getExprId().asInt();
slotToAccessPaths.put(slotId, new CollectAccessPathResult(path,
context.bottomFilter, context.type));
@@ -187,7 +187,7 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
return continueCollectAccessPath(struct, context);
}
}
- context.accessPathBuilder.addPrefix(((Literal)
fieldName).getStringValue());
+ context.accessPathBuilder.addPrefix(((Literal)
fieldName).getStringValue().toLowerCase());
return continueCollectAccessPath(struct, context);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
index 0d56260629b..de22e9d2101 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -44,6 +44,8 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.TreeMultimap;
import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Comparator;
@@ -68,29 +70,36 @@ import java.util.Optional;
* </li>
*/
public class NestedColumnPruning implements CustomRewriter {
+ public static final Logger LOG =
LogManager.getLogger(NestedColumnPruning.class);
+
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
- StatementContext statementContext =
jobContext.getCascadesContext().getStatementContext();
- SessionVariable sessionVariable =
statementContext.getConnectContext().getSessionVariable();
- if (!sessionVariable.enablePruneNestedColumns ||
!statementContext.hasNestedColumns()) {
- return plan;
- }
+ try {
+ StatementContext statementContext =
jobContext.getCascadesContext().getStatementContext();
+ SessionVariable sessionVariable =
statementContext.getConnectContext().getSessionVariable();
+ if (!sessionVariable.enablePruneNestedColumns ||
!statementContext.hasNestedColumns()) {
+ return plan;
+ }
- AccessPathPlanCollector collector = new AccessPathPlanCollector();
- Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths =
collector.collect(plan, statementContext);
- Map<Integer, AccessPathInfo> slotToResult =
pruneDataType(slotToAccessPaths);
+ AccessPathPlanCollector collector = new AccessPathPlanCollector();
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths =
collector.collect(plan, statementContext);
+ Map<Integer, AccessPathInfo> slotToResult =
pruneDataType(slotToAccessPaths);
- if (!slotToResult.isEmpty()) {
- Map<Integer, AccessPathInfo> slotIdToPruneType =
Maps.newLinkedHashMap();
- for (Entry<Integer, AccessPathInfo> kv : slotToResult.entrySet()) {
- Integer slotId = kv.getKey();
- AccessPathInfo accessPathInfo = kv.getValue();
- slotIdToPruneType.put(slotId, accessPathInfo);
+ if (!slotToResult.isEmpty()) {
+ Map<Integer, AccessPathInfo> slotIdToPruneType =
Maps.newLinkedHashMap();
+ for (Entry<Integer, AccessPathInfo> kv :
slotToResult.entrySet()) {
+ Integer slotId = kv.getKey();
+ AccessPathInfo accessPathInfo = kv.getValue();
+ slotIdToPruneType.put(slotId, accessPathInfo);
+ }
+ SlotTypeReplacer typeReplacer = new
SlotTypeReplacer(slotIdToPruneType);
+ return plan.accept(typeReplacer, null);
}
- SlotTypeReplacer typeReplacer = new
SlotTypeReplacer(slotIdToPruneType);
- return plan.accept(typeReplacer, null);
+ return plan;
+ } catch (Throwable t) {
+ LOG.warn("NestedColumnPruning failed.", t);
+ return plan;
}
- return plan;
}
private static Map<Integer, AccessPathInfo> pruneDataType(
@@ -340,7 +349,7 @@ public class NestedColumnPruning implements CustomRewriter {
}
if (this.type.isStructType()) {
- String fieldName = path.get(accessIndex);
+ String fieldName = path.get(accessIndex).toLowerCase();
DataTypeAccessTree child = children.get(fieldName);
if (child != null) {
child.setAccessByPath(path, accessIndex + 1, pathType);
@@ -381,7 +390,7 @@ public class NestedColumnPruning implements CustomRewriter {
return;
}
} else if (isRoot) {
- children.get(path.get(accessIndex)).setAccessByPath(path,
accessIndex + 1, pathType);
+
children.get(path.get(accessIndex).toLowerCase()).setAccessByPath(path,
accessIndex + 1, pathType);
return;
}
throw new AnalysisException("unsupported data type: " + this.type);
@@ -390,7 +399,7 @@ public class NestedColumnPruning implements CustomRewriter {
public static DataTypeAccessTree ofRoot(Slot slot, TAccessPathType
pathType) {
DataTypeAccessTree child = of(slot.getDataType(), pathType);
DataTypeAccessTree root = new DataTypeAccessTree(true,
NullType.INSTANCE, pathType);
- root.children.put(slot.getName(), child);
+ root.children.put(slot.getName().toLowerCase(), child);
return root;
}
@@ -400,7 +409,7 @@ public class NestedColumnPruning implements CustomRewriter {
if (type instanceof StructType) {
StructType structType = (StructType) type;
for (Entry<String, StructField> kv :
structType.getNameToFields().entrySet()) {
- root.children.put(kv.getKey(),
of(kv.getValue().getDataType(), pathType));
+ root.children.put(kv.getKey().toLowerCase(),
of(kv.getValue().getDataType(), pathType));
}
} else if (type instanceof ArrayType) {
root.children.put("*", of(((ArrayType) type).getItemType(),
pathType));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
index 88317682c4c..63e94bc369b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.types;
import org.apache.doris.nereids.util.Utils;
-import java.util.Locale;
import java.util.Objects;
/**
@@ -41,7 +40,7 @@ public class StructField {
* @param nullable Indicates if values of this field can be `null` values
*/
public StructField(String name, DataType dataType, boolean nullable,
String comment) {
- this.name = Objects.requireNonNull(name, "name should not be
null").toLowerCase(Locale.ROOT);
+ this.name = Objects.requireNonNull(name, "name should not be
null").toLowerCase();
this.dataType = Objects.requireNonNull(dataType, "dataType should not
be null");
this.nullable = nullable;
this.comment = Objects.requireNonNull(comment, "comment should not be
null");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 934df6bc3f6..8e8d7c44d16 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -91,6 +91,15 @@ public class PruneNestedColumnTest extends TestWithFeService
implements MemoPatt
connectContext.getSessionVariable().enableNereidsTimeout = false;
}
+ @Test
+ public void testCaseInsensitive() throws Exception {
+ assertColumn("select struct_element(MAP_VALUES(struct_element(S,
'DATA')[1])[1], 'B') from tbl",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ );
+ }
+
@Test
public void testMap() throws Exception {
assertColumn("select MAP_KEYS(struct_element(s, 'data')[0])[1] from
tbl",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
