This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fc3a7ab7c75 [fix](meta path) Keep whole-value TRY_CAST semantics under
nested column pruning (#67718)
fc3a7ab7c75 is described below
commit fc3a7ab7c7561bab88513d4e42a1e0f7f26731c4
Author: minghong <[email protected]>
AuthorDate: Wed Sep 23 11:13:59 2026 +0800
[fix](meta path) Keep whole-value TRY_CAST semantics under nested column
pruning (#67718)
### What problem does this PR solve?
Issue Number: N/A
Problem Summary:
Nested column pruning rewrites a struct-field access over `TRY_CAST`
into a plain `CAST` and narrows both the read and the conversion to the
accessed field, dropping the whole-value semantics of TRY_CAST.
Reproduction (single FE/BE, `enable_strict_cast=true`):
```sql
CREATE TABLE t (id INT, s STRUCT<a:STRING,b:STRING> NULL)
DISTRIBUTED BY HASH(id) BUCKETS 1
PROPERTIES ("replication_num"="1");
INSERT INTO t VALUES
(1,named_struct('a','bad','b','2')),
(2,named_struct('a','10','b','bad')),(3,NULL);
-- with nested column pruning enabled (default)
SELECT id, TRY_CAST(s AS STRUCT<a:INT,b:INT>)['a'] AS a_i FROM t;
```
- id=2: `b='bad'` makes the whole-struct TRY_CAST fail and should yield
NULL, but pruning drops field `b`, the conversion "succeeds" on
`a='10'`, and the query returns 10.
- id=1: the rebuilt plain CAST (instead of TRY_CAST) throws
`[INVALID_ARGUMENT] parse number fail, string: 'bad'` under
`enable_strict_cast=true` instead of returning NULL.
- Root cause: `AccessPathExpressionCollector.visitCast` handles
`TryCast` (a `Cast` subclass) like a plain cast and translates the
narrowed nested access path through it, so
`SlotTypeReplacer.rewriteCast` later rebuilds the expression as `new
Cast(...)` with a pruned target type.
Fix: do not translate a narrowed nested access path through a
`TRY_CAST`; the collector then falls back to reading the whole child
value, so the struct keeps all fields and the expression keeps its
TRY_CAST identity and full target type. Field-by-field pruning of plain
`CAST` over nested types is unchanged.
### Release note
None
### Check List (For Author)
- Test:
- [x] Unit Test:
`PruneNestedColumnTest#testTryCastNotPrunedThroughFieldAccess` (whole
class: 62 tests pass)
- [x] Manual test (detailed steps): reproduced the SQL above end-to-end
on a local FE+BE cluster built from this change. With
`enable_prune_nested_column` on and off all three rows return NULL (no
10, no exception under `enable_strict_cast=true`); `EXPLAIN VERBOSE`
keeps `element_at(TRY_CAST(s AS struct<a:int,b:int>), 'a')` with `all
access paths: [s]`, while plain `CAST` over the same struct is still
pruned to `struct<a:int>` with `[s.a]`.
- Behavior changed:
- [x] Yes: enabling/disabling nested column pruning no longer changes
TRY_CAST results or error semantics.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../rewrite/AccessPathExpressionCollector.java | 16 +++
.../nereids/rules/rewrite/SlotTypeReplacer.java | 10 +-
.../rules/rewrite/PruneNestedColumnTest.java | 64 ++++++++++++
.../column_pruning/try_cast_nested_pruning.out | 25 +++++
.../column_pruning/try_cast_nested_pruning.groovy | 107 +++++++++++++++++++++
5 files changed, 221 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index cfd22ee5eea..b3440c5a8bd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.TryCast;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
@@ -344,6 +345,21 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
);
}
+ @Override
+ public Void visitTryCast(TryCast tryCast, CollectorContext context) {
+ // TRY_CAST semantics cover the WHOLE value: for a composite type any
element
+ // conversion failure makes the entire cast NULL. Narrowing the
read/type down to
+ // only the fields an outer expression accesses would drop the
conversion attempts
+ // of the other fields and silently change the result (e.g.
element_at(try_cast(s as
+ // struct<a:int,b:int>), 'a') must still yield NULL when only field b
is unparsable,
+ // even though field a alone converts fine). Plain Cast over nested
types is pruned
+ // field-by-field on purpose, a TryCast never is: read the whole child
value and keep
+ // the cast identity and target type intact.
+ return tryCast.child(0).accept(this,
+ new CollectorContext(context.statementContext,
context.bottomFilter)
+ );
+ }
+
// array element at
@Override
public Void visitElementAt(ElementAt elementAt, CollectorContext context) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
index ca8a3b5ba4c..102a87dfe37 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -624,7 +624,15 @@ public class SlotTypeReplacer extends
DefaultPlanRewriter<Void> {
newType = prunedTree.pruneCastType(originTree, castTree);
}
- return new Cast(newChild, newType);
+ // Rebuild through withChildren/withTargetType so the concrete cast
kind survives. A
+ // TRY_CAST that reaches here has kept its whole immediate child value
(see
+ // AccessPathExpressionCollector.visitTryCast, which never translates
an access path
+ // through the cast), but that child can still be rebuilt: for
+ // element_at(try_cast(element_at(wrapper, 'f') as struct<...>), 'g')
the collector
+ // records [wrapper, f], so pruning may drop a sibling of wrapper and
replace the
+ // inner element_at. Constructing a Cast here would silently downgrade
the whole-value
+ // TRY_CAST into a strict CAST, turning its NULL result into a cast
error.
+ return
cast.withChildren(ImmutableList.of(newChild)).withTargetType(newType);
}
private List<ColumnAccessPath> replaceAccessPathToFieldId(
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 17d59219a34..b83ad8af87f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -29,6 +29,7 @@ import
org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.Coll
import
org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -156,6 +157,17 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
+ " s struct<`outer`: struct<`a`: string, `inner_f`:
string>>\n"
+ ") properties ('replication_num'='1')");
+ // Struct holding a nested struct plus a sibling field. The sibling
makes the column
+ // prunable, while element_at(wrapper, 'payload') gives the TRY_CAST a
derived child that
+ // is rebuilt (not just re-typed) when the sibling is pruned away.
+ createTable("create table try_cast_nested_tbl(\n"
+ + " id int,\n"
+ + " wrapper struct<\n"
+ + " payload: struct<good: string, bad: string>,\n"
+ + " unused: int\n"
+ + " >\n"
+ + ") properties ('replication_num'='1')");
+
// Tables for outer-join nullability test: verifying that synthetic
nullability
// from outer join does NOT cause META NULL paths on physically NOT
NULL columns.
createTable("create table driving_tbl(\n"
@@ -582,6 +594,38 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
);
}
+ @Test
+ public void testTryCastNotPrunedThroughFieldAccess() throws Exception {
+ // TRY_CAST of a composite type keeps whole-value semantics: the
conversion fails
+ // (and TRY_CAST returns NULL) if ANY field conversion fails. An outer
field access
+ // element_at(try_cast(s as struct<k:int,...>), 'k') must therefore
not narrow the
+ // underlying read to the accessed field only — the whole struct is
read and the
+ // cast target/identity are unchanged.
+ assertColumn(
+ "select element_at(try_cast(s as
struct<k:int,l:array<map<int,struct<a:int,b:double>>>>), 'k')"
+ + " from tbl",
+
"struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of());
+ }
+
+ @Test
+ public void testTryCastKeptWhenDerivedChildIsRebuilt() throws Exception {
+ // Unlike testTryCastNotPrunedThroughFieldAccess, the TRY_CAST child
here is not the slot
+ // itself but a derived element_at. The whole-value context opened by
visitTryCast records
+ // [wrapper, payload], which only pins the payload value: the sibling
field wrapper.unused
+ // is still pruned away, the wrapper slot type changes, and
SlotTypeReplacer therefore
+ // rebuilds the inner element_at. Rebuilding the cast over that
changed child must not
+ // downgrade the whole-value TRY_CAST into a strict CAST.
+ String sql = "select element_at(try_cast(element_at(wrapper,
'payload') as struct<good:int,bad:int>),"
+ + " 'good') from try_cast_nested_tbl";
+ assertColumn(sql,
+ "struct<payload:struct<good:text,bad:text>>",
+ ImmutableList.of(path("wrapper", "payload")),
+ ImmutableList.of());
+ assertTryCastPreserved(sql, "STRUCT<good:INT,bad:INT>");
+ }
+
@Test
public void testPruneArrayLambda() throws Exception {
// map_values(element_at(s, 'data').*)[0].a
@@ -1512,6 +1556,26 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
assertColumns(sql, expectType == null ? null :
ImmutableList.of(Triple.of(expectType, expectAllAccessPaths,
expectPredicateAccessPaths)));
}
+ /** Assert the physical plan contains a TRY_CAST of the given target type
and no plain CAST of it. */
+ private void assertTryCastPreserved(String sql, String expectTargetType)
throws Exception {
+ PhysicalPlan physicalPlan = collectComplexSlots(sql).first;
+ List<String> casts = new ArrayList<>();
+ physicalPlan.foreachUp(plan -> {
+ for (Expression expression : ((PhysicalPlan)
plan).getExpressions()) {
+ expression.foreach(e -> {
+ if (e instanceof Cast) {
+ Cast cast = (Cast) e;
+ casts.add(cast.getClass().getSimpleName() + " -> " +
cast.getDataType().toSql());
+ }
+ });
+ }
+ });
+ Assertions.assertTrue(casts.contains("TryCast -> " + expectTargetType),
+ "expected TRY_CAST -> " + expectTargetType + " in the physical
plan, but found " + casts);
+ Assertions.assertFalse(casts.contains("Cast -> " + expectTargetType),
+ "TRY_CAST was downgraded to a plain CAST: " + casts);
+ }
+
private void assertAllAccessPathsContain(String sql,
List<ColumnAccessPath> expectContainAllAccessPaths,
List<ColumnAccessPath> expectNotContainAllAccessPaths) throws
Exception {
Pair<PhysicalPlan, List<SlotDescriptor>> result =
collectComplexSlots(sql);
diff --git
a/regression-test/data/nereids_rules_p0/column_pruning/try_cast_nested_pruning.out
b/regression-test/data/nereids_rules_p0/column_pruning/try_cast_nested_pruning.out
new file mode 100644
index 00000000000..c7f5ea4e11f
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/column_pruning/try_cast_nested_pruning.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !direct_slot_try_cast --
+1 10
+2 \N
+3 \N
+4 \N
+
+-- !derived_child_try_cast --
+1 10
+2 \N
+3 \N
+4 \N
+
+-- !direct_slot_try_cast_pruning_off --
+1 10
+2 \N
+3 \N
+4 \N
+
+-- !derived_child_try_cast_pruning_off --
+1 10
+2 \N
+3 \N
+4 \N
+
diff --git
a/regression-test/suites/nereids_rules_p0/column_pruning/try_cast_nested_pruning.groovy
b/regression-test/suites/nereids_rules_p0/column_pruning/try_cast_nested_pruning.groovy
new file mode 100644
index 00000000000..883b780d34c
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/column_pruning/try_cast_nested_pruning.groovy
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression tests for TRY_CAST under nested column pruning.
+//
+// TRY_CAST of a composite type keeps whole-value semantics: the conversion
fails (and the
+// result is NULL) if ANY field conversion fails. Nested column pruning may
narrow which
+// columns a scan reads, but it must never narrow the value a TRY_CAST
converts, and it must
+// never downgrade a TRY_CAST into a strict CAST while rebuilding the
expressions whose slots
+// were narrowed.
+//
+// Three shapes are covered:
+// 1. the TRY_CAST child is the slot itself: the whole struct stays read
([s]);
+// 2. the TRY_CAST child is a derived struct access: the container's sibling
field is still
+// pruned ([wrapper.payload]), so the derived element_at is rebuilt and
the TRY_CAST must
+// survive that rebuild. Without the fix the rebuilt expression becomes a
plain CAST and
+// raises "parse number fail" under enable_strict_cast instead of
returning NULL;
+// 3. plain CAST over a nested type: still pruned field by field ([s.a]),
unchanged.
+
+suite("try_cast_nested_pruning") {
+ sql "set enable_prune_nested_column = true"
+ sql "set enable_strict_cast = true"
+ sql "DROP TABLE IF EXISTS try_cast_pruning_tbl"
+ sql """
+ CREATE TABLE try_cast_pruning_tbl (
+ id INT,
+ s STRUCT<a: VARCHAR(10), b: VARCHAR(10)>,
+ wrapper STRUCT<payload: STRUCT<good: VARCHAR(10), bad:
VARCHAR(10)>, unused: INT>
+ ) ENGINE = OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ """
+
+ sql """
+ INSERT INTO try_cast_pruning_tbl VALUES
+ (1, named_struct('a', '10', 'b', '20'),
+ named_struct('payload', named_struct('good', '10', 'bad',
'20'), 'unused', 1)),
+ (2, named_struct('a', '10', 'b', 'bad'),
+ named_struct('payload', named_struct('good', '10', 'bad',
'bad'), 'unused', 2)),
+ (3, NULL, NULL),
+ (4, named_struct('a', 'bad', 'b', '20'),
+ named_struct('payload', named_struct('good', 'bad', 'bad',
'20'), 'unused', 4))
+ """
+
+ // ── 1. TRY_CAST over the slot itself
────────────────────────────────────────
+ // The whole struct is read: narrowing to s.a would drop the failing b
conversion and
+ // silently turn the NULL of row 2 into 10.
+ explain {
+ sql "select element_at(try_cast(s as struct<a:int,b:int>), 'a') from
try_cast_pruning_tbl"
+ contains "all access paths: [s]"
+ }
+
+ order_qt_direct_slot_try_cast """
+ SELECT id, element_at(TRY_CAST(s AS STRUCT<a:INT,b:INT>), 'a') AS a_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+
+ // ── 2. TRY_CAST over a derived struct access
────────────────────────────────
+ // The whole-value context only pins payload, so the sibling field
wrapper.unused is still
+ // pruned away and the inner element_at is rebuilt on the narrowed wrapper
slot. The cast
+ // must keep both its TRY_CAST identity and its full target type across
that rebuild.
+ explain {
+ sql """
+ select element_at(try_cast(element_at(wrapper, 'payload') as
struct<good:int,bad:int>), 'good')
+ from try_cast_pruning_tbl
+ """
+ contains "all access paths: [wrapper.payload]"
+ }
+
+ order_qt_derived_child_try_cast """
+ SELECT id, element_at(TRY_CAST(element_at(wrapper, 'payload') AS
STRUCT<good:INT,bad:INT>), 'good') AS good_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+
+ // ── 3. Plain CAST over a nested type is still pruned field by field
─────────
+ explain {
+ sql "select element_at(cast(s as struct<a:int,b:int>), 'a') from
try_cast_pruning_tbl"
+ contains "all access paths: [s.a]"
+ }
+
+ // ── 4. Toggling pruning must not change any result
──────────────────────────
+ sql "set enable_prune_nested_column = false"
+ order_qt_direct_slot_try_cast_pruning_off """
+ SELECT id, element_at(TRY_CAST(s AS STRUCT<a:INT,b:INT>), 'a') AS a_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+ order_qt_derived_child_try_cast_pruning_off """
+ SELECT id, element_at(TRY_CAST(element_at(wrapper, 'payload') AS
STRUCT<good:INT,bad:INT>), 'good') AS good_i
+ FROM try_cast_pruning_tbl ORDER BY id
+ """
+ sql "set enable_prune_nested_column = true"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
