This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b68bb92c1ba [fix](maxcompute)Fix the NereidsException caused by
non-existent table columns when pushing down mc predicates. (#55635)
b68bb92c1ba is described below
commit b68bb92c1baf65cb3b0b9bb838266ff5fc6d125d
Author: daidai <[email protected]>
AuthorDate: Sun Sep 7 12:22:41 2025 +0800
[fix](maxcompute)Fix the NereidsException caused by non-existent table
columns when pushing down mc predicates. (#55635)
### What problem does this PR solve?
Problem Summary:
fix: maxcompute catalog push down predicates when column not exist in
table.
```
Caused by: java.lang.NullPointerException: Cannot invoke
"com.aliyun.odps.Column.getType()" because the return value of
"java.util.Map.get(Object)" is null
at
org.apache.doris.datasource.maxcompute.source.MaxComputeScanNode.convertExprToOdpsPredicate(MaxComputeScanNode.java:422)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.datasource.maxcompute.source.MaxComputeScanNode.convertPredicate(MaxComputeScanNode.java:303)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.datasource.FileQueryScanNode.doFinalize(FileQueryScanNode.java:221)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.datasource.FileQueryScanNode.finalizeForNereids(FileQueryScanNode.java:213)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator.translatePlan(PhysicalPlanTranslator.java:285)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.splitFragments(NereidsPlanner.java:441)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.distribute(NereidsPlanner.java:527)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.lambda$plan$0(NereidsPlanner.java:149)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.planWithLock(NereidsPlanner.java:225)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.NereidsPlanner.plan(NereidsPlanner.java:145)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand.run(InsertOverwriteTableCommand.java:133)
~[doris-fe.jar:1.2-SNAPSHOT]
at
org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:767)
~[doris-fe.jar:1.2-SNAPSHOT]
... 12 more
```
---
.../maxcompute/source/MaxComputeScanNode.java | 17 +++++++++--------
.../maxcompute/test_external_catalog_maxcompute.out | Bin 4745 -> 6428 bytes
.../test_external_catalog_maxcompute.groovy | 9 +++++++++
3 files changed, 18 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index 4931659b95e..8732c783ca7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.maxcompute.source;
import org.apache.doris.analysis.BinaryPredicate;
-import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.CompoundPredicate.Operator;
import org.apache.doris.analysis.DateLiteral;
@@ -295,7 +294,7 @@ public class MaxComputeScanNode extends FileQueryScanNode {
for (Expr dorisPredicate : conjuncts) {
try {
odpsPredicates.add(convertExprToOdpsPredicate(dorisPredicate));
- } catch (AnalysisException e) {
+ } catch (Exception e) {
Log.warn("Failed to convert predicate " +
dorisPredicate.toString() + "Reason: "
+ e.getMessage());
}
@@ -355,11 +354,13 @@ public class MaxComputeScanNode extends FileQueryScanNode
{
:
com.aliyun.odps.table.optimizer.predicate.InPredicate.Operator.NOT_IN;
String columnName = convertSlotRefToColumnName(expr.getChild(0));
+ if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
+ throw new AnalysisException("Column " + columnName + " not
found in table, can not push "
+ + "down predicate to MaxCompute " + table.getName());
+ }
com.aliyun.odps.OdpsType odpsType =
table.getColumnNameToOdpsColumn().get(columnName).getType();
StringBuilder stringBuilder = new StringBuilder();
-
-
stringBuilder.append(columnName);
stringBuilder.append(" ");
stringBuilder.append(odpsOp.getDescription());
@@ -413,6 +414,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
if (odpsOp != null) {
String columnName =
convertSlotRefToColumnName(expr.getChild(0));
+ if
(!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
+ throw new AnalysisException("Column " + columnName + " not
found in table, can not push "
+ + "down predicate to MaxCompute " +
table.getName());
+ }
com.aliyun.odps.OdpsType odpsType =
table.getColumnNameToOdpsColumn().get(columnName).getType();
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(columnName);
@@ -448,10 +453,6 @@ public class MaxComputeScanNode extends FileQueryScanNode {
private String convertSlotRefToColumnName(Expr expr) throws
AnalysisException {
if (expr instanceof SlotRef) {
return ((SlotRef) expr).getColumnName();
- } else if (expr instanceof CastExpr) {
- if (expr.getChild(0) instanceof SlotRef) {
- return ((SlotRef) expr.getChild(0)).getColumnName();
- }
}
throw new AnalysisException("Do not support convert ["
diff --git
a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
index c388e4eddf6..af7b8dd3f12 100644
Binary files
a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
and
b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
differ
diff --git
a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
index 6843a586d86..827ed963873 100644
---
a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
+++
b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
@@ -447,5 +447,14 @@ suite("test_external_catalog_maxcompute",
"p2,external,maxcompute,external_remot
order_qt_part2_q8 """ SELECT audit_flag FROM `mc_parts2` WHERE `ds` !=
'2027-01-09';"""
qt_part2_q9 """ desc mc_parts2 """
+
+ qt_where_1 """ WITH active_us_sites AS (SELECT web_site_sk,
web_site_id, web_name, web_open_date_sk,
web_close_date_sk, web_company_name, web_city,
+ web_state, web_country, CONCAT_WS(' ',
web_street_number, web_street_name, web_street_type, web_suite_number) AS
full_address, COUNT(1) OVER (PARTITION BY web_site_id) AS a FROM web_site )
+ SELECT web_site_sk, web_site_id, web_name,
web_company_name, full_address, CONCAT(web_city, ', ', web_state) AS
city_state,
+ web_country , a FROM active_us_sites where a>=1 ORDER BY
web_site_sk;"""
+
+ qt_where_2 """select * from web_site where web_mkt_id >
100000000000000 ORDER BY web_site_sk;"""
+ qt_where_3 """ select * from web_site where web_close_date_sk >
CURRENT_DATE() ORDER BY web_site_sk; """
+ qt_where_4 """ select * from web_site where web_rec_end_date <
CURRENT_DATE() ORDER BY web_site_sk; """
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
