This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 090091a92aa branch-3.0: [fix](maxcompute)Fix the NereidsException
caused by non-existent table columns when pushing down mc predicates. #55635
(#55744)
090091a92aa is described below
commit 090091a92aaefbf5322ab14b769f1795c4cd66bf
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Sep 9 10:05:29 2025 +0800
branch-3.0: [fix](maxcompute)Fix the NereidsException caused by
non-existent table columns when pushing down mc predicates. #55635 (#55744)
Cherry-picked from #55635
Co-authored-by: daidai <[email protected]>
---
.../maxcompute/source/MaxComputeScanNode.java | 17 +++++++++--------
.../maxcompute/test_external_catalog_maxcompute.out | Bin 4745 -> 6428 bytes
.../test_external_catalog_maxcompute.groovy | 9 +++++++++
3 files changed, 18 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
index 063aeea68eb..588b9968735 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java
@@ -18,7 +18,6 @@
package org.apache.doris.datasource.maxcompute.source;
import org.apache.doris.analysis.BinaryPredicate;
-import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.CompoundPredicate.Operator;
import org.apache.doris.analysis.DateLiteral;
@@ -301,7 +300,7 @@ public class MaxComputeScanNode extends FileQueryScanNode {
for (Expr dorisPredicate : conjuncts) {
try {
odpsPredicates.add(convertExprToOdpsPredicate(dorisPredicate));
- } catch (AnalysisException e) {
+ } catch (Exception e) {
Log.warn("Failed to convert predicate " +
dorisPredicate.toString() + "Reason: "
+ e.getMessage());
}
@@ -361,11 +360,13 @@ public class MaxComputeScanNode extends FileQueryScanNode
{
:
com.aliyun.odps.table.optimizer.predicate.InPredicate.Operator.NOT_IN;
String columnName = convertSlotRefToColumnName(expr.getChild(0));
+ if (!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
+ throw new AnalysisException("Column " + columnName + " not
found in table, can not push "
+ + "down predicate to MaxCompute " + table.getName());
+ }
com.aliyun.odps.OdpsType odpsType =
table.getColumnNameToOdpsColumn().get(columnName).getType();
StringBuilder stringBuilder = new StringBuilder();
-
-
stringBuilder.append(columnName);
stringBuilder.append(" ");
stringBuilder.append(odpsOp.getDescription());
@@ -419,6 +420,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
if (odpsOp != null) {
String columnName =
convertSlotRefToColumnName(expr.getChild(0));
+ if
(!table.getColumnNameToOdpsColumn().containsKey(columnName)) {
+ throw new AnalysisException("Column " + columnName + " not
found in table, can not push "
+ + "down predicate to MaxCompute " +
table.getName());
+ }
com.aliyun.odps.OdpsType odpsType =
table.getColumnNameToOdpsColumn().get(columnName).getType();
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(columnName);
@@ -454,10 +459,6 @@ public class MaxComputeScanNode extends FileQueryScanNode {
private String convertSlotRefToColumnName(Expr expr) throws
AnalysisException {
if (expr instanceof SlotRef) {
return ((SlotRef) expr).getColumnName();
- } else if (expr instanceof CastExpr) {
- if (expr.getChild(0) instanceof SlotRef) {
- return ((SlotRef) expr.getChild(0)).getColumnName();
- }
}
throw new AnalysisException("Do not support convert ["
diff --git
a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
index c388e4eddf6..af7b8dd3f12 100644
Binary files
a/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
and
b/regression-test/data/external_table_p2/maxcompute/test_external_catalog_maxcompute.out
differ
diff --git
a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
index 6843a586d86..827ed963873 100644
---
a/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
+++
b/regression-test/suites/external_table_p2/maxcompute/test_external_catalog_maxcompute.groovy
@@ -447,5 +447,14 @@ suite("test_external_catalog_maxcompute",
"p2,external,maxcompute,external_remot
order_qt_part2_q8 """ SELECT audit_flag FROM `mc_parts2` WHERE `ds` !=
'2027-01-09';"""
qt_part2_q9 """ desc mc_parts2 """
+
+ qt_where_1 """ WITH active_us_sites AS (SELECT web_site_sk,
web_site_id, web_name, web_open_date_sk,
web_close_date_sk, web_company_name, web_city,
+ web_state, web_country, CONCAT_WS(' ',
web_street_number, web_street_name, web_street_type, web_suite_number) AS
full_address, COUNT(1) OVER (PARTITION BY web_site_id) AS a FROM web_site )
+ SELECT web_site_sk, web_site_id, web_name,
web_company_name, full_address, CONCAT(web_city, ', ', web_state) AS
city_state,
+ web_country , a FROM active_us_sites where a>=1 ORDER BY
web_site_sk;"""
+
+ qt_where_2 """select * from web_site where web_mkt_id >
100000000000000 ORDER BY web_site_sk;"""
+ qt_where_3 """ select * from web_site where web_close_date_sk >
CURRENT_DATE() ORDER BY web_site_sk; """
+ qt_where_4 """ select * from web_site where web_rec_end_date <
CURRENT_DATE() ORDER BY web_site_sk; """
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
