This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 352e7f173d58ede195cf6cd63cea8ec174d715b9
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Fri Jul 7 17:10:50 2023 +0800
[fix](catalog) wrong required slot info causing BE crash (#21598)
For file scan node, this is a special field `requiredSlot`, this field is
set depends on the `isMaterialized` info of slot.
But `isMaterialized` info can be changed during the plan process, so we
must update the `requiredSlot`
in `finalize` phase of scan node, otherwise, it may causing BE crash due to
mismatching slot info.
---
.../java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 5 +++--
.../java/org/apache/doris/planner/external/FileQueryScanNode.java | 8 +++++++-
.../external_table_emr_p2/hive/test_external_catalog_hive.out | 3 +++
.../external_table_emr_p2/hive/test_external_catalog_hive.groovy | 2 ++
4 files changed, 15 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 0b8e9d5bc2..1b5ed54ed1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -389,7 +389,8 @@ public class HiveMetaStoreCache {
throw e;
}
}
- result.setPartitionValues(partitionValues);
+ // Must copy the partitionValues to avoid concurrent modification of
key and value
+ result.setPartitionValues(Lists.newArrayList(partitionValues));
return result;
}
@@ -924,7 +925,7 @@ public class HiveMetaStoreCache {
return dummyKey.equals(((FileCacheKey) obj).dummyKey);
}
return location.equals(((FileCacheKey) obj).location)
- && partitionValues.equals(((FileCacheKey)
obj).partitionValues);
+ && Objects.equals(partitionValues, ((FileCacheKey)
obj).partitionValues);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
index 39727a6f04..d366cbe90e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -81,7 +81,7 @@ import java.util.Set;
/**
* FileQueryScanNode for querying the file access type of catalog, now only
support
- * hive,hudi, iceberg and TVF.
+ * hive, hudi, iceberg and TVF.
*/
public abstract class FileQueryScanNode extends FileScanNode {
private static final Logger LOG =
LogManager.getLogger(FileQueryScanNode.class);
@@ -163,6 +163,10 @@ public abstract class FileQueryScanNode extends
FileScanNode {
@Override
public void updateRequiredSlots(PlanTranslatorContext
planTranslatorContext,
Set<SlotId> requiredByProjectSlotIdSet) throws UserException {
+ updateRequiredSlots();
+ }
+
+ private void updateRequiredSlots() throws UserException {
params.unsetRequiredSlots();
for (SlotDescriptor slot : desc.getSlots()) {
if (!slot.isMaterialized()) {
@@ -196,6 +200,7 @@ public abstract class FileQueryScanNode extends
FileScanNode {
// Create scan range locations and the statistics.
protected void doFinalize() throws UserException {
createScanRangeLocations();
+ updateRequiredSlots();
}
private void setColumnPositionMapping()
@@ -415,3 +420,4 @@ public abstract class FileQueryScanNode extends
FileScanNode {
return Optional.empty();
}
}
+
diff --git
a/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
b/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
index b7a66d44d4..54dc2d9a40 100644
---
a/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
+++
b/regression-test/data/external_table_emr_p2/hive/test_external_catalog_hive.out
@@ -93,3 +93,6 @@ moccasin steel bisque cornsilk lace
-- !q25 --
Z6n2t4XA2n7CXTECJ,PE,iBbsCh0RE1Dd2A,z48
+-- !pr21598 --
+5
+
diff --git
a/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
b/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
index 7df45eab5d..df06c7e246 100644
---
a/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
+++
b/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
@@ -86,6 +86,8 @@ suite("test_external_catalog_hive", "p2") {
sql """ use tpch_1000_orc; """
q03()
+ // test #21598
+ qt_pr21598 """select count(*) from( (SELECT r_regionkey AS key1,
r_name AS name, pday AS pday FROM (SELECT r_regionkey, r_name,
replace(r_comment, ' ', 'aaaa') AS pday FROM
${catalog_name}.tpch_1000_parquet.region) t2))x;"""
// test remember last used database after switch / rename catalog
sql """switch ${catalog_name};"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
