This is an automated email from the ASF dual-hosted git repository.

kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.0 by this push:
     new 646958dfa89 [improvement](hudi)Obtain partition information through 
HMS's API for 2.0 (#31087)
646958dfa89 is described below

commit 646958dfa89d38572391764a4382f2d7eb576183
Author: wuwenchi <[email protected]>
AuthorDate: Mon Feb 19 21:21:21 2024 +0800

    [improvement](hudi)Obtain partition information through HMS's API for 2.0 
(#31087)
---
 docs/en/docs/lakehouse/multi-catalog/hudi.md       |  6 ++++++
 docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md    |  6 ++++++
 .../doris/catalog/external/HMSExternalTable.java   | 10 +++++++++
 .../hudi/HudiCachedPartitionProcessor.java         | 25 +++++++++++++---------
 .../doris/planner/external/hudi/HudiScanNode.java  |  8 +++++--
 5 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/docs/en/docs/lakehouse/multi-catalog/hudi.md 
b/docs/en/docs/lakehouse/multi-catalog/hudi.md
index 52892db2df2..a52c2370ced 100644
--- a/docs/en/docs/lakehouse/multi-catalog/hudi.md
+++ b/docs/en/docs/lakehouse/multi-catalog/hudi.md
@@ -55,6 +55,12 @@ CREATE CATALOG hudi PROPERTIES (
 );
 ```
 
+Optional configuration parameters:
+
+|name|description|default|
+|---|---|---|
+|use_hive_sync_partition|Use hms synchronized partition data|false|
+
 ## Column Type Mapping
 
 Same as that in Hive Catalogs. See the relevant section in [Hive](./hive.md).
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md 
b/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md
index b619283cacf..38bb26d3bc7 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md
@@ -55,6 +55,12 @@ CREATE CATALOG hudi PROPERTIES (
 );
 ```
 
+可选配置参数:
+
+|参数名|说明|默认值|
+|---|---|---|
+|use_hive_sync_partition|使用hms已同步的分区数据|false|
+
 ## 列类型映射
 
 和 Hive Catalog 一致,可参阅 [Hive Catalog](./hive.md) 中 **列类型映射** 一节。
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index bbadac6ecbf..961dc6240d6 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -89,6 +89,8 @@ public class HMSExternalTable extends ExternalTable {
 
     private static final String NUM_ROWS = "numRows";
 
+    private static final String USE_HIVE_SYNC_PARTITION = 
"use_hive_sync_partition";
+
     static {
         SUPPORTED_HIVE_FILE_FORMATS = Sets.newHashSet();
         
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");
@@ -194,6 +196,14 @@ public class HMSExternalTable extends ExternalTable {
         return inputFormatName != null && 
SUPPORTED_HUDI_FILE_FORMATS.contains(inputFormatName);
     }
 
+    /**
+     * Some data lakes (such as Hudi) will synchronize their partition 
information to HMS,
+     * then we can quickly obtain the partition information of the table from 
HMS.
+     */
+    public boolean useHiveSyncPartition() {
+        return 
Boolean.parseBoolean(catalog.getProperties().getOrDefault(USE_HIVE_SYNC_PARTITION,
 "false"));
+    }
+
     public boolean isHoodieCowTable() {
         if (remoteTable.getSd() == null) {
             return false;
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
index f18c133f393..065dd04a39e 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
@@ -85,7 +85,7 @@ public class HudiCachedPartitionProcessor extends 
HudiPartitionProcessor {
     }
 
     public TablePartitionValues getSnapshotPartitionValues(HMSExternalTable 
table,
-            HoodieTableMetaClient tableMetaClient, String timestamp) {
+            HoodieTableMetaClient tableMetaClient, String timestamp, boolean 
useHiveSyncPartition) {
         Preconditions.checkState(catalogId == table.getCatalog().getId());
         Option<String[]> partitionColumns = 
tableMetaClient.getTableConfig().getPartitionFields();
         if (!partitionColumns.isPresent()) {
@@ -98,7 +98,7 @@ public class HudiCachedPartitionProcessor extends 
HudiPartitionProcessor {
         }
         long lastTimestamp = Long.parseLong(lastInstant.get().getTimestamp());
         if (Long.parseLong(timestamp) == lastTimestamp) {
-            return getPartitionValues(table, tableMetaClient);
+            return getPartitionValues(table, tableMetaClient, 
useHiveSyncPartition);
         }
         List<String> partitionNameAndValues = 
getPartitionNamesBeforeOrEquals(timeline, timestamp);
         List<String> partitionNames = Arrays.asList(partitionColumns.get());
@@ -109,7 +109,8 @@ public class HudiCachedPartitionProcessor extends 
HudiPartitionProcessor {
         return partitionValues;
     }
 
-    public TablePartitionValues getPartitionValues(HMSExternalTable table, 
HoodieTableMetaClient tableMetaClient)
+    public TablePartitionValues getPartitionValues(HMSExternalTable table, 
HoodieTableMetaClient tableMetaClient,
+                                                   boolean 
useHiveSyncPartition)
             throws CacheException {
         Preconditions.checkState(catalogId == table.getCatalog().getId());
         Option<String[]> partitionColumns = 
tableMetaClient.getTableConfig().getPartitionFields();
@@ -143,13 +144,17 @@ public class HudiCachedPartitionProcessor extends 
HudiPartitionProcessor {
                 }
                 HMSExternalCatalog catalog = (HMSExternalCatalog) 
table.getCatalog();
                 List<String> partitionNames;
-                // When a Hudi table is synchronized to HMS, the partition 
information is also synchronized,
-                // so even if the metastore is not enabled in the Hudi table
-                //     (for example, if the Metastore is false for a Hudi 
table created with Flink),
-                // we can still obtain the partition information through the 
HMS API.
-                partitionNames = 
catalog.getClient().listPartitionNames(table.getDbName(), table.getName());
-                if (partitionNames.size() == 0) {
-                    LOG.warn("Failed to get partitions from hms api, switch it 
from hudi api.");
+                if (useHiveSyncPartition) {
+                    // When a Hudi table is synchronized to HMS, the partition 
information is also synchronized,
+                    // so even if the metastore is not enabled in the Hudi 
table
+                    //     (for example, if the Metastore is false for a Hudi 
table created with Flink),
+                    // we can still obtain the partition information through 
the HMS API.
+                    partitionNames = 
catalog.getClient().listPartitionNames(table.getDbName(), table.getName());
+                    if (partitionNames.size() == 0) {
+                        LOG.warn("Failed to get partitions from hms api, 
switch it from hudi api.");
+                        partitionNames = getAllPartitionNames(tableMetaClient);
+                    }
+                } else {
                     partitionNames = getAllPartitionNames(tableMetaClient);
                 }
                 List<String> partitionColumnsList = 
Arrays.asList(partitionColumns.get());
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
index 9d601e71daa..803dadae03d 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
@@ -82,6 +82,8 @@ public class HudiScanNode extends HiveScanNode {
 
     private final AtomicLong noLogsSplitNum = new AtomicLong(0);
 
+    private final boolean useHiveSyncPartition;
+
     /**
      * External file scan node for Query Hudi table
      * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check 
column priv
@@ -97,6 +99,7 @@ public class HudiScanNode extends HiveScanNode {
         } else {
             LOG.debug("Hudi table {} is a mor table, and will use JNI to read 
data in BE", hmsTable.getName());
         }
+        useHiveSyncPartition = hmsTable.useHiveSyncPartition();
     }
 
     @Override
@@ -166,9 +169,10 @@ public class HudiScanNode extends HiveScanNode {
                     
.getExtMetaCacheMgr().getHudiPartitionProcess(hmsTable.getCatalog());
             TablePartitionValues partitionValues;
             if (snapshotTimestamp.isPresent()) {
-                partitionValues = 
processor.getSnapshotPartitionValues(hmsTable, metaClient, 
snapshotTimestamp.get());
+                partitionValues = processor.getSnapshotPartitionValues(
+                    hmsTable, metaClient, snapshotTimestamp.get(), 
useHiveSyncPartition);
             } else {
-                partitionValues = processor.getPartitionValues(hmsTable, 
metaClient);
+                partitionValues = processor.getPartitionValues(hmsTable, 
metaClient, useHiveSyncPartition);
             }
             if (partitionValues != null) {
                 // 2. prune partitions by expr


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to