This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new fc45da6c347 [improvement](hudi)Obtain partition information through
HMS's API #30962 #31008
fc45da6c347 is described below
commit fc45da6c347592a343b6fcf49a233f8f6a50a235
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Fri Feb 9 16:50:08 2024 +0800
[improvement](hudi)Obtain partition information through HMS's API #30962
#31008
bp #30962
Co-authored-by: wuwenchi <wuwenchi...@hotmail.com>
---
.../external/hudi/HudiCachedPartitionProcessor.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
index ba793ecf407..f18c133f393 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
@@ -20,6 +20,7 @@ package org.apache.doris.planner.external.hudi;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.Config;
import org.apache.doris.datasource.CacheException;
+import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.planner.external.TablePartitionValues;
import
org.apache.doris.planner.external.TablePartitionValues.TablePartitionKey;
@@ -31,6 +32,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.Option;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.List;
@@ -39,6 +42,7 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
public class HudiCachedPartitionProcessor extends HudiPartitionProcessor {
+ private static final Logger LOG =
LoggerFactory.getLogger(HudiCachedPartitionProcessor.class);
private final long catalogId;
private final Executor executor;
private final LoadingCache<TablePartitionKey, TablePartitionValues>
partitionCache;
@@ -137,7 +141,17 @@ public class HudiCachedPartitionProcessor extends
HudiPartitionProcessor {
if (lastTimestamp <= lastUpdateTimestamp) {
return partitionValues;
}
- List<String> partitionNames =
getAllPartitionNames(tableMetaClient);
+ HMSExternalCatalog catalog = (HMSExternalCatalog)
table.getCatalog();
+ List<String> partitionNames;
+ // When a Hudi table is synchronized to HMS, the partition
information is also synchronized,
+ // so even if the metastore is not enabled in the Hudi table
+ // (for example, if the Metastore is false for a Hudi
table created with Flink),
+ // we can still obtain the partition information through the
HMS API.
+ partitionNames =
catalog.getClient().listPartitionNames(table.getDbName(), table.getName());
+ if (partitionNames.size() == 0) {
+ LOG.warn("Failed to get partitions from hms api, switch it
from hudi api.");
+ partitionNames = getAllPartitionNames(tableMetaClient);
+ }
List<String> partitionColumnsList =
Arrays.asList(partitionColumns.get());
partitionValues.cleanPartitions();
partitionValues.addPartitions(partitionNames,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
