This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch export
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a9197fa16c457f19dfdbecc39446283df4f42a13
Author: caiconghui1 <caicongh...@jd.com>
AuthorDate: Tue May 21 09:06:42 2024 +0800
[enhancement](export) filter empty partition before export table to remote
storage
---
.../main/java/org/apache/doris/load/ExportJob.java | 27 ++++++++++++++--------
1 file changed, 18 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index f4cc1b7c997..f7140c086d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -511,15 +511,25 @@ public class ExportJob implements Writable {
// get partitions
// user specifies partitions, already checked in ExportCommand
if (!this.partitionNames.isEmpty()) {
- this.partitionNames.forEach(partitionName ->
partitions.add(table.getPartition(partitionName)));
+ this.partitionNames.forEach(partitionName -> {
+ Partition partition = table.getPartition(partitionName);
+ if (partition.hasData()) {
+ partitions.add(table.getPartition(partitionName));
+ }
+ });
} else {
- if (table.getPartitions().size() >
Config.maximum_number_of_export_partitions) {
- throw new UserException("The partitions number of this
export job is larger than the maximum number"
- + " of partitions allowed by a export job");
- }
- partitions.addAll(table.getPartitions());
+ table.getPartitions().forEach(
+ partition -> {
+ if (partition.hasData()) {
+ partitions.add(partition);
+ }
+ }
+ );
+ }
+ if (partitions.size() >
Config.maximum_number_of_export_partitions) {
+ throw new UserException("The partitions number of this export
job is larger than the maximum number"
+ + " of partitions allowed by a export job");
}
-
// get tablets
for (Partition partition : partitions) {
// Partition data consistency is not need to verify partition
version.
@@ -589,8 +599,7 @@ public class ExportJob implements Writable {
List<Long> tabletsList = new
ArrayList<>(flatTabletIdList.subList(start, start + tabletsNum));
List<List<Long>> tablets = new ArrayList<>();
for (int i = 0; i < tabletsList.size(); i +=
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT) {
- int end = i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT <
tabletsList.size()
- ? i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT :
tabletsList.size();
+ int end = Math.min(i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT,
tabletsList.size());
tablets.add(new ArrayList<>(tabletsList.subList(i, end)));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
