This is an automated email from the ASF dual-hosted git repository.
caiconghui pushed a commit to branch export
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1d2c4b5342e9c3c71454183a951d613d6d15701b
Author: caiconghui1 <caicongh...@jd.com>
AuthorDate: Tue May 21 09:06:42 2024 +0800
[enhancement](export) filter empty partition before export table to remote
storage
---
.../main/java/org/apache/doris/load/ExportJob.java | 50 ++++++++++++++--------
1 file changed, 33 insertions(+), 17 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index f4cc1b7c997..2988a953ffb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -407,6 +407,13 @@ public class ExportJob implements Writable {
ExportTaskExecutor executor = new ExportTaskExecutor(selectStmts,
this);
jobExecutorList.add(executor);
}
+
+ // add empty task to make export job could be finished finally if
jobExecutorList is empty
+ // which means that export table without data
+ if (jobExecutorList.isEmpty()) {
+ ExportTaskExecutor executor = new
ExportTaskExecutor(Lists.newArrayList(), this);
+ jobExecutorList.add(executor);
+ }
}
/**
@@ -511,15 +518,23 @@ public class ExportJob implements Writable {
// get partitions
// user specifies partitions, already checked in ExportCommand
if (!this.partitionNames.isEmpty()) {
- this.partitionNames.forEach(partitionName ->
partitions.add(table.getPartition(partitionName)));
+ this.partitionNames.forEach(partitionName -> {
+ Partition partition = table.getPartition(partitionName);
+ if (partition.hasData()) {
+ partitions.add(partition);
+ }
+ });
} else {
- if (table.getPartitions().size() >
Config.maximum_number_of_export_partitions) {
- throw new UserException("The partitions number of this
export job is larger than the maximum number"
- + " of partitions allowed by a export job");
- }
- partitions.addAll(table.getPartitions());
+ table.getPartitions().forEach(partition -> {
+ if (partition.hasData()) {
+ partitions.add(partition);
+ }
+ });
+ }
+ if (partitions.size() >
Config.maximum_number_of_export_partitions) {
+ throw new UserException("The partitions number of this export
job is larger than the maximum number"
+ + " of partitions allowed by a export job");
}
-
// get tablets
for (Partition partition : partitions) {
// Partition data consistency is not need to verify partition
version.
@@ -589,8 +604,7 @@ public class ExportJob implements Writable {
List<Long> tabletsList = new
ArrayList<>(flatTabletIdList.subList(start, start + tabletsNum));
List<List<Long>> tablets = new ArrayList<>();
for (int i = 0; i < tabletsList.size(); i +=
MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT) {
- int end = i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT <
tabletsList.size()
- ? i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT :
tabletsList.size();
+ int end = Math.min(i + MAXIMUM_TABLETS_OF_OUTFILE_IN_EXPORT,
tabletsList.size());
tablets.add(new ArrayList<>(tabletsList.subList(i, end)));
}
@@ -708,15 +722,17 @@ public class ExportJob implements Writable {
throw new JobException("Job [{}] has been cancelled, can not
finish this task: {}", id, taskId);
}
- allOutfileInfo.add(outfileInfoList);
- ++finishedTaskCount;
+ if (taskId != -1L && outfileInfoList != null) {
+ allOutfileInfo.add(outfileInfoList);
+ ++finishedTaskCount;
- // calculate progress
- int tmpProgress = finishedTaskCount * 100 / jobExecutorList.size();
- if (finishedTaskCount * 100 / jobExecutorList.size() >= 100) {
- progress = 99;
- } else {
- progress = tmpProgress;
+ // calculate progress
+ int tmpProgress = finishedTaskCount * 100 / jobExecutorList.size();
+ if (finishedTaskCount * 100 / jobExecutorList.size() >= 100) {
+ progress = 99;
+ } else {
+ progress = tmpProgress;
+ }
}
// if all task finished
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
