[doris] branch master updated: [fix](spark load)Temp partition with spark load (#14648)

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new facb7cf4e2 [fix](spark load)Temp partition with spark load (#14648)
facb7cf4e2 is described below

commit facb7cf4e2629bb3eb4463ea5b1fcf280e8bed78
Author: xiaoDjun <36905671+xiaod...@users.noreply.github.com>
AuthorDate: Tue Nov 29 15:21:44 2022 +0800

    [fix](spark load)Temp partition with spark load (#14648)
    
    * [fix](spark load)losing temporary partition item entry
    
    * [fix](spark load)Temp partition with spark load
---
 .../java/org/apache/doris/catalog/RangePartitionInfo.java  | 14 ++++++++++++++
 .../org/apache/doris/load/loadv2/SparkLoadPendingTask.java |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
index 678763b02b..14508c077a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
@@ -25,6 +25,7 @@ import org.apache.doris.common.util.RangeUtils;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Range;
 
 import java.io.DataInput;
@@ -77,6 +78,19 @@ public class RangePartitionInfo extends PartitionInfo {
         return itemEntryList;
     }
 
+    public List<Map.Entry<Long, PartitionItem>> 
getAllPartitionItemEntryList(boolean isSorted) {
+        Map<Long, PartitionItem> tmpMap = Maps.newHashMap();
+
+        tmpMap.putAll(idToItem);
+        tmpMap.putAll(idToTempItem);
+
+        List<Map.Entry<Long, PartitionItem>> itemEntryList = 
Lists.newArrayList(tmpMap.entrySet());
+        if (isSorted) {
+            Collections.sort(itemEntryList, 
RangeUtils.RANGE_MAP_ENTRY_COMPARATOR);
+        }
+        return itemEntryList;
+    }
+
     // create a new range and check it.
     private Range<PartitionKey> createAndCheckNewRange(PartitionKeyDesc 
partKeyDesc, boolean isTemp)
             throws AnalysisException, DdlException {
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
 
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
index 622a82d2af..c8f1518270 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadPendingTask.java
@@ -334,7 +334,7 @@ public class SparkLoadPendingTask extends LoadTask {
                 partitionColumnRefs.add(column.getName());
             }
 
-            for (Map.Entry<Long, PartitionItem> entry : 
rangePartitionInfo.getPartitionItemEntryList(false, true)) {
+            for (Map.Entry<Long, PartitionItem> entry : 
rangePartitionInfo.getAllPartitionItemEntryList(true)) {
                 long partitionId = entry.getKey();
                 if (!partitionIds.contains(partitionId)) {
                     continue;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org