(doris) branch master updated: [enhance](catalog)Allow parallel running of insert overwrite on the external table (#41575)

[morningman]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 5d4ab4d9f95 [enhance](catalog)Allow parallel running of insert 
overwrite on the external table (#41575)
5d4ab4d9f95 is described below

commit 5d4ab4d9f95b421499c5fca705308cf4d9e56e74
Author: zhangdong <493738...@qq.com>
AuthorDate: Fri Oct 11 11:48:19 2024 +0800

    [enhance](catalog)Allow parallel running of insert overwrite on the 
external table (#41575)
    
    pre pr: https://github.com/apache/doris/pull/40558
    
    Allow parallel running of insert overwrite on the external table
---
 .../insertoverwrite/InsertOverwriteManager.java    |  8 +++++
 .../InsertOverwriteManagerTest.java                | 34 ++++++++++++++++------
 2 files changed, 33 insertions(+), 9 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
 
b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
index a00107c76a7..df16b8f1be2 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
@@ -289,6 +289,14 @@ public class InsertOverwriteManager extends MasterDaemon 
implements Writable {
      * @param table Run the table for insert overwrite
      */
     public void recordRunningTableOrException(DatabaseIf db, TableIf table) {
+        // The logic of OlapTable executing insert overwrite is to create 
temporary partitions,
+        // replace partitions, etc.
+        // If executed in parallel, it may cause problems such as not being 
able to find temporary partitions.
+        // But in terms of external table, we don't care the internal logic of 
execution,
+        // so there's no need to keep records
+        if (!(table instanceof OlapTable)) {
+            return;
+        }
         long dbId = db.getId();
         long tableId = table.getId();
         runningLock.writeLock().lock();
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
 
b/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
index 4bf6c9f12d5..026f8213522 100644
--- 
a/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
+++ 
b/fe/fe-core/src/test/java/org/apache/doris/insertoverwrite/InsertOverwriteManagerTest.java
@@ -18,23 +18,27 @@
 package org.apache.doris.insertoverwrite;
 
 import org.apache.doris.catalog.DatabaseIf;
-import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.hive.HMSExternalTable;
 
 import mockit.Expectations;
 import mockit.Mocked;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
 
 public class InsertOverwriteManagerTest {
     @Mocked
     private DatabaseIf db;
 
     @Mocked
-    private TableIf table;
+    private OlapTable table;
+
+    @Mocked
+    private HMSExternalTable hmsExternalTable;
 
     @Before
     public void setUp()
@@ -57,6 +61,14 @@ public class InsertOverwriteManagerTest {
                 table.getName();
                 minTimes = 0;
                 result = "table1";
+
+                hmsExternalTable.getId();
+                minTimes = 0;
+                result = 3L;
+
+                hmsExternalTable.getName();
+                minTimes = 0;
+                result = "hmsTable";
             }
         };
     }
@@ -65,13 +77,17 @@ public class InsertOverwriteManagerTest {
     public void testParallel() {
         InsertOverwriteManager manager = new InsertOverwriteManager();
         manager.recordRunningTableOrException(db, table);
-        try {
-            manager.recordRunningTableOrException(db, table);
-        } catch (Exception e) {
-            Assert.assertTrue(e.getMessage().contains("Not allowed"));
-        }
+        
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
+                () -> manager.recordRunningTableOrException(db, table));
         manager.dropRunningRecord(db.getId(), table.getId());
-        manager.recordRunningTableOrException(db, table);
+        Assertions.assertDoesNotThrow(() -> 
manager.recordRunningTableOrException(db, table));
     }
 
+    @Test
+    public void testHmsTableParallel() {
+        InsertOverwriteManager manager = new InsertOverwriteManager();
+        manager.recordRunningTableOrException(db, hmsExternalTable);
+        Assertions.assertDoesNotThrow(() -> 
manager.recordRunningTableOrException(db, hmsExternalTable));
+        manager.dropRunningRecord(db.getId(), hmsExternalTable.getId());
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org