[doris] 26/33: [zhongjin](config) add Config.min_file_split_size_mb in FE

[morningman]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch doris-for-zhongjin
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 740a2f4ff613e86ba44a97f06f718c32c182e663
Author: morningman <morning...@163.com>
AuthorDate: Sat Apr 8 20:24:03 2023 +0800

    [zhongjin](config) add Config.min_file_split_size_mb in FE
---
 fe/fe-common/src/main/java/org/apache/doris/common/Config.java      | 3 +++
 .../main/java/org/apache/doris/planner/external/HiveSplitter.java   | 6 ++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java 
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 75954dc408..e43587610c 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2104,5 +2104,8 @@ public class Config extends ConfigBase {
      */
     @ConfField
     public static long lock_reporting_threshold_ms = 500L;
+
+    @ConfField(mutable = true)
+    public static long min_file_split_size_mb = 64;
 }
 
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveSplitter.java 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveSplitter.java
index b17704251a..86dd4fab1d 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveSplitter.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveSplitter.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.ListPartitionItem;
 import org.apache.doris.catalog.PartitionItem;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.catalog.external.HMSExternalTable;
+import org.apache.doris.common.Config;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.HMSExternalCatalog;
@@ -161,8 +162,8 @@ public class HiveSplitter implements Splitter {
         if (splitSize <= 0) {
             splitSize = status.getBlockSize();
         }
-        // Min split size is DEFAULT_SPLIT_SIZE(128MB).
-        splitSize = splitSize > DEFAULT_SPLIT_SIZE ? splitSize : 
DEFAULT_SPLIT_SIZE;
+        splitSize = splitSize > Config.min_file_split_size_mb * 1024L * 1204L ?
+                splitSize : Config.min_file_split_size_mb * 1024L * 1024L;
         BlockLocation[] blockLocations = status.getBlockLocations();
         long length = status.getLength();
         long bytesRemaining;
@@ -216,3 +217,4 @@ public class HiveSplitter implements Splitter {
         throw new IllegalArgumentException(String.format("Offset %d is outside 
of file (0..%d)", offset, fileLength));
     }
 }
+


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org