[doris] 02/06: [improvememt](file-cache) increase virtual node number to make file cache more even (#24143)

[kxiao]

This is an automated email from the ASF dual-hosted git repository.

kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git

commit f393de48e37209f5310a8c329aef27fcb92991ad
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Sun Sep 10 19:56:53 2023 +0800

    [improvememt](file-cache) increase virtual node number to make file cache 
more even (#24143)
    
    The origin virtual number is Math.max(Math.min(512 / backends.size(), 32), 
2);, which is too small,
    causing uneven cache distribution when enabling file cache.
---
 .../src/main/java/org/apache/doris/common/Config.java        | 12 ++++++++++++
 .../doris/planner/external/FederationBackendPolicy.java      |  3 +--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java 
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 9ca45ffd2a..b5715ed881 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2116,4 +2116,16 @@ public class Config extends ConfigBase {
     })
     public static long query_metadata_name_ids_timeout = 3;
 
+    @ConfField(mutable = true, masterOnly = true, description = {
+            "是否禁止LocalDeployManager删除节点",
+            "Whether to disable LocalDeployManager drop node"})
+    public static boolean disable_local_deploy_manager_drop_node = true;
+
+    @ConfField(mutable = true, description = {
+            "开启 file cache 后，一致性哈希算法中，每个节点的虚拟节点数。"
+                    + "该值越大，哈希算法的分布越均匀，但是会增加内存开销。",
+            "When file cache is enabled, the number of virtual nodes of each 
node in the consistent hash algorithm. "
+                    + "The larger the value, the more uniform the distribution 
of the hash algorithm, "
+                    + "but it will increase the memory overhead."})
+    public static int virtual_node_number = 2048;
 }
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
index eb2545affd..04c39fbf96 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
@@ -96,9 +96,8 @@ public class FederationBackendPolicy {
             throw new UserException("No available backends");
         }
         
backendMap.putAll(backends.stream().collect(Collectors.groupingBy(Backend::getHost)));
-        int virtualNumber = Math.max(Math.min(512 / backends.size(), 32), 2);
         consistentHash = new ConsistentHash<>(Hashing.murmur3_128(), new 
ScanRangeHash(),
-                new BackendHash(), backends, virtualNumber);
+                new BackendHash(), backends, Config.virtual_node_number);
     }
 
     public Backend getNextBe() {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org