This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9ca7fdf Make MAX_SCHEDULING_TABLETS and MAX_BALANCING_TABLETS
configurable (#2670)
9ca7fdf is described below
commit 9ca7fdfe1ca08055de7c5a044c45d9e73034a59e
Author: kangkaisen <kangkai...@apache.org>
AuthorDate: Mon Jan 6 20:15:38 2020 +0800
Make MAX_SCHEDULING_TABLETS and MAX_BALANCING_TABLETS configurable (#2670)
---
fe/src/main/java/org/apache/doris/clone/TabletChecker.java | 7 ++++---
.../main/java/org/apache/doris/clone/TabletScheduler.java | 14 ++++----------
fe/src/main/java/org/apache/doris/common/Config.java | 10 ++++++++++
3 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/clone/TabletChecker.java
b/fe/src/main/java/org/apache/doris/clone/TabletChecker.java
index fdf7612..1d9ac6d 100644
--- a/fe/src/main/java/org/apache/doris/clone/TabletChecker.java
+++ b/fe/src/main/java/org/apache/doris/clone/TabletChecker.java
@@ -31,6 +31,7 @@ import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.Tablet.TabletStatus;
import org.apache.doris.clone.TabletScheduler.AddResult;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
@@ -157,10 +158,10 @@ public class TabletChecker extends MasterDaemon {
protected void runAfterCatalogReady() {
int pendingNum = tabletScheduler.getPendingNum();
int runningNum = tabletScheduler.getRunningNum();
- if (pendingNum > TabletScheduler.MAX_SCHEDULING_TABLETS
- || runningNum > TabletScheduler.MAX_SCHEDULING_TABLETS) {
+ if (pendingNum > Config.max_scheduling_tablets
+ || runningNum > Config.max_scheduling_tablets) {
LOG.info("too many tablets are being scheduled. pending: {},
running: {}, limit: {}. skip check",
- pendingNum, runningNum,
TabletScheduler.MAX_SCHEDULING_TABLETS);
+ pendingNum, runningNum, Config.max_scheduling_tablets);
return;
}
diff --git a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 006e6d8..e8dbee3 100644
--- a/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -96,13 +96,6 @@ public class TabletScheduler extends MasterDaemon {
public static final int BALANCE_SLOT_NUM_FOR_PATH = 2;
- // if the number of scheduled tablets in TabletScheduler exceed this
threshold,
- // skip checking.
- public static final int MAX_SCHEDULING_TABLETS = 2000;
- // if the number of balancing tablets in TabletScheduler exceed this
threshold,
- // no more balance check
- public static final int MAX_BALANCING_TABLETS = 100;
-
/*
* Tablet is added to pendingTablets as well it's id in allTabletIds.
* TabletScheduler will take tablet from pendingTablets but will not
remove it's id from allTabletIds when
@@ -220,7 +213,8 @@ public class TabletScheduler extends MasterDaemon {
// and number of scheduling tablets exceed the limit,
// refuse to add.
if (tablet.getType() != TabletSchedCtx.Type.BALANCE && !force
- && (pendingTablets.size() > MAX_SCHEDULING_TABLETS ||
runningTablets.size() > MAX_SCHEDULING_TABLETS)) {
+ && (pendingTablets.size() > Config.max_scheduling_tablets
+ || runningTablets.size() > Config.max_scheduling_tablets)) {
return AddResult.LIMIT_EXCEED;
}
@@ -976,9 +970,9 @@ public class TabletScheduler extends MasterDaemon {
}
long numOfBalancingTablets = getBalanceTabletsNumber();
- if (numOfBalancingTablets > MAX_BALANCING_TABLETS) {
+ if (numOfBalancingTablets > Config.max_balancing_tablets) {
LOG.info("number of balancing tablets {} exceed limit: {}, skip
selecting tablets for balance",
- numOfBalancingTablets, MAX_BALANCING_TABLETS);
+ numOfBalancingTablets, Config.max_balancing_tablets);
return;
}
diff --git a/fe/src/main/java/org/apache/doris/common/Config.java
b/fe/src/main/java/org/apache/doris/common/Config.java
index 44fa9eb..48cfee1 100644
--- a/fe/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/src/main/java/org/apache/doris/common/Config.java
@@ -846,6 +846,16 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = true)
public static boolean disable_balance = false;
+
+ // if the number of scheduled tablets in TabletScheduler exceed
max_scheduling_tablets
+ // skip checking.
+ @ConfField(mutable = true, masterOnly = true)
+ public static int max_scheduling_tablets = 2000;
+
+ // if the number of balancing tablets in TabletScheduler exceed
max_balancing_tablets,
+ // no more balance check
+ @ConfField(mutable = true, masterOnly = true)
+ public static int max_balancing_tablets = 100;
// This threshold is to avoid piling up too many report task in FE, which
may cause OOM exception.
// In some large Doris cluster, eg: 100 Backends with ten million
replicas, a tablet report may cost
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
