This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 85cc044aaa [feature](create-table) support setting replication num for
creating table opertaion globally (#21848)
85cc044aaa is described below
commit 85cc044aaa3c0127fa06b23c924b1e9fc299a00c
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Fri Jul 21 19:36:04 2023 +0800
[feature](create-table) support setting replication num for creating table
opertaion globally (#21848)
Add a new FE config `force_olap_table_replication_num`.
If this config is larger than 0, when doing creating table operation, the
replication num of table will
forcibly be this value.
Default is 0, which make no effect.
This config will only effect the creating olap table operation, other
operation such as `add partition`,
`modify table properties` will not be effect.
The motivation of this config is that the most regression test cases are
creating table will single replica,
this will be the regression test running well in p0, p1 pipeline.
But we also need to run these cases in multi backend Doris cluster, so we
need test cases will multi replicas.
But it is hard to modify each test cases. So I add this config, so that we
can simply set it to create all tables with
specified replication number.
---
.../main/java/org/apache/doris/common/Config.java | 11 ++++++++
.../org/apache/doris/analysis/CreateTableStmt.java | 31 ++++++++++++++++++++++
.../org/apache/doris/catalog/CreateTableTest.java | 30 +++++++++++++++++++++
3 files changed, 72 insertions(+)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 068b6a4ca7..424736bead 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2049,4 +2049,15 @@ public class Config extends ConfigBase {
@ConfField
public static long statistics_sql_mem_limit_in_bytes = 2L * 1024 * 1024 *
1024;
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "用于强制设定内表的副本数，如果改参数大于零，则用户在建表时指定的副本数将被忽略，而使用本参数设置的值。"
+ +
"同时，建表语句中指定的副本标签等参数会被忽略。该参数不影响包括创建分区、修改表属性的操作。该参数建议仅用于测试环境",
+ "Used to force the number of replicas of the internal table. If
the config is greater than zero, "
+ + "the number of replicas specified by the user when
creating the table will be ignored, "
+ + "and the value set by this parameter will be used. At
the same time, the replica tags "
+ + "and other parameters specified in the create table
statement will be ignored. "
+ + "This config does not effect the operations including
creating partitions "
+ + "and modifying table properties. "
+ + "This config is recommended to be used only in the test
environment"})
+ public static int force_olap_table_replication_num = 0;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 556bc03a03..183963f4fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@@ -535,6 +536,8 @@ public class CreateTableStmt extends DdlStmt {
}
if (engineName.equals("olap")) {
+ // before analyzing partition, handle the replication allocation
info
+ properties = rewriteReplicaAllocationProperties(properties);
// analyze partition
if (partitionDesc != null) {
if (partitionDesc instanceof ListPartitionDesc ||
partitionDesc instanceof RangePartitionDesc
@@ -624,6 +627,34 @@ public class CreateTableStmt extends DdlStmt {
}
}
+ private Map<String, String> rewriteReplicaAllocationProperties(Map<String,
String> properties) {
+ if (Config.force_olap_table_replication_num <= 0) {
+ return properties;
+ }
+ // if force_olap_table_replication_num is set, use this value to
rewrite the replication_num or
+ // replication_allocation properties
+ Map<String, String> newProperties = properties;
+ if (newProperties == null) {
+ newProperties = Maps.newHashMap();
+ }
+ boolean rewrite = false;
+ if
(newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM)) {
+ newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
+ String.valueOf(Config.force_olap_table_replication_num));
+ rewrite = true;
+ }
+ if
(newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION))
{
+
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
+ new ReplicaAllocation((short)
Config.force_olap_table_replication_num).toCreateStmt());
+ rewrite = true;
+ }
+ if (!rewrite) {
+ newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
+ String.valueOf(Config.force_olap_table_replication_num));
+ }
+ return newProperties;
+ }
+
private void analyzeEngineName() throws AnalysisException {
if (Strings.isNullOrEmpty(engineName)) {
engineName = "olap";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 2d5226d8e0..195405f9e9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog;
+import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.common.AnalysisException;
@@ -25,6 +26,7 @@ import org.apache.doris.common.ConfigBase;
import org.apache.doris.common.ConfigException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
+import org.apache.doris.common.UserException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
@@ -67,6 +69,11 @@ public class CreateTableTest {
Env.getCurrentEnv().createTable(createTableStmt);
}
+ private static void alterTable(String sql) throws Exception {
+ AlterTableStmt alterTableStmt = (AlterTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Env.getCurrentEnv().alterTable(alterTableStmt);
+ }
+
@Test
public void testDuplicateCreateTable() throws Exception {
// test
@@ -739,4 +746,27 @@ public class CreateTableTest {
Assert.assertEquals(ScalarType.MAX_VARCHAR_LENGTH,
tb.getColumn("k3").getStrLen());
Assert.assertEquals(10, tb.getColumn("k4").getStrLen());
}
+
+ @Test
+ public void testCreateTableWithForceReplica() throws DdlException {
+ Config.force_olap_table_replication_num = 1;
+ // no need to specify replication_num, the table can still be created.
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.test_replica\n" + "(k1 int, k2 int)
partition by range(k1)\n" + "(\n"
+ + "partition p1 values less than(\"10\"),\n" + "partition
p2 values less than(\"20\")\n" + ")\n"
+ + "distributed by hash(k2) buckets 1;");
+ });
+
+ // can still set replication_num manually.
+ ExceptionChecker.expectThrowsWithMsg(UserException.class, "Failed to
find enough host with tag",
+ () -> {
+ alterTable("alter table test.test_replica modify partition
p1 set ('replication_num' = '3')");
+ });
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrDdlException("default_cluster:test");
+ OlapTable tb = (OlapTable) db.getTableOrDdlException("test_replica");
+ Partition p1 = tb.getPartition("p1");
+ Assert.assertEquals(1,
tb.getPartitionInfo().getReplicaAllocation(p1.getId()).getTotalReplicaNum());
+ Assert.assertEquals(1,
tb.getTableProperty().getReplicaAllocation().getTotalReplicaNum());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
