This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cf24f301e94 [fix](sql-block-rule) fix block rule missing issue after
replay (#37243)
cf24f301e94 is described below
commit cf24f301e94efb8fcbae323cea1e178db46830e6
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Thu Jul 4 20:56:27 2024 +0800
[fix](sql-block-rule) fix block rule missing issue after replay (#37243)
Intro from #30784
the `sqlBlockRulesSplit` field in `CommonUserProperties` is missing
after
read metadata from image.
After changing meta serde to Gson, some "after work" should be done
by implementing `GsonPostProcessable` interface.
Also change 2 things:
1. Shorten the serde name of some fields.
2. When setting sql block rules for user, no longer check if the rule
exist.
Just like grant/revoke, when grant/revoke, we don't check if db or table
exists.
Decouple these 2 logics to make code simple.
---
.../mysql/privilege/CommonUserProperties.java | 34 ++++++----
.../apache/doris/mysql/privilege/UserProperty.java | 47 ++++++--------
.../doris/blockrule/SqlBlockRuleMgrTest.java | 4 +-
.../org/apache/doris/persist/UserPropertyTest.java | 72 ++++++++++++++++++++++
4 files changed, 114 insertions(+), 43 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
index 63365e1280c..db838a91a56 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
@@ -19,10 +19,12 @@ package org.apache.doris.mysql.privilege;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.resource.Tag;
import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
+import com.google.common.base.Strings;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
@@ -36,35 +38,35 @@ import java.util.Set;
/**
* Used in
*/
-public class CommonUserProperties implements Writable {
+public class CommonUserProperties implements Writable, GsonPostProcessable {
private static final Logger LOG =
LogManager.getLogger(CommonUserProperties.class);
// The max connections allowed for a user on one FE
- @SerializedName("maxConn")
+ @SerializedName(value = "mc", alternate = {"maxConn"})
private long maxConn = 100;
// The maximum total number of query instances that the user is allowed to
send from this FE
- @SerializedName("maxQueryInstances")
+ @SerializedName(value = "mqi", alternate = {"maxQueryInstances"})
private long maxQueryInstances = -1;
- @SerializedName("parallelFragmentExecInstanceNum")
+ @SerializedName(value = "pfei", alternate =
{"parallelFragmentExecInstanceNum"})
private int parallelFragmentExecInstanceNum = -1;
- @SerializedName("sqlBlockRules")
+ @SerializedName(value = "sbr", alternate = {"sqlBlockRule"})
private String sqlBlockRules = "";
- @SerializedName("cpuResourceLimit")
+ @SerializedName(value = "crl", alternate = {"cpuResourceLimit"})
private int cpuResourceLimit = -1;
// The tag of the resource that the user is allowed to use
- @SerializedName("resourceTags")
+ @SerializedName(value = "rt", alternate = {"resourceTag"})
private Set<Tag> resourceTags = Sets.newHashSet();
// user level exec_mem_limit, if > 0, will overwrite the exec_mem_limit in
session variable
- @SerializedName("execMemLimit")
+ @SerializedName(value = "eml", alternate = {"execMemLimit"})
private long execMemLimit = -1;
- @SerializedName("queryTimeout")
+ @SerializedName(value = "qt", alternate = {"queryTimeout"})
private int queryTimeout = -1;
- @SerializedName("insertTimeout")
+ @SerializedName(value = "it", alternate = {"insertTimeout"})
private int insertTimeout = -1;
- @SerializedName("workloadGroup")
+ @SerializedName(value = "wg", alternate = {"workloadGroup"})
private String workloadGroup = WorkloadGroupMgr.DEFAULT_GROUP_NAME;
private String[] sqlBlockRulesSplit = {};
@@ -162,11 +164,10 @@ public class CommonUserProperties implements Writable {
this.workloadGroup = workloadGroup;
}
+ @Deprecated
public static CommonUserProperties read(DataInput in) throws IOException {
String json = Text.readString(in);
CommonUserProperties commonUserProperties =
GsonUtils.GSON.fromJson(json, CommonUserProperties.class);
- // trigger split
-
commonUserProperties.setSqlBlockRulesSplit(commonUserProperties.getSqlBlockRules());
return commonUserProperties;
}
@@ -175,4 +176,11 @@ public class CommonUserProperties implements Writable {
String json = GsonUtils.GSON.toJson(this);
Text.writeString(out, json);
}
+
+ @Override
+ public void gsonPostProcess() throws IOException {
+ if (!Strings.isNullOrEmpty(sqlBlockRules)) {
+ setSqlBlockRulesSplit(sqlBlockRules);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
index 8b4b6d08285..ccc21b58660 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
@@ -64,24 +64,24 @@ import java.util.regex.Pattern;
public class UserProperty implements Writable {
private static final Logger LOG = LogManager.getLogger(UserProperty.class);
// advanced properties
- private static final String PROP_MAX_USER_CONNECTIONS =
"max_user_connections";
- private static final String PROP_MAX_QUERY_INSTANCES =
"max_query_instances";
- private static final String PROP_PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM =
"parallel_fragment_exec_instance_num";
- private static final String PROP_RESOURCE_TAGS = "resource_tags";
- private static final String PROP_RESOURCE = "resource";
- private static final String PROP_SQL_BLOCK_RULES = "sql_block_rules";
- private static final String PROP_CPU_RESOURCE_LIMIT = "cpu_resource_limit";
- private static final String PROP_EXEC_MEM_LIMIT = "exec_mem_limit";
- private static final String PROP_USER_QUERY_TIMEOUT = "query_timeout";
-
- private static final String PROP_USER_INSERT_TIMEOUT = "insert_timeout";
+ public static final String PROP_MAX_USER_CONNECTIONS =
"max_user_connections";
+ public static final String PROP_MAX_QUERY_INSTANCES =
"max_query_instances";
+ public static final String PROP_PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM =
"parallel_fragment_exec_instance_num";
+ public static final String PROP_RESOURCE_TAGS = "resource_tags";
+ public static final String PROP_RESOURCE = "resource";
+ public static final String PROP_SQL_BLOCK_RULES = "sql_block_rules";
+ public static final String PROP_CPU_RESOURCE_LIMIT = "cpu_resource_limit";
+ public static final String PROP_EXEC_MEM_LIMIT = "exec_mem_limit";
+ public static final String PROP_USER_QUERY_TIMEOUT = "query_timeout";
+
+ public static final String PROP_USER_INSERT_TIMEOUT = "insert_timeout";
// advanced properties end
- private static final String PROP_LOAD_CLUSTER = "load_cluster";
- private static final String PROP_QUOTA = "quota";
- private static final String PROP_DEFAULT_LOAD_CLUSTER =
"default_load_cluster";
+ public static final String PROP_LOAD_CLUSTER = "load_cluster";
+ public static final String PROP_QUOTA = "quota";
+ public static final String PROP_DEFAULT_LOAD_CLUSTER =
"default_load_cluster";
- private static final String PROP_WORKLOAD_GROUP = "default_workload_group";
+ public static final String PROP_WORKLOAD_GROUP = "default_workload_group";
public static final String DEFAULT_CLOUD_CLUSTER = "default_cloud_cluster";
@@ -90,20 +90,20 @@ public class UserProperty implements Writable {
// for normal user
public static final Set<Pattern> COMMON_PROPERTIES = Sets.newHashSet();
- @SerializedName(value = "qualifiedUser")
+ @SerializedName(value = "qu", alternate = {"qualifiedUser"})
private String qualifiedUser;
- @SerializedName(value = "commonProperties")
+ @SerializedName(value = "cp", alternate = {"commonProperties"})
private CommonUserProperties commonProperties = new CommonUserProperties();
// load cluster
- @SerializedName(value = "defaultLoadCluster")
+ @SerializedName(value = "dlc", alternate = {"defaultLoadCluster"})
private String defaultLoadCluster = null;
- @SerializedName(value = "clusterToDppConfig")
+ @SerializedName(value = "cdc", alternate = {"clusterToDppConfig"})
private Map<String, DppConfig> clusterToDppConfig = Maps.newHashMap();
- @SerializedName(value = "defaultCloudCluster")
+ @SerializedName(value = "dcc", alternate = {"defaultCloudCluster"})
private String defaultCloudCluster = null;
/*
@@ -290,13 +290,6 @@ public class UserProperty implements Writable {
if (keyArr.length != 1) {
throw new DdlException(PROP_SQL_BLOCK_RULES + " format
error");
}
-
- // check if sql_block_rule has already exist
- for (String ruleName : value.replaceAll(" ", "").split(",")) {
- if (!ruleName.equals("") &&
!Env.getCurrentEnv().getSqlBlockRuleMgr().existRule(ruleName)) {
- throw new DdlException("the sql block rule " +
ruleName + " not exist");
- }
- }
sqlBlockRules = value;
} else if (keyArr[0].equalsIgnoreCase(PROP_CPU_RESOURCE_LIMIT)) {
// set property "cpu_resource_limit" = "2";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java
index 2f93ee5beaa..3dc9f8b59e1 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/blockrule/SqlBlockRuleMgrTest.java
@@ -207,10 +207,8 @@ public class SqlBlockRuleMgrTest extends TestWithFeService
{
SetUserPropertyStmt setUserPropertyStmt = (SetUserPropertyStmt)
UtFrameUtils.parseAndAnalyzeStmt(setPropertyStr,
connectContext);
- ExceptionChecker.expectThrowsWithMsg(DdlException.class,
- String.format("the sql block rule %s not exist", ruleName),
+ ExceptionChecker.expectThrowsNoException(
() ->
Env.getCurrentEnv().getAuth().updateUserProperty(setUserPropertyStmt));
-
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/UserPropertyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/UserPropertyTest.java
new file mode 100644
index 00000000000..5b7cc9d861e
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/UserPropertyTest.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.persist;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
+import org.apache.doris.mysql.privilege.UserProperty;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class UserPropertyTest {
+ @Test
+ public void testSerialization() throws IOException, UserException {
+ // 1. Write objects to file
+ final Path path = Files.createTempFile("propertiesInfo", "tmp");
+ DataOutputStream out = new
DataOutputStream(Files.newOutputStream(path));
+
+ List<Pair<String, String>> properties = Lists.newArrayList();
+ properties.add(Pair.of(UserProperty.PROP_MAX_USER_CONNECTIONS, "100"));
+ properties.add(Pair.of(UserProperty.PROP_MAX_QUERY_INSTANCES, "2"));
+
properties.add(Pair.of(UserProperty.PROP_PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM,
"8"));
+ properties.add(Pair.of(UserProperty.PROP_SQL_BLOCK_RULES, "r1,r2"));
+
+ UserProperty prop = new UserProperty();
+ prop.update(properties);
+
+ prop.write(out);
+ out.flush();
+ out.close();
+
+ // 2. Read objects from file
+ DataInputStream in = new DataInputStream(Files.newInputStream(path));
+
+ UserProperty prop2 = UserProperty.read(in);
+
+ Assert.assertEquals(100, prop2.getMaxConn());
+ Assert.assertEquals(2, prop2.getMaxQueryInstances());
+ Assert.assertEquals(8, prop2.getParallelFragmentExecInstanceNum());
+ Assert.assertEquals(Lists.newArrayList("r1", "r2"),
+
Arrays.stream(prop2.getSqlBlockRules()).sorted().collect(Collectors.toList()));
+
+ // 3. delete files
+ in.close();
+ Files.delete(path);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
