This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 77374bef320 [Enhancement](audit log) Add print audit log sesssion
variable (#38419)
77374bef320 is described below
commit 77374bef320bf61efc62ba03dc3a66616a72ef52
Author: abmdocrt <yukang.lian2...@gmail.com>
AuthorDate: Wed Jul 31 11:38:33 2024 +0800
[Enhancement](audit log) Add print audit log sesssion variable (#38419)
## Proposed changes
For the `insert into` statements during group commit load via JDBC.
Printing audit logs can severely impact performance. Therefore, we have
introduced a session variable to control whether to print audit logs. It
is recommended to turn off audit logs only during group commit load via
JDBC.
<!--Describe your changes.-->
---
.../org/apache/doris/qe/MysqlConnectProcessor.java | 28 +++++++++++++++-------
.../java/org/apache/doris/qe/SessionVariable.java | 10 ++++++++
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
index b772da388db..165767c8611 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
@@ -136,18 +136,26 @@ public class MysqlConnectProcessor extends
ConnectProcessor {
executor = new StmtExecutor(ctx, executeStmt);
ctx.setExecutor(executor);
executor.execute();
- PrepareStmtContext preparedStmtContext =
ConnectContext.get().getPreparedStmt(String.valueOf(stmtId));
- if (preparedStmtContext != null) {
- stmtStr = executeStmt.toSql();
+ //For the `insert into` statements during group commit load via
JDBC.
+ //Printing audit logs can severely impact performance.
+ //Therefore, we have introduced a session variable to control
whether to print audit logs.
+ //It is recommended to turn off audit logs only during group
commit load via JDBC.
+ if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
+ PrepareStmtContext preparedStmtContext =
ConnectContext.get().getPreparedStmt(String.valueOf(stmtId));
+ if (preparedStmtContext != null) {
+ stmtStr = executeStmt.toSql();
+ }
}
- } catch (Throwable e) {
+ } catch (Throwable e) {
// Catch all throwable.
// If reach here, maybe doris bug.
LOG.warn("Process one query failed because unknown reason: ", e);
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
e.getClass().getSimpleName() + ", msg: " + e.getMessage());
}
- auditAfterExec(stmtStr, executor.getParsedStmt(),
executor.getQueryStatisticsForAuditLog(), true);
+ if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
+ auditAfterExec(stmtStr, executor.getParsedStmt(),
executor.getQueryStatisticsForAuditLog(), true);
+ }
}
private void handleExecute(PrepareCommand prepareCommand, long stmtId,
PreparedStatementContext prepCtx) {
@@ -199,15 +207,19 @@ public class MysqlConnectProcessor extends
ConnectProcessor {
executor = new StmtExecutor(ctx, stmt);
ctx.setExecutor(executor);
executor.execute();
- stmtStr = executeStmt.toSql();
- } catch (Throwable e) {
+ if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
+ stmtStr = executeStmt.toSql();
+ }
+ } catch (Throwable e) {
// Catch all throwable.
// If reach here, maybe doris bug.
LOG.warn("Process one query failed because unknown reason: ", e);
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
e.getClass().getSimpleName() + ", msg: " + e.getMessage());
}
- auditAfterExec(stmtStr, executor.getParsedStmt(),
executor.getQueryStatisticsForAuditLog(), true);
+ if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
+ auditAfterExec(stmtStr, executor.getParsedStmt(),
executor.getQueryStatisticsForAuditLog(), true);
+ }
}
// process COM_EXECUTE, parse binary row data
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 04b7b2a1f17..6af6d7f6753 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -473,8 +473,11 @@ public class SessionVariable implements Serializable,
Writable {
public static final String EXTERNAL_TABLE_ANALYZE_PART_NUM =
"external_table_analyze_part_num";
public static final String ENABLE_STRONG_CONSISTENCY =
"enable_strong_consistency_read";
+
public static final String GROUP_COMMIT = "group_commit";
+ public static final String ENABLE_PREPARED_STMT_AUDIT_LOG =
"enable_prepared_stmt_audit_log";
+
public static final String PARALLEL_SYNC_ANALYZE_TASK_NUM =
"parallel_sync_analyze_task_num";
public static final String TRUNCATE_CHAR_OR_VARCHAR_COLUMNS =
"truncate_char_or_varchar_columns";
@@ -1676,6 +1679,9 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = GROUP_COMMIT, needForward = true)
public String groupCommit = "off_mode";
+ @VariableMgr.VarAttr(name = ENABLE_PREPARED_STMT_AUDIT_LOG, needForward =
true)
+ public boolean enablePreparedStmtAuditLog = true;
+
@VariableMgr.VarAttr(name = INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD,
description =
{"在match_all中求取多个倒排索引的交集时,如果最大的倒排索引中的总数是最小倒排索引中的总数的整数倍,"
+ "则使用跳表来优化交集操作。",
@@ -4106,6 +4112,10 @@ public class SessionVariable implements Serializable,
Writable {
return groupCommit;
}
+ public boolean isEnablePreparedStmtAuditLog() {
+ return enablePreparedStmtAuditLog;
+ }
+
public boolean isEnableMaterializedViewRewrite() {
return enableMaterializedViewRewrite;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
