This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f94222a04ead1ff65fc8f4f28003fb4fb28c1b76
Author: deardeng <565620...@qq.com>
AuthorDate: Thu May 9 20:54:53 2024 +0800
[fix](log) Support fe log rollover size strategy (#34446)
---
.../main/java/org/apache/doris/common/Config.java | 6 +
.../java/org/apache/doris/common/Log4jConfig.java | 172 ++++++++++++---------
2 files changed, 102 insertions(+), 76 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index f5888b048fd..5f1d1047812 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2599,6 +2599,12 @@ public class Config extends ConfigBase {
//==========================================================================
// begin of cloud config
//==========================================================================
+ @ConfField(description = {"是否启用FE
日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的log。默认为按照时间策略删除",
+ "Whether to enable the FE log file deletion policy based on size, "
+ + "where logs exceeding the specified size are deleted. "
+ + "It is disabled by default and follows a time-based deletion
policy."},
+ options = {"age", "size"})
+ public static String log_rollover_strategy = "age";
@ConfField public static int info_sys_accumulated_file_size = 4;
@ConfField public static int warn_sys_accumulated_file_size = 2;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
index 320cdd6114b..8c840a876bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
@@ -39,81 +39,101 @@ import java.util.Map;
public class Log4jConfig extends XmlConfiguration {
private static final long serialVersionUID = 1L;
- // CHECKSTYLE OFF
- private static String xmlConfTemplate = "<?xml version=\"1.0\"
encoding=\"utf-8\"?>\n"
- + "\n<!-- Auto Generated. DO NOT MODIFY IT! -->\n"
- + "<Configuration status=\"info\"
packages=\"org.apache.doris.common\">\n"
- + " <Appenders>\n"
- + " <Console name=\"Console\" target=\"SYSTEM_OUT\">"
- + " <PatternLayout charset=\"UTF-8\">\n"
- + " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
- + " </PatternLayout>\n"
- + " </Console>"
- + " <RollingFile name=\"Sys\"
fileName=\"${sys_log_dir}/fe.log\"
filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i${sys_file_postfix}\"
immediateFlush=\"${immediate_flush_flag}\">\n"
- + " <PatternLayout charset=\"UTF-8\">\n"
- + " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
- + " </PatternLayout>\n"
- + " <Policies>\n"
- + " <TimeBasedTriggeringPolicy/>\n"
- + " <SizeBasedTriggeringPolicy
size=\"${sys_roll_maxsize}MB\"/>\n"
- + " </Policies>\n"
- + " <DefaultRolloverStrategy max=\"${sys_roll_num}\"
fileIndex=\"max\">\n"
- + " <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n"
- + " <IfFileName glob=\"fe.log.*\" />\n"
- + " <IfAny>\n"
- + " <IfAccumulatedFileSize
exceeds=\"${info_sys_accumulated_file_size}GB\"/>\n"
- + " </IfAny>\n"
- + " </Delete>\n"
- + " </DefaultRolloverStrategy>\n"
- + " </RollingFile>\n"
- + " <RollingFile name=\"SysWF\"
fileName=\"${sys_log_dir}/fe.warn.log\"
filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i${sys_file_postfix}\"
immediateFlush=\"${immediate_flush_flag}\">\n"
- + " <PatternLayout charset=\"UTF-8\">\n"
- + " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
- + " </PatternLayout>\n"
- + " <Policies>\n"
- + " <TimeBasedTriggeringPolicy/>\n"
- + " <SizeBasedTriggeringPolicy
size=\"${sys_roll_maxsize}MB\"/>\n"
- + " </Policies>\n"
- + " <DefaultRolloverStrategy max=\"${sys_roll_num}\"
fileIndex=\"max\">\n"
- + " <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n"
- + " <IfFileName glob=\"fe.warn.log.*\" />\n"
- + " <IfAny>\n"
- + " <IfAccumulatedFileSize
exceeds=\"${warn_sys_accumulated_file_size}GB\"/>\n"
- + " </IfAny>\n"
- + " </Delete>\n"
- + " </DefaultRolloverStrategy>\n"
- + " </RollingFile>\n"
- + " <RollingFile name=\"Auditfile\"
fileName=\"${audit_log_dir}/fe.audit.log\"
filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i${audit_file_postfix}\">\n"
- + " <PatternLayout charset=\"UTF-8\">\n"
- + " <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}]
%m%n</Pattern>\n"
- + " </PatternLayout>\n"
- + " <Policies>\n"
- + " <TimeBasedTriggeringPolicy/>\n"
- + " <SizeBasedTriggeringPolicy
size=\"${audit_roll_maxsize}MB\"/>\n"
- + " </Policies>\n"
- + " <DefaultRolloverStrategy max=\"${audit_roll_num}\"
fileIndex=\"max\">\n"
- + " <Delete basePath=\"${audit_log_dir}/\"
maxDepth=\"1\">\n"
- + " <IfFileName glob=\"fe.audit.log.*\" />\n"
- + " <IfAny>\n"
- + " <IfAccumulatedFileSize
exceeds=\"${audit_sys_accumulated_file_size}GB\"/>\n"
- + " </IfAny>\n"
- + " </Delete>\n"
- + " </DefaultRolloverStrategy>\n"
- + " </RollingFile>\n"
- + " </Appenders>\n"
- + " <Loggers>\n"
- + " <Root level=\"${sys_log_level}\"
includeLocation=\"${include_location_flag}\">\n"
- + " <AppenderRef ref=\"Sys\"/>\n"
- + " <AppenderRef ref=\"SysWF\" level=\"WARN\"/>\n"
- + " <!--REPLACED BY Console Logger-->\n"
- + " </Root>\n"
- + " <Logger name=\"audit\" level=\"ERROR\"
additivity=\"false\">\n"
- + " <AppenderRef ref=\"Auditfile\"/>\n"
- + " </Logger>\n"
- + " <!--REPLACED BY AUDIT AND VERBOSE MODULE NAMES-->\n"
- + " </Loggers>\n"
- + "</Configuration>";
- // CHECKSTYLE ON
+ private static StringBuilder xmlConfTemplateBuilder = new StringBuilder();
+
+ private static void getXmlConfByStrategy(final String size, final String
age) {
+ if (Config.log_rollover_strategy.equalsIgnoreCase("size")) {
+ xmlConfTemplateBuilder
+ .append(" <IfAny>\n")
+ .append(" <IfAccumulatedFileSize
exceeds=\"${").append(size).append("}GB\"/>\n")
+ .append(" </IfAny>\n");
+ } else {
+ // default age
+ xmlConfTemplateBuilder
+ .append(" <IfLastModified
age=\"${").append(age).append("}\" />\n");
+ }
+ }
+
+ static {
+ // CHECKSTYLE OFF
+ xmlConfTemplateBuilder.append("<?xml version=\"1.0\"
encoding=\"utf-8\"?>\n")
+ .append("\n<!-- Auto Generated. DO NOT MODIFY IT! -->\n")
+ .append("<Configuration status=\"info\"
packages=\"org.apache.doris.common\">\n")
+ .append(" <Appenders>\n")
+ .append(" <Console name=\"Console\" target=\"SYSTEM_OUT\">")
+ .append(" <PatternLayout charset=\"UTF-8\">\n")
+ .append(" <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
+ .append(" </PatternLayout>\n")
+ .append(" </Console>\n")
+ .append(" <RollingFile name=\"Sys\"
fileName=\"${sys_log_dir}/fe.log\"
filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i${sys_file_postfix}\"
immediateFlush=\"${immediate_flush_flag}\">\n")
+ .append(" <PatternLayout charset=\"UTF-8\">\n")
+ .append(" <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
+ .append(" </PatternLayout>\n")
+ .append(" <Policies>\n")
+ .append(" <TimeBasedTriggeringPolicy/>\n")
+ .append(" <SizeBasedTriggeringPolicy
size=\"${sys_roll_maxsize}MB\"/>\n")
+ .append(" </Policies>\n")
+ .append(" <DefaultRolloverStrategy max=\"${sys_roll_num}\"
fileIndex=\"max\">\n")
+ .append(" <Delete basePath=\"${sys_log_dir}/\"
maxDepth=\"1\">\n")
+ .append(" <IfFileName glob=\"fe.log.*\" />\n");
+
+ getXmlConfByStrategy("info_sys_accumulated_file_size",
"sys_log_delete_age");
+
+ xmlConfTemplateBuilder
+ .append(" </Delete>\n")
+ .append(" </DefaultRolloverStrategy>\n")
+ .append(" </RollingFile>\n")
+ .append(" <RollingFile name=\"SysWF\"
fileName=\"${sys_log_dir}/fe.warn.log\"
filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i${sys_file_postfix}\"
immediateFlush=\"${immediate_flush_flag}\">\n")
+ .append(" <PatternLayout charset=\"UTF-8\">\n")
+ .append(" <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p
(%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
+ .append(" </PatternLayout>\n")
+ .append(" <Policies>\n")
+ .append(" <TimeBasedTriggeringPolicy/>\n")
+ .append(" <SizeBasedTriggeringPolicy
size=\"${sys_roll_maxsize}MB\"/>\n")
+ .append(" </Policies>\n")
+ .append(" <DefaultRolloverStrategy max=\"${sys_roll_num}\"
fileIndex=\"max\">\n")
+ .append(" <Delete basePath=\"${sys_log_dir}/\"
maxDepth=\"1\">\n")
+ .append(" <IfFileName glob=\"fe.warn.log.*\" />\n");
+
+ getXmlConfByStrategy("warn_sys_accumulated_file_size",
"sys_log_delete_age");
+
+ xmlConfTemplateBuilder
+ .append(" </Delete>\n")
+ .append(" </DefaultRolloverStrategy>\n")
+ .append(" </RollingFile>\n")
+ .append(" <RollingFile name=\"Auditfile\"
fileName=\"${audit_log_dir}/fe.audit.log\"
filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i${audit_file_postfix}\">\n")
+ .append(" <PatternLayout charset=\"UTF-8\">\n")
+ .append(" <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}]
%m%n</Pattern>\n")
+ .append(" </PatternLayout>\n")
+ .append(" <Policies>\n")
+ .append(" <TimeBasedTriggeringPolicy/>\n")
+ .append(" <SizeBasedTriggeringPolicy
size=\"${audit_roll_maxsize}MB\"/>\n")
+ .append(" </Policies>\n")
+ .append(" <DefaultRolloverStrategy max=\"${audit_roll_num}\"
fileIndex=\"max\">\n")
+ .append(" <Delete basePath=\"${audit_log_dir}/\"
maxDepth=\"1\">\n")
+ .append(" <IfFileName glob=\"fe.audit.log.*\" />\n");
+
+ getXmlConfByStrategy("audit_sys_accumulated_file_size",
"audit_log_delete_age");
+
+ xmlConfTemplateBuilder
+ .append(" </Delete>\n")
+ .append(" </DefaultRolloverStrategy>\n")
+ .append(" </RollingFile>\n")
+ .append(" </Appenders>\n")
+ .append(" <Loggers>\n")
+ .append(" <Root level=\"${sys_log_level}\"
includeLocation=\"${include_location_flag}\">\n")
+ .append(" <AppenderRef ref=\"Sys\"/>\n")
+ .append(" <AppenderRef ref=\"SysWF\" level=\"WARN\"/>\n")
+ .append(" <!--REPLACED BY Console Logger-->\n")
+ .append(" </Root>\n")
+ .append(" <Logger name=\"audit\" level=\"ERROR\"
additivity=\"false\">\n")
+ .append(" <AppenderRef ref=\"Auditfile\"/>\n")
+ .append(" </Logger>\n")
+ .append(" <!--REPLACED BY AUDIT AND VERBOSE MODULE NAMES-->\n")
+ .append(" </Loggers>\n")
+ .append("</Configuration>");
+ // CHECKSTYLE ON
+ }
private static StrSubstitutor strSub;
private static String sysLogLevel;
@@ -134,7 +154,7 @@ public class Log4jConfig extends XmlConfiguration {
public static boolean foreground = false;
private static void reconfig() throws IOException {
- String newXmlConfTemplate = xmlConfTemplate;
+ String newXmlConfTemplate = xmlConfTemplateBuilder.toString();
// sys log config
// ATTN, sys_log_dir is deprecated, use LOG_DIR instead
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
