This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new fbc82e0253f [opt](log) refine the BE logger (#35942) (#35988)
fbc82e0253f is described below
commit fbc82e0253fb3972f43e32abe2eeb971709c8d55
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Thu Jun 6 22:25:22 2024 +0800
[opt](log) refine the BE logger (#35942) (#35988)
bp #35942
---
be/CMakeLists.txt | 3 +++
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 5 +++++
be/src/common/logconfig.cpp | 39 ++++++++++++++++++++++++++++++---
bin/start_be.sh | 45 ++++++++++++++++++++++++---------------
regression-test/framework/pom.xml | 2 +-
6 files changed, 75 insertions(+), 21 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index f5d99c69962..0305e886ef9 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -66,6 +66,9 @@ else()
add_definitions(-DBOOST_STACKTRACE_USE_NOOP)
endif()
+# enable glog custom prefix
+add_definitions(-DGLOG_CUSTOM_PREFIX_SUPPORT)
+
# Options
option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries."
ON)
option(USE_LIBCPP "Use libc++" OFF)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 299f38cd7ca..8d33fa8a4f1 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1228,6 +1228,8 @@ DEFINE_Bool(enable_jvm_monitor, "false");
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
DEFINE_mBool(skip_loading_stale_rowset_meta, "false");
+DEFINE_Bool(enable_file_logger, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 5fccbec6285..55bee9e17ce 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1308,6 +1308,11 @@ DECLARE_Bool(enable_jvm_monitor);
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
DECLARE_mBool(skip_loading_stale_rowset_meta);
+// Whether to use file to record log. When starting BE with --console,
+// all logs will be written to both standard output and file.
+// Disable this option will no longer use file to record log.
+// Only works when starting BE with --console.
+DECLARE_Bool(enable_file_logger);
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp
index bf408272776..0865e8ce471 100644
--- a/be/src/common/logconfig.cpp
+++ b/be/src/common/logconfig.cpp
@@ -21,6 +21,7 @@
#include <cerrno>
#include <cstdlib>
#include <cstring>
+#include <iomanip>
#include <iostream>
#include <mutex>
#include <string>
@@ -48,6 +49,28 @@ static bool iequals(const std::string& a, const std::string&
b) {
return true;
}
+void custom_prefix(std::ostream& s, const google::LogMessageInfo& l, void*) {
+ // Add prefix "RuntimeLogger ".
+ s << "RuntimeLogger ";
+ // Same as in fe.log
+ // The following is same as default log format. eg:
+ // I20240605 15:25:15.677153 1763151 wal_manager.cpp:481] msg...
+ s << l.severity[0];
+ s << std::setw(4) << 1900 + l.time.year();
+ s << std::setw(2) << 1 + l.time.month();
+ s << std::setw(2) << l.time.day();
+ s << ' ';
+ s << std::setw(2) << l.time.hour() << ':';
+ s << std::setw(2) << l.time.min() << ':';
+ s << std::setw(2) << l.time.sec() << ".";
+ s << std::setw(6) << l.time.usec();
+ s << ' ';
+ s << std::setfill(' ') << std::setw(5);
+ s << l.thread_id << std::setfill('0');
+ s << ' ';
+ s << l.filename << ':' << l.line_number << "]";
+}
+
bool init_glog(const char* basename) {
std::lock_guard<std::mutex> logging_lock(logging_mutex);
@@ -55,8 +78,13 @@ bool init_glog(const char* basename) {
return true;
}
- if (getenv("DORIS_LOG_TO_STDERR") != nullptr) {
- FLAGS_alsologtostderr = true;
+ bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr);
+ if (log_to_console) {
+ if (config::enable_file_logger) {
+ FLAGS_alsologtostderr = true;
+ } else {
+ FLAGS_logtostderr = true;
+ }
}
// don't log to stderr except fatal level
@@ -144,7 +172,12 @@ bool init_glog(const char* basename) {
}
}
- google::InitGoogleLogging(basename);
+ if (log_to_console) {
+ // Only add prefix if log output to stderr
+ google::InitGoogleLogging(basename, &custom_prefix);
+ } else {
+ google::InitGoogleLogging(basename);
+ }
logging_initialized = true;
diff --git a/bin/start_be.sh b/bin/start_be.sh
index 06679eb1b37..8fcc152d624 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -140,7 +140,7 @@ export
CLASSPATH="${DORIS_HOME}/conf/:${DORIS_CLASSPATH}:${CLASSPATH}"
# DORIS_CLASSPATH is for self-managed jni
export DORIS_CLASSPATH="-Djava.class.path=${DORIS_CLASSPATH}"
-#echo ${DORIS_CLASSPATH}
+# log ${DORIS_CLASSPATH}
export
LD_LIBRARY_PATH="${DORIS_HOME}/lib/hadoop_hdfs/native:${LD_LIBRARY_PATH}"
@@ -181,16 +181,6 @@ PID_DIR="$(
)"
export PID_DIR
-# set odbc conf path
-export ODBCSYSINI="${DORIS_HOME}/conf"
-
-# support utf8 for oracle database
-export NLS_LANG='AMERICAN_AMERICA.AL32UTF8'
-
-# filter known leak.
-export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/lsan_suppr.conf"
-export ASAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf"
-
while read -r line; do
envline="$(echo "${line}" |
sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
@@ -203,6 +193,27 @@ while read -r line; do
fi
done <"${DORIS_HOME}/conf/be.conf"
+STDOUT_LOGGER="${LOG_DIR}/be.out"
+log() {
+ # same datetime format as in fe.log: 2024-06-03 14:54:41,478
+ cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)")
+ if [[ "${RUN_CONSOLE}" -eq 1 ]]; then
+ echo "StdoutLogger ${cur_date} $1"
+ else
+ echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}"
+ fi
+}
+
+# set odbc conf path
+export ODBCSYSINI="${DORIS_HOME}/conf"
+
+# support utf8 for oracle database
+export NLS_LANG='AMERICAN_AMERICA.AL32UTF8'
+
+# filter known leak.
+export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/lsan_suppr.conf"
+export ASAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf"
+
if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then
# shellcheck disable=1091
source "${DORIS_HOME}/bin/palo_env.sh"
@@ -220,7 +231,7 @@ fi
for var in http_proxy HTTP_PROXY https_proxy HTTPS_PROXY; do
if [[ -n ${!var} ]]; then
- echo "env '${var}' = '${!var}', need unset it using 'unset ${var}'"
+ log "env '${var}' = '${!var}', need unset it using 'unset ${var}'"
exit 1
fi
done
@@ -241,7 +252,7 @@ if [[ -f "${pidfile}" ]]; then
fi
chmod 550 "${DORIS_HOME}/lib/doris_be"
-echo "start time: $(date)" >>"${LOG_DIR}/be.out"
+log "Start time: $(date)"
if [[ ! -f '/bin/limit3' ]]; then
LIMIT=''
@@ -287,7 +298,7 @@ set_tcmalloc_heap_limit() {
fi
if [[ "${mem_limit_mb}" -gt "${total_mem_mb}" ]]; then
- echo "mem_limit is larger than whole memory of the server.
${mem_limit_mb} > ${total_mem_mb}."
+ log "mem_limit is larger than the total memory of the server.
${mem_limit_mb} > ${total_mem_mb}"
return 1
fi
export TCMALLOC_HEAP_LIMIT_MB=${mem_limit_mb}
@@ -342,9 +353,9 @@ fi
# set LIBHDFS_OPTS for hadoop libhdfs
export LIBHDFS_OPTS="${final_java_opt}"
-#echo "CLASSPATH: ${CLASSPATH}"
-#echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
-#echo "LIBHDFS_OPTS: ${LIBHDFS_OPTS}"
+# log "CLASSPATH: ${CLASSPATH}"
+# log "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
+# log "LIBHDFS_OPTS: ${LIBHDFS_OPTS}"
if [[ -z ${JEMALLOC_CONF} ]]; then
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
diff --git a/regression-test/framework/pom.xml
b/regression-test/framework/pom.xml
index 79b76a684b4..5d2cce687ee 100644
--- a/regression-test/framework/pom.xml
+++ b/regression-test/framework/pom.xml
@@ -230,7 +230,7 @@ under the License.
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>31.0.1-jre</version>
+ <version>32.1.2-jre</version>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
