This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b10c5336c02 [opt](log) refine the cloud logger (#39488)
b10c5336c02 is described below
commit b10c5336c025b7f7856809bf66824eb2c1ca5674
Author: walter <w41te...@gmail.com>
AuthorDate: Tue Aug 20 09:54:54 2024 +0800
[opt](log) refine the cloud logger (#39488)
Followup https://github.com/apache/doris/pull/35679 and #24556
# Background
Previously, the cloud logs were written to files. The main cloud logs
include meta_service.INFO and meta_service.WARNING, doris_cloud.out.
In a K8s deployment environment, logs usually must be output to standard
output, and other components process the log stream.
# Solution
This PR made the following changes:
Modified the glog config:
- When started with --daemon, logs are still written to various files,
and the format remains unchanged.
- When started with --console, meta_service.INFO's log is output to
standard output and marked with the prefix RuntimeLogger.
Examples are as follows:
```
RuntimeLogger I20240605 23:41:20.426553 4137369
runtime_query_statistics_mgr.cpp:245] Report profile thread stopped
```
Added a new cloud config: `enable_file_logger`, which defaults to true.
This indicates that logs will be recorded in files regardless of the
startup method. For example, if it is started with `--console`, the log
will be output to both the file and the standard output. If it is false,
the log will not be recorded in the file regardless of the startup
method.
---
cloud/CMakeLists.txt | 3 +++
cloud/script/start.sh | 6 ++++++
cloud/src/common/config.h | 5 +++++
cloud/src/common/logging.cpp | 47 +++++++++++++++++++++++++++++++++++++++-----
4 files changed, 56 insertions(+), 5 deletions(-)
diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt
index bc3f5664282..bb697d791e7 100644
--- a/cloud/CMakeLists.txt
+++ b/cloud/CMakeLists.txt
@@ -457,6 +457,9 @@ if (NOT EXISTS ${THIRDPARTY_DIR}/include/foundationdb)
execute_process(COMMAND "tar" "xf" "${THIRDPARTY_SRC}/${FDB_LIB}" "-C"
"${THIRDPARTY_DIR}/")
endif ()
+# enable glog custom prefix
+add_definitions(-DGLOG_CUSTOM_PREFIX_SUPPORT)
+
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lfdb_c
-L${THIRDPARTY_DIR}/lib")
add_subdirectory(${SRC_DIR}/common)
diff --git a/cloud/script/start.sh b/cloud/script/start.sh
index ebe2f88e16c..5a61d3534b5 100644
--- a/cloud/script/start.sh
+++ b/cloud/script/start.sh
@@ -34,12 +34,14 @@ fi
RUN_DAEMON=0
RUN_VERSION=0
+RUN_CONSOLE=0
for arg; do
shift
[[ "${arg}" = "--daemonized" ]] && RUN_DAEMON=1 && continue
[[ "${arg}" = "-daemonized" ]] && RUN_DAEMON=1 && continue
[[ "${arg}" = "--daemon" ]] && RUN_DAEMON=1 && continue
[[ "${arg}" = "--version" ]] && RUN_VERSION=1 && continue
+ [[ "${arg}" = "--console" ]] && RUN_CONSOLE=1 && continue
set -- "$@" "${arg}"
done
# echo "$@" "daemonized=${daemonized}"}
@@ -137,6 +139,10 @@ if [[ "${RUN_DAEMON}" -eq 1 ]]; then
tail -n10 "${DORIS_HOME}/log/${process}.out" | grep 'working directory'
-B1 -A10
echo "please check process log for more details"
echo ""
+elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then
+ export DORIS_LOG_TO_STDERR=1
+ date
+ "${bin}" "$@" 2>&1
else
"${bin}" "$@"
fi
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index cb4bee9648e..2b00e3d2245 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -47,6 +47,11 @@ CONF_Int32(warn_log_filenum_quota, "1");
CONF_Bool(log_immediate_flush, "false");
CONF_Strings(log_verbose_modules, ""); // Comma seprated list: a.*,b.*
CONF_Int32(log_verbose_level, "5");
+// Whether to use file to record log. When starting Cloud with --console,
+// all logs will be written to both standard output and file.
+// Disable this option will no longer use file to record log.
+// Only works when starting Cloud with --console.
+CONF_Bool(enable_file_logger, "true");
// recycler config
CONF_mInt64(recycle_interval_seconds, "3600");
diff --git a/cloud/src/common/logging.cpp b/cloud/src/common/logging.cpp
index 838e8892633..65f9048a4df 100644
--- a/cloud/src/common/logging.cpp
+++ b/cloud/src/common/logging.cpp
@@ -22,6 +22,7 @@
#include <glog/logging.h>
#include <glog/vlog_is_on.h>
+#include <iomanip>
#include <iostream>
#include <mutex>
@@ -72,6 +73,28 @@ void AnnotateTag::format_tag_list(std::ostream& stream) {
}
}
+void custom_prefix(std::ostream& s, const google::LogMessageInfo& l, void*) {
+ // Add prefix "RuntimeLogger ".
+ s << "RuntimeLogger ";
+ // Same as in fe.log
+ // The following is same as default log format. eg:
+ // I20240605 15:25:15.677153 1763151 meta_service_txn.cpp:481] msg...
+ s << l.severity[0];
+ s << std::setw(4) << 1900 + l.time.year();
+ s << std::setw(2) << 1 + l.time.month();
+ s << std::setw(2) << l.time.day();
+ s << ' ';
+ s << std::setw(2) << l.time.hour() << ':';
+ s << std::setw(2) << l.time.min() << ':';
+ s << std::setw(2) << l.time.sec() << ".";
+ s << std::setw(6) << l.time.usec();
+ s << ' ';
+ s << std::setfill(' ') << std::setw(5);
+ s << l.thread_id << std::setfill('0');
+ s << ' ';
+ s << l.filename << ':' << l.line_number << "]";
+}
+
/**
* @param basename the basename of log file
* @return true for success
@@ -82,10 +105,19 @@ bool init_glog(const char* basename) {
std::lock_guard<std::mutex> logging_lock(mtx);
if (inited) return true;
- FLAGS_alsologtostderr = false;
- // Don't log to stderr except fatal level
- // so fatal log can output to be.out .
- FLAGS_stderrthreshold = google::ERROR;
+ bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr);
+ if (log_to_console) {
+ if (config::enable_file_logger) {
+ FLAGS_alsologtostderr = true;
+ } else {
+ FLAGS_logtostderr = true;
+ }
+ } else {
+ FLAGS_alsologtostderr = false;
+ // Don't log to stderr except fatal level
+ // so fatal log can output to be.out .
+ FLAGS_stderrthreshold = google::ERROR;
+ }
// Set glog log dir
FLAGS_log_dir = config::log_dir;
@@ -128,7 +160,12 @@ bool init_glog(const char* basename) {
if (i.empty()) continue;
google::SetVLOGLevel(i.c_str(), config::log_verbose_level);
}
- google::InitGoogleLogging(basename);
+ if (log_to_console) {
+ // Only add prefix if log output to stderr
+ google::InitGoogleLogging(basename, &custom_prefix);
+ } else {
+ google::InitGoogleLogging(basename);
+ }
inited = true;
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
