This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8545ca4d8d7 [improve] Add PULSAR_LOG_FORMAT env var for easy JSON
logging (#25515)
8545ca4d8d7 is described below
commit 8545ca4d8d7258d31d6cfbd462e2a1ab6e69a402
Author: Matteo Merli <[email protected]>
AuthorDate: Tue Apr 14 22:15:52 2026 -0700
[improve] Add PULSAR_LOG_FORMAT env var for easy JSON logging (#25515)
---
bin/bookkeeper | 7 ++--
bin/pulsar | 10 ++++--
bin/pulsar-admin-common.cmd | 8 +++--
bin/pulsar-admin-common.sh | 7 ++--
bin/pulsar-perf | 7 ++--
conf/log4j2.yaml | 45 +++++++++++++++++-------
conf/pulsar_env.sh | 3 ++
distribution/server/src/assemble/LICENSE.bin.txt | 2 +-
distribution/shell/src/assemble/LICENSE.bin.txt | 2 +-
gradle/libs.versions.toml | 2 +-
10 files changed, 67 insertions(+), 26 deletions(-)
diff --git a/bin/bookkeeper b/bin/bookkeeper
index 11445f2f57a..2785c0c621c 100755
--- a/bin/bookkeeper
+++ b/bin/bookkeeper
@@ -226,7 +226,9 @@ OPTS="$OPTS $BOOKIE_MEM $BOOKIE_GC $BOOKIE_GC_LOG
$BOOKIE_EXTRA_OPTS"
# log directory & file
BOOKIE_LOG_APPENDER=${BOOKIE_LOG_APPENDER:-"Console"}
-BOOKIE_LOG_CONSOLE_JSON_TEMPLATE=${BOOKIE_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}
+# BOOKIE_LOG_CONSOLE_JSON_TEMPLATE is deprecated — use BOOKIE_LOG_JSON_TEMPLATE
+BOOKIE_LOG_JSON_TEMPLATE=${BOOKIE_LOG_JSON_TEMPLATE:-${BOOKIE_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}}
+BOOKIE_LOG_FORMAT=${BOOKIE_LOG_FORMAT:-"text"}
# send bookie shell output to console unconditionally
if [ $COMMAND == "shell" ]; then
@@ -240,7 +242,8 @@ fi
#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$BOOKIE_LOG_APPENDER"
-OPTS="$OPTS
-Dpulsar.log.console.json.template=$BOOKIE_LOG_CONSOLE_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.json.template=$BOOKIE_LOG_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.format=$BOOKIE_LOG_FORMAT"
OPTS="$OPTS -Dbk.log.appender=$BOOKIE_LOG_APPENDER"
OPTS="$OPTS -Dbk.log.level=error"
OPTS="$OPTS -Dpulsar.log.dir=$BOOKIE_LOG_DIR"
diff --git a/bin/pulsar b/bin/pulsar
index bae6168bf26..a10bbcc3019 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -329,8 +329,13 @@ else
fi
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"}
-PULSAR_LOG_CONSOLE_JSON_TEMPLATE=${PULSAR_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}
+# PULSAR_LOG_CONSOLE_JSON_TEMPLATE is deprecated — use PULSAR_LOG_JSON_TEMPLATE
+PULSAR_LOG_JSON_TEMPLATE=${PULSAR_LOG_JSON_TEMPLATE:-${PULSAR_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}}
PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}
+# Log format: "text" (default) or "json" (ECS JSON) — selects the layout for
both
+# the Console and RollingFile appenders via Log4j2 SystemPropertyArbiter
+PULSAR_LOG_FORMAT=${PULSAR_LOG_FORMAT:-"text"}
+
if [ ! -d "$PULSAR_LOG_DIR" ]; then
mkdir -p "$PULSAR_LOG_DIR"
fi
@@ -338,7 +343,8 @@
PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
-OPTS="$OPTS
-Dpulsar.log.console.json.template=$PULSAR_LOG_CONSOLE_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.json.template=$PULSAR_LOG_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.format=$PULSAR_LOG_FORMAT"
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
if [ -n "$PULSAR_LOG_LEVEL" ]; then
OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
diff --git a/bin/pulsar-admin-common.cmd b/bin/pulsar-admin-common.cmd
index f2082f9bd8f..28e1e3c67cc 100644
--- a/bin/pulsar-admin-common.cmd
+++ b/bin/pulsar-admin-common.cmd
@@ -113,12 +113,16 @@ set "OPTS=%OPTS% %PULSAR_EXTRA_OPTS%"
if "%PULSAR_LOG_DIR%" == "" set "PULSAR_LOG_DIR=%PULSAR_HOME%\logs"
if "%PULSAR_LOG_APPENDER%" == "" set "PULSAR_LOG_APPENDER=RoutingAppender"
-if "%PULSAR_LOG_CONSOLE_JSON_TEMPLATE%" == "" set
"PULSAR_LOG_CONSOLE_JSON_TEMPLATE=classpath:EcsLayout.json"
+REM PULSAR_LOG_CONSOLE_JSON_TEMPLATE is deprecated -- use
PULSAR_LOG_JSON_TEMPLATE
+if "%PULSAR_LOG_JSON_TEMPLATE%" == "" set
"PULSAR_LOG_JSON_TEMPLATE=%PULSAR_LOG_CONSOLE_JSON_TEMPLATE%"
+if "%PULSAR_LOG_JSON_TEMPLATE%" == "" set
"PULSAR_LOG_JSON_TEMPLATE=classpath:EcsLayout.json"
+if "%PULSAR_LOG_FORMAT%" == "" set "PULSAR_LOG_FORMAT=text"
if "%PULSAR_ROUTING_APPENDER_DEFAULT%" == "" set
"PULSAR_ROUTING_APPENDER_DEFAULT=Console"
if "%PULSAR_LOG_IMMEDIATE_FLUSH%" == "" set "PULSAR_LOG_IMMEDIATE_FLUSH=false"
set "OPTS=%OPTS% -Dpulsar.log.appender=%PULSAR_LOG_APPENDER%"
-set "OPTS=%OPTS%
-Dpulsar.log.console.json.template=%PULSAR_LOG_CONSOLE_JSON_TEMPLATE%"
+set "OPTS=%OPTS% -Dpulsar.log.json.template=%PULSAR_LOG_JSON_TEMPLATE%"
+set "OPTS=%OPTS% -Dpulsar.log.format=%PULSAR_LOG_FORMAT%"
set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%"
if not "%PULSAR_LOG_LEVEL%" == "" set "OPTS=%OPTS%
-Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
if not "%PULSAR_LOG_ROOT_LEVEL%" == "" (
diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh
index e8264795ae5..673534b76f8 100755
--- a/bin/pulsar-admin-common.sh
+++ b/bin/pulsar-admin-common.sh
@@ -150,7 +150,9 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS"
# log directory & file
PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"}
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"}
-PULSAR_LOG_CONSOLE_JSON_TEMPLATE=${PULSAR_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}
+# PULSAR_LOG_CONSOLE_JSON_TEMPLATE is deprecated — use PULSAR_LOG_JSON_TEMPLATE
+PULSAR_LOG_JSON_TEMPLATE=${PULSAR_LOG_JSON_TEMPLATE:-${PULSAR_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}}
+PULSAR_LOG_FORMAT=${PULSAR_LOG_FORMAT:-"text"}
PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"}
PULSAR_LOG_ROOT_LEVEL=${PULSAR_LOG_ROOT_LEVEL:-"${PULSAR_LOG_LEVEL}"}
PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"}
@@ -158,7 +160,8 @@
PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
-OPTS="$OPTS
-Dpulsar.log.console.json.template=$PULSAR_LOG_CONSOLE_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.json.template=$PULSAR_LOG_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.format=$PULSAR_LOG_FORMAT"
OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR"
OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
OPTS="$OPTS -Dpulsar.log.root.level=$PULSAR_LOG_ROOT_LEVEL"
diff --git a/bin/pulsar-perf b/bin/pulsar-perf
index 9f92a1ec9c5..383cac5d6a4 100755
--- a/bin/pulsar-perf
+++ b/bin/pulsar-perf
@@ -145,13 +145,16 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS"
# log directory & file
PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"}
-PULSAR_LOG_CONSOLE_JSON_TEMPLATE=${PULSAR_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}
+# PULSAR_LOG_CONSOLE_JSON_TEMPLATE is deprecated — use PULSAR_LOG_JSON_TEMPLATE
+PULSAR_LOG_JSON_TEMPLATE=${PULSAR_LOG_JSON_TEMPLATE:-${PULSAR_LOG_CONSOLE_JSON_TEMPLATE:-"classpath:EcsLayout.json"}}
+PULSAR_LOG_FORMAT=${PULSAR_LOG_FORMAT:-"text"}
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-perftest.log"}
PULSAR_LOG_IMMEDIATE_FLUSH="${PULSAR_LOG_IMMEDIATE_FLUSH:-"false"}"
#Configure log configuration system properties
OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER"
-OPTS="$OPTS
-Dpulsar.log.console.json.template=$PULSAR_LOG_CONSOLE_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.json.template=$PULSAR_LOG_JSON_TEMPLATE"
+OPTS="$OPTS -Dpulsar.log.format=$PULSAR_LOG_FORMAT"
if [ -n "$PULSAR_LOG_LEVEL" ]; then
OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL"
fi
diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml
index 05923816647..f3ea2199660 100644
--- a/conf/log4j2.yaml
+++ b/conf/log4j2.yaml
@@ -40,7 +40,9 @@ Configuration:
value: "info"
- name: "pulsar.routing.appender.default"
value: "Console"
- - name: "pulsar.log.console.json.template"
+ - name: "pulsar.log.format"
+ value: "text"
+ - name: "pulsar.log.json.template"
value: "classpath:EcsLayout.json"
# Example: logger-filter script
@@ -55,14 +57,17 @@ Configuration:
# Console
Console:
- - name: Console
- target: SYSTEM_OUT
- PatternLayout:
- Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level
%logger{36} - %msg %X%n"
- - name: ConsoleJson
- target: SYSTEM_OUT
- JsonTemplateLayout:
- eventTemplateUri: ${sys:pulsar.log.console.json.template}
+ name: Console
+ target: SYSTEM_OUT
+ SystemPropertyArbiter:
+ - propertyName: pulsar.log.format
+ propertyValue: json
+ JsonTemplateLayout:
+ eventTemplateUri: ${sys:pulsar.log.json.template}
+ - propertyName: pulsar.log.format
+ propertyValue: text
+ PatternLayout:
+ Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level
%logger{36} - %msg %X%n"
# Rolling file appender configuration
RollingFile:
@@ -70,8 +75,15 @@ Configuration:
fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}"
filePattern:
"${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz"
immediateFlush: ${sys:pulsar.log.immediateFlush}
- PatternLayout:
- Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36}
- %msg %X%n"
+ SystemPropertyArbiter:
+ - propertyName: pulsar.log.format
+ propertyValue: json
+ JsonTemplateLayout:
+ eventTemplateUri: ${sys:pulsar.log.json.template}
+ - propertyName: pulsar.log.format
+ propertyValue: text
+ PatternLayout:
+ Pattern: "%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level
%logger{36} - %msg %X%n"
Policies:
TimeBasedTriggeringPolicy:
interval: 1
@@ -108,8 +120,15 @@ Configuration:
name: "Rolling-${ctx:function}"
fileName :
"${sys:pulsar.log.dir}/functions/${ctx:function}/${ctx:functionname}-${ctx:instance}.log"
filePattern :
"${sys:pulsar.log.dir}/functions/${sys:pulsar.log.file}-${ctx:instance}-%d{MM-dd-yyyy}-%i.log.gz"
- PatternLayout:
- Pattern: "%d{ABSOLUTE} %level{length=5} [%thread]
[instance: %X{instance}] %logger{1} - %msg%n"
+ SystemPropertyArbiter:
+ - propertyName: pulsar.log.format
+ propertyValue: json
+ JsonTemplateLayout:
+ eventTemplateUri: ${sys:pulsar.log.json.template}
+ - propertyName: pulsar.log.format
+ propertyValue: text
+ PatternLayout:
+ Pattern: "%d{ABSOLUTE} %level{length=5} [%thread]
[instance: %X{instance}] %logger{1} - %msg%n"
Policies:
TimeBasedTriggeringPolicy:
interval: 1
diff --git a/conf/pulsar_env.sh b/conf/pulsar_env.sh
index 530a08c0ed7..b84011efd68 100755
--- a/conf/pulsar_env.sh
+++ b/conf/pulsar_env.sh
@@ -29,6 +29,9 @@
# Logs location
# PULSAR_LOG_DIR=
+# Log format: "text" (default) or "json" (ECS JSON format, useful for log
aggregators)
+# PULSAR_LOG_FORMAT=json
+
# Configuration file of settings used in broker server
# PULSAR_BROKER_CONF=
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt
b/distribution/server/src/assemble/LICENSE.bin.txt
index 351d4884548..42fb9f60937 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -277,7 +277,7 @@ The Apache Software License, Version 2.0
- io.swagger-swagger-annotations-1.6.2.jar
- io.swagger-swagger-core-1.6.2.jar
- io.swagger-swagger-models-1.6.2.jar
- * slog -- io.github.merlimat.slog-slog-0.9.5.jar
+ * slog -- io.github.merlimat.slog-slog-0.9.7.jar
* DataSketches
- com.yahoo.datasketches-memory-0.8.3.jar
- com.yahoo.datasketches-sketches-core-0.8.3.jar
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt
b/distribution/shell/src/assemble/LICENSE.bin.txt
index 350f38dcdad..6b1f764b666 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -392,7 +392,7 @@ The Apache Software License, Version 2.0
- opentelemetry-common-1.56.0.jar
- opentelemetry-context-1.56.0.jar
* Slog
- - slog-0.9.5.jar
+ - slog-0.9.7.jar
* BookKeeper
- bookkeeper-common-allocator-4.17.3.jar
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index b5a5aca4a67..ac458d5d607 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -34,7 +34,7 @@ jackson-annotations = "2.21"
protobuf = "3.25.5"
grpc = "1.75.0"
slf4j = "2.0.17"
-slog = "0.9.5"
+slog = "0.9.7"
log4j2 = "2.25.3"
lombok = "1.18.42"
# OpenTelemetry
