This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7062685609 HDDS-12805. Use slf4j for HTTP request log (#8255)
7062685609 is described below
commit 70626856099498442cf571791c591f4d9189dc50
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Apr 11 00:20:16 2025 +0200
HDDS-12805. Use slf4j for HTTP request log (#8255)
---
.../hadoop/hdds/server/http/HttpRequestLog.java | 106 ---------------------
.../hdds/server/http/HttpRequestLogAppender.java | 62 ------------
.../hadoop/hdds/server/http/HttpServer2.java | 22 +++--
.../hdds/server/http/TestHttpRequestLog.java | 51 ----------
.../server/http/TestHttpRequestLogAppender.java | 40 --------
hadoop-ozone/dist/src/shell/conf/log4j.properties | 19 ++++
6 files changed, 34 insertions(+), 266 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLog.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLog.java
deleted file mode 100644
index e06653ed4b..0000000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLog.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.server.http;
-
-import java.util.HashMap;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogConfigurationException;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.log4j.Appender;
-import org.eclipse.jetty.server.AsyncRequestLogWriter;
-import org.eclipse.jetty.server.CustomRequestLog;
-import org.eclipse.jetty.server.RequestLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * RequestLog object for use with Http.
- */
-public final class HttpRequestLog {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(HttpRequestLog.class);
- private static final HashMap<String, String> SERVER_TO_COMPONENT;
-
- private HttpRequestLog() {
- }
-
- static {
- SERVER_TO_COMPONENT = new HashMap<String, String>();
- SERVER_TO_COMPONENT.put("cluster", "resourcemanager");
- SERVER_TO_COMPONENT.put("hdfs", "namenode");
- SERVER_TO_COMPONENT.put("node", "nodemanager");
- }
-
- public static RequestLog getRequestLog(String name) {
-
- String lookup = SERVER_TO_COMPONENT.get(name);
- if (lookup != null) {
- name = lookup;
- }
- String loggerName = "http.requests." + name;
- String appenderName = name + "requestlog";
- Log logger = LogFactory.getLog(loggerName);
-
- boolean isLog4JLogger;
-
- try {
- isLog4JLogger = logger instanceof Log4JLogger;
- } catch (NoClassDefFoundError err) {
- // In some dependent projects, log4j may not even be on the classpath at
- // runtime, in which case the above instanceof check will throw
- // NoClassDefFoundError.
- LOG.debug("Could not load Log4JLogger class", err);
- isLog4JLogger = false;
- }
- if (isLog4JLogger) {
- Log4JLogger httpLog4JLog = (Log4JLogger) logger;
- org.apache.log4j.Logger httpLogger = httpLog4JLog.getLogger();
- Appender appender = null;
-
- try {
- appender = httpLogger.getAppender(appenderName);
- } catch (LogConfigurationException e) {
- LOG.warn("Http request log for {} could not be created", loggerName);
- throw e;
- }
-
- if (appender == null) {
- LOG.info("Http request log for {} is not defined", loggerName);
- return null;
- }
-
- if (appender instanceof HttpRequestLogAppender) {
- HttpRequestLogAppender requestLogAppender
- = (HttpRequestLogAppender) appender;
- AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter();
- logWriter.setFilename(requestLogAppender.getFilename());
- logWriter.setRetainDays(requestLogAppender.getRetainDays());
- return new CustomRequestLog(logWriter,
- CustomRequestLog.EXTENDED_NCSA_FORMAT);
- } else {
- LOG.warn("Jetty request log for {} was of the wrong class",
loggerName);
- return null;
- }
- } else {
- LOG.warn("Jetty request log can only be enabled using Log4j");
- return null;
- }
- }
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLogAppender.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLogAppender.java
deleted file mode 100644
index 9777e2d9b5..0000000000
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLogAppender.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.server.http;
-
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.spi.LoggingEvent;
-
-/**
- * Log4j Appender adapter for HttpRequestLog.
- */
-public class HttpRequestLogAppender extends AppenderSkeleton {
-
- private String filename;
- private int retainDays;
-
- public HttpRequestLogAppender() {
- }
-
- public void setRetainDays(int retainDays) {
- this.retainDays = retainDays;
- }
-
- public int getRetainDays() {
- return retainDays;
- }
-
- public void setFilename(String filename) {
- this.filename = filename;
- }
-
- public String getFilename() {
- return filename;
- }
-
- @Override
- public void append(LoggingEvent event) {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public boolean requiresLayout() {
- return false;
- }
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
index 241fb5ab81..3b76325226 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
@@ -89,6 +89,7 @@
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.ConnectionFactory;
import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.CustomRequestLog;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -96,6 +97,7 @@
import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.Slf4jRequestLogWriter;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.HandlerCollection;
@@ -618,14 +620,13 @@ private void initializeWebServer(Builder builder) throws
IOException {
handler.getSessionCookieConfig().setSecure(true);
ContextHandlerCollection contexts = new ContextHandlerCollection();
- RequestLog requestLog = HttpRequestLog.getRequestLog(builder.name);
-
handlers.addHandler(contexts);
- if (requestLog != null) {
- RequestLogHandler requestLogHandler = new RequestLogHandler();
- requestLogHandler.setRequestLog(requestLog);
- handlers.addHandler(requestLogHandler);
- }
+
+ RequestLog requestLog = getRequestLog(builder.name);
+ RequestLogHandler requestLogHandler = new RequestLogHandler();
+ requestLogHandler.setRequestLog(requestLog);
+ handlers.addHandler(requestLogHandler);
+
handlers.addHandler(webAppContext);
final String appDir = getWebAppsPath(builder.name);
if (!builder.skipDefaultApps) {
@@ -1794,4 +1795,11 @@ public static void setHttpBaseDir(OzoneConfiguration
ozoneConfiguration)
tmpMetaDir.getAbsolutePath());
}
}
+
+ private static RequestLog getRequestLog(String name) {
+ String loggerName = "http.requests." + name;
+ Slf4jRequestLogWriter writer = new Slf4jRequestLogWriter();
+ writer.setLoggerName(loggerName);
+ return new CustomRequestLog(writer, CustomRequestLog.EXTENDED_NCSA_FORMAT);
+ }
}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLog.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLog.java
deleted file mode 100644
index eb596acc7a..0000000000
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLog.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.server.http;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-
-import org.apache.log4j.Logger;
-import org.eclipse.jetty.server.CustomRequestLog;
-import org.eclipse.jetty.server.RequestLog;
-import org.junit.jupiter.api.Test;
-
-/**
- * Testing HttpRequestLog.
- */
-public class TestHttpRequestLog {
-
- @Test
- public void testAppenderUndefined() {
- RequestLog requestLog = HttpRequestLog.getRequestLog("test");
- assertNull(requestLog, "RequestLog should be null");
- }
-
- @Test
- public void testAppenderDefined() {
- HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender();
- requestLogAppender.setName("testrequestlog");
- Logger.getLogger("http.requests.test").addAppender(requestLogAppender);
- RequestLog requestLog = HttpRequestLog.getRequestLog("test");
- Logger.getLogger("http.requests.test").removeAppender(requestLogAppender);
- assertNotNull(requestLog, "RequestLog should not be null");
- assertEquals(CustomRequestLog.class, requestLog.getClass(),
- "Class mismatch");
- }
-}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java
deleted file mode 100644
index 418f921584..0000000000
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.server.http;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import org.junit.jupiter.api.Test;
-
-/**
- * Test Http request log appender.
- */
-public class TestHttpRequestLogAppender {
-
- @Test
- public void testParameterPropagation() {
-
- HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender();
- requestLogAppender.setFilename("jetty-namenode-yyyy_mm_dd.log");
- requestLogAppender.setRetainDays(17);
- assertEquals("jetty-namenode-yyyy_mm_dd.log",
- requestLogAppender.getFilename(), "Filename mismatch");
- assertEquals(17, requestLogAppender.getRetainDays(),
- "Retain days mismatch");
- }
-}
diff --git a/hadoop-ozone/dist/src/shell/conf/log4j.properties
b/hadoop-ozone/dist/src/shell/conf/log4j.properties
index aa3d0b4bf4..ae3c6f51cf 100644
--- a/hadoop-ozone/dist/src/shell/conf/log4j.properties
+++ b/hadoop-ozone/dist/src/shell/conf/log4j.properties
@@ -122,6 +122,25 @@ log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
+#
+# HTTP request logs
+#
+log4j.appender.HttpAccess=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.HttpAccess.File=${hadoop.log.dir}/access.log
+log4j.appender.HttpAccess.DatePattern=.yyyy-MM-dd
+log4j.appender.HttpAccess.layout=org.apache.log4j.PatternLayout
+log4j.appender.HttpAccess.layout.ConversionPattern=%m%n
+
+log4j.additivity.http.requests=false
+log4j.logger.http.requests=INFO,HttpAccess
+# Create separate appender for each co-hosted component if needed, then enable
distinct logger configs:
+#log4j.logger.http.requests.hddsDatanode=INFO,HttpAccess
+#log4j.logger.http.requests.ozoneManager=INFO,HttpAccess
+#log4j.logger.http.requests.recon=INFO,HttpAccess
+#log4j.logger.http.requests.s3gateway=INFO,HttpAccess
+#log4j.logger.http.requests.s3g-web=INFO,HttpAccess
+#log4j.logger.http.requests.scm=INFO,HttpAccess
+#log4j.logger.http.requests.webhdfs=INFO,HttpAccess
# Custom Logging levels
# AWS SDK & S3A FileSystem
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
