This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 3114baac5 RANGER-4336: added configurations to enable status logging
in audit framework
3114baac5 is described below
commit 3114baac5e4aa0f461152de74354ab0ab2d9e258
Author: Madhan Neethiraj <[email protected]>
AuthorDate: Thu Jul 27 01:15:34 2023 -0700
RANGER-4336: added configurations to enable status logging in audit
framework
---
.../ranger/audit/provider/BaseAuditHandler.java | 36 +++++++++++++++++-----
.../apache/ranger/audit/queue/AuditAsyncQueue.java | 18 +++++++++++
.../java/org/apache/ranger/audit/TestConsumer.java | 26 ++++++++--------
3 files changed, 61 insertions(+), 19 deletions(-)
diff --git
a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
index 8511ce9cb..68c33c90d 100644
---
a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
+++
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -38,6 +38,11 @@ public abstract class BaseAuditHandler implements
AuditHandler {
static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP =
"xasecure.audit.log.failure.report.min.interval.ms";
+ static final String AUDIT_LOG_STATUS_LOG_ENABLED =
"xasecure.audit.log.status.log.enabled";
+ static final String AUDIT_LOG_STATUS_LOG_INTERVAL_SEC =
"xasecure.audit.log.status.log.interval.sec";
+ static final boolean DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED = false;
+ static final long DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = 5 *
60; // 5 minutes
+
public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE
= "xasecure.policymgr.clientssl.keystore";
public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE
= "xasecure.policymgr.clientssl.keystore.type";
public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL
= "xasecure.policymgr.clientssl.keystore.credential.file";
@@ -90,8 +95,10 @@ public abstract class BaseAuditHandler implements
AuditHandler {
long lastStashedCount = 0;
long lastDeferredCount = 0;
- long lastStatusLogTime = System.currentTimeMillis();
- long statusLogIntervalMS = 1 * 60 * 1000;
+ boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
+ long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC
* 1000;
+ long lastStatusLogTime = System.currentTimeMillis();
+ long nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
protected Properties props = null;
protected Map<String, String> configProps = new HashMap<String,
String>();
@@ -138,6 +145,19 @@ public abstract class BaseAuditHandler implements
AuditHandler {
mLogFailureReportMinIntervalInMs =
MiscUtil.getIntProperty(props,
AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60
* 1000);
+ boolean globalStatusLogEnabled =
MiscUtil.getBooleanProperty(props, AUDIT_LOG_STATUS_LOG_ENABLED,
DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED);
+ long globalStatusLogIntervalSec =
MiscUtil.getLongProperty(props, AUDIT_LOG_STATUS_LOG_INTERVAL_SEC,
DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC);
+
+ statusLogEnabled = MiscUtil.getBooleanProperty(props,
basePropertyName + ".status.log.enabled", globalStatusLogEnabled);
+ statusLogIntervalMS = MiscUtil.getLongProperty(props,
basePropertyName + ".status.log.interval.sec", globalStatusLogIntervalSec) *
1000;
+
+ nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
+
+ LOG.info(AUDIT_LOG_STATUS_LOG_ENABLED + "=" +
globalStatusLogEnabled);
+ LOG.info(AUDIT_LOG_STATUS_LOG_INTERVAL_SEC + "=" +
globalStatusLogIntervalSec);
+ LOG.info(basePropertyName + ".status.log.enabled=" +
statusLogEnabled);
+ LOG.info(basePropertyName + ".status.log.interval.sec=" +
(statusLogIntervalMS / 1000));
+
String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG +
".";
for (Object propNameObj : props.keySet()) {
String propName = propNameObj.toString();
@@ -275,9 +295,10 @@ public abstract class BaseAuditHandler implements
AuditHandler {
return lastDeferredCount;
}
+ public boolean isStatusLogEnabled() { return statusLogEnabled; }
+
public void logStatusIfRequired() {
- long currTime = System.currentTimeMillis();
- if ((currTime - lastStatusLogTime) > statusLogIntervalMS) {
+ if (System.currentTimeMillis() > nextStatusLogTime) {
logStatus();
}
}
@@ -285,9 +306,10 @@ public abstract class BaseAuditHandler implements
AuditHandler {
public void logStatus() {
try {
long currTime = System.currentTimeMillis();
-
long diffTime = currTime - lastStatusLogTime;
+
lastStatusLogTime = currTime;
+ nextStatusLogTime = currTime + statusLogIntervalMS;
long diffCount = totalCount - lastIntervalCount;
long diffSuccess = totalSuccessCount -
lastIntervalSuccessCount;
@@ -306,7 +328,7 @@ public abstract class BaseAuditHandler implements
AuditHandler {
lastStashedCount = totalStashedCount;
lastDeferredCount = totalDeferredCount;
- if (LOG.isDebugEnabled()) {
+ if (statusLogEnabled) {
String finalPath = "";
String tFinalPath = getFinalPath();
if (!getName().equals(tFinalPath)) {
@@ -336,7 +358,7 @@ public abstract class BaseAuditHandler implements
AuditHandler {
: "")
+ (totalDeferredCount > 0 ? (",
totalDeferredCount=" + totalDeferredCount)
: "");
- LOG.debug(msg);
+ LOG.info(msg);
}
} catch (Throwable t) {
LOG.error("Error while printing stats. auditProvider="
+ getName());
diff --git
a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
index 68527d37d..b226b4e20 100644
---
a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
+++
b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
@@ -56,8 +56,13 @@ public class AuditAsyncQueue extends AuditQueue implements
Runnable {
*/
@Override
public boolean log(AuditEventBase event) {
+ logStatusIfRequired();
+
+ addTotalCount(1);
+
// Add to the queue and return ASAP
if (queue.size() >= getMaxQueueSize()) {
+ addFailedCount(1);
return false;
}
queue.add(event);
@@ -134,6 +139,17 @@ public class AuditAsyncQueue extends AuditQueue implements
Runnable {
}
}
+ @Override
+ public void logStatus() {
+ super.logStatus();
+
+ if (isStatusLogEnabled()) {
+ logger.info("AuditAsyncQueue.log(name={}):
totalCount={}, currentQueueLength={}", getName(), getTotalCount(),
queue.size());
+ }
+ }
+
+ public int size() { return queue.size(); }
+
public void runLogAudit() {
while (true) {
try {
@@ -150,6 +166,8 @@ public class AuditAsyncQueue extends AuditQueue implements
Runnable {
eventList.add(event);
queue.drainTo(eventList, MAX_DRAIN - 1);
consumer.log(eventList);
+
+ logStatusIfRequired();
}
} catch (InterruptedException e) {
logger.info("Caught exception in consumer
thread. Shutdown might be in progress");
diff --git
a/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java
b/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java
index 579485663..09386d231 100644
--- a/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java
+++ b/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java
@@ -19,9 +19,7 @@
package org.apache.ranger.audit;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.List;
import java.util.Properties;
import org.apache.ranger.audit.destination.AuditDestination;
@@ -37,11 +35,11 @@ public class TestConsumer extends AuditDestination {
int countTotal = 0;
int sumTotal = 0;
int batchCount = 0;
+ AuthzAuditEvent lastEvent = null;
+ AuthzAuditEvent lastOutOfSeqEvent = null;
String providerName = getClass().getName();
boolean isDown = false;
- List<AuthzAuditEvent> eventList = new ArrayList<AuthzAuditEvent>();
-
/*
* (non-Javadoc)
*
@@ -58,7 +56,8 @@ public class TestConsumer extends AuditDestination {
AuthzAuditEvent azEvent = (AuthzAuditEvent) event;
sumTotal += azEvent.getEventCount();
logger.info("EVENT:" + event);
- eventList.add(azEvent);
+
+ processEvent(azEvent);
}
return true;
}
@@ -85,7 +84,7 @@ public class TestConsumer extends AuditDestination {
AuthzAuditEvent.class);
sumTotal += event.getEventCount();
logger.info("JSON:" + jsonStr);
- eventList.add(event);
+ processEvent(event);
return true;
}
@@ -198,13 +197,16 @@ public class TestConsumer extends AuditDestination {
// Local methods
public AuthzAuditEvent isInSequence() {
- long lastSeq = -1;
- for (AuthzAuditEvent event : eventList) {
- if (event.getSeqNum() <= lastSeq) {
- return event;
+ return lastOutOfSeqEvent;
+ }
+
+ private void processEvent(AuthzAuditEvent azEvent) {
+ if (lastEvent == null) {
+ lastEvent = azEvent;
+ } else if (lastOutOfSeqEvent == null) {
+ if (azEvent.getSeqNum() <= lastEvent.getSeqNum()) {
+ lastOutOfSeqEvent = azEvent;
}
- lastSeq = event.getSeqNum();
}
- return null;
}
}
