This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 04c93b3df RANGER-4565: Enhance Ranger's performance tracing module to
optionally collect statistical information
04c93b3df is described below
commit 04c93b3df9577c7f6e4f91f573a87c046311e15c
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Tue Nov 28 09:47:03 2023 -0800
RANGER-4565: Enhance Ranger's performance tracing module to optionally
collect statistical information
---
.../ranger/plugin/service/RangerBasePlugin.java | 6 ++
.../ranger/plugin/util/PerfDataRecorder.java | 77 +++++++++++++++++-----
.../plugin/util/RangerPerfCollectorTracer.java | 2 +-
.../ranger/plugin/util/RangerPerfTracer.java | 4 +-
4 files changed, 71 insertions(+), 18 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index 2f4af9763..5d6c3d97c 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -86,6 +86,12 @@ public class RangerBasePlugin {
this.pluginConfig = pluginConfig;
this.pluginContext = new RangerPluginContext(pluginConfig);
+ boolean usePerfDataRecorder =
pluginConfig.getBoolean("ranger.perf.aggregate.data", false);
+ int perfDataDumpInterval =
pluginConfig.getInt("ranger.perf.aggregate.data.dump.interval", 0);
+ boolean usePerfDataLock =
pluginConfig.getBoolean("ranger.perf.aggregate.data.lock.enabled", false);
+
+ PerfDataRecorder.initialize(usePerfDataRecorder,
perfDataDumpInterval, usePerfDataLock, null);
+
Set<String> superUsers =
toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.users"));
Set<String> superGroups =
toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() + ".super.groups"));
Set<String> auditExcludeUsers =
toSet(pluginConfig.get(pluginConfig.getPropertyPrefix() +
".audit.exclude.users"));
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
index dce60b0ba..a1df53fac 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
@@ -37,13 +37,27 @@ public class PerfDataRecorder {
private static final Logger PERF =
RangerPerfTracer.getPerfLogger(PerfDataRecorder.class);
private static volatile PerfDataRecorder instance;
- private Map<String, PerfStatistic> perfStatistics = new HashMap<>();
+ final private Map<String, PerfStatistic> perfStatistics =
Collections.synchronizedMap(new HashMap<>());
+ private RangerReadWriteLock lock = null;
public static void initialize(List<String> names) {
- if (instance == null) {
- synchronized (PerfDataRecorder.class) {
- if (instance == null) {
- instance = new PerfDataRecorder(names);
+ initialize(true, 0, false, names);
+ }
+
+ public static void initialize(final boolean useRecorder, final int
collectionIntervalInSeconds, final boolean usePerfDataLock, List<String> names)
{
+ if (useRecorder) {
+ if (instance == null) {
+ synchronized (PerfDataRecorder.class) {
+ if (instance == null) {
+ instance = new
PerfDataRecorder(names);
+ instance.lock = new
RangerReadWriteLock(usePerfDataLock);
+ if (collectionIntervalInSeconds
> 0) {
+ Thread statDumper = new
StatisticsDumper(collectionIntervalInSeconds);
+
statDumper.setName("Perf-Statistics-Dumper");
+
statDumper.setDaemon(true);
+ statDumper.start();
+ }
+ }
}
}
}
@@ -61,7 +75,9 @@ public class PerfDataRecorder {
public static void clearStatistics() {
if (instance != null) {
- instance.clear();
+ try (RangerReadWriteLock.RangerLock writeLock =
instance.lock.getWriteLock()) {
+ instance.clear();
+ }
}
}
@@ -72,7 +88,11 @@ public class PerfDataRecorder {
}
private void dumpStatistics() {
- List<String> tags = new ArrayList<>(perfStatistics.keySet());
+ List<String> tags;
+
+ try (RangerReadWriteLock.RangerLock readLock =
lock.getReadLock()) {
+ tags = new ArrayList<>(perfStatistics.keySet());
+ }
Collections.sort(tags);
@@ -111,20 +131,24 @@ public class PerfDataRecorder {
}
private void record(String tag, long cpuTime, long elapsedTime) {
- PerfStatistic perfStatistic = perfStatistics.get(tag);
+ try (RangerReadWriteLock.RangerLock writeLock =
lock.getWriteLock()) {
+
+ PerfStatistic perfStatistic = perfStatistics.get(tag);
- if (perfStatistic == null) {
- synchronized (PerfDataRecorder.class) {
- perfStatistic = perfStatistics.get(tag);
+ if (perfStatistic == null) {
+ synchronized (PerfDataRecorder.class) {
+ perfStatistic = perfStatistics.get(tag);
- if(perfStatistic == null) {
- perfStatistic = new PerfStatistic();
- perfStatistics.put(tag, perfStatistic);
+ if (perfStatistic == null) {
+ perfStatistic = new
PerfStatistic();
+ perfStatistics.put(tag,
perfStatistic);
+ }
}
}
- }
- perfStatistic.addPerfDataItem(cpuTime, elapsedTime);
+ perfStatistic.addPerfDataItem(cpuTime, elapsedTime);
+
+ }
}
private PerfDataRecorder(List<String> names) {
@@ -206,4 +230,25 @@ public class PerfDataRecorder {
return maxTimeSpent.get();
}
}
+
+ private static class StatisticsDumper extends Thread {
+ final int collectionIntervalInSeconds;
+ StatisticsDumper(final int collectionIntervalInSeconds) {
+ this.collectionIntervalInSeconds =
collectionIntervalInSeconds;
+ }
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ sleep(collectionIntervalInSeconds *
1000);
+ printStatistics();
+ clearStatistics();
+ } catch (InterruptedException exception) {
+ printStatistics();
+ LOG.warn("Thread[" + this.getName() +
"] was interrupted. Returning from thread. Performance statistics will NOT be
dumped periodically!!");
+ break;
+ }
+ }
+ }
+ }
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
index ebfebdae8..bc27ab24a 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfCollectorTracer.java
@@ -37,7 +37,7 @@ public class RangerPerfCollectorTracer extends
RangerPerfTracer {
long reportingThreshold = threadInfo == null ? 0L :
(1000000/1000 - 1); // just about a microsecond
if (elapsedTime > reportingThreshold) {
- logger.debug("[PERF]:" + (threadInfo != null ?
threadInfo.getThreadName() : "") + ":" + tag + data + ":" + getElapsedCpuTime()
+ ":" + getElapsedUserTime());
+ PerfDataRecorder.recordStatistic(tag,
(getElapsedCpuTime()+500)/1000, (getElapsedUserTime() + 500)/1000);
}
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
index c8804739f..040b340e3 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
@@ -55,7 +55,9 @@ public class RangerPerfTracer {
int indexOfTagEndMarker = StringUtils.indexOf(tag,
tagEndMarker);
if (indexOfTagEndMarker != -1) {
realTag = StringUtils.substring(tag, 0,
indexOfTagEndMarker);
- data = StringUtils.substring(tag,
indexOfTagEndMarker);
+ if (!PerfDataRecorder.collectStatistics()) {
+ data = StringUtils.substring(tag,
indexOfTagEndMarker);
+ }
} else {
realTag = tag;
}
