This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new e76101d28 RANGER-4379: Assorted debugging help : save policy-cache at
Ranger-admin and policy-cache as well as downloaded policy-deltas on plugin side
e76101d28 is described below
commit e76101d28b001217f81ffcbd0647714a07fe68c3
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Mon Sep 25 07:59:44 2023 -0700
RANGER-4379: Assorted debugging help : save policy-cache at Ranger-admin
and policy-cache as well as downloaded policy-deltas on plugin side
---
.../plugin/policyengine/RangerResourceTrie.java | 12 ++--
.../ranger/plugin/service/RangerBasePlugin.java | 7 ++-
.../apache/ranger/plugin/util/PolicyRefresher.java | 66 +++++++++++++++++++---
.../ranger/common/RangerServicePoliciesCache.java | 44 +++++++++++++++
4 files changed, 116 insertions(+), 13 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
index 647059203..2f725036d 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
@@ -1305,14 +1305,14 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
void toString(StringBuilder sb) {
String nodeValue = this.str;
- sb.append("nodeValue=").append(nodeValue);
+ sb.append("nodeValue=").append(nodeValue == null ? "ROOT" :
nodeValue);
sb.append("; isSetup=").append(isSetup);
sb.append(";
isSharingParentWildcardEvaluators=").append(isSharingParentWildcardEvaluators);
sb.append("; childCount=").append(children.size());
- sb.append("; evaluators=[ ");
+ sb.append("; evaluators=[");
if (evaluators != null) {
for (U evaluator : evaluators) {
- sb.append(evaluator.getId()).append(" ");
+ sb.append(evaluator.getId()).append(",");
}
}
sb.append("]");
@@ -1320,7 +1320,7 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
sb.append("; wildcardEvaluators=[ ");
if (wildcardEvaluators != null) {
for (U evaluator : wildcardEvaluators) {
- sb.append(evaluator.getId()).append(" ");
+ sb.append(evaluator.getId()).append(",");
}
}
sb.append("]");
@@ -1329,6 +1329,10 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
void toString(String prefix, StringBuilder sb) {
String nodeValue = prefix + (str != null ? str : "");
+ if (!nodeValue.equals(prefix)) {
+ prefix = prefix + "|";
+ }
+
sb.append(prefix);
toString(sb);
sb.append("]\n");
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index f1eb08e4e..2f4af9763 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -410,7 +410,9 @@ public class RangerBasePlugin {
newPolicyEngine.setTrustedProxyAddresses(pluginConfig.getTrustedProxyAddresses());
}
+ LOG.info("Switching policy engine from
[" + getPolicyVersion() + "]");
this.policyEngine =
newPolicyEngine;
+ LOG.info("Switched policy engine to ["
+ getPolicyVersion() + "]");
this.currentAuthContext =
pluginContext.getAuthContext();
pluginContext.notifyAuthContextChanged();
@@ -516,7 +518,6 @@ public class RangerBasePlugin {
if (resultProcessor != null) {
resultProcessor.processResult(ret);
}
-
return ret;
}
@@ -1327,4 +1328,8 @@ public class RangerBasePlugin {
return ret;
}
+
+ public Long getPolicyVersion() {
+ return this.policyEngine == null ? -1L :
this.policyEngine.getPolicyVersion();
+ }
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
index c130309ea..aa0c80119 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java
@@ -20,10 +20,14 @@
package org.apache.ranger.plugin.util;
import java.io.File;
+import java.io.FileFilter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.Reader;
import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
import java.util.Timer;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -449,14 +453,15 @@ public class PolicyRefresher extends Thread {
} catch (Exception excp) {
LOG.error("failed to save policies to cache
file '" + cacheFile.getAbsolutePath() + "'", excp);
} finally {
- if(writer != null) {
- try {
- writer.close();
- } catch(Exception excp) {
- LOG.error("error while closing
opened cache file '" + cacheFile.getAbsolutePath() + "'", excp);
- }
- }
- }
+ if (writer != null) {
+ try {
+ writer.close();
+
deleteOldestVersionCacheFileInCacheDirectory(cacheFile.getParentFile());
+ } catch (Exception excp) {
+ LOG.error("error while
closing opened cache file '" + cacheFile.getAbsolutePath() + "'", excp);
+ }
+ }
+ }
RangerPerfTracer.log(perf);
@@ -490,6 +495,51 @@ public class PolicyRefresher extends Thread {
}
}
+ private void deleteOldestVersionCacheFileInCacheDirectory(File
cacheDirectory) {
+ int maxVersionsToPreserve =
plugIn.getConfig().getInt(plugIn.getConfig().getPropertyPrefix() +
"max.versions.to.preserve", 1);
+ FileFilter logFileFilter = (file) ->
file.getName().matches(".+json_.+");
+
+ File[] filesInParent = cacheDirectory.listFiles(logFileFilter);
+ List<Long> policyVersions = new ArrayList<>();
+
+ if (filesInParent != null && filesInParent.length > 0) {
+ for (File f : filesInParent) {
+ String fileName = f.getName();
+ // Extract the part after json_
+ int policyVersionIdx =
fileName.lastIndexOf("json_");
+ String policyVersionStr =
fileName.substring(policyVersionIdx + 5);
+ Long policyVersion =
Long.valueOf(policyVersionStr);
+ policyVersions.add(policyVersion);
+ }
+ } else {
+ LOG.info("No files matching '.+json_*' found");
+ }
+
+ if (!policyVersions.isEmpty()) {
+ policyVersions.sort(new Comparator<Long>() {
+ @Override
+ public int compare(Long o1, Long o2) {
+ if (o1.equals(o2)) return 0;
+ return o1 < o2 ? -1 : 1;
+ }
+ });
+ }
+
+ if (policyVersions.size() > maxVersionsToPreserve) {
+ String fileName = this.cacheFileName + "_" +
Long.toString(policyVersions.get(0));
+ String pathName = cacheDirectory.getAbsolutePath() +
File.separator + fileName;
+ File toDelete = new File(pathName);
+ if (toDelete.exists()) {
+ boolean isDeleted = toDelete.delete();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("file :[" + pathName + "] is
deleted");
+ }
+ } else {
+ LOG.info("File: " + pathName + " does not
exist!");
+ }
+ }
+ }
+
private void disableCache() {
if (LOG.isDebugEnabled()) {
LOG.debug("==>
PolicyRefresher.disableCache(serviceName=" + serviceName + ")");
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
index 9fda659ac..c768f1c34 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
@@ -33,9 +33,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.FileFilter;
import java.io.FileWriter;
import java.io.Writer;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -203,6 +205,7 @@ public class RangerServicePoliciesCache {
if (policies != null) {
RangerAdminConfig config =
RangerAdminConfig.getInstance();
boolean doSaveToDisk =
config.getBoolean("ranger.admin.policy.save.to.disk", false);
+ int maxVersionsToSaveToDisk =
config.getInt("ranger.admin.policy.max.versions.to.save.to.disk", 1);
if (doSaveToDisk) {
File cacheFile = null;
@@ -237,6 +240,47 @@ public class RangerServicePoliciesCache {
} catch (Exception excp) {
LOG.error("failed to save
policies to cache file '" + cacheFile.getAbsolutePath() + "'", excp);
}
+ String serviceDefName =
policies.getServiceDef().getName();
+ String serviceName =
policies.getServiceName();
+
+ File parentFile =
cacheFile.getParentFile();
+ FileFilter logFileFilter = (file) ->
file.getName().matches(serviceDefName +"_.+json_.+");
+ File[] filesInParent =
parentFile.listFiles(logFileFilter);
+ List<Long> policyVersions = new
ArrayList<>();
+ if (filesInParent != null &&
filesInParent.length > 0) {
+ for (File f : filesInParent) {
+ String fileName =
f.getName();
+ // Extract the part
after json_
+ int policyVersionIdx =
fileName.lastIndexOf("json_");
+ String policyVersionStr
= fileName.substring(policyVersionIdx + 5);
+ Long policyVersion =
Long.valueOf(policyVersionStr);
+
policyVersions.add(policyVersion);
+ }
+ } else {
+ LOG.info("No files matching '"
+ serviceDefName + "_.+json_*' found");
+ }
+ if (!policyVersions.isEmpty()) {
+ policyVersions.sort(new
Comparator<Long>() {
+ @Override
+ public int compare(Long
o1, Long o2) {
+ if
(o1.equals(o2)) return 0;
+ return o1 < o2
? -1 : 1;
+ }
+ });
+ }
+
+ if (policyVersions.size() >
maxVersionsToSaveToDisk) {
+ String fileName =
serviceDefName + "_" + serviceName + ".json_" +
Long.toString(policyVersions.get(0));
+ String pathName =
parentFile.getAbsolutePath() + File.separator + fileName;
+ File toDelete = new
File(pathName);
+ if (toDelete.exists()) {
+ //LOG.info("Deleting
file :[" + pathName + "]");
+ boolean isDeleted =
toDelete.delete();
+ //LOG.info("file :[" +
pathName + "] is deleted");
+ } else {
+ LOG.info("File: " +
pathName + " does not exist!");
+ }
+ }
}
}
} else {
