This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch ranger-2.4
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.4 by this push:
new 0f930c905 RANGER-4100: Efficient computation of the smallest set of
evaluators returned by search of multiple Trie trees - #2
0f930c905 is described below
commit 0f930c9053d080555544f5e515059e8d292cc22d
Author: Madhan Neethiraj <[email protected]>
AuthorDate: Wed Feb 22 18:59:47 2023 -0800
RANGER-4100: Efficient computation of the smallest set of evaluators
returned by search of multiple Trie trees - #2
(cherry picked from commit 93e888f7c0d12bd18426a49719fa996f8874d162)
---
.../policyengine/RangerPolicyRepository.java | 153 +++++++--------------
.../plugin/policyengine/RangerResourceTrie.java | 4 +-
.../plugin/policyengine/TestPolicyEngine.java | 6 +-
3 files changed, 52 insertions(+), 111 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 85a3afd01..fce2b1052 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -42,6 +42,7 @@ import
org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import
org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator.RangerPolicyResourceEvaluator;
import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.util.RangerPerfTracer;
+import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever;
import org.apache.ranger.plugin.util.ServiceDefUtil;
import org.apache.ranger.plugin.util.ServicePolicies;
import org.slf4j.Logger;
@@ -72,30 +73,30 @@ public class RangerPolicyRepository {
AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT
}
- private final String serviceName;
- private final String zoneName;
- private final String appId;
- private final RangerPolicyEngineOptions options;
- private final RangerPluginContext pluginContext;
- private final RangerServiceDef serviceDef;
- private /*final*/ List<RangerPolicy> policies;
- private final long policyVersion;
- private /*final*/ List<RangerContextEnricher> contextEnrichers;
- private final AuditModeEnum auditModeEnum;
- private final Map<String, AuditInfo> accessAuditCache;
- private final String componentServiceName;
- private final RangerServiceDef componentServiceDef;
- private final Map<String, RangerResourceTrie> policyResourceTrie;
- private final Map<String, RangerResourceTrie> dataMaskResourceTrie;
- private final Map<String, RangerResourceTrie> rowFilterResourceTrie;
- private final Map<String, RangerResourceTrie> auditFilterResourceTrie;
- private List<RangerPolicyEvaluator> policyEvaluators;
- private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
- private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
- private final List<RangerPolicyEvaluator> auditPolicyEvaluators;
- private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
- private boolean isContextEnrichersShared =
false;
- private boolean isPreCleaned =
false;
+ private final String
serviceName;
+ private final String
zoneName;
+ private final String
appId;
+ private final RangerPolicyEngineOptions
options;
+ private final RangerPluginContext
pluginContext;
+ private final RangerServiceDef
serviceDef;
+ private /*final*/ List<RangerPolicy>
policies;
+ private final long
policyVersion;
+ private /*final*/ List<RangerContextEnricher>
contextEnrichers;
+ private final AuditModeEnum
auditModeEnum;
+ private final Map<String, AuditInfo>
accessAuditCache;
+ private final String
componentServiceName;
+ private final RangerServiceDef
componentServiceDef;
+ private final Map<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> policyResourceTrie;
+ private final Map<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> dataMaskResourceTrie;
+ private final Map<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> rowFilterResourceTrie;
+ private final Map<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> auditFilterResourceTrie;
+ private List<RangerPolicyEvaluator>
policyEvaluators;
+ private List<RangerPolicyEvaluator>
dataMaskPolicyEvaluators;
+ private List<RangerPolicyEvaluator>
rowFilterPolicyEvaluators;
+ private final List<RangerPolicyEvaluator>
auditPolicyEvaluators;
+ private Map<Long, RangerPolicyEvaluator>
policyEvaluatorsMap;
+ private boolean
isContextEnrichersShared = false;
+ private boolean
isPreCleaned = false;
RangerPolicyRepository(final RangerPolicyRepository other, final
List<RangerPolicyDelta> deltas, long policyVersion) {
this.serviceName = other.serviceName;
@@ -117,8 +118,8 @@ public class RangerPolicyRepository {
if (other.policyResourceTrie != null) {
this.policyResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry :
other.policyResourceTrie.entrySet()) {
- policyResourceTrie.put(entry.getKey(), new
RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> entry :
other.policyResourceTrie.entrySet()) {
+ policyResourceTrie.put(entry.getKey(), new
RangerResourceTrie<>(entry.getValue()));
}
} else {
this.policyResourceTrie = null;
@@ -127,8 +128,8 @@ public class RangerPolicyRepository {
if (other.dataMaskResourceTrie != null) {
this.dataMaskResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry :
other.dataMaskResourceTrie.entrySet()) {
- dataMaskResourceTrie.put(entry.getKey(), new
RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> entry :
other.dataMaskResourceTrie.entrySet()) {
+ dataMaskResourceTrie.put(entry.getKey(), new
RangerResourceTrie<>(entry.getValue()));
}
} else {
this.dataMaskResourceTrie = null;
@@ -137,8 +138,8 @@ public class RangerPolicyRepository {
if (other.rowFilterResourceTrie != null) {
this.rowFilterResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry :
other.rowFilterResourceTrie.entrySet()) {
- rowFilterResourceTrie.put(entry.getKey(), new
RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> entry :
other.rowFilterResourceTrie.entrySet()) {
+ rowFilterResourceTrie.put(entry.getKey(), new
RangerResourceTrie<>(entry.getValue()));
}
} else {
this.rowFilterResourceTrie = null;
@@ -147,8 +148,8 @@ public class RangerPolicyRepository {
if (other.auditFilterResourceTrie != null) {
this.auditFilterResourceTrie = new HashMap<>();
- for (Map.Entry<String, RangerResourceTrie> entry :
other.auditFilterResourceTrie.entrySet()) {
- auditFilterResourceTrie.put(entry.getKey(), new
RangerResourceTrie(entry.getValue()));
+ for (Map.Entry<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> entry :
other.auditFilterResourceTrie.entrySet()) {
+ auditFilterResourceTrie.put(entry.getKey(), new
RangerResourceTrie<>(entry.getValue()));
}
} else {
this.auditFilterResourceTrie = null;
@@ -157,7 +158,7 @@ public class RangerPolicyRepository {
if (other.accessAuditCache != null) {
int auditResultCacheSize = other.accessAuditCache.size();
- this.accessAuditCache = Collections.synchronizedMap(new
CacheMap<String, AuditInfo>(auditResultCacheSize));
+ this.accessAuditCache = Collections.synchronizedMap(new
CacheMap<>(auditResultCacheSize));
} else {
this.accessAuditCache = null;
}
@@ -217,7 +218,7 @@ public class RangerPolicyRepository {
final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 *
1024;
int auditResultCacheSize =
pluginContext.getConfig().getInt(propertyName,
RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE);
- accessAuditCache = Collections.synchronizedMap(new
CacheMap<String, AuditInfo>(auditResultCacheSize));
+ accessAuditCache = Collections.synchronizedMap(new
CacheMap<>(auditResultCacheSize));
} else {
accessAuditCache = null;
}
@@ -587,7 +588,7 @@ public class RangerPolicyRepository {
if (CollectionUtils.isNotEmpty(tags) && getServiceDef() != null) {
- ret = new ArrayList<PolicyEvaluatorForTag>();
+ ret = new ArrayList<>();
for (RangerTagForEval tag : tags) {
if (tag.isApplicable(accessTime)) {
@@ -691,7 +692,7 @@ public class RangerPolicyRepository {
return auditFilterResourceTrie == null ||
StringUtils.isEmpty(resourceStr) ? getAuditPolicyEvaluators() :
getLikelyMatchPolicyEvaluators(auditFilterResourceTrie, request);
}
- private List<RangerPolicyEvaluator>
getLikelyMatchPolicyEvaluators(Map<String, RangerResourceTrie> resourceTrie,
RangerAccessRequest request) {
+ private List<RangerPolicyEvaluator>
getLikelyMatchPolicyEvaluators(Map<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> resourceTrie,
RangerAccessRequest request) {
List<RangerPolicyEvaluator> ret = Collections.EMPTY_LIST;
RangerAccessResource resource = request.getResource();
@@ -702,67 +703,7 @@ public class RangerPolicyRepository {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG,
"RangerPolicyRepository.getLikelyMatchEvaluators(resource=" +
resource.getAsString() + ")");
}
- List<String> resourceKeys = resource == null ?
null :
options.getServiceDefHelper().getOrderedResourceNames(resource.getKeys());
- Set<RangerPolicyResourceEvaluator> smallestList = null;
-
- if (CollectionUtils.isNotEmpty(resourceKeys)) {
-
- for (String resourceName : resourceKeys) {
- RangerResourceTrie<RangerPolicyResourceEvaluator> trie =
resourceTrie.get(resourceName);
-
- if (trie == null) { // if no trie exists for this resource
level, ignore and continue to next level
- continue;
- }
-
- Set<RangerPolicyResourceEvaluator>
serviceResourceMatchersForResource =
trie.getEvaluatorsForResource(resource.getValue(resourceName),
request.getResourceMatchingScope());
- Set<RangerPolicyResourceEvaluator> inheritedResourceMatchers
= trie.getInheritedEvaluators();
-
- if (smallestList != null) {
- if (CollectionUtils.isEmpty(inheritedResourceMatchers) &&
CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
- smallestList = null;
- } else if
(CollectionUtils.isEmpty(inheritedResourceMatchers)) {
-
smallestList.retainAll(serviceResourceMatchersForResource);
- } else if
(CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
- smallestList.retainAll(inheritedResourceMatchers);
- } else {
- Set<RangerPolicyResourceEvaluator> smaller, bigger;
- if (serviceResourceMatchersForResource.size() <
inheritedResourceMatchers.size()) {
- smaller = serviceResourceMatchersForResource;
- bigger = inheritedResourceMatchers;
- } else {
- smaller = inheritedResourceMatchers;
- bigger = serviceResourceMatchersForResource;
- }
- Set<RangerPolicyResourceEvaluator> tmp = new
HashSet<>();
- if (smallestList.size() < smaller.size()) {
-
smallestList.stream().filter(smaller::contains).forEach(tmp::add);
-
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
- } else {
-
smaller.stream().filter(smallestList::contains).forEach(tmp::add);
- if (smallestList.size() < bigger.size()) {
-
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
- } else {
-
bigger.stream().filter(smallestList::contains).forEach(tmp::add);
- }
- }
- smallestList = tmp;
- }
- } else {
- if (CollectionUtils.isEmpty(inheritedResourceMatchers) ||
CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
- Set<RangerPolicyResourceEvaluator> tmp =
CollectionUtils.isEmpty(inheritedResourceMatchers) ?
serviceResourceMatchersForResource : inheritedResourceMatchers;
- smallestList = resourceKeys.size() == 1 ||
CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
- } else {
- smallestList = new
HashSet<>(serviceResourceMatchersForResource);
- smallestList.addAll(inheritedResourceMatchers);
- }
- }
-
- if (CollectionUtils.isEmpty(smallestList)) {// no tags for
this resource, bail out
- smallestList = null;
- break;
- }
- }
- }
+ Collection<RangerPolicyResourceEvaluator> smallestList =
RangerResourceEvaluatorsRetriever.getEvaluators(resourceTrie,
resource.getAsMap(), request.getResourceMatchingScope());
if (smallestList != null) {
if (smallestList.size() == 0) {
@@ -1224,8 +1165,8 @@ public class RangerPolicyRepository {
return ret;
}
- private Map<String, RangerResourceTrie> createResourceTrieMap(List<?
extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval,
boolean optimizeTrieForSpace) {
- final Map<String, RangerResourceTrie> ret;
+ private Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>>
createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean
optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
+ final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>>
ret;
if (serviceDef != null &&
CollectionUtils.isNotEmpty(serviceDef.getResources())) {
ret = new HashMap<>();
@@ -1240,7 +1181,7 @@ public class RangerPolicyRepository {
return ret;
}
- private void updateTrie(Map<String, RangerResourceTrie> trieMap, Integer
policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator
newEvaluator) {
+ private void updateTrie(Map<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap, Integer
policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator
newEvaluator) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyRepository.updateTrie(policyDeltaType="
+ policyDeltaType + "): ");
}
@@ -1436,7 +1377,7 @@ public class RangerPolicyRepository {
break;
}
- Map<String, RangerResourceTrie> trieMap = getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap
= getTrie(policyType);
if (trieMap != null) {
updateTrie(trieMap, changeType, currentEvaluator, newEvaluator);
@@ -1457,8 +1398,8 @@ public class RangerPolicyRepository {
return ret;
}
- Map<String, RangerResourceTrie> getTrie(final int policyType) {
- final Map<String, RangerResourceTrie> ret;
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>>
getTrie(final int policyType) {
+ final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>>
ret;
switch (policyType) {
case RangerPolicy.POLICY_TYPE_ACCESS:
ret = policyResourceTrie;
@@ -1583,10 +1524,10 @@ public class RangerPolicyRepository {
for (int policyType = 0; policyType < flags.length; policyType++) {
if (flags[policyType]) {
- Map<String, RangerResourceTrie> trie = getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>>
trie = getTrie(policyType);
if (trie != null) {
- for (Map.Entry<String, RangerResourceTrie> entry :
trie.entrySet()) {
+ for (Map.Entry<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : trie.entrySet()) {
entry.getValue().wrapUpUpdate();
}
}
@@ -1594,7 +1535,7 @@ public class RangerPolicyRepository {
}
if (auditFilterResourceTrie != null) {
- for (Map.Entry<String, RangerResourceTrie> entry :
auditFilterResourceTrie.entrySet()) {
+ for (Map.Entry<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> entry :
auditFilterResourceTrie.entrySet()) {
entry.getValue().wrapUpUpdate();
}
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
index 504acd3a2..f89d51e35 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
@@ -105,7 +105,7 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
this(resourceDef, evaluators, isOptimizedForRetrieval, false,
pluginContext);
}
- public RangerResourceTrie(RangerResourceDef resourceDef, List<T>
evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace,
RangerPluginContext pluginContext) {
+ public <T extends RangerResourceEvaluator, E>
RangerResourceTrie(RangerResourceDef resourceDef, List<E> evaluators, boolean
isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext
pluginContext) {
if(LOG.isDebugEnabled()) {
LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ",
evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" +
isOptimizedForRetrieval + ", isOptimizedForSpace=" + isOptimizedForSpace + ")");
}
@@ -154,7 +154,7 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
this.isOptimizedForRetrieval = !isOptimizedForSpace &&
isOptimizedForRetrieval; // isOptimizedForSpace takes precedence
this.separatorChar =
ServiceDefUtil.getCharOption(matcherOptions, OPTION_PATH_SEPARATOR,
DEFAULT_PATH_SEPARATOR_CHAR);
- TrieNode<T> tmpRoot = buildTrie(resourceDef, evaluators,
builderThreadCount);
+ final TrieNode tmpRoot = buildTrie(resourceDef, evaluators,
builderThreadCount);
if (builderThreadCount > 1 && tmpRoot == null) { // if multi-threaded
trie-creation failed, build using a single thread
this.root = buildTrie(resourceDef, evaluators, 1);
diff --git
a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
index 632d7cf7e..f4f7aaafb 100644
---
a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
+++
b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
@@ -995,13 +995,13 @@ public class TestPolicyEngine {
public static boolean compareTrie(final int policyType,
RangerPolicyRepository me, RangerPolicyRepository other) {
boolean ret;
- Map<String, RangerResourceTrie> myTrie =
me.getTrie(policyType);
- Map<String, RangerResourceTrie> otherTrie =
other.getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>>
myTrie = me.getTrie(policyType);
+ Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>>
otherTrie = other.getTrie(policyType);
ret = myTrie.size() == otherTrie.size();
if (ret) {
- for (Map.Entry<String, RangerResourceTrie> entry :
myTrie.entrySet()) {
+ for (Map.Entry<String,
RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : myTrie.entrySet()) {
RangerResourceTrie myResourceTrie =
entry.getValue();
RangerResourceTrie otherResourceTrie =
otherTrie.get(entry.getKey());
