This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new c0480ed72 RANGER-4905:Reduce memory needed to create Ranger policy
engine
c0480ed72 is described below
commit c0480ed72656d56cf2e5ab03bbea37e372363081
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Thu Aug 15 09:57:56 2024 -0700
RANGER-4905:Reduce memory needed to create Ranger policy engine
---
.../plugin/contextenricher/RangerTagEnricher.java | 13 ++--
.../validation/RangerSecurityZoneValidator.java | 2 +-
.../validation/RangerZoneResourceMatcher.java | 8 ++-
.../ranger/plugin/policyengine/PolicyEngine.java | 1 +
.../plugin/policyengine/RangerPluginContext.java | 75 ++++++++++++++++++++--
.../policyengine/RangerPolicyEngineOptions.java | 10 +++
.../policyengine/RangerSecurityZoneMatcher.java | 2 +-
.../policyengine/gds/GdsDataShareEvaluator.java | 5 +-
.../plugin/policyengine/gds/GdsPolicyEngine.java | 2 +-
.../gds/GdsSharedResourceEvaluator.java | 8 ++-
.../RangerAbstractPolicyEvaluator.java | 1 +
.../RangerDefaultPolicyResourceMatcher.java | 51 ++++++++++-----
.../RangerPolicyResourceMatcher.java | 3 +
.../org/apache/ranger/sizing/RangerMemSizing.java | 22 ++++---
14 files changed, 155 insertions(+), 48 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 2fa24eba6..a8fbc0215 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -29,11 +29,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
-import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
-import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
-import org.apache.ranger.plugin.policyengine.RangerAccessResource;
-import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
-import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
+import org.apache.ranger.plugin.policyengine.*;
import
org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
import
org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
import org.apache.ranger.plugin.util.DownloadTrigger;
@@ -437,7 +433,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
for (ListIterator<RangerServiceResource> iter =
serviceResources.listIterator(); iter.hasNext(); ) {
RangerServiceResource serviceResource
= iter.next();
- RangerServiceResourceMatcher
serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource,
serviceDefHelper, hierarchies);
+ RangerServiceResourceMatcher
serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource,
serviceDefHelper, hierarchies, getPluginContext());
if (serviceResourceMatcher != null) {
resourceMatchers.add(serviceResourceMatcher);
@@ -484,7 +480,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
if (removedOldServiceResource) {
if
(!StringUtils.isEmpty(serviceResource.getResourceSignature())) {
- RangerServiceResourceMatcher
resourceMatcher = createRangerServiceResourceMatcher(serviceResource,
serviceDefHelper, hierarchies);
+ RangerServiceResourceMatcher
resourceMatcher = createRangerServiceResourceMatcher(serviceResource,
serviceDefHelper, hierarchies, getPluginContext());
if (resourceMatcher != null) {
for
(RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
@@ -613,7 +609,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
return ret;
}
- static public RangerServiceResourceMatcher
createRangerServiceResourceMatcher(RangerServiceResource serviceResource,
RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies) {
+ static public RangerServiceResourceMatcher
createRangerServiceResourceMatcher(RangerServiceResource serviceResource,
RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies,
RangerPluginContext pluginContext) {
if (LOG.isDebugEnabled()) {
LOG.debug("==>
createRangerServiceResourceMatcher(serviceResource=" + serviceResource + ")");
@@ -644,6 +640,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
matcher.setServiceDef(serviceDefHelper.getServiceDef());
matcher.setPolicyResources(serviceResource.getResourceElements(), policyType);
+ matcher.setPluginContext(pluginContext);
if (LOG.isDebugEnabled()) {
LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with
(resource=" + serviceResource
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
index 360426198..b88ac21a6 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
@@ -354,7 +354,7 @@ public class RangerSecurityZoneValidator extends
RangerValidator {
policyResources.put(resourceDefName, policyResource);
}
- RangerZoneResourceMatcher matcher = new
RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDefHelper);
+ RangerZoneResourceMatcher matcher = new
RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDefHelper,
null);
matchers.add(matcher);
resourceNames.addAll(policyResources.keySet());
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
index bf4247660..1a8a867a0 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.model.validation;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import
org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
import
org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
import org.apache.ranger.plugin.policyresourcematcher.RangerResourceEvaluator;
@@ -41,17 +42,18 @@ public class RangerZoneResourceMatcher implements
RangerResourceEvaluator {
private final RangerPolicyResourceMatcher
policyResourceMatcher;
private RangerServiceDef.RangerResourceDef
leafResourceDef;
- public RangerZoneResourceMatcher(final String securityZoneName, final
Map<String, RangerPolicy.RangerPolicyResource> policyResource, final
RangerServiceDef serviceDef) {
- this(securityZoneName, policyResource, new
RangerServiceDefHelper(serviceDef));
+ public RangerZoneResourceMatcher(final String securityZoneName, final
Map<String, RangerPolicy.RangerPolicyResource> policyResource, final
RangerServiceDef serviceDef, RangerPluginContext pluginContext) {
+ this(securityZoneName, policyResource, new
RangerServiceDefHelper(serviceDef), pluginContext);
}
- public RangerZoneResourceMatcher(final String securityZoneName, final
Map<String, RangerPolicy.RangerPolicyResource> policyResource, final
RangerServiceDefHelper serviceDefHelper) {
+ public RangerZoneResourceMatcher(final String securityZoneName, final
Map<String, RangerPolicy.RangerPolicyResource> policyResource, final
RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) {
final RangerServiceDef serviceDef =
serviceDefHelper.getServiceDef();
final Collection<String> resourceKeys =
policyResource.keySet();
final RangerDefaultPolicyResourceMatcher matcher = new
RangerDefaultPolicyResourceMatcher();
matcher.setServiceDef(serviceDef);
matcher.setServiceDefHelper(serviceDefHelper);
+ matcher.setPluginContext(pluginContext);
boolean found = false;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
index 704434b8e..2de3cfa0d 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
@@ -197,6 +197,7 @@ public class PolicyEngine {
}
normalizeServiceDefs(servicePolicies);
+ pluginContext.cleanResourceMatchers();
this.pluginContext = pluginContext;
this.lock = new RangerReadWriteLock(isUseReadWriteLock);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java
index 5f086ed49..8a3e43e48 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java
@@ -23,18 +23,26 @@ import org.apache.commons.lang.StringUtils;
import org.apache.ranger.admin.client.RangerAdminClient;
import org.apache.ranger.admin.client.RangerAdminRESTClient;
import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
import org.apache.ranger.plugin.service.RangerAuthContext;
import org.apache.ranger.plugin.service.RangerAuthContextListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
public class RangerPluginContext {
private static final Logger LOG =
LoggerFactory.getLogger(RangerPluginContext.class);
- private final RangerPluginConfig config;
- private RangerAuthContext authContext;
- private RangerAuthContextListener authContextListener;
- private RangerAdminClient adminClient;
+ private final RangerPluginConfig
config;
+ private RangerAuthContext
authContext;
+ private RangerAuthContextListener
authContextListener;
+ private RangerAdminClient
adminClient;
+ private final Map<String, Map<RangerPolicy.RangerPolicyResource,
RangerResourceMatcher>> resourceMatchers = new HashMap<>();
+ private final ReentrantReadWriteLock
lock = new ReentrantReadWriteLock(true); // fair lock
public RangerPluginContext(RangerPluginConfig config) {
@@ -53,6 +61,65 @@ public class RangerPluginContext {
public RangerAuthContext getAuthContext() { return authContext; }
+ public RangerResourceMatcher getResourceMatcher(String resourceDefName,
RangerPolicy.RangerPolicyResource resource) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getResourceMatcher(resourceDefName={},
resource={})", resourceDefName, resource);
+ }
+ RangerResourceMatcher ret = null;
+
+ try {
+ lock.readLock().lock();
+
+ Map<RangerPolicy.RangerPolicyResource,
RangerResourceMatcher> matchersForResource =
resourceMatchers.get(resourceDefName);
+
+ if (matchersForResource != null) {
+ ret = matchersForResource.get(resource);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getResourceMatcher(resourceDefName={},
resource={}) : ret={}", resourceDefName, resource, ret);
+ }
+
+ return ret;
+ }
+
+ public void setResourceMatcher(String resourceDefName,
RangerPolicy.RangerPolicyResource resource, RangerResourceMatcher matcher) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> setResourceMatcher(resourceDefName={},
resource={}, matcher={})", resourceDefName, resource, matcher);
+ }
+ if (config != null &&
config.getPolicyEngineOptions().enableResourceMatcherReuse) {
+ try {
+ lock.writeLock().lock();
+
+ Map<RangerPolicy.RangerPolicyResource,
RangerResourceMatcher> matchersForResource =
resourceMatchers.computeIfAbsent(resourceDefName, k -> new HashMap<>());
+ matchersForResource.put(resource, matcher);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== setResourceMatcher(resourceDefName={},
resource={}, matcher={})", resourceDefName, resource, matcher);
+ }
+ }
+
+ void cleanResourceMatchers() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> cleanResourceMatchers()");
+ }
+ try {
+ lock.writeLock().lock();
+
+ resourceMatchers.clear();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== cleanResourceMatchers()");
+ }
+ }
+
public void setAuthContext(RangerAuthContext authContext) {
this.authContext = authContext; }
public void setAuthContextListener(RangerAuthContextListener
authContextListener) { this.authContextListener = authContextListener; }
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
index f881eaa14..251fb41cf 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java
@@ -37,6 +37,7 @@ public class RangerPolicyEngineOptions {
public boolean evaluateDelegateAdminOnly = false;
public boolean enableTagEnricherWithLocalRefresher = false;
public boolean enableUserStoreEnricherWithLocalRefresher = false;
+ public boolean enableResourceMatcherReuse = true;
@Deprecated
public boolean disableAccessEvaluationWithPolicyACLSummary = true;
public boolean optimizeTrieForRetrieval = false;
@@ -63,6 +64,7 @@ public class RangerPolicyEngineOptions {
this.evaluateDelegateAdminOnly =
other.evaluateDelegateAdminOnly;
this.enableTagEnricherWithLocalRefresher =
other.enableTagEnricherWithLocalRefresher;
this.enableUserStoreEnricherWithLocalRefresher =
other.enableUserStoreEnricherWithLocalRefresher;
+ this.enableResourceMatcherReuse =
other.enableResourceMatcherReuse;
this.optimizeTrieForRetrieval = other.optimizeTrieForRetrieval;
this.disableRoleResolution = other.disableRoleResolution;
this.serviceDefHelper = null;
@@ -88,6 +90,7 @@ public class RangerPolicyEngineOptions {
disableGdsInfoRetriever = conf.getBoolean(propertyPrefix +
".policyengine.option.disable.gdsinfo.retriever", false);
cacheAuditResults = conf.getBoolean(propertyPrefix +
".policyengine.option.cache.audit.results", true);
+ enableResourceMatcherReuse = conf.getBoolean(propertyPrefix +
".policyengine.option.enable.resourcematcher.reuse", true);
if (!disableTrieLookupPrefilter) {
cacheAuditResults = false;
@@ -119,6 +122,7 @@ public class RangerPolicyEngineOptions {
enableUserStoreEnricherWithLocalRefresher = false;
optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix +
".policyengine.option.optimize.trie.for.retrieval", false);
disableRoleResolution = conf.getBoolean(propertyPrefix +
".policyengine.option.disable.role.resolution", true);
+ enableResourceMatcherReuse = conf.getBoolean(propertyPrefix +
".policyengine.option.enable.resourcematcher.reuse", true);
}
public void configureDelegateAdmin(Configuration conf, String
propertyPrefix) {
@@ -131,6 +135,7 @@ public class RangerPolicyEngineOptions {
disableUserStoreRetriever = conf.getBoolean(propertyPrefix +
".policyengine.option.disable.userstore.retriever", true);
disableGdsInfoRetriever = conf.getBoolean(propertyPrefix +
".policyengine.option.disable.gdsinfo.retriever", true);
optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix +
".policyengine.option.optimize.trie.for.retrieval", false);
+ enableResourceMatcherReuse = conf.getBoolean(propertyPrefix +
".policyengine.option.enable.resourcematcher.reuse", true);
cacheAuditResults = false;
evaluateDelegateAdminOnly = true;
@@ -157,6 +162,7 @@ public class RangerPolicyEngineOptions {
optimizeTrieForSpace = conf.getBoolean(propertyPrefix +
".policyengine.option.optimize.trie.for.space", false);
optimizeTagTrieForRetrieval = conf.getBoolean(propertyPrefix +
".policyengine.option.optimize.tag.trie.for.retrieval", false);
optimizeTagTrieForSpace = conf.getBoolean(propertyPrefix +
".policyengine.option.optimize.tag.trie.for.space", true);
+ enableResourceMatcherReuse = conf.getBoolean(propertyPrefix +
".policyengine.option.enable.resourcematcher.reuse", true);
}
public RangerServiceDefHelper getServiceDefHelper() {
@@ -194,6 +200,7 @@ public class RangerPolicyEngineOptions {
&& this.optimizeTrieForSpace ==
that.optimizeTrieForSpace
&& this.optimizeTagTrieForRetrieval ==
that.optimizeTagTrieForRetrieval
&& this.optimizeTagTrieForSpace ==
that.optimizeTagTrieForSpace
+ && this.enableResourceMatcherReuse ==
that.enableResourceMatcherReuse
;
}
return ret;
@@ -236,6 +243,8 @@ public class RangerPolicyEngineOptions {
ret *= 2;
ret += optimizeTagTrieForSpace ? 1 : 0;
ret *= 2;
+ ret += enableResourceMatcherReuse ? 1 : 0;
+ ret *= 2;
return ret;
}
@@ -260,6 +269,7 @@ public class RangerPolicyEngineOptions {
", optimizeTrieForSpace: " +
optimizeTrieForSpace +
", optimizeTagTrieForRetrieval: " +
optimizeTagTrieForRetrieval +
", optimizeTagTrieForSpace: " +
optimizeTagTrieForSpace +
+ ", enableResourceMatcherReuse: " +
enableResourceMatcherReuse +
" }";
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java
index 822bb3902..0d44f7109 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java
@@ -171,7 +171,7 @@ public class RangerSecurityZoneMatcher {
policyResources.put(resourceDefName, new
RangerPolicyResource(resourceValues, false, isRecursive));
}
- matchers.add(new RangerZoneResourceMatcher(zoneName,
policyResources, serviceDef));
+ matchers.add(new RangerZoneResourceMatcher(zoneName,
policyResources, serviceDef, pluginContext));
if (LOG.isDebugEnabled()) {
LOG.debug("Built matcher for resource:[{}] in
zone:[{}]", resource, zoneName);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java
index 1c608aa3b..df4d06018 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsDataShareEvaluator.java
@@ -24,6 +24,7 @@ import
org.apache.ranger.plugin.conditionevaluator.RangerConditionEvaluator;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import org.apache.ranger.plugin.policyengine.RangerResourceACLs;
import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
import org.apache.ranger.plugin.policyevaluator.RangerCustomConditionEvaluator;
@@ -48,7 +49,7 @@ public class GdsDataShareEvaluator {
private final Map<String, RangerResourceTrie<GdsSharedResourceEvaluator>>
resourceTries;
private final List<GdsDshidEvaluator>
dsidEvaluators = new ArrayList<>();
- public GdsDataShareEvaluator(DataShareInfo dsh, List<SharedResourceInfo>
resources, RangerServiceDefHelper serviceDefHelper) {
+ public GdsDataShareEvaluator(DataShareInfo dsh, List<SharedResourceInfo>
resources, RangerServiceDefHelper serviceDefHelper, RangerPluginContext
pluginContext) {
LOG.debug("==> GdsDataShareEvaluator({}, {})", dsh, resources);
this.dsh = dsh;
@@ -63,7 +64,7 @@ public class GdsDataShareEvaluator {
resourceTries = new HashMap<>();
for (SharedResourceInfo resource : resources) {
- GdsSharedResourceEvaluator evaluator = new
GdsSharedResourceEvaluator(resource, dsh.getDefaultAccessTypes(),
serviceDefHelper);
+ GdsSharedResourceEvaluator evaluator = new
GdsSharedResourceEvaluator(resource, dsh.getDefaultAccessTypes(),
serviceDefHelper, pluginContext);
evaluators.add(evaluator);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java
index 53843136c..a1593daaf 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java
@@ -261,7 +261,7 @@ public class GdsPolicyEngine {
if (gdsInfo.getDataShares() != null) {
for (DataShareInfo dsh : gdsInfo.getDataShares()) {
- GdsDataShareEvaluator dshEvaluator = new
GdsDataShareEvaluator(dsh, dshResources.get(dsh.getId()), serviceDefHelper);
+ GdsDataShareEvaluator dshEvaluator = new
GdsDataShareEvaluator(dsh, dshResources.get(dsh.getId()), serviceDefHelper,
pluginContext);
List<GdsDataShareEvaluator> zoneEvaluators =
zoneDataShares.computeIfAbsent(dshEvaluator.getZoneName(), k -> new
ArrayList<>());
zoneEvaluators.add(dshEvaluator);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
index 9785d4b25..c2773c9e0 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
@@ -29,6 +29,7 @@ import
org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import org.apache.ranger.plugin.policyengine.RangerResourceACLs;
import org.apache.ranger.plugin.policyevaluator.RangerCustomConditionEvaluator;
import
org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
@@ -55,7 +56,7 @@ public class GdsSharedResourceEvaluator implements
RangerResourceEvaluator {
private final RangerResourceDef leafResourceDef;
private final Set<String> allowedAccessTypes;
- public GdsSharedResourceEvaluator(SharedResourceInfo resource, Set<String>
defaultAccessTypes, RangerServiceDefHelper serviceDefHelper) {
+ public GdsSharedResourceEvaluator(SharedResourceInfo resource, Set<String>
defaultAccessTypes, RangerServiceDefHelper serviceDefHelper,
RangerPluginContext pluginContext) {
this.resource = resource;
this.conditionEvaluator =
RangerCustomConditionEvaluator.getInstance().getExpressionEvaluator(resource.getConditionExpr(),
serviceDefHelper.getServiceDef());
@@ -71,7 +72,7 @@ public class GdsSharedResourceEvaluator implements
RangerResourceEvaluator {
this.policyResource = resource.getResource();
}
- this.policyResourceMatcher = initPolicyResourceMatcher(policyResource,
serviceDefHelper);
+ this.policyResourceMatcher = initPolicyResourceMatcher(policyResource,
serviceDefHelper, pluginContext);
this.leafResourceDef =
ServiceDefUtil.getLeafResourceDef(serviceDefHelper.getServiceDef(),
policyResource);
this.allowedAccessTypes =
serviceDefHelper.expandImpliedAccessGrants(resource.getAccessTypes() != null ?
resource.getAccessTypes() : defaultAccessTypes);
@@ -170,12 +171,13 @@ public class GdsSharedResourceEvaluator implements
RangerResourceEvaluator {
return resource.getSubResourceMasks() != null ?
resource.getSubResourceMasks().get(subResourceName) : null;
}
- private static RangerPolicyResourceMatcher
initPolicyResourceMatcher(Map<String, RangerPolicyResource> policyResource,
RangerServiceDefHelper serviceDefHelper) {
+ private static RangerPolicyResourceMatcher
initPolicyResourceMatcher(Map<String, RangerPolicyResource> policyResource,
RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) {
RangerDefaultPolicyResourceMatcher matcher = new
RangerDefaultPolicyResourceMatcher();
matcher.setServiceDefHelper(serviceDefHelper);
matcher.setServiceDef(serviceDefHelper.getServiceDef());
matcher.setPolicyResources(policyResource,
RangerPolicy.POLICY_TYPE_ACCESS);
+ matcher.setPluginContext(pluginContext);
matcher.init();
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index 07fe6a38d..549dc8f5a 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -542,6 +542,7 @@ public abstract class RangerAbstractPolicyEvaluator
implements RangerPolicyEvalu
this.resourceMatcher.setPolicyResources(resource,
policyType);
this.resourceMatcher.setServiceDef(serviceDef);
this.resourceMatcher.setServiceDefHelper(serviceDefHelper);
+ this.resourceMatcher.setPluginContext(pluginContext);
this.resourceMatcher.init();
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
index f16157ce6..0c377b357 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java
@@ -37,6 +37,7 @@ import
org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
import
org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
+import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher;
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
import org.apache.ranger.plugin.util.RangerPerfTracer;
@@ -60,6 +61,7 @@ public class RangerDefaultPolicyResourceMatcher implements
RangerPolicyResourceM
private List<RangerResourceDef> validResourceHierarchy;
private boolean isInitialized = false;
private RangerServiceDefHelper serviceDefHelper;
+ private RangerPluginContext pluginContext = null;
private final boolean forceEnableWildcardMatch;
@@ -113,6 +115,9 @@ public class RangerDefaultPolicyResourceMatcher implements
RangerPolicyResourceM
this.serviceDefHelper = serviceDefHelper;
}
+ @Override
+ public void setPluginContext(RangerPluginContext pluginContext) {
this.pluginContext = pluginContext; }
+
public int getPolicyType() { return policyType; }
public RangerServiceDefHelper getServiceDefHelper() {
@@ -812,29 +817,41 @@ public class RangerDefaultPolicyResourceMatcher
implements RangerPolicyResourceM
String resName = resourceDef.getName();
String clsName = resourceDef.getMatcher();
- if (!StringUtils.isEmpty(clsName)) {
- try {
- @SuppressWarnings("unchecked")
- Class<RangerResourceMatcher> matcherClass =
(Class<RangerResourceMatcher>) Class.forName(clsName);
+ if (pluginContext != null) {
+ ret = pluginContext.getResourceMatcher(resName, resource);
+ }
+
+ if (ret == null) {
+ if (!StringUtils.isEmpty(clsName)) {
+ try {
+ @SuppressWarnings("unchecked")
Class<RangerResourceMatcher> matcherClass = (Class<RangerResourceMatcher>)
Class.forName(clsName);
- ret = matcherClass.newInstance();
- } catch (Exception excp) {
- LOG.error("failed to instantiate resource matcher '" +
clsName + "' for '" + resName + "'. Default resource matcher will be used",
excp);
+ ret = matcherClass.newInstance();
+ } catch (Exception excp) {
+ LOG.error("failed to instantiate resource matcher '" +
clsName + "' for '" + resName + "'. Default resource matcher will be used",
excp);
+ }
}
- }
- if (ret == null) {
- ret = new RangerDefaultResourceMatcher();
- }
+ if (ret == null) {
+ ret = new RangerDefaultResourceMatcher();
+ }
- if (forceEnableWildcardMatch &&
!Boolean.parseBoolean(resourceDef.getMatcherOptions().get(OPTION_WILD_CARD))) {
- resourceDef =
serviceDefHelper.getWildcardEnabledResourceDef(resourceDef.getName(),
policyType);
- }
+ if (forceEnableWildcardMatch &&
!Boolean.parseBoolean(resourceDef.getMatcherOptions().get(OPTION_WILD_CARD))) {
+ resourceDef =
serviceDefHelper.getWildcardEnabledResourceDef(resourceDef.getName(),
policyType);
+ }
- ret.setResourceDef(resourceDef);
- ret.setPolicyResource(resource);
- ret.init();
+ ret.setResourceDef(resourceDef);
+ ret.setPolicyResource(resource);
+ ret.init();
+ if (pluginContext != null) {
+ pluginContext.setResourceMatcher(resName, resource, ret);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Did not create a fresh matcher - used matcher
from pluginContext");
+ }
+ }
} else {
LOG.error("RangerDefaultPolicyResourceMatcher: RangerResourceDef
is null");
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
index e1cd89b70..ad6869ad0 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java
@@ -28,6 +28,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
import
org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
+import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
public interface RangerPolicyResourceMatcher {
@@ -48,6 +49,8 @@ public interface RangerPolicyResourceMatcher {
void setServiceDefHelper(RangerServiceDefHelper serviceDefHelper);
+ void setPluginContext(RangerPluginContext pluginContext);
+
RangerServiceDef getServiceDef();
RangerResourceMatcher getResourceMatcher(String resourceName);
diff --git
a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
index e4ff6eeca..c8e9e9a70 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
@@ -85,6 +85,7 @@ public class RangerMemSizing {
private final boolean deDup;
private final boolean deDupStrings;
private final String optimizationMode;
+ private final boolean reuseResourceMatchers;
private final PrintStream out;
public RangerMemSizing(CommandLine cmdLine) {
@@ -100,6 +101,7 @@ public class RangerMemSizing {
this.deDup = Boolean.parseBoolean(cmdLine.getOptionValue("d",
"true"));
this.deDupStrings = this.deDup;
this.optimizationMode =
StringUtils.startsWithIgnoreCase(cmdLine.getOptionValue('o', "space"), "s") ?
OPT_MODE_SPACE : OPT_MODEL_RETRIEVAL;
+ this.reuseResourceMatchers =
Boolean.parseBoolean(cmdLine.getOptionValue('m', "true"));
}
public void run() {
@@ -131,31 +133,32 @@ public class RangerMemSizing {
out.println();
out.println("Parameters:");
if (policies != null) {
- out.println(" Policies: file=" + policyFile + ", size=" + new
File(policyFile).length() + ", " + toSummaryStr(policies));
+ out.println(" Policies: file=" + policyFile + ", size=" + new
File(policyFile).length() + ", " + toSummaryStr(policies));
}
if (tags != null) {
- out.println(" Tags: file=" + tagFile + ", size=" + new
File(tagFile).length() + ", " + toSummaryStr(tags));
+ out.println(" Tags: file=" + tagFile + ", size=" + new
File(tagFile).length() + ", " + toSummaryStr(tags));
}
if (roles != null) {
- out.println(" Roles: file=" + rolesFile + ", size=" + new
File(rolesFile).length() + ", " + toSummaryStr(roles));
+ out.println(" Roles: file=" + rolesFile + ", size=" + new
File(rolesFile).length() + ", " + toSummaryStr(roles));
}
if (userStore != null) {
- out.println(" UserStore: file=" + userStoreFile + ", size=" + new
File(userStoreFile).length() + ", " + toSummaryStr(userStore));
+ out.println(" UserStore: file=" + userStoreFile + ", size=" + new
File(userStoreFile).length() + ", " + toSummaryStr(userStore));
}
if (genRequestsFile != null) {
- out.println(" GenReq: file=" + genRequestsFile + ", requestCount=" +
genReqCount);
+ out.println(" GenReq: file=" + genRequestsFile + ",
requestCount=" + genReqCount);
}
if (evalRequestsFile != null) {
- out.println(" EvalReq: file=" + evalRequestsFile + ", requestCount="
+ evalReqCount + ", avgTimeTaken=" + evalAvgTimeNs + "ns, clientCount=" +
evalClientsCount);
+ out.println(" EvalReq: file=" + evalRequestsFile + ",
requestCount=" + evalReqCount + ", avgTimeTaken=" + evalAvgTimeNs + "ns,
clientCount=" + evalClientsCount);
}
- out.println(" DeDup: " + deDup);
- out.println(" OptMode: " + optimizationMode);
+ out.println(" DeDup: " + deDup);
+ out.println(" OptMode: " + optimizationMode);
+ out.println(" ReuseMatchers: " + reuseResourceMatchers);
out.println();
out.println("Results:");
@@ -561,6 +564,7 @@ public class RangerMemSizing {
Option evalClients = new Option("c", "evalClients", true, "eval clients
count");
Option gdsInfo = new Option("g", "gdsInfo", true, "gdsInfo file");
Option optimizeMode = new Option("o", "optMode", true, "optimization mode:
space|retrieval");
+ Option reuseResourceMatchers = new Option("m", "reuseResourceMatchers",
true, "reuse resource matchers: true|false");
Options options = new Options();
@@ -575,6 +579,7 @@ public class RangerMemSizing {
options.addOption(gdsInfo);
options.addOption(deDup);
options.addOption(optimizeMode);
+ options.addOption(reuseResourceMatchers);
try {
CommandLine cmdLine = new DefaultParser().parse(options, args);
@@ -612,6 +617,7 @@ public class RangerMemSizing {
ret.optimizeTrieForRetrieval = !ret.optimizeTrieForSpace;
ret.optimizeTagTrieForSpace = ret.optimizeTrieForSpace;
ret.optimizeTagTrieForRetrieval = ret.optimizeTrieForRetrieval;
+ ret.enableResourceMatcherReuse = reuseResourceMatchers;
return ret;
}
