This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 4ecb2f854 RANGER-4609:Support in File-based Tag Retriever to provide
tag-deltas
4ecb2f854 is described below
commit 4ecb2f854497a7379654685f8c3049d13a1f39a9
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Thu Dec 14 12:00:44 2023 -0800
RANGER-4609:Support in File-based Tag Retriever to provide tag-deltas
---
.../RangerFileBasedTagRetriever.java | 199 ++++++++++++++-------
1 file changed, 133 insertions(+), 66 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
index 448c665fc..df2c7ccf1 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java
@@ -29,7 +29,7 @@ import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
public class RangerFileBasedTagRetriever extends RangerTagRetriever {
@@ -40,7 +40,9 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
private String serviceTagsFileName;
private Gson gsonBuilder;
private boolean deDupTags;
-
+ int tagFilesCount = 0;
+ int currentTagFileIndex = 0;
+ boolean isInitial = true;
@Override
public void init(Map<String, String> options) {
@@ -55,11 +57,9 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
String serviceTagsFileNameProperty = "serviceTagsFileName";
String serviceTagsDefaultFileName =
"/testdata/test_servicetags_hive.json";
String deDupTagsProperty = "deDupTags";
+ String tagFilesCountProperty = "tagFileCount";
if (StringUtils.isNotBlank(serviceName) && serviceDef != null
&& StringUtils.isNotBlank(appId)) {
- InputStream serviceTagsFileStream = null;
-
-
// Open specified file from options- it should contain
service-tags
serviceTagsFileName = options != null?
options.get(serviceTagsFileNameProperty) : null;
@@ -67,51 +67,22 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
deDupTags =
Boolean.parseBoolean(deDupTagsVal);
serviceTagsFileName = serviceTagsFileName == null ?
serviceTagsDefaultFileName : serviceTagsFileName;
-
- File f = new File(serviceTagsFileName);
-
- if (f.exists() && f.isFile() && f.canRead()) {
- try {
- serviceTagsFileStream = new
FileInputStream(f);
- serviceTagsFileURL = f.toURI().toURL();
- } catch (FileNotFoundException exception) {
- LOG.error("Error processing input
file:" + serviceTagsFileName + " or no privilege for reading file " +
serviceTagsFileName, exception);
- } catch (MalformedURLException
malformedException) {
- LOG.error("Error processing input
file:" + serviceTagsFileName + " cannot be converted to URL " +
serviceTagsFileName, malformedException);
- }
- } else {
- URL fileURL =
getClass().getResource(serviceTagsFileName);
- if (fileURL == null &&
!serviceTagsFileName.startsWith("/")) {
- fileURL = getClass().getResource("/" +
serviceTagsFileName);
- }
-
- if (fileURL == null) {
- fileURL =
ClassLoader.getSystemClassLoader().getResource(serviceTagsFileName);
- if (fileURL == null &&
!serviceTagsFileName.startsWith("/")) {
- fileURL =
ClassLoader.getSystemClassLoader().getResource("/" + serviceTagsFileName);
- }
- }
-
- if (fileURL != null) {
+ if (options != null) {
+ String tagFilesCountStr =
options.get(tagFilesCountProperty);
+ if (!StringUtils.isNotEmpty(tagFilesCountStr)) {
try {
- serviceTagsFileStream =
fileURL.openStream();
- serviceTagsFileURL = fileURL;
- } catch (Exception exception) {
- LOG.error(serviceTagsFileName +
" is not a file", exception);
+ tagFilesCount =
Integer.parseInt(tagFilesCountStr);
+ } catch (Exception e) {
+ LOG.error("Exception while
parsing tagFileCount option value:[" + tagFilesCountStr + "]");
+ LOG.error("Setting
tagFilesCount to 0");
}
- } else {
- LOG.warn("Error processing input file:
URL not found for " + serviceTagsFileName + " or no privilege for reading file
" + serviceTagsFileName);
}
}
- if (serviceTagsFileStream != null) {
- try {
- serviceTagsFileStream.close();
- } catch (Exception e) {
- // Ignore
- }
+ if (StringUtils.isNotBlank(serviceTagsFileName)) {
+ serviceTagsFileURL =
getTagFileURL(serviceTagsFileName);
}
-
+ isInitial = true;
} else {
LOG.error("FATAL: Cannot find
service/serviceDef/serviceTagsFile to use for retrieving tags. Will NOT be able
to retrieve tags.");
}
@@ -123,44 +94,140 @@ public class RangerFileBasedTagRetriever extends
RangerTagRetriever {
@Override
public ServiceTags retrieveTags(long lastKnownVersion, long
lastActivationTimeInMillis) throws Exception {
-
if (LOG.isDebugEnabled()) {
LOG.debug("==> retrieveTags(lastKnownVersion=" +
lastKnownVersion + ", lastActivationTimeInMillis=" + lastActivationTimeInMillis
+ ", serviceTagsFilePath=" + serviceTagsFileName);
}
- ServiceTags serviceTags = null;
+ ServiceTags serviceTags = readFromFile();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== retrieveTags(lastKnownVersion=" +
lastKnownVersion + ", lastActivationTimeInMillis=" +
lastActivationTimeInMillis);
+ }
+
+ return serviceTags;
+ }
+
+ URL getTagFileURL(String fileName) {
+ URL fileURL = null;
+
+ InputStream tagFileStream = null;
+
+ File f = new File(fileName);
+
+ if (f.exists() && f.isFile() && f.canRead()) {
+ try {
+ tagFileStream = new FileInputStream(f);
+ fileURL = f.toURI().toURL();
+ } catch (FileNotFoundException exception) {
+ LOG.error("Error processing input file:" +
fileName + " or no privilege for reading file " + fileName, exception);
+ } catch (MalformedURLException malformedException) {
+ LOG.error("Error processing input file:" +
fileName + " cannot be converted to URL " + fileName, malformedException);
+ }
+ } else {
+
+ fileURL = getClass().getResource(fileName);
+ if (fileURL == null) {
+ if (!fileName.startsWith("/")) {
+ fileURL = getClass().getResource("/" +
fileName);
+ }
+ }
+
+ if (fileURL == null) {
+ fileURL =
ClassLoader.getSystemClassLoader().getResource(fileName);
+ if (fileURL == null) {
+ if (!fileName.startsWith("/")) {
+ fileURL =
ClassLoader.getSystemClassLoader().getResource("/" + fileName);
+ }
+ }
+ }
+
+ if (fileURL != null) {
+
+ try {
+ tagFileStream = fileURL.openStream();
+ } catch (Exception exception) {
+ fileURL = null;
+ LOG.error(fileName + " is not a file",
exception);
+ }
+ } else {
+ LOG.warn("Error processing input file: URL not
found for " + fileName + " or no privilege for reading file " + fileName);
+ }
+ }
+
+ if (tagFileStream != null) {
+ try {
+ tagFileStream.close();
+ } catch (Exception e) {
+ // Ignore
+ }
+ }
+ return fileURL;
+ }
+
+ private ServiceTags readFromFile() {
- if (serviceTagsFileURL != null) {
- try (
- InputStream serviceTagsFileStream =
serviceTagsFileURL.openStream();
- Reader reader = new
InputStreamReader(serviceTagsFileStream, Charset.forName("UTF-8"))
- ) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==>
RangerFileBasedTagRetriever.readFromFile: sourceFileName=" +
serviceTagsFileName);
+ }
- serviceTags = gsonBuilder.fromJson(reader,
ServiceTags.class);
+ ServiceTags ret = null;
- if (serviceTags.getTagVersion() <=
lastKnownVersion) {
- // No change in serviceTags
- serviceTags = null;
- } else {
+ String fileName;
+
+ fileName = serviceTagsFileName;
+
+ if (isInitial) {
+ isInitial = false;
+ if (serviceTagsFileURL != null) {
+ try (
+ InputStream fileStream =
serviceTagsFileURL.openStream();
+ Reader reader = new
InputStreamReader(fileStream, StandardCharsets.UTF_8)
+ ) {
+
+ ret = gsonBuilder.fromJson(reader,
ServiceTags.class);
if (deDupTags) {
- final int countOfDuplicateTags
= serviceTags.dedupTags();
- LOG.info("Number of duplicate
tags removed from the received serviceTags:[" + countOfDuplicateTags + "].
Number of tags in the de-duplicated serviceTags :[" +
serviceTags.getTags().size() + "].");
+ final int countOfDuplicateTags
= ret.dedupTags();
+ LOG.info("Number of duplicate
tags removed from the received serviceTags:[" + countOfDuplicateTags + "].
Number of tags in the de-duplicated serviceTags :[" + ret.getTags().size() +
"].");
}
+
+ } catch (IOException e) {
+ LOG.warn("Error processing input file:
or no privilege for reading file " + fileName, e);
}
- } catch (IOException e) {
- LOG.warn("Error processing input file: or no
privilege for reading file " + serviceTagsFileName);
- throw e;
+ } else {
+ LOG.error("Error reading file: " + fileName);
}
- } else {
- LOG.error("Error reading file: " + serviceTagsFileName);
- throw new Exception("serviceTagsFileURL is null!");
+
+ } else if (tagFilesCount > 0) {
+
+ currentTagFileIndex = currentTagFileIndex %
tagFilesCount;
+ fileName = serviceTagsFileName + "_" +
currentTagFileIndex + ".json";
+ URL fileURL = getTagFileURL(fileName);
+ if (fileURL != null) {
+ try (
+ InputStream fileStream =
fileURL.openStream();
+ Reader reader = new
InputStreamReader(fileStream, StandardCharsets.UTF_8)
+ ) {
+
+ ret = gsonBuilder.fromJson(reader,
ServiceTags.class);
+ currentTagFileIndex++;
+ if (deDupTags) {
+ final int countOfDuplicateTags
= ret.dedupTags();
+ LOG.info("Number of duplicate
tags removed from the received serviceTags:[" + countOfDuplicateTags + "].
Number of tags in the de-duplicated serviceTags :[" + ret.getTags().size() +
"].");
+ }
+ } catch (IOException e) {
+ LOG.warn("Error processing input file:
or no privilege for reading file " + fileName, e);
+ }
+ } else {
+ LOG.error("Error reading file: " + fileName);
+ }
+
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== retrieveTags(lastKnownVersion=" +
lastKnownVersion + ", lastActivationTimeInMillis=" +
lastActivationTimeInMillis);
+ LOG.debug("<==
RangerFileBasedTagRetriever.readFromFile: sourceFileName=" + fileName);
}
- return serviceTags;
+ return ret;
}
}
