This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new fde454390 RANGER-4353: Introduce option in Ranger to control retention
period of x_trx_log table data
fde454390 is described below
commit fde454390e99cb60db125a0ba2241e9807f77b39
Author: Pradeep AgrawaL <[email protected]>
AuthorDate: Mon Aug 21 16:41:17 2023 +0530
RANGER-4353: Introduce option in Ranger to control retention period of
x_trx_log table data
Change-Id: Icd3bdade26181afaee2abb54fb866e10a24217e4
---
.../java/org/apache/ranger/biz/ServiceDBStore.java | 34 ++++++++++++++++++++++
.../org/apache/ranger/db/XXAuthSessionDao.java | 17 ++++++-----
.../java/org/apache/ranger/db/XXTrxLogDao.java | 12 ++++++++
.../java/org/apache/ranger/rest/ServiceREST.java | 8 +++--
.../main/resources/META-INF/jpa_named_queries.xml | 7 +++++
.../main/resources/conf.dist/ranger-admin-site.xml | 9 ++++++
6 files changed, 78 insertions(+), 9 deletions(-)
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index ed1ea0376..9b02229e1 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -253,6 +253,8 @@ public class ServiceDBStore extends AbstractServiceStore {
public static Integer TAG_RETENTION_PERIOD_IN_DAYS = 3;
public static boolean SUPPORTS_PURGE_LOGIN_RECORDS = false;
public static Integer LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = 0;
+ public static boolean SUPPORTS_PURGE_TRANSACTION_RECORDS = false;
+ public static Integer TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS = 0;
private static final String RANGER_PLUGIN_CONFIG_PREFIX =
"ranger.plugin.";
public static final String RANGER_PLUGIN_AUDIT_FILTERS =
"ranger.plugin.audit.filters";
@@ -398,6 +400,8 @@ public class ServiceDBStore extends AbstractServiceStore {
SUPPORTS_PURGE_LOGIN_RECORDS
= config.getBoolean("ranger.admin.init.purge.login_records", false);
LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS
= config.getInt("ranger.admin.init.purge.login_records.retention.days", 0);
+ SUPPORTS_PURGE_TRANSACTION_RECORDS
= config.getBoolean("ranger.admin.init.purge.transaction_records", false);
+
TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS =
config.getInt("ranger.admin.init.purge.transaction_records.retention.days", 0);
isRolesDownloadedByService =
config.getBoolean("ranger.support.for.service.specific.role.download", false);
SUPPORTS_IN_PLACE_POLICY_UPDATES =
SUPPORTS_POLICY_DELTAS && config.getBoolean("ranger.admin" +
RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES,
RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT);
@@ -407,6 +411,8 @@ public class ServiceDBStore extends AbstractServiceStore {
LOG.info("TAG_RETENTION_PERIOD_IN_DAYS=" + TAG_RETENTION_PERIOD_IN_DAYS);
LOG.info("SUPPORTS_PURGE_LOGIN_RECORDS=" + SUPPORTS_PURGE_LOGIN_RECORDS);
LOG.info("LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS=" +
LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS);
+
LOG.info("SUPPORTS_PURGE_TRANSACTION_RECORDS=" +
SUPPORTS_PURGE_TRANSACTION_RECORDS);
+
LOG.info("TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS=" +
TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS);
LOG.info("isRolesDownloadedByService="
+ isRolesDownloadedByService);
LOG.info("SUPPORTS_IN_PLACE_POLICY_UPDATES=" +
SUPPORTS_IN_PLACE_POLICY_UPDATES);
@@ -427,6 +433,9 @@ public class ServiceDBStore extends AbstractServiceStore {
if
(SUPPORTS_PURGE_LOGIN_RECORDS) {
removeAuthSessions(LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS);
}
+ if
(SUPPORTS_PURGE_TRANSACTION_RECORDS) {
+
removeTransactionLogs(TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS);
+ }
//createUnzonedSecurityZone();
initRMSDaos();
return null;
@@ -5342,6 +5351,31 @@ public class ServiceDBStore extends AbstractServiceStore
{
}
}
+ public void removeTransactionLogs(int retentionInDays) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> removeTransactionLogs(" +
retentionInDays + ")");
+ }
+
+ if (retentionInDays > 0) {
+ long rowsCount = daoMgr.getXXTrxLog().getAllCount();
+ long rowsDeleted =
daoMgr.getXXTrxLog().deleteOlderThan(retentionInDays);
+ LOG.info("Deleted " + rowsDeleted + " records from
x_trx_log that are older than " + retentionInDays + " days");
+ List<XXTrxLog> trxLogList = new ArrayList<XXTrxLog>();
+ XXTrxLog xxTrxLog = new XXTrxLog();
+ xxTrxLog.setAction("Deleted Transaction records");
+
xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_AUTH_SESS);
+ xxTrxLog.setPreviousValue("Total Records : "+rowsCount);
+ xxTrxLog.setNewValue("Deleted Records : "+rowsDeleted);
+ trxLogList.add(xxTrxLog);
+ bizUtil.createTrxLog(trxLogList);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== removeTransactionLogs(" +
retentionInDays + ")");
+
+ }
+ }
+
public List<String> getPolicyLabels(SearchFilter searchFilter) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.getPolicyLabels()");
diff --git
a/security-admin/src/main/java/org/apache/ranger/db/XXAuthSessionDao.java
b/security-admin/src/main/java/org/apache/ranger/db/XXAuthSessionDao.java
index f69b8d2bb..645c27cbd 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXAuthSessionDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXAuthSessionDao.java
@@ -95,14 +95,17 @@ public class XXAuthSessionDao extends
BaseDao<XXAuthSession> {
batchDeleteByIds("XXAuthSession.deleteByIds", ids, "ids");
}
- public long deleteOlderThan(int olderThanInDays) {
- Date since = new Date(System.currentTimeMillis() -
TimeUnit.DAYS.toMillis(olderThanInDays));
- LOG.info("Deleting x_auth_sess records that are older than " +
olderThanInDays + " days, that is, older than " + since);
+ public long deleteOlderThan(int olderThanInDays) {
+ Date since = new Date(System.currentTimeMillis() -
TimeUnit.DAYS.toMillis(olderThanInDays));
- long ret =
getEntityManager().createNamedQuery("XXAuthSession.deleteOlderThan").setParameter("olderThan",
since).executeUpdate();
+ LOG.info("Deleting x_auth_sess records that are older than " +
olderThanInDays + " days, that is, older than " + since);
+ long ret =
getEntityManager().createNamedQuery("XXAuthSession.deleteOlderThan").setParameter("olderThan",
since).executeUpdate();
+ LOG.info("Deleted " + ret + " x_auth_sess records");
- LOG.info("Deleted " + ret + " x_auth_sess records");
- return ret;
- }
+ LOG.info("Updating x_trx_log.sess_id with null which are older
than " + olderThanInDays + " days, that is, older than " + since);
+ long updated =
getEntityManager().createNamedQuery("XXTrxLog.updateSessIdWithNull").setParameter("olderThan",
since).executeUpdate();
+ LOG.info("Updated " + updated + " x_trx_log records");
+ return ret;
+ }
}
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTrxLogDao.java
b/security-admin/src/main/java/org/apache/ranger/db/XXTrxLogDao.java
index a83e91f5b..13372ab4a 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXTrxLogDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXTrxLogDao.java
@@ -20,7 +20,9 @@
package org.apache.ranger.db;
import java.util.ArrayList;
+import java.util.Date;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import javax.persistence.NoResultException;
@@ -102,5 +104,15 @@ public class XXTrxLogDao extends BaseDao<XXTrxLog> {
return rowAffected;
}
+ public long deleteOlderThan(int olderThanInDays) {
+ Date since = new Date(System.currentTimeMillis() -
TimeUnit.DAYS.toMillis(olderThanInDays));
+ logger.info("Deleting x_trx_log records that are older than " +
olderThanInDays + " days, that is, older than " + since);
+
+ long ret =
getEntityManager().createNamedQuery("XXTrxLog.deleteOlderThan").setParameter("olderThan",
since).executeUpdate();
+
+ logger.info("Deleted " + ret + " x_trx_log records");
+ return ret;
+ }
+
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index d2d76733e..de8a15823 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -3951,7 +3951,7 @@ public class ServiceREST {
LOG.debug("==> ServiceREST.purgeRecords(" + recordType
+ ", " + olderThan + ")");
}
- if (StringUtils.isEmpty(recordType) ||
!"login_records".equalsIgnoreCase(recordType)) {
+ if (!"login_records".equalsIgnoreCase(recordType) &&
!"trx_records".equalsIgnoreCase(recordType)) {
throw
restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, "Invalid
record type - " + recordType, true);
}
@@ -3966,7 +3966,11 @@ public class ServiceREST {
perf = RangerPerfTracer.getPerfTracer(PERF_LOG,
"ServiceREST.purgeRecords(recordType=" + recordType + ", olderThan=" +
olderThan + ")");
}
- svcStore.removeAuthSessions(olderThan);
+ if ("login_records".equalsIgnoreCase(recordType)) {
+ svcStore.removeAuthSessions(olderThan);
+ } else if ("trx_records".equalsIgnoreCase(recordType)) {
+ svcStore.removeTransactionLogs(olderThan);
+ }
} catch (WebApplicationException excp) {
throw excp;
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index 1e8e4e2c5..cf6ebad95 100755
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -1529,6 +1529,13 @@
AND obj.attributeName = :attrName
</query>
</named-query>
+ <named-query name="XXTrxLog.deleteOlderThan">
+ <query>delete from XXTrxLog obj where obj.createTime <
:olderThan</query>
+ </named-query>
+ <named-query name="XXTrxLog.updateSessIdWithNull">
+ <query>UPDATE XXTrxLog obj SET obj.sessionId = null WHERE
obj.createTime < :olderThan</query>
+ </named-query>
+
<named-query name="XXAuthSession.getAuthSessionByUserId">
<query>SELECT obj FROM XXAuthSession obj WHERE obj.userId =
:userId
</query>
diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
index d6bf174e9..f3dbb777b 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
@@ -393,4 +393,13 @@
<name>ranger.admin.init.purge.login_records.retention.days</name>
<value>0</value>
</property>
+ <property>
+ <name>ranger.admin.init.purge.transaction_records</name>
+ <value>false</value>
+ </property>
+ <property>
+
<name>ranger.admin.init.purge.transaction_records.retention.days
+ </name>
+ <value>0</value>
+ </property>
</configuration>
