This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 756873e ATLAS-4339: Atlas should support skip temporary tables using
config property
756873e is described below
commit 756873eabdd5fa8dbc47682c045b0dff381d4e52
Author: Radhika Kundam <rkun...@cloudera.com>
AuthorDate: Wed Jun 23 14:18:28 2021 -0700
ATLAS-4339: Atlas should support skip temporary tables using config property
Signed-off-by: Sarath Subramanian <sar...@apache.org>
(cherry picked from commit 77b03aa1407ec6d78cc0fcaf9a06654191d42d73)
---
.../org/apache/atlas/hive/hook/AtlasHiveHookContext.java | 16 +++++++++++-----
.../main/java/org/apache/atlas/hive/hook/HiveHook.java | 15 +++++++++++----
.../apache/atlas/hive/hook/HiveMetastoreHookImpl.java | 4 ++--
.../org/apache/atlas/hive/hook/events/AlterTable.java | 2 +-
.../org/apache/atlas/hive/hook/events/BaseHiveEvent.java | 4 +++-
.../apache/atlas/hive/hook/events/CreateHiveProcess.java | 5 +----
.../org/apache/atlas/hive/hook/events/CreateTable.java | 5 +----
7 files changed, 30 insertions(+), 21 deletions(-)
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index ab8e183..14cc2f2 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -58,19 +58,20 @@ public class AtlasHiveHookContext {
private boolean isSkippedInputEntity;
private boolean isSkippedOutputEntity;
+ private boolean skipTempTables;
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation,
HookContext hiveContext,
- HiveHookObjectNamesCache knownObjects) throws
Exception {
- this(hook, hiveOperation, hiveContext, knownObjects, null, null);
+ HiveHookObjectNamesCache knownObjects, boolean
skipTempTables) throws Exception {
+ this(hook, hiveOperation, hiveContext, knownObjects, null, null,
skipTempTables);
}
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation,
HiveHookObjectNamesCache knownObjects,
- HiveMetastoreHook metastoreHook, ListenerEvent
listenerEvent) throws Exception {
- this(hook, hiveOperation, null, knownObjects, metastoreHook,
listenerEvent);
+ HiveMetastoreHook metastoreHook, ListenerEvent
listenerEvent, boolean skipTempTables) throws Exception {
+ this(hook, hiveOperation, null, knownObjects, metastoreHook,
listenerEvent, skipTempTables);
}
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation,
HookContext hiveContext, HiveHookObjectNamesCache knownObjects,
- HiveMetastoreHook metastoreHook, ListenerEvent
listenerEvent) throws Exception {
+ HiveMetastoreHook metastoreHook, ListenerEvent
listenerEvent, boolean skipTempTables) throws Exception {
this.hook = hook;
this.hiveOperation = hiveOperation;
this.hiveContext = hiveContext;
@@ -79,6 +80,7 @@ public class AtlasHiveHookContext {
this.metastoreHook = metastoreHook;
this.metastoreEvent = listenerEvent;
this.metastoreHandler = (listenerEvent != null) ?
metastoreEvent.getIHMSHandler() : null;
+ this.skipTempTables = skipTempTables;
init();
}
@@ -131,6 +133,10 @@ public class AtlasHiveHookContext {
}
}
+ public boolean isSkipTempTables() {
+ return skipTempTables;
+ }
+
public LineageInfo getLineageInfo() {
return hiveContext != null ? hiveContext.getLinfo() : null;
}
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index bf500ca..2db970a 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -70,7 +70,8 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
public static final String HOOK_HIVE_TABLE_PRUNE_PATTERN
= CONF_PREFIX + "hive_table.prune.pattern";
public static final String HOOK_HIVE_TABLE_CACHE_SIZE
= CONF_PREFIX + "hive_table.cache.size";
public static final String HOOK_HIVE_IGNORE_DDL_OPERATIONS
= CONF_PREFIX + "hs2.ignore.ddl.operations";
- public static final String
HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX +
"hs2.filter.entity.additional.types.to.retain";
+ public static final String
HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX +
"hs2.filter.entity.additional.types.to.retain";
+ public static final String HOOK_HIVE_SKIP_TEMP_TABLES
= CONF_PREFIX + "skip.temp.tables";
public static final String DEFAULT_HOST_NAME = "localhost";
private static final Map<String, HiveOperation> OPERATION_MAP = new
HashMap<>();
@@ -93,6 +94,7 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
private static final boolean
hiveProcessPopulateDeprecatedAttributes;
private static HiveHookObjectNamesCache knownObjects = null;
private static String hostName;
+ private static boolean skipTempTables = true;
static {
for (HiveOperation hiveOperation : HiveOperation.values()) {
@@ -154,6 +156,7 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
ignoreDummyDatabaseName =
atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name",
defaultDummyDatabase);
ignoreDummyTableName =
atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name",
defaultDummyTable);
ignoreValuesTmpTableNamePrefix =
atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix",
"Values__Tmp__Table__");
+ skipTempTables =
atlasProperties.getBoolean(HOOK_HIVE_SKIP_TEMP_TABLES, true);
try {
hostName = InetAddress.getLocalHost().getHostName();
@@ -177,7 +180,7 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
try {
HiveOperation oper =
OPERATION_MAP.get(hookContext.getOperationName());
- AtlasHiveHookContext context = new AtlasHiveHookContext(this,
oper, hookContext, getKnownObjects());
+ AtlasHiveHookContext context = new AtlasHiveHookContext(this,
oper, hookContext, getKnownObjects(), isSkipTempTables());
BaseHiveEvent event = null;
switch (oper) {
@@ -196,7 +199,7 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
break;
case CREATETABLE:
- event = new CreateTable(context, true);
+ event = new CreateTable(context);
break;
case DROPTABLE:
@@ -213,7 +216,7 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
case EXPORT:
case IMPORT:
case QUERY:
- event = new CreateHiveProcess(context, true);
+ event = new CreateHiveProcess(context);
break;
case ALTERTABLE_FILEFORMAT:
@@ -292,6 +295,10 @@ public class HiveHook extends AtlasHook implements
ExecuteWithHookContext {
return hiveProcessPopulateDeprecatedAttributes;
}
+ public static boolean isSkipTempTables() {
+ return skipTempTables;
+ }
+
public PreprocessAction getPreprocessActionForHiveTable(String
qualifiedName) {
PreprocessAction ret = PreprocessAction.NONE;
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
index 3c0f0c1..a371d44 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
@@ -115,7 +115,7 @@ public class HiveMetastoreHookImpl extends
MetaStoreEventListener {
try {
HiveOperation oper = operContext.getOperation();
- AtlasHiveHookContext context = new
AtlasHiveHookContext(hiveHook, oper, hiveHook.getKnownObjects(), this,
listenerEvent);
+ AtlasHiveHookContext context = new
AtlasHiveHookContext(hiveHook, oper, hiveHook.getKnownObjects(), this,
listenerEvent, hiveHook.isSkipTempTables());
BaseHiveEvent event = null;
switch (oper) {
@@ -132,7 +132,7 @@ public class HiveMetastoreHookImpl extends
MetaStoreEventListener {
break;
case CREATETABLE:
- event = new CreateTable(context, true);
+ event = new CreateTable(context);
break;
case DROPTABLE:
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
index e164370..d2f09cc 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
@@ -29,7 +29,7 @@ import java.util.List;
public class AlterTable extends CreateTable {
public AlterTable(AtlasHiveHookContext context) {
- super(context, true);
+ super(context);
}
@Override
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index aefe9e9..8f6cbcc 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -165,6 +165,7 @@ public abstract class BaseHiveEvent {
public static final Map<Integer, String> OWNER_TYPE_TO_ENUM_VALUE = new
HashMap<>();
+ protected final boolean skipTempTables;
static {
OWNER_TYPE_TO_ENUM_VALUE.put(1, "USER");
@@ -176,7 +177,8 @@ public abstract class BaseHiveEvent {
protected BaseHiveEvent(AtlasHiveHookContext context) {
- this.context = context;
+ this.context = context;
+ this.skipTempTables = context.isSkipTempTables();
}
public AtlasHiveHookContext getContext() {
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index e5295f4..5787c93 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -49,12 +49,9 @@ import java.util.Set;
public class CreateHiveProcess extends BaseHiveEvent {
private static final Logger LOG =
LoggerFactory.getLogger(CreateHiveProcess.class);
- private final boolean skipTempTables;
- public CreateHiveProcess(AtlasHiveHookContext context, boolean
skipTempTables) {
+ public CreateHiveProcess(AtlasHiveHookContext context) {
super(context);
-
- this.skipTempTables = skipTempTables;
}
@Override
diff --git
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
index 63fc894..91611de 100644
---
a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
+++
b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
@@ -42,12 +42,9 @@ import static org.apache.hadoop.hive.ql.plan.HiveOperation.*;
public class CreateTable extends BaseHiveEvent {
private static final Logger LOG =
LoggerFactory.getLogger(CreateTable.class);
- private final boolean skipTempTables;
- public CreateTable(AtlasHiveHookContext context, boolean skipTempTables) {
+ public CreateTable(AtlasHiveHookContext context) {
super(context);
-
- this.skipTempTables = skipTempTables;
}
@Override
