This is an automated email from the ASF dual-hosted git repository.
chaitalithombare pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new d285ddb20 ATLAS-4974: impala-bridge, impala-bridge-shim,
impala-hook-api modules: update for code readability improvements (#381)
d285ddb20 is described below
commit d285ddb205dd100227bdf9b7448fe4eead54bea9
Author: chaitalicod <36201417+chaitali...@users.noreply.github.com>
AuthorDate: Wed Jun 18 14:21:23 2025 +0530
ATLAS-4974: impala-bridge, impala-bridge-shim, impala-hook-api modules:
update for code readability improvements (#381)
Co-authored-by: chaitalithombare <chaitalithomb...@apache.org>
---
.../atlas/impala/hook/ImpalaIdentifierParser.java | 4 +-
.../apache/atlas/impala/model/ImpalaDataType.java | 1 -
.../apache/atlas/impala/ImpalaLineageITBase.java | 93 ++++----
.../apache/atlas/impala/ImpalaLineageToolIT.java | 243 ++++++++++-----------
.../atlas/impala/hook/ImpalaLineageHookIT.java | 58 +++--
.../atlas/impala/hook/ImpalaLineageHookTest.java | 15 +-
6 files changed, 198 insertions(+), 216 deletions(-)
diff --git
a/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/ImpalaIdentifierParser.java
b/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/ImpalaIdentifierParser.java
index 9be87d62b..729fc5fcc 100644
---
a/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/ImpalaIdentifierParser.java
+++
b/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/ImpalaIdentifierParser.java
@@ -35,9 +35,7 @@ public class ImpalaIdentifierParser {
// add "." to allow <dbName>.<tableName>
public static final String VALID_IMPALA_IDENTIFIER_REGEX =
"^[a-zA-Z][a-zA-Z0-9_.]{0,127}$";
- private ImpalaIdentifierParser() {
- throw new UnsupportedOperationException("ImpalaIdentifierParser");
- }
+ private ImpalaIdentifierParser() {}
public static boolean isTableNameValid(String inTableName) {
if (StringUtils.isEmpty(inTableName)) {
diff --git
a/addons/impala-bridge/src/main/java/org/apache/atlas/impala/model/ImpalaDataType.java
b/addons/impala-bridge/src/main/java/org/apache/atlas/impala/model/ImpalaDataType.java
index 4e0d47870..dfe2d71ad 100644
---
a/addons/impala-bridge/src/main/java/org/apache/atlas/impala/model/ImpalaDataType.java
+++
b/addons/impala-bridge/src/main/java/org/apache/atlas/impala/model/ImpalaDataType.java
@@ -21,7 +21,6 @@ package org.apache.atlas.impala.model;
* Data types used for Impala bridge
*/
public enum ImpalaDataType {
-
IMPALA_PROCESS,
IMPALA_PROCESS_EXECUTION,
IMPALA_COLUMN_LINEAGE,
diff --git
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageITBase.java
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageITBase.java
index ef23a26d1..9f552e72e 100644
---
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageITBase.java
+++
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageITBase.java
@@ -18,20 +18,6 @@
package org.apache.atlas.impala;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_QUALIFIED_NAME;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_QUERY_TEXT;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_RECENT_QUERIES;
-import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.HIVE_TYPE_DB;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.HIVE_TYPE_TABLE;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClientV2;
import org.apache.atlas.impala.hook.AtlasImpalaHookContext;
@@ -49,11 +35,24 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.testng.annotations.BeforeClass;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_QUALIFIED_NAME;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_QUERY_TEXT;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_RECENT_QUERIES;
+import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.HIVE_TYPE_DB;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.HIVE_TYPE_TABLE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.fail;
public class ImpalaLineageITBase {
private static final Logger LOG =
LoggerFactory.getLogger(ImpalaLineageITBase.class);
@@ -77,7 +76,6 @@ public class ImpalaLineageITBase {
protected static SessionState ss;
protected static HiveConf conf;
-
@BeforeClass
public void setUp() throws Exception {
//Set-up hive session
@@ -90,42 +88,40 @@ public class ImpalaLineageITBase {
driverWithoutContext = new Driver(conf);
Configuration configuration = ApplicationProperties.get();
-
String[] atlasEndPoint =
configuration.getStringArray(ImpalaLineageHook.ATLAS_ENDPOINT);
if (atlasEndPoint == null || atlasEndPoint.length == 0) {
- atlasEndPoint = new String[]{DGI_URL};
+ atlasEndPoint = new String[] {DGI_URL};
}
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
- atlasClientV2 = new AtlasClientV2(atlasEndPoint, new
String[]{"admin", "admin"});
+ atlasClientV2 = new AtlasClientV2(atlasEndPoint, new String[]
{"admin", "admin"});
} else {
atlasClientV2 = new AtlasClientV2(atlasEndPoint);
}
-
}
// return guid of the entity
protected String assertEntityIsRegistered(final String typeName, final
String property, final String value,
- final AssertPredicate assertPredicate) throws Exception {
+ final AssertPredicate assertPredicate) throws Exception {
waitFor(100000, new Predicate() {
@Override
public void evaluate() throws Exception {
AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.getEntityByAttribute(typeName, Collections
- .singletonMap(property,value));
+ .singletonMap(property, value));
AtlasEntity entity = atlasEntityWithExtInfo.getEntity();
assertNotNull(entity);
- if (assertPredicate != null) {
- assertPredicate.assertOnEntity(entity);
- }
+ if (assertPredicate != null) {
+ assertPredicate.assertOnEntity(entity);
+ }
}
});
- AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.getEntityByAttribute(typeName,
Collections.singletonMap(property,value));
+ AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.getEntityByAttribute(typeName, Collections.singletonMap(property,
value));
AtlasEntity entity = atlasEntityWithExtInfo.getEntity();
return (String) entity.getGuid();
}
protected String assertEntityIsRegistered(final String typeName,
List<String> processQFNames,
- final AssertPredicates assertPredicates) throws Exception {
+ final AssertPredicates assertPredicates) throws Exception {
List<Map<String, String>> attributesList = new ArrayList<>();
for (String processName : processQFNames) {
@@ -148,7 +144,7 @@ public class ImpalaLineageITBase {
}
protected String assertEntityIsRegisteredViaGuid(String guid,
- final AssertPredicate assertPredicate) throws Exception {
+ final AssertPredicate assertPredicate) throws Exception {
waitFor(80000, new Predicate() {
@Override
public void evaluate() throws Exception {
@@ -158,7 +154,6 @@ public class ImpalaLineageITBase {
if (assertPredicate != null) {
assertPredicate.assertOnEntity(entity);
}
-
}
});
AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.getEntityByGuid(guid);
@@ -166,7 +161,6 @@ public class ImpalaLineageITBase {
return (String) entity.getGuid();
}
-
protected String assertProcessIsRegistered(List<String> processQFNames,
String queryString) throws Exception {
try {
Thread.sleep(5000);
@@ -178,17 +172,17 @@ public class ImpalaLineageITBase {
public String assertOnEntities(final List<AtlasEntity>
entities) throws Exception {
for (AtlasEntity entity : entities) {
List<String> recentQueries = (List<String>) entity
- .getAttribute(ATTRIBUTE_RECENT_QUERIES);
+ .getAttribute(ATTRIBUTE_RECENT_QUERIES);
- if (queryString.equalsIgnoreCase(recentQueries.get(0)))
+ if
(queryString.equalsIgnoreCase(recentQueries.get(0))) {
return entity.getGuid();
-
+ }
}
throw new IllegalStateException("Not found entity with
matching query");
}
});
- } catch(Exception e) {
+ } catch (Exception e) {
LOG.error("Exception : ", e);
throw e;
}
@@ -205,10 +199,10 @@ public class ImpalaLineageITBase {
public void assertOnEntity(final AtlasEntity entity) throws
Exception {
List<String> recentQueries = (List<String>)
entity.getAttribute(ATTRIBUTE_RECENT_QUERIES);
- Assert.assertEquals(recentQueries.get(0),
lower(queryString));
+ assertEquals(recentQueries.get(0), lower(queryString));
}
});
- } catch(Exception e) {
+ } catch (Exception e) {
LOG.error("Exception : ", e);
throw e;
}
@@ -220,10 +214,9 @@ public class ImpalaLineageITBase {
String guid = "";
List<AtlasObjectId> processExecutions =
toAtlasObjectIdList(impalaProcess.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS_EXECUTIONS));
+ BaseImpalaEvent.ATTRIBUTE_PROCESS_EXECUTIONS));
for (AtlasObjectId processExecution : processExecutions) {
- AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.
- getEntityByGuid(processExecution.getGuid());
+ AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.getEntityByGuid(processExecution.getGuid());
AtlasEntity entity = atlasEntityWithExtInfo.getEntity();
if
(String.valueOf(entity.getAttribute(ATTRIBUTE_QUERY_TEXT)).equals(queryString.toLowerCase().trim()))
{
@@ -236,10 +229,10 @@ public class ImpalaLineageITBase {
@Override
public void assertOnEntity(final AtlasEntity entity) throws
Exception {
String queryText = (String)
entity.getAttribute(ATTRIBUTE_QUERY_TEXT);
- Assert.assertEquals(queryText,
queryString.toLowerCase().trim());
+ assertEquals(queryText, queryString.toLowerCase().trim());
}
});
- } catch(Exception e) {
+ } catch (Exception e) {
LOG.error("Exception : ", e);
throw e;
}
@@ -291,7 +284,6 @@ public class ImpalaLineageITBase {
return ret;
}
-
protected String assertDatabaseIsRegistered(String dbName) throws
Exception {
return assertDatabaseIsRegistered(dbName, null);
}
@@ -300,7 +292,7 @@ public class ImpalaLineageITBase {
LOG.debug("Searching for database: {}", dbName);
String dbQualifiedName = dbName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
- CLUSTER_NAME;
+ CLUSTER_NAME;
dbQualifiedName = dbQualifiedName.toLowerCase();
@@ -327,7 +319,7 @@ public class ImpalaLineageITBase {
LOG.debug("Searching for table {}", fullTableName);
String tableQualifiedName = (fullTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE).toLowerCase() +
- CLUSTER_NAME;
+ CLUSTER_NAME;
return assertEntityIsRegistered(HIVE_TYPE_TABLE,
REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName,
assertPredicate);
@@ -427,7 +419,7 @@ public class ImpalaLineageITBase {
try {
predicate.evaluate();
return;
- } catch(Error | Exception e) {
+ } catch (Error | Exception e) {
if (System.currentTimeMillis() >= mustEnd) {
fail("Assertions failed. Failing after waiting for timeout
" + timeout + " msecs", e);
}
@@ -450,7 +442,7 @@ public class ImpalaLineageITBase {
while (true) {
try {
return predicate.evaluate();
- } catch(Error | Exception e) {
+ } catch (Error | Exception e) {
if (System.currentTimeMillis() >= mustEnd) {
fail("Assertions failed. Failing after waiting for timeout
" + timeout + " msecs", e);
}
@@ -491,5 +483,8 @@ public class ImpalaLineageITBase {
protected String tableName() {
return "table_" + random();
}
- protected String dbName() {return "db_" + random();}
+
+ protected String dbName() {
+ return "db_" + random();
+ }
}
diff --git
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java
index 4f8c2da3b..3eb15d23b 100644
---
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java
+++
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java
@@ -17,26 +17,26 @@
*/
package org.apache.atlas.impala;
-import static
org.apache.atlas.impala.hook.AtlasImpalaHookContext.QNAME_SEP_PROCESS;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_END_TIME;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_NAME;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_QUERY_TEXT;
-
-import java.util.ArrayList;
-import java.util.List;
import org.apache.atlas.impala.hook.AtlasImpalaHookContext;
import org.apache.atlas.impala.hook.ImpalaLineageHook;
import org.apache.atlas.impala.hook.events.BaseImpalaEvent;
import org.apache.atlas.impala.model.ImpalaQuery;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
-import org.testng.Assert;
import org.testng.annotations.Test;
+import java.util.ArrayList;
+import java.util.List;
+
+import static
org.apache.atlas.impala.hook.AtlasImpalaHookContext.QNAME_SEP_PROCESS;
import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_DDL_QUERIES;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_END_TIME;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_NAME;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_QUERY_TEXT;
import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_START_TIME;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
public class ImpalaLineageToolIT extends ImpalaLineageITBase {
public static final long TABLE_CREATE_TIME_SOURCE = 1554750070;
@@ -53,8 +53,8 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
public void testCreateViewFromFile() {
// this file contains a single lineage record for "create view".
// It has table vertex with createTime
- String IMPALA = dir + "impalaCreateView.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaCreateView.json";
+ String impalaWal = dir + "WALimpala.wal";
List<ImpalaQuery> lineageList = new ArrayList<>();
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -67,21 +67,21 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)",
false);
+ createTable(dbName, sourceTableName, "(id string, count int)",
false);
String targetTableName = "view_1";
- createTable(dbName, targetTableName,"(count int, id string)",
false);
+ createTable(dbName, targetTableName, "(count int, id string)",
false);
// process lineage record, and send corresponding notification to
Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_3
- String createTime = new Long((long)(1554750072)*1000).toString();
+ String createTime = new Long((long) (1554750072) *
1000).toString();
String processQFName =
- "db_1.view_1" +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
+ "db_1.view_1" +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime;
processQFName = processQFName.toLowerCase();
@@ -92,9 +92,9 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS
+ processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS));
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
assertEquals(processQFName,
processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
assertEquals(processExecutionQFName,
processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
@@ -119,8 +119,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
public void testCreateViewWithCommentSpacesFromFile() {
// this file contains a single lineage record for "create view".
// It has table vertex with createTime
- String IMPALA = dir + "impalaCreateViewWithCommentSpaces.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaCreateViewWithCommentSpaces.json";
+ String impalaWal = dir + "WALimpala.wal";
List<ImpalaQuery> lineageList = new ArrayList<>();
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -133,19 +133,19 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)",
false);
+ createTable(dbName, sourceTableName, "(id string, count int)",
false);
String targetTableName = "view_1";
- createTable(dbName, targetTableName,"(count int, id string)",
false);
+ createTable(dbName, targetTableName, "(count int, id string)",
false);
// process lineage record, and send corresponding notification to
Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_3
- String createTime = new Long((long)(1554750072)*1000).toString();
+ String createTime = new Long((long) (1554750072) *
1000).toString();
String processQFName =
"db_8.view_1" +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME +
AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime;
@@ -157,8 +157,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS
+ processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
@@ -188,8 +188,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
// this file contains a single lineage record for "create view".
// there is no table vertex with createTime, which is lineage record
generated by Impala
// originally. The table create time is hard-coded before Impala fixes
this issue.
- String IMPALA = dir + "impalaCreateViewNoCreateTime.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaCreateViewNoCreateTime.json";
+ String impalaWal = dir + "WALimpala.wal";
List<ImpalaQuery> lineageList = new ArrayList<>();
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -202,25 +202,25 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)",
false);
+ createTable(dbName, sourceTableName, "(id string, count int)",
false);
String targetTableName = "view_1";
- createTable(dbName, targetTableName,"(count int, id string)",
false);
+ createTable(dbName, targetTableName, "(count int, id string)",
false);
// process lineage record, and send corresponding notification to
Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
Long beforeCreateTime = System.currentTimeMillis() /
BaseImpalaEvent.MILLIS_CONVERT_FACTOR;
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
Long afterCreateTime = System.currentTimeMillis() /
BaseImpalaEvent.MILLIS_CONVERT_FACTOR;
String processQFNameWithoutTime =
- dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
+ dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS;
processQFNameWithoutTime = processQFNameWithoutTime.toLowerCase();
List<String> processQFNames = new ArrayList<>();
- String createTime = new
Long(beforeCreateTime.longValue()*1000).toString();
+ String createTime = new Long(beforeCreateTime.longValue() *
1000).toString();
processQFNames.add(processQFNameWithoutTime + createTime);
if (beforeCreateTime != afterCreateTime) {
@@ -235,9 +235,9 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processEntity1 = validateProcess(processQFNames,
queryString);
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS));
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String guid = assertTableIsRegistered(dbName,
targetTableName);
AtlasEntity entity =
atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -259,8 +259,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
*/
@Test
public void testCreateTableAsSelectFromFile() throws Exception {
- String IMPALA = dir + "impalaCreateTableAsSelect.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaCreateTableAsSelect.json";
+ String impalaWal = dir + "WALimpala.wal";
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -271,21 +271,21 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)", false);
+ createTable(dbName, sourceTableName, "(id string, count int)", false);
String targetTableName = "table_2";
- createTable(dbName, targetTableName,"(count int, id string)", false);
+ createTable(dbName, targetTableName, "(count int, id string)", false);
// process lineage record, and send corresponding notification to Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_4.
- String createTime = new Long(TABLE_CREATE_TIME*1000).toString();
+ String createTime = new Long(TABLE_CREATE_TIME * 1000).toString();
String processQFName =
- dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
+ dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime;
processQFName = processQFName.toLowerCase();
@@ -294,9 +294,9 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processEntity1 = validateProcess(processQFName,
queryString);
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS));
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
@@ -320,8 +320,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
*/
@Test
public void testCreateTableAsSelectWithCommentSpacesFromFile() throws
Exception {
- String IMPALA = dir +
"impalaCreateTableAsSelectWithCommentSpaces.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir +
"impalaCreateTableAsSelectWithCommentSpaces.json";
+ String impalaWal = dir + "WALimpala.wal";
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -332,19 +332,19 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)", false);
+ createTable(dbName, sourceTableName, "(id string, count int)", false);
String targetTableName = "table_2";
- createTable(dbName, targetTableName,"(count int, id string)", false);
+ createTable(dbName, targetTableName, "(count int, id string)", false);
// process lineage record, and send corresponding notification to Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_4.
- String createTime = new Long(TABLE_CREATE_TIME*1000).toString();
+ String createTime = new Long(TABLE_CREATE_TIME * 1000).toString();
String processQFName =
dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME +
AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime;
@@ -357,15 +357,14 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
assertEquals(processQFName,
processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
assertEquals(processExecutionQFName,
processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
-
String guid = assertTableIsRegistered(dbName,
targetTableName);
AtlasEntity entity =
atlasClientV2.getEntityByGuid(guid).getEntity();
List ddlQueries = (List)
entity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
@@ -383,8 +382,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
*/
@Test
public void testAlterViewAsSelectFromFile() throws Exception {
- String IMPALA = dir + "impalaAlterViewAsSelect.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaAlterViewAsSelect.json";
+ String impalaWal = dir + "WALimpala.wal";
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -395,21 +394,21 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)", false);
+ createTable(dbName, sourceTableName, "(id string, count int)", false);
String targetTableName = "view_1";
- createTable(dbName, targetTableName,"(count int, id string)", false);
+ createTable(dbName, targetTableName, "(count int, id string)", false);
// process lineage record, and send corresponding notification to Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_4.
- String createTime = new Long(TABLE_CREATE_TIME*1000).toString();
+ String createTime = new Long(TABLE_CREATE_TIME * 1000).toString();
String processQFName =
- dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
+ dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime;
processQFName = processQFName.toLowerCase();
@@ -418,9 +417,9 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processEntity1 = validateProcess(processQFName,
queryString);
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS));
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
@@ -444,8 +443,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
*/
@Test
public void testAlterViewAsSelectWithCommentSpacesFromFile() throws
Exception {
- String IMPALA = dir + "impalaAlterViewAsSelectWithCommentSpaces.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaAlterViewAsSelectWithCommentSpaces.json";
+ String impalaWal = dir + "WALimpala.wal";
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -456,19 +455,19 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)", false);
+ createTable(dbName, sourceTableName, "(id string, count int)", false);
String targetTableName = "view_1";
- createTable(dbName, targetTableName,"(count int, id string)", false);
+ createTable(dbName, targetTableName, "(count int, id string)", false);
// process lineage record, and send corresponding notification to Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_4.
- String createTime = new Long(TABLE_CREATE_TIME*1000).toString();
+ String createTime = new Long(TABLE_CREATE_TIME * 1000).toString();
String processQFName =
dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME +
AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime;
@@ -481,8 +480,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
@@ -506,8 +505,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
*/
@Test
public void testInsertIntoAsSelectFromFile() throws Exception {
- String IMPALA = dir + "impalaInsertIntoAsSelect.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaInsertIntoAsSelect.json";
+ String impalaWal = dir + "WALimpala.wal";
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -518,36 +517,35 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)", false);
+ createTable(dbName, sourceTableName, "(id string, count int)", false);
String targetTableName = "table_2";
- createTable(dbName, targetTableName,"(count int, id string, int_col
int)", false);
+ createTable(dbName, targetTableName, "(count int, id string, int_col
int)", false);
// process lineage record, and send corresponding notification to Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_4.
- String createTime1 = new
Long(TABLE_CREATE_TIME_SOURCE*1000).toString();
- String createTime2 = new Long(TABLE_CREATE_TIME*1000).toString();
+ String createTime1 = new Long(TABLE_CREATE_TIME_SOURCE *
1000).toString();
+ String createTime2 = new Long(TABLE_CREATE_TIME * 1000).toString();
String sourceQFName = dbName + "." + sourceTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
- CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime1;
+ CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime1;
String targetQFName = dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
- CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime2;
+ CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime2;
String processQFName = "QUERY:" + sourceQFName.toLowerCase() +
"->:INSERT:" + targetQFName.toLowerCase();
-
String queryString = "insert into table " + dbName + "." +
targetTableName + " (count, id) select count, id from " + dbName + "." +
sourceTableName;
AtlasEntity processEntity1 = validateProcess(processQFName,
queryString);
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS));
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
assertEquals(processQFName,
processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
assertEquals(processExecutionQFName,
processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
@@ -568,8 +566,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
*/
@Test
public void testMultipleInsertIntoAsSelectFromFile() throws Exception {
- String IMPALA = dir + "impalaMultipleInsertIntoAsSelect1.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir + "impalaMultipleInsertIntoAsSelect1.json";
+ String impalaWal = dir + "WALimpala.wal";
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -580,30 +578,30 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "table_1";
- createTable(dbName, sourceTableName,"(id string, count int)", false);
+ createTable(dbName, sourceTableName, "(id string, count int)", false);
String targetTableName = "table_2";
- createTable(dbName, targetTableName,"(count int, id string, int_col
int)", false);
+ createTable(dbName, targetTableName, "(count int, id string, int_col
int)", false);
// process lineage record, and send corresponding notification to Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// re-run the same lineage record, should have the same process entity
and another process execution entity
Thread.sleep(5000);
- IMPALA = dir + "impalaMultipleInsertIntoAsSelect2.json";
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ impala = dir + "impalaMultipleInsertIntoAsSelect2.json";
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
Thread.sleep(5000);
// verify the process is saved in Atlas
// the value is from info in IMPALA_4.
- String createTime1 = new
Long(TABLE_CREATE_TIME_SOURCE*1000).toString();
- String createTime2 = new Long(TABLE_CREATE_TIME*1000).toString();
+ String createTime1 = new Long(TABLE_CREATE_TIME_SOURCE *
1000).toString();
+ String createTime2 = new Long(TABLE_CREATE_TIME * 1000).toString();
String sourceQFName = dbName + "." + sourceTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
- CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime1;
+ CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime1;
String targetQFName = dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
- CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime2;
+ CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime2;
String processQFName = "QUERY:" + sourceQFName.toLowerCase() +
"->:INSERT:" + targetQFName.toLowerCase();
String queryString = "insert into table " + dbName + "." +
targetTableName + " (count, id) select count, id from " + dbName + "." +
sourceTableName;
@@ -614,17 +612,16 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processEntity1 = validateProcess(processQFName,
queryString);
List<AtlasObjectId> processExecutions =
toAtlasObjectIdList(processEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS_EXECUTIONS));
- Assert.assertEquals(processExecutions.size(), 2);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS_EXECUTIONS));
+ assertEquals(processExecutions.size(), 2);
for (AtlasObjectId processExecutionId : processExecutions) {
- AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.
- getEntityByGuid(processExecutionId.getGuid());
+ AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo =
atlasClientV2.getEntityByGuid(processExecutionId.getGuid());
AtlasEntity processExecutionEntity =
atlasEntityWithExtInfo.getEntity();
String entityQueryText =
String.valueOf(processExecutionEntity.getAttribute(ATTRIBUTE_QUERY_TEXT)).toLowerCase().trim();
if (!(queryString.equalsIgnoreCase(entityQueryText) ||
queryString2.equalsIgnoreCase(entityQueryText))) {
String errorMessage = String.format("process query text '%s'
does not match expected value of '%s' or '%s'", entityQueryText, queryString,
queryString2);
- Assert.assertTrue(false, errorMessage);
+ assertTrue(false, errorMessage);
}
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS
+ processExecutionEntity.getAttribute(ATTRIBUTE_START_TIME).toString() +
@@ -651,8 +648,8 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
*/
@Test
public void testCreateTableAsSelectVertexIdNoTableNameFromFile() throws
Exception {
- String IMPALA = dir +
"impalaCreateTableAsSelectVertexIdNoTableName.json";
- String IMPALA_WAL = dir + "WALimpala.wal";
+ String impala = dir +
"impalaCreateTableAsSelectVertexIdNoTableName.json";
+ String impalaWal = dir + "WALimpala.wal";
ImpalaLineageHook impalaLineageHook = new ImpalaLineageHook();
@@ -663,21 +660,21 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
createDatabase(dbName);
String sourceTableName = "sales_asia";
- createTable(dbName, sourceTableName,"(id string, name string)", false);
+ createTable(dbName, sourceTableName, "(id string, name string)",
false);
String targetTableName = "sales_china";
- createTable(dbName, targetTableName,"(id string, name string)", false);
+ createTable(dbName, targetTableName, "(id string, name string)",
false);
// process lineage record, and send corresponding notification to Atlas
- String[] args = new String[]{"-d", "./", "-p", "impala"};
+ String[] args = new String[] {"-d", "./", "-p", "impala"};
ImpalaLineageTool toolInstance = new ImpalaLineageTool(args);
- toolInstance.importHImpalaEntities(impalaLineageHook, IMPALA,
IMPALA_WAL);
+ toolInstance.importHImpalaEntities(impalaLineageHook, impala,
impalaWal);
// verify the process is saved in Atlas
// the value is from info in IMPALA_4.
- String createTime = new Long((long)1560885039*1000).toString();
+ String createTime = new Long((long) 1560885039 * 1000).toString();
String processQFName =
- dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
+ dbName + "." + targetTableName +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime;
processQFName = processQFName.toLowerCase();
@@ -686,9 +683,9 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
AtlasEntity processEntity1 = validateProcess(processQFName,
queryString);
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryString);
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS));
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String processExecutionQFName = processQFName + QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
@@ -703,4 +700,4 @@ public class ImpalaLineageToolIT extends
ImpalaLineageITBase {
assertNotNull(ddlQueries);
assertEquals(ddlQueries.size(), 1);
}
-}
\ No newline at end of file
+}
diff --git
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java
index e0277c884..d025e956b 100644
---
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java
+++
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java
@@ -18,36 +18,32 @@
package org.apache.atlas.impala.hook;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
import org.apache.atlas.impala.ImpalaLineageITBase;
import org.apache.atlas.impala.hook.events.BaseImpalaEvent;
import org.apache.atlas.impala.model.ImpalaDependencyType;
+import org.apache.atlas.impala.model.ImpalaQuery;
import org.apache.atlas.impala.model.ImpalaVertexType;
import org.apache.atlas.impala.model.LineageEdge;
-import org.apache.atlas.impala.model.ImpalaQuery;
import org.apache.atlas.impala.model.LineageVertex;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_START_TIME;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_DDL_QUERIES;
import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_END_TIME;
import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_NAME;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_DDL_QUERIES;
+import static
org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_START_TIME;
+import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
public class ImpalaLineageHookIT extends ImpalaLineageITBase {
private static final Logger LOG =
LoggerFactory.getLogger(ImpalaLineageHookIT.class);
@@ -82,56 +78,56 @@ public class ImpalaLineageHookIT extends
ImpalaLineageITBase {
queryObj.setQueryId("3a441d0c130962f8:7f634aec00000000");
queryObj.setHash("64ff0425ccdfaada53e3f2fd76f566f7");
queryObj.setUser("admin");
- queryObj.setTimestamp((long)1554750072);
- queryObj.setEndTime((long)1554750554);
+ queryObj.setTimestamp((long) 1554750072);
+ queryObj.setEndTime((long) 1554750554);
LineageEdge edge1 = new LineageEdge();
- edge1.setSources( Arrays.asList((long)1));
- edge1.setTargets( Arrays.asList((long)0));
+ edge1.setSources(Arrays.asList((long) 1));
+ edge1.setTargets(Arrays.asList((long) 0));
edge1.setEdgeType(ImpalaDependencyType.PROJECTION);
edges.add(edge1);
LineageEdge edge2 = new LineageEdge();
- edge2.setSources( Arrays.asList((long)3));
- edge2.setTargets( Arrays.asList((long)2));
+ edge2.setSources(Arrays.asList((long) 3));
+ edge2.setTargets(Arrays.asList((long) 2));
edge2.setEdgeType(ImpalaDependencyType.PROJECTION);
edges.add(edge2);
queryObj.setEdges(edges);
LineageVertex vertex1 = new LineageVertex();
- vertex1.setId((long)0);
+ vertex1.setId((long) 0);
vertex1.setVertexType(ImpalaVertexType.COLUMN);
vertex1.setVertexId(viewName + ".count");
vertices.add(vertex1);
LineageVertex vertex2 = new LineageVertex();
- vertex2.setId((long)1);
+ vertex2.setId((long) 1);
vertex2.setVertexType(ImpalaVertexType.COLUMN);
vertex2.setVertexId(tableName + ".count");
vertices.add(vertex2);
LineageVertex vertex3 = new LineageVertex();
- vertex3.setId((long)2);
+ vertex3.setId((long) 2);
vertex3.setVertexType(ImpalaVertexType.COLUMN);
vertex3.setVertexId(viewName + ".id");
vertices.add(vertex3);
LineageVertex vertex4 = new LineageVertex();
- vertex4.setId((long)3);
+ vertex4.setId((long) 3);
vertex4.setVertexType(ImpalaVertexType.COLUMN);
vertex4.setVertexId(tableName + ".id");
vertices.add(vertex4);
LineageVertex vertex5 = new LineageVertex();
- vertex5.setId((long)4);
+ vertex5.setId((long) 4);
vertex5.setVertexType(ImpalaVertexType.TABLE);
vertex5.setVertexId(viewName);
vertex5.setCreateTime(System.currentTimeMillis() / 1000);
vertices.add(vertex5);
LineageVertex vertex6 = new LineageVertex();
- vertex6.setId((long)5);
+ vertex6.setId((long) 5);
vertex6.setVertexType(ImpalaVertexType.TABLE);
vertex6.setVertexId(tableName);
vertex6.setCreateTime(System.currentTimeMillis() / 1000);
@@ -143,7 +139,7 @@ public class ImpalaLineageHookIT extends
ImpalaLineageITBase {
impalaHook.process(queryObj);
String createTime = new
Long(BaseImpalaEvent.getTableCreateTime(vertex5)).toString();
String processQFName =
- vertex5.getVertexId() +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
+ vertex5.getVertexId() +
AtlasImpalaHookContext.QNAME_SEP_METADATA_NAMESPACE +
CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS +
createTime;
processQFName = processQFName.toLowerCase();
@@ -152,9 +148,9 @@ public class ImpalaLineageHookIT extends
ImpalaLineageITBase {
AtlasEntity processEntity1 = validateProcess(processQFName,
queryObj.getQueryText());
AtlasEntity processExecutionEntity1 =
validateProcessExecution(processEntity1, queryObj.getQueryText());
AtlasObjectId process1 =
toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
- BaseImpalaEvent.ATTRIBUTE_PROCESS));
- Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
- Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ BaseImpalaEvent.ATTRIBUTE_PROCESS));
+ assertEquals(process1.getGuid(), processEntity1.getGuid());
+ assertEquals(numberOfProcessExecutions(processEntity1), 1);
String processExecutionQFName = processQFName +
AtlasImpalaHookContext.QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
AtlasImpalaHookContext.QNAME_SEP_PROCESS +
processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
diff --git
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookTest.java
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookTest.java
index 5a1e1da0b..4b4041b14 100644
---
a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookTest.java
+++
b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookTest.java
@@ -25,16 +25,13 @@ import org.slf4j.LoggerFactory;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import java.util.HashMap;
-import java.util.Map;
-
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
public class ImpalaLineageHookTest {
private static final Logger LOG =
LoggerFactory.getLogger(ImpalaLineageHookTest.class);
- @Test(dataProvider = "queryDataProvider")
+ @Test(dataProvider = "queryDataProvider")
public void testAllImpalaOperationTypes(String query, ImpalaOperationType
expectedOperationType) {
try {
ImpalaOperationType operationType =
ImpalaOperationParser.getImpalaOperationType(query);
@@ -50,11 +47,11 @@ public class ImpalaLineageHookTest {
String table2 = "table_" + random();
return new Object[][] {
- { "CREATE VIEW my_view AS SELECT id, name FROM " + table1,
ImpalaOperationType.CREATEVIEW },
- { "CREATE TABLE " + table1 + " AS SELECT id, name FROM " +
table1, ImpalaOperationType.CREATETABLE_AS_SELECT },
- { "ALTER VIEW my_view AS SELECT id, name FROM " + table1,
ImpalaOperationType.ALTERVIEW_AS },
- { "INSERT INTO " + table1 + " SELECT id, name FROM " + table1,
ImpalaOperationType.QUERY },
- { "WITH filtered_data AS (SELECT id, name, amount FROM " +
table1 + " WHERE amount > 100) " +
+ {"CREATE VIEW my_view AS SELECT id, name FROM " + table1,
ImpalaOperationType.CREATEVIEW },
+ {"CREATE TABLE " + table1 + " AS SELECT id, name FROM " +
table1, ImpalaOperationType.CREATETABLE_AS_SELECT },
+ {"ALTER VIEW my_view AS SELECT id, name FROM " + table1,
ImpalaOperationType.ALTERVIEW_AS },
+ {"INSERT INTO " + table1 + " SELECT id, name FROM " + table1,
ImpalaOperationType.QUERY },
+ {"WITH filtered_data AS (SELECT id, name, amount FROM " +
table1 + " WHERE amount > 100) " +
"INSERT INTO " + table2 + " SELECT id, name, amount
FROM filtered_data", ImpalaOperationType.QUERY_WITH_CLAUSE }
};
}
