Repository: atlas Updated Branches: refs/heads/master f28d0f54d -> 8c9d3da2c
ATLAS-2357: Fix IT failures in hive-bridge module Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8c9d3da2 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8c9d3da2 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8c9d3da2 Branch: refs/heads/master Commit: 8c9d3da2cecccd043dd8257c4102b8b147e4f6b7 Parents: f28d0f5 Author: Sarath Subramanian <[email protected]> Authored: Fri Jan 12 11:18:06 2018 -0800 Committer: Sarath Subramanian <[email protected]> Committed: Fri Jan 12 11:18:06 2018 -0800 ---------------------------------------------------------------------- addons/hive-bridge/pom.xml | 49 ++++++- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 3 +- .../java/org/apache/atlas/hive/HiveITBase.java | 9 +- .../org/apache/atlas/hive/hook/HiveHookIT.java | 9 +- .../test/resources/atlas-application.properties | 135 +++++++++++++++++ .../src/test/resources/atlas-log4j.xml | 137 +++++++++++++++++ .../src/test/resources/hive-site.xml | 4 +- .../src/test/resources/policy-store.txt | 9 ++ .../test/resources/users-credentials.properties | 3 + pom.xml | 23 +-- .../apache/atlas/runner/LocalSolrRunner.java | 2 +- webapp/pom.xml | 31 +++- .../test/resources/atlas-application.properties | 146 +++++++++++++++++++ webapp/src/test/resources/policy-store.txt | 9 ++ .../test/resources/users-credentials.properties | 3 + 15 files changed, 528 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index 9421e32..8f7024b 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -346,16 +346,16 @@ </httpConnector> <war>../../webapp/target/atlas-webapp-${project.version}.war</war> <daemon>true</daemon> + <webAppSourceDirectory>../../webapp/src/test/webapp</webAppSourceDirectory> <webApp> <contextPath>/</contextPath> <descriptor>${project.basedir}/../../webapp/src/test/webapp/WEB-INF/web.xml</descriptor> - <extraClasspath>${project.basedir}/../../webapp/target/test-classes/</extraClasspath> </webApp> <useTestScope>true</useTestScope> <systemProperties> <systemProperty> <name>log4j.configuration</name> - <value>file://${project.basedir}/../../distro/src/conf/atlas-log4j.xml</value> + <value>${project.basedir}/target/test-classes/atlas-log4j.xml</value> </systemProperty> <systemProperty> <name>atlas.log.file</name> @@ -363,18 +363,35 @@ </systemProperty> <systemProperty> <name>atlas.log.dir</name> - <value>${project.build.directory}/logs</value> + <value>${project.basedir}/target/logs</value> + </systemProperty> + <systemProperty> + <name>atlas.graphdb.backend</name> + <value>${graphdb.backend.impl}</value> + </systemProperty> + <systemProperty> + <name>keystore.file</name> + <value>${project.build.directory}/atlas.keystore + </value> + </systemProperty> + <systemProperty> + <name>truststore.file</name> + <value>${project.build.directory}/atlas.keystore</value> + </systemProperty> + <systemProperty> + <name>atlas.home</name> + <value>${project.basedir}/target</value> </systemProperty> <systemProperty> <name>atlas.data</name> - <value>${project.build.directory}/data</value> + <value>${project.basedir}/target/data</value> </systemProperty> <systemProperty> <key>atlas.conf</key> - <value>${project.build.directory}/../../../intg/target/test-classes</value> + <value>${project.basedir}/target/test-classes</value> </systemProperty> <systemProperty> - <key>atlas.home</key> + <key>embedded.solr.directory</key> <value>${project.basedir}/target</value> </systemProperty> </systemProperties> @@ -458,6 +475,26 @@ </execution> </executions> </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <executions> + <execution> + <id>copy-solr-resources</id> + <phase>validate</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/solr</outputDirectory> + <resources> + <resource> + <directory>${basedir}/../../test-tools/src/main/resources/solr</directory> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index 2d2a9bf..7cca48f 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -94,6 +94,7 @@ public class HiveMetaStoreBridge { public static final String CREATE_TIME = "createTime"; public static final String LAST_ACCESS_TIME = "lastAccessTime"; public static final String HDFS_PATH = "hdfs_path"; + public static final String SEP = ":".intern(); private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class); @@ -409,7 +410,7 @@ public class HiveMetaStoreBridge { public static String getTableProcessQualifiedName(String clusterName, Table table) { String tableQualifiedName = getTableQualifiedName(clusterName, table); Date createdTime = getTableCreatedTime(table); - return tableQualifiedName + HiveHook.SEP + createdTime.getTime(); + return tableQualifiedName + SEP + createdTime.getTime(); } /** http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java index b4ea3f9..01418cf 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java @@ -127,6 +127,13 @@ public class HiveITBase { return "pfile://" + mkdir(path); } + protected String file(String tag) throws Exception { + String filename = System.getProperty("user.dir") + "/target/" + tag + "-data-" + random(); + File file = new File(filename); + file.createNewFile(); + return file.getAbsolutePath(); + } + protected String mkdir(String tag) throws Exception { String filename = "./target/" + tag + "-data-" + random(); File file = new File(filename); @@ -155,7 +162,7 @@ public class HiveITBase { protected String assertEntityIsRegistered(final String typeName, final String property, final String value, final HiveHookIT.AssertPredicate assertPredicate) throws Exception { - waitFor(2000, new HiveHookIT.Predicate() { + waitFor(80000, new HiveHookIT.Predicate() { @Override public void evaluate() throws Exception { Referenceable entity = atlasClient.getEntity(typeName, property, value); http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 262f7dc..1b1ebec 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -720,13 +720,6 @@ public class HiveHookIT extends HiveITBase { //TODO -Add update test case } - private String file(String tag) throws Exception { - String filename = "./target/" + tag + "-data-" + random(); - File file = new File(filename); - file.createNewFile(); - return file.getAbsolutePath(); - } - @Test public void testExportImportUnPartitionedTable() throws Exception { String tableName = createTable(false); @@ -1812,7 +1805,7 @@ public class HiveHookIT extends HiveITBase { } private void assertEntityIsNotRegistered(final String typeName, final String property, final String value) throws Exception { - waitFor(1000, new Predicate() { + waitFor(80000, new Predicate() { @Override public void evaluate() throws Exception { try { http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/atlas-application.properties b/addons/hive-bridge/src/test/resources/atlas-application.properties new file mode 100644 index 0000000..eff0aa4 --- /dev/null +++ b/addons/hive-bridge/src/test/resources/atlas-application.properties @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#system property +atlas.data=${sys:user.dir}/target/data + + + +#re-use existing property +atlas.graph.data=${atlas.data}/graph + +#plain property +atlas.service=atlas + +######### Atlas Server Configs ######### +atlas.rest.address=http://localhost:31000 + +######### Graph Database Configs ######### + + +# Graph database implementation. Value inserted by maven. +atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase + +# Graph Storage +atlas.graph.storage.backend=berkeleyje + +# Entity repository implementation +atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + +# Graph Search Index Backend +atlas.graph.index.search.backend=solr + +#Berkeley storage directory +atlas.graph.storage.directory=${sys:atlas.data}/berkley + +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 + +atlas.graph.storage.hostname=${graph.storage.hostname} +atlas.graph.storage.hbase.regions-per-server=1 +atlas.graph.storage.lock.wait-time=10000 + +#ElasticSearch +atlas.graph.index.search.directory=${sys:atlas.data}/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +# Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} +atlas.graph.index.search.solr.embedded=true +atlas.graph.index.search.max-result-set-size=150 + +######### Hive Lineage Configs ######### +## Schema +atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns +atlas.lineage.schema.query.hive_table_v1=hive_table_v1 where __guid='%s'\, columns + +######### Notification Configs ######### +atlas.notification.embedded=true + +atlas.kafka.zookeeper.connect=localhost:19026 +atlas.kafka.bootstrap.servers=localhost:19027 +atlas.kafka.data=${sys:atlas.data}/kafka +atlas.kafka.zookeeper.session.timeout.ms=4000 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.consumer.timeout.ms=4000 +atlas.kafka.auto.commit.interval.ms=100 +atlas.kafka.hook.group.id=atlas +atlas.kafka.entities.group.id=atlas_entities +#atlas.kafka.auto.commit.enable=false + +atlas.kafka.enable.auto.commit=false +atlas.kafka.auto.offset.reset=earliest +atlas.kafka.session.timeout.ms=30000 + + + +######### Entity Audit Configs ######### +atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS +atlas.audit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=localhost +atlas.audit.hbase.zookeeper.property.clientPort=19026 + +######### Security Properties ######### + +# SSL config +atlas.enableTLS=false +atlas.server.https.port=31443 + +######### Security Properties ######### + +hbase.security.authentication=simple + +atlas.hook.falcon.synchronous=true + +######### JAAS Configuration ######## + +atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.KafkaClient.loginModuleControlFlag = required +atlas.jaas.KafkaClient.option.useKeyTab = true +atlas.jaas.KafkaClient.option.storeKey = true +atlas.jaas.KafkaClient.option.serviceName = kafka +atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab +atlas.jaas.KafkaClient.option.principal = atlas/[email protected] + +######### High Availability Configuration ######## +atlas.server.ha.enabled=false +#atlas.server.ids=id1 +#atlas.server.address.id1=localhost:21000 + +#########POLICY FILE PATH ######### +# atlas.auth.policy.file=policy-store.txt + +atlas.authentication.method.file=true +atlas.authentication.method.ldap.type=none +# atlas.authentication.method.file.filename=users-credentials.properties +atlas.authentication.method.kerberos=false \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/atlas-log4j.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/atlas-log4j.xml b/addons/hive-bridge/src/test/resources/atlas-log4j.xml new file mode 100755 index 0000000..a0f9629 --- /dev/null +++ b/addons/hive-bridge/src/test/resources/atlas-log4j.xml @@ -0,0 +1,137 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/> + </layout> + </appender> + + <appender name="FILE" class="org.apache.log4j.RollingFileAppender"> + <param name="File" value="${atlas.log.dir}/${atlas.log.file}"/> + <param name="Append" value="true"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/> + <param name="maxFileSize" value="100MB" /> + <param name="maxBackupIndex" value="20" /> + </layout> + </appender> + + <appender name="AUDIT" class="org.apache.log4j.RollingFileAppender"> + <param name="File" value="${atlas.log.dir}/audit.log"/> + <param name="Append" value="true"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %x %m%n"/> + <param name="maxFileSize" value="100MB" /> + <param name="maxBackupIndex" value="20" /> + </layout> + </appender> + + <appender name="METRICS" class="org.apache.log4j.RollingFileAppender"> + <param name="File" value="${atlas.log.dir}/metric.log"/> + <param name="Append" value="true"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %x %m%n"/> + <param name="maxFileSize" value="100MB" /> + </layout> + </appender> + + <appender name="FAILED" class="org.apache.log4j.RollingFileAppender"> + <param name="File" value="${atlas.log.dir}/failed.log"/> + <param name="Append" value="true"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d %m"/> + <param name="maxFileSize" value="100MB" /> + <param name="maxBackupIndex" value="20" /> + </layout> + </appender> + + <!-- Uncomment the following for perf logs --> + <!-- + <appender name="perf_appender" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="file" value="${atlas.log.dir}/atlas_perf.log" /> + <param name="datePattern" value="'.'yyyy-MM-dd" /> + <param name="append" value="true" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d|%t|%m%n" /> + </layout> + </appender> + + <logger name="org.apache.atlas.perf" additivity="false"> + <level value="debug" /> + <appender-ref ref="perf_appender" /> + </logger> + --> + + <logger name="org.apache.atlas" additivity="false"> + <level value="info"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="com.thinkaurelius.titan" additivity="false"> + <level value="warn"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="org.springframework" additivity="false"> + <level value="warn"/> + <appender-ref ref="console"/> + </logger> + + <logger name="org.eclipse" additivity="false"> + <level value="warn"/> + <appender-ref ref="console"/> + </logger> + + <logger name="com.sun.jersey" additivity="false"> + <level value="warn"/> + <appender-ref ref="console"/> + </logger> + + <!-- to avoid logs - The configuration log.flush.interval.messages = 1 was supplied but isn't a known config --> + <logger name="org.apache.kafka.common.config.AbstractConfig" additivity="false"> + <level value="error"/> + <appender-ref ref="FILE"/> + </logger> + + <logger name="AUDIT" additivity="false"> + <level value="info"/> + <appender-ref ref="AUDIT"/> + </logger> + + <logger name="METRICS" additivity="false"> + <level value="debug"/> + <appender-ref ref="METRICS"/> + </logger> + + <logger name="FAILED" additivity="false"> + <level value="info"/> + <appender-ref ref="AUDIT"/> + </logger> + + <root> + <priority value="warn"/> + <appender-ref ref="FILE"/> + </root> + +</log4j:configuration> http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/hive-site.xml b/addons/hive-bridge/src/test/resources/hive-site.xml index 876f41d..ff98668 100644 --- a/addons/hive-bridge/src/test/resources/hive-site.xml +++ b/addons/hive-bridge/src/test/resources/hive-site.xml @@ -43,12 +43,12 @@ <property> <name>hive.metastore.warehouse.dir</name> - <value>${user.dir}/target/metastore</value> + <value>${project.basedir}/target/metastore</value> </property> <property> <name>javax.jdo.option.ConnectionURL</name> - <value>jdbc:derby:${user.dir}/target/metastore_db;create=true</value> + <value>jdbc:derby:${project.basedir}/target/metastore_db;create=true</value> </property> <property> http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/policy-store.txt ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/policy-store.txt b/addons/hive-bridge/src/test/resources/policy-store.txt new file mode 100644 index 0000000..048affe --- /dev/null +++ b/addons/hive-bridge/src/test/resources/policy-store.txt @@ -0,0 +1,9 @@ +##Policy Format +##r-READ, w-WRITE, u-UPDATE, d-DELETE +##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name +## +adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:* +dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:* +dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:* +hadoopPolicy;;;;hadoop:rwud;;type:*,entity:*,operation:* +rangerTagSyncPolicy;;;;RANGER_TAG_SYNC:r;;type:*,entity:* http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/users-credentials.properties ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/users-credentials.properties b/addons/hive-bridge/src/test/resources/users-credentials.properties new file mode 100644 index 0000000..3fc3bb1 --- /dev/null +++ b/addons/hive-bridge/src/test/resources/users-credentials.properties @@ -0,0 +1,3 @@ +#username=group::sha256-password +admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 +rangertagsync=RANGER_TAG_SYNC::e3f67240f5117d1753c940dae9eea772d36ed5fe9bd9c94a300e40413f1afb9d http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c64b26e..3f5c042 100644 --- a/pom.xml +++ b/pom.xml @@ -1839,6 +1839,7 @@ <user.dir>${project.basedir}</user.dir> <atlas.data>${project.build.directory}/data</atlas.data> <log4j.configuration>atlas-log4j.xml</log4j.configuration> + <embedded.solr.directory>${project.basedir}/target</embedded.solr.directory> </systemProperties> <skipTests>${skipTests}</skipTests> <forkCount>${surefire.forkCount}</forkCount> @@ -1868,6 +1869,7 @@ <projectBaseDir>${projectBaseDir}</projectBaseDir> <atlas.data>${project.build.directory}/data</atlas.data> <log4j.configuration>atlas-log4j.xml</log4j.configuration> + <embedded.solr.directory>${project.basedir}/target</embedded.solr.directory> </systemPropertyVariables> <redirectTestOutputToFile>true</redirectTestOutputToFile> <argLine>-Djava.awt.headless=true -Dproject.version=${project.version} @@ -2125,27 +2127,6 @@ <skipAssembly>true</skipAssembly> </configuration> </plugin> - - <plugin> - <artifactId>maven-resources-plugin</artifactId> - <executions> - <execution> - <id>copy-resources</id> - <phase>validate</phase> - <goals> - <goal>copy-resources</goal> - </goals> - <configuration> - <outputDirectory>${project.build.directory}/solr</outputDirectory> - <resources> - <resource> - <directory>${basedir}/test-tools/src/main/resources/solr</directory> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java ---------------------------------------------------------------------- diff --git a/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java b/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java index cdcac64..ed8e9e1 100644 --- a/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java +++ b/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java @@ -37,7 +37,7 @@ import java.util.stream.Collectors; public class LocalSolrRunner { protected static final String[] COLLECTIONS = readCollections(); - private static final String TARGET_DIRECTORY = System.getProperty("user.dir") + File.separator + "target"; + private static final String TARGET_DIRECTORY = System.getProperty("embedded.solr.directory"); private static final String COLLECTIONS_FILE = "collections.txt"; private static final String SOLR_XML = "solr.xml"; private static final String TEMPLATE_DIRECTORY = "core-template"; http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index 2cf6e10..30711db 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -602,7 +602,7 @@ </systemProperty> <systemProperty> <name>atlas.log.dir</name> - <value>${project.build.directory}/logs</value> + <value>${project.basedir}/target/logs</value> </systemProperty> <systemProperty> <name>atlas.graphdb.backend</name> @@ -619,15 +619,19 @@ </systemProperty> <systemProperty> <name>atlas.home</name> - <value>${project.build.directory}</value> + <value>${project.basedir}/target</value> </systemProperty> <systemProperty> <name>atlas.data</name> - <value>${project.build.directory}/data</value> + <value>${project.basedir}/target/data</value> </systemProperty> <systemProperty> <key>atlas.conf</key> - <value>${project.build.directory}/../../intg/target/test-classes</value> + <value>${project.basedir}/target/test-classes</value> + </systemProperty> + <systemProperty> + <key>embedded.solr.directory</key> + <value>${project.basedir}/target</value> </systemProperty> </systemProperties> <stopKey>atlas-stop</stopKey> @@ -706,6 +710,25 @@ <skipEnunciate>${skipEnunciate}</skipEnunciate> </configuration> </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <phase>post-integration-test</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <delete dir="${project.build.directory}/data"/> + <delete dir="${project.build.directory}/logs"/> + </tasks> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/atlas-application.properties b/webapp/src/test/resources/atlas-application.properties new file mode 100644 index 0000000..077a90f --- /dev/null +++ b/webapp/src/test/resources/atlas-application.properties @@ -0,0 +1,146 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#system property +atlas.data=${sys:user.dir}/target/data + + + +#re-use existing property +atlas.graph.data=${atlas.data}/graph + +#plain property +atlas.service=atlas + +#invalid system property +atlas.db=${atlasdb} + +atlas.TypeSystem.impl=org.apache.atlas.typesystem.types.TypeSystem + + + +######### Atlas Server Configs ######### +atlas.rest.address=http://localhost:31000 + +######### Graph Database Configs ######### + + +# Graph database implementation. Value inserted by maven. +atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase + +# Graph Storage +atlas.graph.storage.backend=berkeleyje + +# Entity repository implementation +atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository + +# Graph Search Index Backend +atlas.graph.index.search.backend=solr + +#Berkeley storage directory +atlas.graph.storage.directory=${sys:atlas.data}/berkley + +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 + +atlas.graph.storage.hostname=${graph.storage.hostname} +atlas.graph.storage.hbase.regions-per-server=1 +atlas.graph.storage.lock.wait-time=10000 + +#ElasticSearch +atlas.graph.index.search.directory=${sys:atlas.data}/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + +# Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} +atlas.graph.index.search.solr.embedded=true +atlas.graph.index.search.max-result-set-size=150 + +######### Hive Lineage Configs ######### +## Schema +atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns +atlas.lineage.schema.query.hive_table_v1=hive_table_v1 where __guid='%s'\, columns + +######### Notification Configs ######### +atlas.notification.embedded=true + +atlas.kafka.zookeeper.connect=localhost:19026 +atlas.kafka.bootstrap.servers=localhost:19027 +atlas.kafka.data=${sys:atlas.data}/kafka +atlas.kafka.zookeeper.session.timeout.ms=4000 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.kafka.consumer.timeout.ms=4000 +atlas.kafka.auto.commit.interval.ms=100 +atlas.kafka.hook.group.id=atlas +atlas.kafka.entities.group.id=atlas_entities +#atlas.kafka.auto.commit.enable=false + +atlas.kafka.enable.auto.commit=false +atlas.kafka.auto.offset.reset=earliest +atlas.kafka.session.timeout.ms=30000 + + + +######### Entity Audit Configs ######### +atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS +atlas.audit.zookeeper.session.timeout.ms=1000 +atlas.audit.hbase.zookeeper.quorum=localhost +atlas.audit.hbase.zookeeper.property.clientPort=19026 + +######### Security Properties ######### + +# SSL config +atlas.enableTLS=false +atlas.server.https.port=31443 + +######### Security Properties ######### + +hbase.security.authentication=simple + +atlas.hook.falcon.synchronous=true + +######### JAAS Configuration ######## + +atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule +atlas.jaas.KafkaClient.loginModuleControlFlag = required +atlas.jaas.KafkaClient.option.useKeyTab = true +atlas.jaas.KafkaClient.option.storeKey = true +atlas.jaas.KafkaClient.option.serviceName = kafka +atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab +atlas.jaas.KafkaClient.option.principal = atlas/[email protected] + +######### High Availability Configuration ######## +atlas.server.ha.enabled=false +#atlas.server.ids=id1 +#atlas.server.address.id1=localhost:21000 + +#########POLICY FILE PATH ######### +# atlas.auth.policy.file=policy-store.txt + +atlas.authentication.method.file=true +atlas.authentication.method.ldap.type=none +# atlas.authentication.method.file.filename=users-credentials.properties +atlas.authentication.method.kerberos=false + +######### Gremlin Search Configuration ######### +# Set to false to disable gremlin search. +atlas.search.gremlin.enable=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/src/test/resources/policy-store.txt ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/policy-store.txt b/webapp/src/test/resources/policy-store.txt new file mode 100644 index 0000000..048affe --- /dev/null +++ b/webapp/src/test/resources/policy-store.txt @@ -0,0 +1,9 @@ +##Policy Format +##r-READ, w-WRITE, u-UPDATE, d-DELETE +##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name +## +adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:* +dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:* +dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:* +hadoopPolicy;;;;hadoop:rwud;;type:*,entity:*,operation:* +rangerTagSyncPolicy;;;;RANGER_TAG_SYNC:r;;type:*,entity:* http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/src/test/resources/users-credentials.properties ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/users-credentials.properties b/webapp/src/test/resources/users-credentials.properties new file mode 100644 index 0000000..3fc3bb1 --- /dev/null +++ b/webapp/src/test/resources/users-credentials.properties @@ -0,0 +1,3 @@ +#username=group::sha256-password +admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 +rangertagsync=RANGER_TAG_SYNC::e3f67240f5117d1753c940dae9eea772d36ed5fe9bd9c94a300e40413f1afb9d
