This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new c8e89b4 ATLAS-3733: upgraded JanusGraph version to 0.5.1, and Tinkerpop to 3.4.6 c8e89b4 is described below commit c8e89b47e33ec7cc73987ccfce76ac6513eb3add Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Fri Apr 17 10:48:56 2020 -0700 ATLAS-3733: upgraded JanusGraph version to 0.5.1, and Tinkerpop to 3.4.6 (cherry picked from commit 750646e75f870be1f5d1c67aa23933c6bb453c40) --- .../graphdb/janus/query/NativeJanusGraphQuery.java | 3 +- .../janusgraph/diskstorage/solr/Solr6Index.java | 21 +- .../test/resources/atlas-application.properties | 1 - pom.xml | 4 +- .../userprofile/UserProfileServiceTest.java | 249 +++++++++++---------- 5 files changed, 140 insertions(+), 138 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java index a7a169a..2485938 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java @@ -39,6 +39,7 @@ import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase; import org.apache.atlas.repository.graphdb.janus.AtlasJanusVertex; import org.apache.tinkerpop.gremlin.process.traversal.Compare; import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.janusgraph.graphdb.query.JanusGraphPredicateUtils; import org.janusgraph.graphdb.query.graph.GraphCentricQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -226,7 +227,7 @@ public class NativeJanusGraphQuery implements NativeTinkerpopGraphQuery<AtlasJan JanusGraphPredicate pred; if (op instanceof ComparisionOperator) { Compare c = getGremlinPredicate((ComparisionOperator) op); - pred = JanusGraphPredicate.Converter.convert(c); + pred = JanusGraphPredicateUtils.convert(c); } else { pred = getGremlinPredicate((MatchingOperator)op); } diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java index d0d7fd7..484c161 100644 --- a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java +++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java @@ -56,7 +56,6 @@ import org.apache.http.HttpException; import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; import org.apache.http.client.HttpClient; -import org.apache.http.client.utils.HttpClientUtils; import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.impl.auth.KerberosScheme; import org.apache.http.protocol.HttpContext; @@ -114,7 +113,7 @@ import org.janusgraph.diskstorage.indexing.RawQuery; import org.janusgraph.diskstorage.solr.transform.GeoToWktConverter; import org.janusgraph.diskstorage.util.DefaultTransaction; import org.janusgraph.graphdb.configuration.PreInitializeConfigOptions; -import org.janusgraph.graphdb.database.serialize.AttributeUtil; +import org.janusgraph.graphdb.database.serialize.AttributeUtils; import org.janusgraph.graphdb.internal.Order; import org.janusgraph.graphdb.query.JanusGraphPredicate; import org.janusgraph.graphdb.query.condition.And; @@ -1009,8 +1008,8 @@ public class Solr6Index implements IndexProvider { public boolean supports(KeyInformation information, JanusGraphPredicate predicate) { final Class<?> dataType = information.getDataType(); final Mapping mapping = Mapping.getMapping(information); - if (mapping!=Mapping.DEFAULT && !AttributeUtil.isString(dataType) && - !(mapping==Mapping.PREFIX_TREE && AttributeUtil.isGeo(dataType))) return false; + if (mapping!=Mapping.DEFAULT && !AttributeUtils.isString(dataType) && + !(mapping==Mapping.PREFIX_TREE && AttributeUtils.isGeo(dataType))) return false; if (Number.class.isAssignableFrom(dataType)) { return predicate instanceof Cmp; @@ -1021,7 +1020,7 @@ public class Solr6Index implements IndexProvider { case PREFIX_TREE: return predicate == Geo.INTERSECT || predicate == Geo.WITHIN || predicate == Geo.CONTAINS; } - } else if (AttributeUtil.isString(dataType)) { + } else if (AttributeUtils.isString(dataType)) { switch(mapping) { case DEFAULT: case TEXT: @@ -1049,9 +1048,9 @@ public class Solr6Index implements IndexProvider { if (Number.class.isAssignableFrom(dataType) || dataType == Date.class || dataType == Instant.class || dataType == Boolean.class || dataType == UUID.class) { return mapping == Mapping.DEFAULT; - } else if (AttributeUtil.isString(dataType)) { + } else if (AttributeUtils.isString(dataType)) { return mapping == Mapping.DEFAULT || mapping == Mapping.TEXT || mapping == Mapping.STRING; - } else if (AttributeUtil.isGeo(dataType)) { + } else if (AttributeUtils.isGeo(dataType)) { return mapping == Mapping.DEFAULT || mapping == Mapping.PREFIX_TREE; } return false; @@ -1066,17 +1065,17 @@ public class Solr6Index implements IndexProvider { if (ParameterType.MAPPED_NAME.hasParameter(keyInfo.getParameters())) return key; String postfix; final Class dataType = keyInfo.getDataType(); - if (AttributeUtil.isString(dataType)) { + if (AttributeUtils.isString(dataType)) { final Mapping map = getStringMapping(keyInfo); switch (map) { case TEXT: postfix = "_t"; break; case STRING: postfix = "_s"; break; default: throw new IllegalArgumentException("Unsupported string mapping: " + map); } - } else if (AttributeUtil.isWholeNumber(dataType)) { + } else if (AttributeUtils.isWholeNumber(dataType)) { if (dataType.equals(Long.class)) postfix = "_l"; else postfix = "_i"; - } else if (AttributeUtil.isDecimal(dataType)) { + } else if (AttributeUtils.isDecimal(dataType)) { if (dataType.equals(Float.class)) postfix = "_f"; else postfix = "_d"; } else if (dataType.equals(BigInteger.class)) { @@ -1124,7 +1123,7 @@ public class Solr6Index implements IndexProvider { */ private static Mapping getStringMapping(KeyInformation information) { - assert AttributeUtil.isString(information.getDataType()); + assert AttributeUtils.isString(information.getDataType()); Mapping map = Mapping.getMapping(information); if (map==Mapping.DEFAULT) map = Mapping.TEXT; return map; diff --git a/intg/src/test/resources/atlas-application.properties b/intg/src/test/resources/atlas-application.properties index 159f98b..7e74d51 100644 --- a/intg/src/test/resources/atlas-application.properties +++ b/intg/src/test/resources/atlas-application.properties @@ -45,7 +45,6 @@ atlas.graphdb.backend=${graphdb.backend.impl} # Graph Storage atlas.graph.storage.backend=${graph.storage.backend} -atlas.graph.storage.transactions=false # Entity repository implementation atlas.EntityAuditRepository.impl=${entity.repository.impl} diff --git a/pom.xml b/pom.xml index c7b4702..c7bd951 100644 --- a/pom.xml +++ b/pom.xml @@ -656,8 +656,8 @@ <jersey.version>1.19</jersey.version> <jsr.version>1.1</jsr.version> - <janus.version>0.4.1</janus.version> - <tinkerpop.version>3.4.4</tinkerpop.version> + <janus.version>0.5.1</janus.version> + <tinkerpop.version>3.4.6</tinkerpop.version> <lucene-solr.version>7.3.0</lucene-solr.version> <hadoop.version>3.1.1</hadoop.version> diff --git a/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java index 2410a9c..eeab3bc 100644 --- a/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/userprofile/UserProfileServiceTest.java @@ -54,7 +54,9 @@ import static org.testng.Assert.assertTrue; public class UserProfileServiceTest { private UserProfileService userProfileService; private AtlasTypeDefStore typeDefStore; - private int max_searches = 4; + + private static final int NUM_USERS = 2; + private static final int NUM_SEARCHES = 4; @Inject public void UserProfileServiceTest(AtlasTypeRegistry typeRegistry, @@ -78,142 +80,121 @@ public class UserProfileServiceTest { @Test public void filterInternalType() throws AtlasBaseException { SearchFilter searchFilter = new SearchFilter(); + FilterUtil.addParamsToHideInternalType(searchFilter); + AtlasTypesDef filteredTypeDefs = typeDefStore.searchTypesDef(searchFilter); assertNotNull(filteredTypeDefs); + Optional<AtlasEntityDef> anyInternal = filteredTypeDefs.getEntityDefs().stream().filter(e -> e.getSuperTypes().contains("__internal")).findAny(); + assertFalse(anyInternal.isPresent()); } - @Test + @Test(dependsOnMethods = "filterInternalType") public void createsNewProfile() throws AtlasBaseException { - int i = 0; - assertSaveLoadUserProfile(i++); - assertSaveLoadUserProfile(i); + for (int i = 0; i < NUM_USERS; i++) { + AtlasUserProfile expected = getAtlasUserProfile(i); + AtlasUserProfile actual = userProfileService.saveUserProfile(expected); + + assertNotNull(actual); + assertEquals(expected.getName(), actual.getName()); + assertEquals(expected.getFullName(), actual.getFullName()); + assertNotNull(actual.getGuid()); + } } - @Test(dependsOnMethods = { "createsNewProfile", "savesQueryForAnNonExistentUser" }, expectedExceptions = AtlasBaseException.class) - public void atteptsToAddAlreadyExistingQueryForAnExistingUser() throws AtlasBaseException { - SearchParameters expectedSearchParameter = getActualSearchParameters(); + @Test(dependsOnMethods = "createsNewProfile") + public void saveSearchesForUser() throws AtlasBaseException { + String userName = getIndexBasedUserName(0); + SearchParameters searchParameters = getActualSearchParameters(); - for (int i = 0; i < 2; i++) { - String userName = getIndexBasedUserName(i); + for (int i = 0; i < NUM_SEARCHES; i++) { + userProfileService.addSavedSearch(getDefaultSavedSearch(userName, getIndexBasedQueryName(i), searchParameters)); + } - for (int j = 0; j < max_searches; j++) { - String queryName = getIndexBasedQueryName(j); - AtlasUserSavedSearch expected = getDefaultSavedSearch(userName, queryName, expectedSearchParameter); - AtlasUserSavedSearch actual = userProfileService.addSavedSearch(expected); + for (int i = 0; i < NUM_SEARCHES; i++) { + AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(userName, getIndexBasedQueryName(i)); - assertNotNull(actual); - assertNotNull(actual.getGuid()); - assertEquals(actual.getOwnerName(), expected.getOwnerName()); - assertEquals(actual.getName(), expected.getName()); - assertEquals(actual.getSearchType(), expected.getSearchType()); - assertEquals(actual.getSearchParameters(), expected.getSearchParameters()); - } + assertEquals(savedSearch.getName(), getIndexBasedQueryName(i)); + assertEquals(savedSearch.getSearchParameters(), searchParameters); } } - @Test(dependsOnMethods = { "createsNewProfile", "savesQueryForAnNonExistentUser", "atteptsToAddAlreadyExistingQueryForAnExistingUser" }) - public void savesExistingQueryForAnExistingUser() throws AtlasBaseException { + @Test(dependsOnMethods = "saveSearchesForUser", expectedExceptions = AtlasBaseException.class) + public void attemptToAddExistingSearch() throws AtlasBaseException { + String userName = getIndexBasedUserName(0); SearchParameters expectedSearchParameter = getActualSearchParameters(); - for (int i = 0; i < 2; i++) { - String userName = getIndexBasedUserName(i); - - for (int j = 4; j < max_searches + 6; j++) { - String queryName = getIndexBasedQueryName(j); - AtlasUserSavedSearch actual = userProfileService.addSavedSearch(getDefaultSavedSearch(userName, queryName, expectedSearchParameter)); - assertNotNull(actual); - - AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(userName, queryName); - assertNotNull(savedSearch); - assertEquals(savedSearch.getSearchParameters(), expectedSearchParameter); - } + for (int j = 0; j < NUM_SEARCHES; j++) { + String queryName = getIndexBasedQueryName(j); + AtlasUserSavedSearch expected = getDefaultSavedSearch(userName, queryName, expectedSearchParameter); + AtlasUserSavedSearch actual = userProfileService.addSavedSearch(expected); + + assertNotNull(actual); + assertNotNull(actual.getGuid()); + assertEquals(actual.getOwnerName(), expected.getOwnerName()); + assertEquals(actual.getName(), expected.getName()); + assertEquals(actual.getSearchType(), expected.getSearchType()); + assertEquals(actual.getSearchParameters(), expected.getSearchParameters()); } } - private SearchParameters getActualSearchParameters() { - SearchParameters sp = new SearchParameters(); - sp.setClassification("test-classification"); - sp.setQuery("g.v().has('__guid').__guid.toList()"); - sp.setLimit(10); - sp.setTypeName("some-type"); - - return sp; - } - - @Test(dependsOnMethods = "createsNewProfile") - public void savesQueryForAnNonExistentUser() throws AtlasBaseException { - String expectedUserName = getIndexBasedUserName(0); - String expectedQueryName = "testQuery"; - SearchParameters expectedSearchParam = getActualSearchParameters(); - AtlasUserSavedSearch expectedSavedSearch = getDefaultSavedSearch(expectedUserName, expectedQueryName, expectedSearchParam); - - AtlasUserSavedSearch actual = userProfileService.addSavedSearch(expectedSavedSearch); - assertEquals(actual.getOwnerName(), expectedUserName); - assertEquals(actual.getName(), expectedQueryName); - } + @Test(dependsOnMethods = "attemptToAddExistingSearch") + public void verifySavedSearchesForUser() throws AtlasBaseException { + String userName = getIndexBasedUserName(0); + List<AtlasUserSavedSearch> searches = userProfileService.getSavedSearches(userName); + List<String> names = getIndexBasedQueryNamesList(); - private AtlasUserSavedSearch getDefaultSavedSearch(String userName, String queryName, SearchParameters expectedSearchParam) { - return new AtlasUserSavedSearch(userName, queryName, - BASIC, expectedSearchParam); + for (int i = 0; i < names.size(); i++) { + assertTrue(names.contains(searches.get(i).getName()), searches.get(i).getName() + " failed!"); + } } - @Test(dependsOnMethods = "createsNewProfile") - public void savesMultipleQueriesForUser() throws AtlasBaseException { - final String userName = getIndexBasedUserName(0); - createUserWithSavedQueries(userName); - } + @Test(dependsOnMethods = "verifySavedSearchesForUser") + public void verifyQueryConversionFromJSON() throws AtlasBaseException { + List<AtlasUserSavedSearch> list = userProfileService.getSavedSearches("first-0"); - private void createUserWithSavedQueries(String userName) throws AtlasBaseException { - SearchParameters actualSearchParameter = getActualSearchParameters(); + for (int i = 0; i < NUM_SEARCHES; i++) { + SearchParameters sp = list.get(i).getSearchParameters(); + String json = AtlasType.toJson(sp); - saveQueries(userName, actualSearchParameter); - for (int i = 0; i < max_searches; i++) { - AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(userName, getIndexBasedQueryName(i)); - assertEquals(savedSearch.getName(), getIndexBasedQueryName(i)); - assertEquals(savedSearch.getSearchParameters(), actualSearchParameter); + assertEquals(AtlasType.toJson(getActualSearchParameters()).replace("\n", "").replace(" ", ""), json); } } - private void saveQueries(String userName, SearchParameters sp) throws AtlasBaseException { - for (int i = 0; i < max_searches; i++) { - userProfileService.addSavedSearch(getDefaultSavedSearch(userName, getIndexBasedQueryName(i), sp)); - } - } + @Test(dependsOnMethods = "verifyQueryConversionFromJSON") + public void addAdditionalSearchesForUser() throws AtlasBaseException { + SearchParameters expectedSearchParameter = getActualSearchParameters(); - @Test(dependsOnMethods = {"createsNewProfile", "savesMultipleQueriesForUser"}) - public void verifyQueryNameListForUser() throws AtlasBaseException { - final String userName = getIndexBasedUserName(0); + for (int i = 0; i < NUM_USERS; i++) { + String userName = getIndexBasedUserName(i); - List<AtlasUserSavedSearch> list = userProfileService.getSavedSearches(userName); - List<String> names = getIndexBasedQueryNamesList(); - for (int i = 0; i < names.size(); i++) { - assertTrue(names.contains(list.get(i).getName()), list.get(i).getName() + " failed!"); - } - } + for (int j = 0; j < 6; j++) { + String queryName = getIndexBasedQueryName(NUM_SEARCHES + j); + AtlasUserSavedSearch actual = userProfileService.addSavedSearch(getDefaultSavedSearch(userName, queryName, expectedSearchParameter)); - @Test(dependsOnMethods = {"createsNewProfile", "savesMultipleQueriesForUser"}) - public void verifyQueryConversionFromJSON() throws AtlasBaseException { - List<AtlasUserSavedSearch> list = userProfileService.getSavedSearches("first-0"); + assertNotNull(actual); - for (int i = 0; i < max_searches; i++) { - SearchParameters sp = list.get(i).getSearchParameters(); - String json = AtlasType.toJson(sp); - assertEquals(AtlasType.toJson(getActualSearchParameters()).replace("\n", "").replace(" ", ""), json); + AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(userName, queryName); + + assertNotNull(savedSearch); + assertEquals(savedSearch.getSearchParameters(), expectedSearchParameter); + } } } - @Test(dependsOnMethods = {"createsNewProfile", "savesMultipleQueriesForUser", "verifyQueryConversionFromJSON"}) + @Test(dependsOnMethods = { "addAdditionalSearchesForUser"}) public void updateSearch() throws AtlasBaseException { - final String queryName = getIndexBasedQueryName(0); - String userName = getIndexBasedUserName(0); - AtlasUserSavedSearch expected = userProfileService.getSavedSearch(userName, queryName); + String userName = getIndexBasedUserName(0); + String queryName = getIndexBasedQueryName(0); + AtlasUserSavedSearch expected = userProfileService.getSavedSearch(userName, queryName); + assertNotNull(expected); SearchParameters sp = expected.getSearchParameters(); + sp.setClassification("new-classification"); AtlasUserSavedSearch actual = userProfileService.updateSavedSearch(expected); @@ -223,56 +204,62 @@ public class UserProfileServiceTest { assertEquals(actual.getSearchParameters().getClassification(), expected.getSearchParameters().getClassification()); } - @Test(dependsOnMethods = {"createsNewProfile", "savesMultipleQueriesForUser", "verifyQueryNameListForUser"}, expectedExceptions = AtlasBaseException.class) + @Test(dependsOnMethods = { "updateSearch" }) public void deleteUsingGuid() throws AtlasBaseException { - final String queryName = getIndexBasedQueryName(1); - String userName = getIndexBasedUserName(0); + String userName = getIndexBasedUserName(0); + String queryName = getIndexBasedQueryName(1); + AtlasUserSavedSearch expected = userProfileService.getSavedSearch(userName, queryName); - AtlasUserSavedSearch expected = userProfileService.getSavedSearch(userName, queryName); assertNotNull(expected); userProfileService.deleteSavedSearch(expected.getGuid()); - userProfileService.getSavedSearch(userName, queryName); + + try { + userProfileService.getSavedSearch(userName, queryName); + } catch (AtlasBaseException ex) { + assertEquals(ex.getAtlasErrorCode().name(), AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND.name()); + } } - @Test(dependsOnMethods = {"createsNewProfile", "savesMultipleQueriesForUser", "verifyQueryNameListForUser"}) + @Test(dependsOnMethods = { "deleteUsingGuid" }) public void deleteSavedQuery() throws AtlasBaseException { - final String userName = getIndexBasedUserName(0); + String userName = getIndexBasedUserName(0); AtlasUserProfile expected = userProfileService.getUserProfile(userName); + assertNotNull(expected); - int new_max_searches = expected.getSavedSearches().size(); - String queryNameToBeDeleted = getIndexBasedQueryName(max_searches - 2); + int searchCount = expected.getSavedSearches().size(); + String queryNameToBeDeleted = getIndexBasedQueryName(NUM_SEARCHES - 2); + userProfileService.deleteSearchBySearchName(userName, queryNameToBeDeleted); List<AtlasUserSavedSearch> savedSearchList = userProfileService.getSavedSearches(userName); - assertEquals(savedSearchList.size(), new_max_searches - 1); + + assertEquals(savedSearchList.size(), searchCount - 1); } - @Test(dependsOnMethods = {"createsNewProfile", "savesMultipleQueriesForUser", "verifyQueryNameListForUser"}) + + @Test(dependsOnMethods = { "deleteSavedQuery" }) void deleteUser() throws AtlasBaseException { - String userName = getIndexBasedUserName(1); + String userName = getIndexBasedUserName(0); + AtlasUserProfile userProfile = userProfileService.getUserProfile(userName); + + if (userProfile.getSavedSearches() != null) { + for (AtlasUserSavedSearch savedSearch : userProfile.getSavedSearches()) { + userProfileService.deleteSavedSearch(savedSearch.getGuid()); + } + } userProfileService.deleteUserProfile(userName); + try { userProfileService.getUserProfile(userName); - } - catch(AtlasBaseException ex) { + } catch (AtlasBaseException ex) { assertEquals(ex.getAtlasErrorCode().name(), AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND.name()); } } - private void assertSaveLoadUserProfile(int i) throws AtlasBaseException { - String s = String.valueOf(i); - AtlasUserProfile expected = getAtlasUserProfile(i); - - AtlasUserProfile actual = userProfileService.saveUserProfile(expected); - assertNotNull(actual); - assertEquals(expected.getName(), actual.getName()); - assertEquals(expected.getFullName(), actual.getFullName()); - assertNotNull(actual.getGuid()); - } - public static AtlasUserProfile getAtlasUserProfile(Integer s) { + private static AtlasUserProfile getAtlasUserProfile(Integer s) { return new AtlasUserProfile(getIndexBasedUserName(s), String.format("first-%s last-%s", s, s)); } @@ -284,9 +271,25 @@ public class UserProfileServiceTest { return String.format("testQuery-%s", i.toString()); } - public List<String> getIndexBasedQueryNamesList() { + private SearchParameters getActualSearchParameters() { + SearchParameters sp = new SearchParameters(); + + sp.setClassification("test-classification"); + sp.setQuery("g.v().has('__guid').__guid.toList()"); + sp.setLimit(10); + sp.setTypeName("some-type"); + + return sp; + } + + private AtlasUserSavedSearch getDefaultSavedSearch(String userName, String queryName, SearchParameters expectedSearchParam) { + return new AtlasUserSavedSearch(userName, queryName, BASIC, expectedSearchParam); + } + + private List<String> getIndexBasedQueryNamesList() { List<String> list = new ArrayList<>(); - for (int i = 0; i < max_searches; i++) { + + for (int i = 0; i < NUM_SEARCHES; i++) { list.add(getIndexBasedQueryName(i)); }