ATLAS-2079: Fix coverity scan issue and IT failures introduced by ATLAS-2062
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/024fd221 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/024fd221 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/024fd221 Branch: refs/heads/master Commit: 024fd221043ae745921d9bfa9e6e689507972f1a Parents: 88eadb9 Author: Sarath Subramanian <[email protected]> Authored: Tue Aug 22 23:06:31 2017 -0700 Committer: Sarath Subramanian <[email protected]> Committed: Tue Aug 22 23:06:31 2017 -0700 ---------------------------------------------------------------------- .../model/instance/EntityMutationResponse.java | 20 ++++++++- .../org/apache/atlas/type/AtlasEntityType.java | 17 ++++--- pom.xml | 4 +- .../atlas/repository/graph/GraphHelper.java | 4 +- .../test/resources/atlas-application.properties | 4 ++ .../NotificationHookConsumerIT.java | 29 ++++++------ .../web/integration/EntityJerseyResourceIT.java | 47 +++++++++++++++++--- 7 files changed, 90 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/024fd221/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java index 7078436..751df27 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/EntityMutationResponse.java @@ -33,6 +33,8 @@ import javax.xml.bind.annotation.XmlRootElement; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.instance.EntityMutations.EntityOperation; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.annotate.JsonAutoDetect; import org.codehaus.jackson.annotate.JsonIgnore; import org.codehaus.jackson.annotate.JsonIgnoreProperties; @@ -200,9 +202,25 @@ public class EntityMutationResponse { mutatedEntities.put(op, opEntities); } - opEntities.add(header); + if (!entityHeaderExists(opEntities, header)) { + opEntities.add(header); + } } + private boolean entityHeaderExists(List<AtlasEntityHeader> entityHeaders, AtlasEntityHeader newEntityHeader) { + boolean ret = false; + + if (CollectionUtils.isNotEmpty(entityHeaders) && newEntityHeader != null) { + for (AtlasEntityHeader entityHeader : entityHeaders) { + if (StringUtils.equals(entityHeader.getGuid(), newEntityHeader.getGuid())) { + ret = true; + break; + } + } + } + + return ret; + } public StringBuilder toString(StringBuilder sb) { if ( sb == null) { http://git-wip-us.apache.org/repos/asf/atlas/blob/024fd221/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java index 2cb8e27..28215fd 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java @@ -621,12 +621,11 @@ public class AtlasEntityType extends AtlasStructType { AtlasEntity entityObj = (AtlasEntity) obj; for (AtlasAttribute attribute : relationshipAttributes.values()) { - String attributeName = attribute.getName(); - if (attribute != null) { - AtlasType dataType = attribute.getAttributeType(); - Object value = entityObj.getAttribute(attributeName); - String fieldName = objName + "." + attributeName; + String attributeName = attribute.getName(); + AtlasType dataType = attribute.getAttributeType(); + Object value = entityObj.getAttribute(attributeName); + String fieldName = objName + "." + attributeName; if (isValidRelationshipType(dataType) && value != null) { ret = dataType.validateValue(value, fieldName, messages) && ret; @@ -638,12 +637,12 @@ public class AtlasEntityType extends AtlasStructType { Map attributes = AtlasTypeUtil.toStructAttributes((Map)obj); for (AtlasAttribute attribute : relationshipAttributes.values()) { - String attributeName = attribute.getName(); if (attribute != null) { - AtlasType dataType = attribute.getAttributeType(); - Object value = attributes.get(attributeName); - String fieldName = objName + "." + attributeName; + String attributeName = attribute.getName(); + AtlasType dataType = attribute.getAttributeType(); + Object value = attributes.get(attributeName); + String fieldName = objName + "." + attributeName; if (isValidRelationshipType(dataType) && value != null) { ret = dataType.validateValue(value, fieldName, messages) && ret; http://git-wip-us.apache.org/repos/asf/atlas/blob/024fd221/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f127774..5adfc9c 100644 --- a/pom.xml +++ b/pom.xml @@ -1922,7 +1922,7 @@ <log4j.configuration>atlas-log4j.xml</log4j.configuration> </systemProperties> <skipTests>${skipTests}</skipTests> - <forkCount>2C</forkCount> + <forkCount>1C</forkCount> <reuseForks>false</reuseForks> <redirectTestOutputToFile>true</redirectTestOutputToFile> <argLine>-Djava.awt.headless=true -Dproject.version=${project.version} @@ -2087,7 +2087,7 @@ <goals> <goal>check</goal> </goals> - <phase>verify</phase> + <phase>validate</phase> </execution> </executions> </plugin> http://git-wip-us.apache.org/repos/asf/atlas/blob/024fd221/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 1ec5a72..0177f7e 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -597,12 +597,12 @@ public final class GraphHelper { } public AtlasEdge getEdgeForGUID(String guid) throws AtlasBaseException { - AtlasEdge ret = null; + AtlasEdge ret; try { ret = findEdge(Constants.GUID_PROPERTY_KEY, guid); } catch (EntityNotFoundException e) { - new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_GUID_NOT_FOUND, guid); + throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_GUID_NOT_FOUND, guid); } return ret; http://git-wip-us.apache.org/repos/asf/atlas/blob/024fd221/typesystem/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/typesystem/src/test/resources/atlas-application.properties b/typesystem/src/test/resources/atlas-application.properties index 7967b76..65dd9a3 100644 --- a/typesystem/src/test/resources/atlas-application.properties +++ b/typesystem/src/test/resources/atlas-application.properties @@ -139,3 +139,7 @@ atlas.authentication.method.file=true atlas.authentication.method.ldap.type=none # atlas.authentication.method.file.filename=users-credentials.properties atlas.authentication.method.kerberos=false + +######### Gremlin Search Configuration ######### +# Set to false to disable gremlin search. +atlas.search.gremlin.enable=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/atlas/blob/024fd221/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java index 9c5597e..d41db3e 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java +++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java @@ -21,6 +21,11 @@ package org.apache.atlas.notification; import org.apache.atlas.EntityAuditEvent; import org.apache.atlas.kafka.NotificationProvider; import org.apache.atlas.notification.hook.HookNotification; +import org.apache.atlas.notification.hook.HookNotification.HookNotificationMessage; +import org.apache.atlas.notification.hook.HookNotification.EntityDeleteRequest; +import org.apache.atlas.notification.hook.HookNotification.EntityPartialUpdateRequest; +import org.apache.atlas.notification.hook.HookNotification.EntityCreateRequest; +import org.apache.atlas.notification.hook.HookNotification.EntityUpdateRequest; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.web.integration.BaseResourceIT; @@ -31,6 +36,7 @@ import org.testng.annotations.Test; import java.util.List; +import static java.lang.Thread.sleep; import static org.testng.Assert.assertEquals; public class NotificationHookConsumerIT extends BaseResourceIT { @@ -54,8 +60,9 @@ public class NotificationHookConsumerIT extends BaseResourceIT { notificationInterface.close(); } - private void sendHookMessage(HookNotification.HookNotificationMessage message) throws NotificationException { + private void sendHookMessage(HookNotificationMessage message) throws NotificationException, InterruptedException { notificationInterface.send(NotificationInterface.NotificationType.HOOK, message); + sleep(1000); } @Test @@ -71,8 +78,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { entity.set(DESCRIPTION, randomString()); entity.set(QUALIFIED_NAME, dbName); entity.set(CLUSTER_NAME, randomString()); - sendHookMessage(new HookNotification.EntityCreateRequest(TEST_USER, entity)); - + sendHookMessage(new EntityCreateRequest(TEST_USER, entity)); waitFor(MAX_WAIT_TIME, new Predicate() { @Override public boolean evaluate() throws Exception { @@ -91,8 +97,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { entity.set(QUALIFIED_NAME, dbName); entity.set(CLUSTER_NAME, randomString()); - sendHookMessage(new HookNotification.EntityCreateRequest(TEST_USER, entity)); - + sendHookMessage(new EntityCreateRequest(TEST_USER, entity)); waitFor(MAX_WAIT_TIME, new Predicate() { @Override public boolean evaluate() throws Exception { @@ -103,8 +108,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { //Assert that user passed in hook message is used in audit Referenceable instance = atlasClientV1.getEntity(DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, (String) entity.get(QUALIFIED_NAME)); - List<EntityAuditEvent> events = - atlasClientV1.getEntityAuditEvents(instance.getId()._getId(), (short) 1); + List<EntityAuditEvent> events = atlasClientV1.getEntityAuditEvents(instance.getId()._getId(), (short) 1); assertEquals(events.size(), 1); assertEquals(events.get(0).getUser(), TEST_USER); } @@ -122,8 +126,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { final Referenceable newEntity = new Referenceable(DATABASE_TYPE_BUILTIN); newEntity.set("owner", randomString()); - sendHookMessage( - new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity)); + sendHookMessage(new EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity)); waitFor(MAX_WAIT_TIME, new Predicate() { @Override public boolean evaluate() throws Exception { @@ -152,8 +155,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { final String newName = "db" + randomString(); newEntity.set(QUALIFIED_NAME, newName); - sendHookMessage( - new HookNotification.EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity)); + sendHookMessage(new EntityPartialUpdateRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName, newEntity)); waitFor(MAX_WAIT_TIME, new Predicate() { @Override public boolean evaluate() throws Exception { @@ -179,8 +181,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { final String dbId = atlasClientV1.createEntity(entity).get(0); - sendHookMessage( - new HookNotification.EntityDeleteRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName)); + sendHookMessage(new EntityDeleteRequest(TEST_USER, DATABASE_TYPE_BUILTIN, QUALIFIED_NAME, dbName)); waitFor(MAX_WAIT_TIME, new Predicate() { @Override public boolean evaluate() throws Exception { @@ -209,7 +210,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { newEntity.set(CLUSTER_NAME, randomString()); //updating unique attribute - sendHookMessage(new HookNotification.EntityUpdateRequest(TEST_USER, newEntity)); + sendHookMessage(new EntityUpdateRequest(TEST_USER, newEntity)); waitFor(MAX_WAIT_TIME, new Predicate() { @Override public boolean evaluate() throws Exception { http://git-wip-us.apache.org/repos/asf/atlas/blob/024fd221/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java index 660f05e..ba5465d 100755 --- a/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java @@ -842,7 +842,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT { @Test - public void testPartialUpdate() throws Exception { + public void testPartialUpdateByGuid() throws Exception { String dbName = "db" + randomString(); String tableName = "table" + randomString(); Referenceable hiveDBInstance = createHiveDBInstanceBuiltIn(dbName); @@ -878,30 +878,63 @@ public class EntityJerseyResourceIT extends BaseResourceIT { LOG.debug("Updating entity= {}", tableUpdated); EntityResult entityResult = atlasClientV1.updateEntity(guid, tableUpdated); - assertEquals(entityResult.getUpdateEntities().size(), 1); - assertEquals(entityResult.getUpdateEntities().get(0), guid); + assertEquals(entityResult.getUpdateEntities().size(), 2); + assertEquals(entityResult.getUpdateEntities().get(1), guid); Referenceable entity = atlasClientV1.getEntity(guid); List<Referenceable> refs = (List<Referenceable>) entity.get("columns"); Assert.assertTrue(refs.get(0).equalsContents(columns.get(0))); + } + + @Test + public void testPartialUpdateByUniqueAttributes() throws Exception { + String dbName = "db" + randomString(); + String tableName = "table" + randomString(); + Referenceable hiveDBInstance = createHiveDBInstanceBuiltIn(dbName); + Id dbId = createInstance(hiveDBInstance); + Referenceable hiveTableInstance = createHiveTableInstanceBuiltIn(dbName, tableName, dbId); + Id tableId = createInstance(hiveTableInstance); + + final String guid = tableId._getId(); + try { + Assert.assertNotNull(UUID.fromString(guid)); + } catch (IllegalArgumentException e) { + Assert.fail("Response is not a guid, " + guid); + } + + String colName = "col1"+randomString(); + final List<Referenceable> columns = new ArrayList<>(); + Map<String, Object> values = new HashMap<>(); + values.put(NAME, colName); + values.put("comment", "col1 comment"); + values.put(QUALIFIED_NAME, "default.table.col1@"+colName); + values.put("comment", "col1 comment"); + values.put("type", "string"); + values.put("owner", "user1"); + values.put("position", 0); + values.put("description", "col1"); + values.put("table", tableId); //table is a required reference, can't be null + + Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values); + columns.add(ref); //Update by unique attribute values.put("type", "int"); ref = new Referenceable(BaseResourceIT.COLUMN_TYPE_BUILTIN, values); columns.set(0, ref); - tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, new HashMap<String, Object>() {{ + Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, new HashMap<String, Object>() {{ put("columns", columns); }}); LOG.debug("Updating entity= {}", tableUpdated); - entityResult = atlasClientV1.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, + EntityResult entityResult = atlasClientV1.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE_BUILTIN, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, (String) hiveTableInstance.get(QUALIFIED_NAME), tableUpdated); assertEquals(entityResult.getUpdateEntities().size(), 2); assertEquals(entityResult.getUpdateEntities().get(1), guid); - entity = atlasClientV1.getEntity(guid); - refs = (List<Referenceable>) entity.get("columns"); + Referenceable entity = atlasClientV1.getEntity(guid); + List<Referenceable> refs = (List<Referenceable>) entity.get("columns"); Assert.assertTrue(refs.get(0).getValuesMap().equals(values)); Assert.assertEquals(refs.get(0).get("type"), "int");
