Repository: atlas Updated Branches: refs/heads/branch-0.8 ac7c5e11d -> c850bcad4
ATLAS-2813: SoftRef implementation. Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/c850bcad Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/c850bcad Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/c850bcad Branch: refs/heads/branch-0.8 Commit: c850bcad4a9adc3b3870e59ac1dc12d483aef89b Parents: ac7c5e1 Author: Ashutosh Mestry <[email protected]> Authored: Mon Aug 13 13:51:17 2018 -0700 Committer: Ashutosh Mestry <[email protected]> Committed: Mon Aug 13 13:51:17 2018 -0700 ---------------------------------------------------------------------- .../atlas/model/typedef/AtlasStructDef.java | 33 ++++- .../store/graph/v1/AtlasStructDefStoreV1.java | 8 ++ .../graph/v1/AttributeMutationContext.java | 4 + .../store/graph/v1/EntityGraphMapper.java | 41 ++++-- .../store/graph/v1/EntityGraphRetriever.java | 76 ++++++++++- .../store/graph/v1/SoftReferenceTest.java | 132 +++++++++++++++++++ .../src/test/resources/json/rdbms-db.json | 70 ++++++++++ .../test/resources/json/typesDef-soft-ref.json | 115 ++++++++++++++++ 8 files changed, 463 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java index c3c85d3..9dba1c0 100644 --- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java +++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java @@ -256,6 +256,9 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { public static class AtlasAttributeDef implements Serializable { private static final long serialVersionUID = 1L; + public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference"; + private final String STRING_TRUE = "true"; + /** * single-valued attribute or multi-valued attribute. */ @@ -272,6 +275,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { private boolean isUnique; private boolean isIndexable; private List<AtlasConstraintDef> constraints; + private Map<String, String> options; public AtlasAttributeDef() { this(null, null); } @@ -281,12 +285,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, List<AtlasConstraintDef> constraints) { - this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, null, constraints, null); + this(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, null, constraints, null, null); } public AtlasAttributeDef(String name, String typeName, boolean isOptional, Cardinality cardinality, int valuesMinCount, int valuesMaxCount, boolean isUnique, boolean isIndexable, String defaultValue, - List<AtlasConstraintDef> constraints, String description) { + List<AtlasConstraintDef> constraints, Map<String,String> options, String description) { setName(name); setTypeName(typeName); setIsOptional(isOptional); @@ -296,6 +300,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { setIsUnique(isUnique); setIsIndexable(isIndexable); setConstraints(constraints); + setOptions(options); } public AtlasAttributeDef(AtlasAttributeDef other) { @@ -309,6 +314,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { setIsUnique(other.getIsUnique()); setIsIndexable(other.getIsIndexable()); setConstraints(other.getConstraints()); + setOptions(other.getOptions()); } } @@ -399,6 +405,23 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { cDefs.add(constraintDef); } + public Map<String, String> getOptions() { + return options; + } + + public void setOptions(Map<String, String> options) { + if (options != null) { + this.options = new HashMap<>(options); + } else { + this.options = null; + } + } + + public boolean isSoftReferenced() { + return this.options != null && + getOptions().containsKey(AtlasAttributeDef.ATTRDEF_OPTION_SOFT_REFERENCE) && + getOptions().get(AtlasAttributeDef.ATTRDEF_OPTION_SOFT_REFERENCE).equals(STRING_TRUE); + } public StringBuilder toString(StringBuilder sb) { if (sb == null) { @@ -414,6 +437,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { sb.append(", valuesMaxCount=").append(valuesMaxCount); sb.append(", isUnique=").append(isUnique); sb.append(", isIndexable=").append(isIndexable); + sb.append(", options='").append(options).append('\''); sb.append(", constraints=["); if (CollectionUtils.isNotEmpty(constraints)) { int i = 0; @@ -444,12 +468,13 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable { Objects.equals(name, that.name) && Objects.equals(typeName, that.typeName) && cardinality == that.cardinality && - Objects.equals(constraints, that.constraints); + Objects.equals(constraints, that.constraints) && + Objects.equals(options, that.options); } @Override public int hashCode() { - return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, constraints); + return Objects.hash(name, typeName, isOptional, cardinality, valuesMinCount, valuesMaxCount, isUnique, isIndexable, constraints, options); } @Override http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java index 137fb30..e59c389 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java @@ -463,6 +463,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1<AtlasStructDe attribInfo.put("isComposite", attribute.isOwnedRef()); attribInfo.put("reverseAttributeName", attribute.getInverseRefAttributeName()); + if(attributeDef.getOptions() != null) { + attribInfo.put("options", AtlasType.toJson(attributeDef.getOptions())); + } + final int lower; final int upper; @@ -501,6 +505,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1<AtlasStructDe ret.setIsUnique((Boolean) attribInfo.get("isUnique")); ret.setIsIndexable((Boolean) attribInfo.get("isIndexable")); + if(attribInfo.get("options") != null) { + ret.setOptions(AtlasType.fromJson((String) attribInfo.get("options"), Map.class)); + } + if ((Boolean)attribInfo.get("isComposite")) { ret.addConstraint(new AtlasConstraintDef(AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF)); } http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java index b6d82dd..b32c092 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java @@ -126,6 +126,10 @@ public class AttributeMutationContext { return value; } + public void setValue(Object value) { + this.value = value; + } + public String getVertexProperty() { return vertexProperty; } public AtlasVertex getReferringVertex() { return referringVertex; } http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java index 9e7a119..3811919 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java @@ -302,6 +302,10 @@ public class EntityGraphMapper { } case OBJECT_ID_TYPE: { + if (ctx.getAttributeDef().isSoftReferenced()) { + return mapSoftRefValue(ctx, context); + } + String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexProperty()); AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel); AtlasEdge newEdge = null; @@ -337,6 +341,19 @@ public class EntityGraphMapper { } } + private Object mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) { + if(ctx.getValue() == null || !(ctx.getValue() instanceof AtlasObjectId)) { + return null; + } + + AtlasObjectId objectId = (AtlasObjectId) ctx.getValue(); + String resolvedGuid = context.getGuidAssignments().get(objectId.getGuid()); + String softRefValue = String.format("%s:%s", objectId.getTypeName(), resolvedGuid); + + ctx.setValue(softRefValue); + return mapPrimitiveValue(ctx); + } + private void addInverseReference(AttributeMutationContext ctx, AtlasAttribute inverseAttribute, AtlasEdge edge) throws AtlasBaseException { AtlasStructType inverseType = inverseAttribute.getDefinedInType(); @@ -515,17 +532,18 @@ public class EntityGraphMapper { if (MapUtils.isNotEmpty(newVal)) { boolean isReference = AtlasGraphUtilsV1.isReference(mapType.getValueType()); + boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); for (Map.Entry<Object, Object> entry : newVal.entrySet()) { String key = entry.getKey().toString(); String propertyName = GraphHelper.getQualifiedNameForMapKey(ctx.getVertexProperty(), GraphHelper.encodePropertyKey(key)); - AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key); + AtlasEdge existingEdge = isSoftReference ? null : getEdgeIfExists(mapType, currentMap, key); AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(), propertyName, mapType.getValueType(), existingEdge); //Add/Update/Remove property value Object newEntry = mapCollectionElementsToVertex(mapCtx, context); - setMapValueProperty(mapType.getValueType(), ctx.getReferringVertex(), propertyName, newEntry); + setMapValueProperty(mapType.getValueType(), isSoftReference, ctx.getReferringVertex(), propertyName, newEntry); newMap.put(key, newEntry); @@ -571,6 +589,7 @@ public class EntityGraphMapper { AtlasType elementType = arrType.getElementType(); List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty()); boolean isReference = AtlasGraphUtilsV1.isReference(elementType); + boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List<Object> newElementsCreated = new ArrayList<>(); @@ -581,7 +600,7 @@ public class EntityGraphMapper { } for (int index = 0; index < newElements.size(); index++) { - AtlasEdge existingEdge = getEdgeAt(currentElements, index, elementType); + AtlasEdge existingEdge = (isSoftReference) ? null : getEdgeAt(currentElements, index, elementType); AttributeMutationContext arrCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), ctx.getAttribute(), newElements.get(index), ctx.getVertexProperty(), elementType, existingEdge); @@ -601,7 +620,7 @@ public class EntityGraphMapper { } // for dereference on way out - setArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated); + setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElementsCreated); if (LOG.isDebugEnabled()) { LOG.debug("<== mapArrayValue({})", ctx); @@ -662,6 +681,10 @@ public class EntityGraphMapper { case OBJECT_ID_TYPE: AtlasEntityType instanceType = getInstanceType(ctx.getValue()); ctx.setElementType(instanceType); + if (ctx.getAttributeDef().isSoftReferenced()) { + return mapSoftRefValue(ctx, context); + } + return mapObjectIdValue(ctx, context); default: @@ -741,8 +764,8 @@ public class EntityGraphMapper { } } - private static void setMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, Object value) { - if (AtlasGraphUtilsV1.isReference(elementType)) { + private static void setMapValueProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String vertexPropertyName, Object value) { + if (AtlasGraphUtilsV1.isReference(elementType) && !isSoftReference) { vertex.setPropertyFromElementId(vertexPropertyName, (AtlasEdge)value); } else { @@ -870,8 +893,8 @@ public class EntityGraphMapper { return Collections.emptyList(); } - private void setArrayElementsProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, List<Object> values) { - if (AtlasGraphUtilsV1.isReference(elementType)) { + private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String vertexPropertyName, List<Object> values) { + if (AtlasGraphUtilsV1.isReference(elementType) && !isSoftReference) { GraphHelper.setListPropertyFromElementIds(vertex, vertexPropertyName, (List) values); } else { @@ -897,7 +920,7 @@ public class EntityGraphMapper { } private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, AtlasMapType mapType, Object val) { - if (mapType.getValueType().getTypeCategory() == TypeCategory.OBJECT_ID_TYPE) { + if (mapType.getValueType().getTypeCategory() == TypeCategory.OBJECT_ID_TYPE && !ctx.getAttributeDef().isSoftReferenced()) { AtlasEdge edge = (AtlasEdge) val; if (ctx.getAttribute().isOwnedRef() && GraphHelper.getStatus(edge) == AtlasEntity.Status.DELETED && http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java index a243fd7..81b553e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java @@ -67,6 +67,10 @@ public final class EntityGraphRetriever { private static final String NAME = "name"; private static final String QUALIFIED_NAME = "qualifiedName"; + private static final String SOFT_REFERENCE_FORMAT_SEPERATOR = ":"; + private static final int SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME = 0; + private static final int SOFT_REFERENCE_FORMAT_INDEX_GUID = 1; + private static final String MAP_VALUE_FORMAT = "%s.%s"; private static final GraphHelper graphHelper = GraphHelper.getInstance(); @@ -446,13 +450,25 @@ public final class EntityGraphRetriever { ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo, isMinExtInfo); break; case OBJECT_ID_TYPE: - ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, isMinExtInfo); + if(attribute.getAttributeDef().isSoftReferenced()) { + ret = mapVertexToObjectIdForSoftRef(entityVertex, vertexPropertyName); + } else { + ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, isMinExtInfo); + } break; case ARRAY: - ret = mapVertexToArray(entityVertex, (AtlasArrayType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, isMinExtInfo); + if(attribute.getAttributeDef().isSoftReferenced()) { + ret = mapVertexToArrayForSoftRef(entityVertex, vertexPropertyName); + } else { + ret = mapVertexToArray(entityVertex, (AtlasArrayType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, isMinExtInfo); + } break; case MAP: - ret = mapVertexToMap(entityVertex, (AtlasMapType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, isMinExtInfo); + if(attribute.getAttributeDef().isSoftReferenced()) { + ret = mapVertexToMapForSoftRef(entityVertex, vertexPropertyName); + } else { + ret = mapVertexToMap(entityVertex, (AtlasMapType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, isMinExtInfo); + } break; case CLASSIFICATION: // do nothing @@ -462,6 +478,60 @@ public final class EntityGraphRetriever { return ret; } + private Object mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) { + List mapKeys = entityVertex.getListProperty(propertyName); + if (CollectionUtils.isEmpty(mapKeys)) { + return null; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Mapping map attribute {} for vertex {}", propertyName, entityVertex); + } + + Map<String, Object> ret = new HashMap<>(mapKeys.size()); + + for (Object mapKey : mapKeys) { + final String keyPropertyName = String.format(MAP_VALUE_FORMAT, propertyName, mapKey); + + Object mapValue = mapVertexToObjectIdForSoftRef(entityVertex, keyPropertyName); + if (mapValue != null) { + ret.put((String) mapKey, mapValue); + } + } + + return ret; + } + + private Object mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) { + List<AtlasObjectId> objectIds = new ArrayList<>(); + List list = entityVertex.getListProperty(propertyName); + for (Object o : list) { + if(!(o instanceof String)) { + continue; + } + + AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat((String) o); + objectIds.add(objectId); + } + + return objectIds; + } + + private Object mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) { + Object rawValue = GraphHelper.getSingleValuedProperty(entityVertex, vertexPropertyName, String.class); + return getAtlasObjectIdFromSoftRefFormat((String) rawValue); + } + + private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String rawValue) { + String[] objectIdParts = StringUtils.split(rawValue, SOFT_REFERENCE_FORMAT_SEPERATOR); + if(objectIdParts.length < 2) { + return null; + } + + return new AtlasObjectId(objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_GUID], + objectIdParts[SOFT_REFERENCE_FORMAT_INDEX_TYPE_NAME]); + } + private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute, boolean isMinExtInfo) throws AtlasBaseException { List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName); http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java new file mode 100644 index 0000000..856f383 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.repository.store.graph.v1; + +import org.apache.atlas.TestModules; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.TestResourceFileUtils; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +@Guice(modules = TestModules.SoftDeleteModule.class) +public class SoftReferenceTest { + private static final String TYPE_RDBMS_DB = "rdbms_db"; + private static final String RDBMS_DB_FILE = "rdbms-db"; + private static final String TYPE_RDBMS_STORAGE = "rdbms_storage"; + private static final String TYPESDEF_FILE_NAME = "typesdef-soft-ref"; + private static final String RDBMS_DB_STORAGE_PROPERTY = "sd"; + private static final String RDBMS_DB_TABLES_PROPERTY = "tables"; + private static final String RDBMS_DB_REGIONS_PROPERTY = "regions"; + private static final String RDBMS_SD_PROPERTY = "rdbms_db.sd"; + private static final String TYPE_RDBMS_TABLES = "rdbms_table"; + + @Inject + AtlasTypeRegistry typeRegistry; + + @Inject + private AtlasTypeDefStore typeDefStore; + + @Inject + private AtlasEntityStore entityStore; + + private AtlasType dbType; + + @Test + public void typeCreationFromFile() throws IOException, AtlasBaseException { + String typesDefJson = TestResourceFileUtils.getJson(TYPESDEF_FILE_NAME); + + AtlasTypesDef typesDef = AtlasType.fromJson(typesDefJson, AtlasTypesDef.class); + assertNotNull(typesDef); + + typeDefStore.createTypesDef(typesDef); + + dbType = typeRegistry.getType(TYPE_RDBMS_DB); + assertNotNull(dbType); + AtlasEntityDef dbType = typeRegistry.getEntityDefByName(TYPE_RDBMS_DB); + assertNotNull(dbType); + assertTrue(dbType.getAttribute(RDBMS_DB_STORAGE_PROPERTY).isSoftReferenced()); + assertTrue(dbType.getAttribute(RDBMS_DB_TABLES_PROPERTY).isSoftReferenced()); + assertTrue(dbType.getAttribute(RDBMS_DB_REGIONS_PROPERTY).isSoftReferenced()); + assertNotNull(typeRegistry.getEntityDefByName(TYPE_RDBMS_STORAGE)); + assertNotNull(typeRegistry.getEntityDefByName(TYPE_RDBMS_TABLES)); + } + + @Test(dependsOnMethods = "typeCreationFromFile") + public void entityCreationUsingSoftRef() throws IOException, AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo dbEntity = AtlasType.fromJson( + TestResourceFileUtils.getJson(RDBMS_DB_FILE), AtlasEntity.AtlasEntityWithExtInfo.class); + + EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); + assertNotNull(response); + assertTrue(response.getCreatedEntities().size() == 6); + assertGraphStructure(response.getCreatedEntities().get(0).getGuid(), + response.getCreatedEntities().get(1).getGuid(), RDBMS_SD_PROPERTY); + } + + private void assertGraphStructure(String dbGuid, String storageGuid, String propertyName) throws AtlasBaseException { + AtlasVertex vertex = AtlasGraphUtilsV1.findByGuid(dbGuid); + Iterator<AtlasEdge> edgesOut = vertex.getEdges(AtlasEdgeDirection.OUT).iterator(); + Iterator<AtlasEdge> edgesIn = vertex.getEdges(AtlasEdgeDirection.IN).iterator(); + + String sd = AtlasGraphUtilsV1.getProperty(vertex, propertyName, String.class); + + assertNotNull(sd); + assertAttribute(dbGuid, storageGuid); + assertFalse(edgesOut.hasNext()); + assertFalse(edgesIn.hasNext()); + assertNotNull(vertex); + } + + private void assertAttribute(String dbGuid, String storageGuid) throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid); + AtlasEntity entity = entityWithExtInfo.getEntity(); + + Object val = entity.getAttribute(RDBMS_DB_STORAGE_PROPERTY); + assertTrue(val instanceof AtlasObjectId); + assertEquals(((AtlasObjectId) val).getTypeName(), TYPE_RDBMS_STORAGE); + assertEquals(((AtlasObjectId) val).getGuid(), storageGuid); + assertNotNull(entity.getAttribute(RDBMS_DB_TABLES_PROPERTY)); + assertEquals(((List) entity.getAttribute(RDBMS_DB_TABLES_PROPERTY)).size(), 2); + assertNotNull(entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY)); + assertEquals(((Map) entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY)).size(), 2); + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/repository/src/test/resources/json/rdbms-db.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/rdbms-db.json b/repository/src/test/resources/json/rdbms-db.json new file mode 100644 index 0000000..a897039 --- /dev/null +++ b/repository/src/test/resources/json/rdbms-db.json @@ -0,0 +1,70 @@ +{ + "entity": { + "attributes": { + "name": "employee", + "sd": { + "guid": "-99288075821829", + "typeName": "rdbms_storage" + }, + "tables": [ + { + "guid": "-99288075821830", + "typeName": "rdbms_table" + }, + { + "guid": "-99288075821831", + "typeName": "rdbms_table" + } + ], + "regions": { + "west": { + "guid": "-99288075821832", + "typeName": "rdbms_table" + }, + "east": { + "guid": "-99288075821833", + "typeName": "rdbms_table" + } + } + }, + "classifications": [], + "typeName": "rdbms_db" + }, + "referredEntities": { + "-99288075821829": { + "guid": "-99288075821829", + "typeName": "rdbms_storage", + "attributes": { + "name": "binary" + } + }, + "-99288075821830": { + "guid": "-99288075821830", + "typeName": "rdbms_table", + "attributes": { + "name": "open" + } + }, + "-99288075821831": { + "guid": "-99288075821831", + "typeName": "rdbms_table", + "attributes": { + "name": "close" + } + }, + "-99288075821832": { + "guid": "-99288075821832", + "typeName": "rdbms_table", + "attributes": { + "name": "west" + } + }, + "-99288075821833": { + "guid": "-99288075821833", + "typeName": "rdbms_table", + "attributes": { + "name": "east" + } + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/c850bcad/repository/src/test/resources/json/typesDef-soft-ref.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/typesDef-soft-ref.json b/repository/src/test/resources/json/typesDef-soft-ref.json new file mode 100644 index 0000000..9715b4b --- /dev/null +++ b/repository/src/test/resources/json/typesDef-soft-ref.json @@ -0,0 +1,115 @@ +{ + "enumDefs": [], + "structDefs": [], + "classificationDefs": [], + "entityDefs": [ + { + "category": "ENTITY", + "name": "rdbms_table", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "name", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false + } + ], + "superTypes": [] + }, + { + "category": "ENTITY", + "name": "rdbms_storage", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "name", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false + } + ], + "superTypes": [] + }, + { + "category": "ENTITY", + "name": "rdbms_db", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "name", + "typeName": "string", + "isOptional": false, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": true, + "isIndexable": true + }, + { + "name": "sd", + "typeName": "rdbms_storage", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "options": { + "isSoftReference": "true" + } + }, + { + "name": "tables", + "typeName": "array<rdbms_table>", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "options": { + "isSoftReference": "true" + } + }, + { + "name": "regions", + "typeName": "map<string,rdbms_table>", + "isOptional": true, + "cardinality": "SINGLE", + "valuesMinCount": -1, + "valuesMaxCount": -1, + "isUnique": false, + "isIndexable": false, + "constraints": [ + { + "type": "ownedRef" + } + ], + "options": { + "isSoftReference": "true" + } + } + ], + "superTypes": [] + } + ] +}
