Repository: atlas Updated Branches: refs/heads/branch-0.8 9bdbb3184 -> b9aa6d5d3
http://git-wip-us.apache.org/repos/asf/atlas/blob/b9aa6d5d/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java index c24a55f..59d98e2 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java @@ -19,6 +19,7 @@ package org.apache.atlas.typesystem.types; import org.apache.atlas.AtlasException; +import org.apache.atlas.type.AtlasType; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -40,6 +41,7 @@ public class AttributeInfo { * that this refers to. */ public final String reverseAttributeName; + public final boolean isSoftRef; private IDataType dataType; public AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws AtlasException { @@ -52,6 +54,7 @@ public class AttributeInfo { this.isUnique = def.isUnique; this.isIndexable = def.isIndexable; this.reverseAttributeName = def.reverseAttributeName; + this.isSoftRef = def.isSoftRef; } public IDataType dataType() { @@ -124,11 +127,29 @@ public class AttributeInfo { public static AttributeDefinition fromJson(String jsonStr) throws JSONException { JSONObject json = new JSONObject(jsonStr); String reverseAttr = null; + boolean isSoftRef = false; if (json.has("reverseAttributeName")) { reverseAttr = json.getString("reverseAttributeName"); } - return new AttributeDefinition(json.getString("name"), json.getString("dataType"), + + AttributeDefinition attributeDefinition = new AttributeDefinition(json.getString("name"), json.getString("dataType"), Multiplicity.fromJson(json.getString("multiplicity")), json.getBoolean("isComposite"), json.getBoolean("isUnique"), json.getBoolean("isIndexable"), reverseAttr); + + if (json.has("options")) { + isSoftRef = getSoftRef(json); + attributeDefinition.setSoftRef(isSoftRef); + } + + return attributeDefinition; + } + + private static boolean getSoftRef(JSONObject json) throws JSONException { + final String SOFT_REF_KEY = "isSoftReference"; + + boolean isSoftRef; + Map map = AtlasType.fromJson(json.getString("options"), Map.class); + isSoftRef = (map != null && map.containsKey(SOFT_REF_KEY)) ? Boolean.parseBoolean((String) map.get(SOFT_REF_KEY)) : false; + return isSoftRef; } } http://git-wip-us.apache.org/repos/asf/atlas/blob/b9aa6d5d/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index d4e7262..7d23125 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -28,12 +28,12 @@ import org.apache.atlas.authorize.AtlasResourceTypes; import org.apache.atlas.authorize.simple.AtlasAuthorizationUtils; import org.apache.atlas.discovery.SearchContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.clusterinfo.AtlasCluster; -import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.impexp.AtlasCluster; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasExportResult; import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.model.impexp.ExportImportAuditEntry; import org.apache.atlas.model.metrics.AtlasMetrics; import org.apache.atlas.repository.impexp.ClusterService; import org.apache.atlas.repository.impexp.ExportImportAuditService; @@ -82,6 +82,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.locks.ReentrantLock; @@ -439,7 +440,6 @@ public class AdminResource { /** * Fetch details of a cluster. * @param clusterName name of target cluster with which it is paired - * @param entityQualifiedName qualified name of top level entity * @return AtlasCluster * @throws AtlasBaseException */ @@ -447,8 +447,7 @@ public class AdminResource { @Path("/cluster/{clusterName}") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasCluster getCluster(@PathParam("clusterName") String clusterName, - @QueryParam("entity") String entityQualifiedName) throws AtlasBaseException { + public AtlasCluster getCluster(@PathParam("clusterName") String clusterName) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -467,22 +466,21 @@ public class AdminResource { @Path("/expimp/audit") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasSearchResult getExportImportAudit(@QueryParam("sourceClusterName") String sourceCluster, - @QueryParam("targetCluster") String targetCluster, - @QueryParam("userName") String userName, - @QueryParam("operation") String operation, - @QueryParam("startTime") String startTime, - @QueryParam("endTime") String endTime, - @QueryParam("limit") int limit, - @QueryParam("offset") int offset) throws AtlasBaseException { + public List<ExportImportAuditEntry> getExportImportAudit(@QueryParam("clusterName") String cluster, + @QueryParam("userName") String userName, + @QueryParam("operation") String operation, + @QueryParam("startTime") String startTime, + @QueryParam("endTime") String endTime, + @QueryParam("limit") int limit, + @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "getExportImportAudit(" + sourceCluster + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "getExportImportAudit(" + cluster + ")"); } - return exportImportAuditService.get(userName, operation, sourceCluster, targetCluster, startTime, endTime, limit, offset); + return exportImportAuditService.get(userName, operation, cluster, startTime, endTime, limit, offset); } finally { AtlasPerfTracer.log(perf); } http://git-wip-us.apache.org/repos/asf/atlas/blob/b9aa6d5d/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java index fc804d2..7acd332 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java @@ -21,13 +21,15 @@ package org.apache.atlas.web.resources; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.model.impexp.AtlasCluster; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.model.impexp.AtlasImportResult; import org.apache.atlas.repository.impexp.ZipSource; import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.atlas.web.integration.BaseResourceIT; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; import org.testng.annotations.Test; import java.io.ByteArrayInputStream; @@ -43,6 +45,13 @@ import static org.testng.Assert.assertTrue; public class AdminExportImportTestIT extends BaseResourceIT { private final String FILE_TO_IMPORT = "stocks-base.zip"; + private final String EXPORT_REQUEST_FILE = "export-incremental"; + private final String SOURCE_CLUSTER_NAME = "cl1"; + + static final String IMPORT_TRANSFORM_CLEAR_ATTRS = + "{ \"Asset\": { \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }"; + static final String IMPORT_TRANSFORM_SET_DELETED = + "{ \"Asset\": { \"*\":[ \"setDeleted\" ] } }"; @Test public void isActive() throws AtlasServiceException { @@ -50,25 +59,34 @@ public class AdminExportImportTestIT extends BaseResourceIT { } @Test(dependsOnMethods = "isActive") - public void importData() throws AtlasServiceException, IOException { + public void importData() throws AtlasServiceException { performImport(FILE_TO_IMPORT); + assertReplicationData("cl1"); } @Test(dependsOnMethods = "importData") public void exportData() throws AtlasServiceException, IOException, AtlasBaseException { - final int EXPECTED_CREATION_ORDER_SIZE = 13; + final int EXPECTED_CREATION_ORDER_SIZE = 10; - AtlasExportRequest request = TestResourceFileUtils.readObjectFromJson(".", "export-incremental", AtlasExportRequest.class); + AtlasExportRequest request = TestResourceFileUtils.readObjectFromJson(".", EXPORT_REQUEST_FILE, AtlasExportRequest.class); byte[] exportedBytes = atlasClientV2.exportData(request); assertNotNull(exportedBytes); ZipSource zs = new ZipSource(new ByteArrayInputStream(exportedBytes)); assertNotNull(zs.getExportResult()); - assertEquals(zs.getCreationOrder().size(), EXPECTED_CREATION_ORDER_SIZE); + assertTrue(zs.getCreationOrder().size() > EXPECTED_CREATION_ORDER_SIZE); } private void performImport(String fileToImport) throws AtlasServiceException { AtlasImportRequest request = new AtlasImportRequest(); + request.getOptions().put(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM, SOURCE_CLUSTER_NAME); + request.getOptions().put(AtlasImportRequest.TRANSFORMS_KEY, IMPORT_TRANSFORM_CLEAR_ATTRS); + + performImport(fileToImport, request); + } + + private void performImport(String fileToImport, AtlasImportRequest request) throws AtlasServiceException { + byte[] fileBytes = new byte[0]; try { fileBytes = Files.readAllBytes(Paths.get(TestResourceFileUtils.getTestFilePath(fileToImport))); @@ -82,4 +100,23 @@ public class AdminExportImportTestIT extends BaseResourceIT { assertNotNull(result.getMetrics()); assertEquals(result.getProcessedEntities().size(), 37); } + + private void assertReplicationData(String clusterName) throws AtlasServiceException { + AtlasCluster cluster = atlasClientV2.getCluster(clusterName); + assertNotNull(cluster); + assertNotNull(cluster.getAdditionalInfo()); + assertTrue(cluster.getAdditionalInfo().size() > 0); + } + + @AfterClass + protected void teardown() { + AtlasImportRequest request = new AtlasImportRequest(); + request.getOptions().put(AtlasImportRequest.TRANSFORMS_KEY, IMPORT_TRANSFORM_SET_DELETED); + + try { + performImport(FILE_TO_IMPORT, request); + } catch (AtlasServiceException e) { + throw new SkipException("performTeardown: failed! Subsequent tests results may be affected.", e); + } + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/b9aa6d5d/webapp/src/test/resources/json/export-incremental.json ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/json/export-incremental.json b/webapp/src/test/resources/json/export-incremental.json index 9710841..2e4bba9 100644 --- a/webapp/src/test/resources/json/export-incremental.json +++ b/webapp/src/test/resources/json/export-incremental.json @@ -2,10 +2,10 @@ "itemsToExport": [ { "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks@cl1" } - } ], "options": { - "fetchType": "full" + "fetchType": "incremental", + "replicatedTo": "cl2" } } http://git-wip-us.apache.org/repos/asf/atlas/blob/b9aa6d5d/webapp/src/test/resources/stocks-base.zip ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/stocks-base.zip b/webapp/src/test/resources/stocks-base.zip index 40c7f37..8663805 100644 Binary files a/webapp/src/test/resources/stocks-base.zip and b/webapp/src/test/resources/stocks-base.zip differ
