Repository: atlas Updated Branches: refs/heads/master 561cdc91a -> 03f2754d1
http://git-wip-us.apache.org/repos/asf/atlas/blob/03f2754d/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java index a73abcd..e3c4085 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java @@ -18,44 +18,57 @@ package org.apache.atlas.repository.impexp; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; public class ImportTransformsTest { - private final String qualifiedName = "qualifiedName"; + private final String ATTR_NAME_QUALIFIED_NAME = "qualifiedName"; private final String lowerCaseCL1 = "@cl1"; private final String lowerCaseCL2 = "@cl2"; - private final String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; - private final String jsonTransforms2 = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + private final String jsonLowerCaseReplace = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + private final String jsonReplaceLowerCase = "{ \"Asset\": { \"qualifiedName\":[ \"replace:@cl1:@cl2\" ] }, \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }"; + private final String jsonReplaceRemoveClassification = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"removeClassification:%s_to_%s\" ] } }"; + private final String jsonReplaceAndAddAttrValue = "{ \"hive_table\": { \"qualifiedName\":[ \"replace:@%s:@%s\"], \"*\":[ \"add:%s=list:%s\" ] } }"; + private final String jsonSingleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster\", \"clearAttrValue:replicatedFromCluster\" ] } }"; + private final String jsonMultipleClearAttrValue = "{ \"hive_table\": { \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }"; + private final String jsonSetDeleted = "{ \"hive_table\": { \"*\":[ \"setDeleted\" ] } }"; private ImportTransforms transform; + private String HIVE_TABLE_ATTR_SYNC_INFO = "hive_table.syncInfo"; + private String HIVE_TABLE_ATTR_REPLICATED_FROM = "replicatedFromCluster"; + private String HIVE_TABLE_ATTR_REPLICATED_TO = "replicatedToCluster"; @BeforeTest - public void setup() throws AtlasBaseException { - transform = ImportTransforms.fromJson(jsonTransforms); + public void setup() { + transform = ImportTransforms.fromJson(jsonLowerCaseReplace); + } + + @BeforeMethod + public void setUp() { } @Test public void transformEntityWith2Transforms() throws AtlasBaseException { AtlasEntity entity = getHiveTableAtlasEntity(); - String attrValue = (String) entity.getAttribute(qualifiedName); + String attrValue = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); transform.apply(entity); - assertEquals(entity.getAttribute(qualifiedName), applyDefaultTransform(attrValue)); + assertEquals(entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), applyDefaultTransform(attrValue)); } @Test @@ -64,15 +77,15 @@ public class ImportTransformsTest { AtlasEntityWithExtInfo entityWithExtInfo = getAtlasEntityWithExtInfo(); AtlasEntity entity = entityWithExtInfo.getEntity(); - String attrValue = (String) entity.getAttribute(qualifiedName); + String attrValue = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME); String[] expectedValues = getExtEntityExpectedValues(entityWithExtInfo); transform.apply(entityWithExtInfo); - assertEquals(entityWithExtInfo.getEntity().getAttribute(qualifiedName), applyDefaultTransform(attrValue)); + assertEquals(entityWithExtInfo.getEntity().getAttribute(ATTR_NAME_QUALIFIED_NAME), applyDefaultTransform(attrValue)); for (int i = 0; i < expectedValues.length; i++) { - assertEquals(entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName), expectedValues[i]); + assertEquals(entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(ATTR_NAME_QUALIFIED_NAME), expectedValues[i]); } } @@ -92,17 +105,92 @@ public class ImportTransformsTest { @Test public void transformFromJsonWithMultipleEntries() { - ImportTransforms t = ImportTransforms.fromJson(jsonTransforms2); + ImportTransforms t = ImportTransforms.fromJson(jsonReplaceLowerCase); assertNotNull(t); assertEquals(t.getTransforms().size(), 2); } + @Test + public void removeClassificationTransform_RemovesSpecifiedClassification() throws AtlasBaseException { + List<AtlasClassification> classifications = new ArrayList<>(); + classifications.add(new AtlasClassification("cl2_to_cl1")); + + String s = String.format(jsonReplaceRemoveClassification, "cl1", "cl2", "cl2", "cl1"); + ImportTransforms t = ImportTransforms.fromJson(s); + + AtlasEntity entity = getHiveTableAtlasEntity(); + String expected_qualifiedName = entity.getAttribute(ATTR_NAME_QUALIFIED_NAME).toString().replace("@cl1", "@cl2"); + entity.setClassifications(classifications); + assertEquals(entity.getClassifications().size(), 1); + + t.apply(entity); + + assertNotNull(t); + assertEquals(entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), expected_qualifiedName); + } + + @Test + public void add_setsValueOfAttribute() throws AtlasBaseException { + final String expected_syncInfo = "cl1:import"; + String s = String.format(jsonReplaceAndAddAttrValue, "cl1", "cl2", HIVE_TABLE_ATTR_SYNC_INFO, expected_syncInfo); + ImportTransforms t = ImportTransforms.fromJson(s); + + AtlasEntity entity = getHiveTableAtlasEntity(); + String expected_qualifiedName = entity.getAttribute(ATTR_NAME_QUALIFIED_NAME).toString().replace("@cl1", "@cl2"); + + t.apply(entity); + + assertNotNull(t); + assertEquals(entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), expected_qualifiedName); + } + + + @Test + public void clearAttrValue_removesValueOfAttribute() throws AtlasBaseException { + AtlasEntity entity = getHiveTableAtlasEntity(); + assertNotNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_FROM)); + assertNotNull(entity.getAttribute(HIVE_TABLE_ATTR_REPLICATED_TO)); + + ImportTransforms t = ImportTransforms.fromJson(jsonSingleClearAttrValue); + + assertTrue(t.getTransforms().size() > 0); + + t.apply(entity); + + assertNotNull(t); + } + + @Test + public void clearAttrValueForMultipleAttributes_removesValueOfAttribute() throws AtlasBaseException { + AtlasEntity entity = getHiveTableAtlasEntity(); + ImportTransforms t = ImportTransforms.fromJson(jsonMultipleClearAttrValue); + + assertTrue(t.getTransforms().size() > 0); + + t.apply(entity); + + assertNotNull(t); + } + + @Test + public void setDeleted_SetsStatusToDeleted() throws AtlasBaseException { + AtlasEntity entity = getHiveTableAtlasEntity(); + assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE); + ImportTransforms t = ImportTransforms.fromJson(jsonSetDeleted); + + assertTrue(t.getTransforms().size() > 0); + + t.apply(entity); + assertNotNull(t); + } + + private String[] getExtEntityExpectedValues(AtlasEntityWithExtInfo entityWithExtInfo) { String[] ret = new String[entityWithExtInfo.getReferredEntities().size()]; for (int i = 0; i < ret.length; i++) { - String attrValue = (String) entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName); + String attrValue = (String) entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(ATTR_NAME_QUALIFIED_NAME); ret[i] = attrValue.replace(lowerCaseCL1, lowerCaseCL2); } @@ -115,9 +203,7 @@ public class ImportTransformsTest { List<ImportTransformer> trList = new ArrayList<>(); trList.add(ImportTransformer.getTransformer(String.format("replace:%s:%s", lowerCaseCL1, lowerCaseCL2))); - - tr.put(qualifiedName, trList); - + tr.put(ATTR_NAME_QUALIFIED_NAME, trList); transform.getTransforms().put("hive_column", tr); } @@ -127,11 +213,15 @@ public class ImportTransformsTest { private AtlasEntity getHiveTableAtlasEntity() { AtlasEntity entity = new AtlasEntity("hive_table"); + entity.setStatus(AtlasEntity.Status.ACTIVE); Map<String, Object> attributes = new HashMap<>(); - attributes.put(qualifiedName, "TABLE1.default" + lowerCaseCL1); + attributes.put(ATTR_NAME_QUALIFIED_NAME, "TABLE1.default" + lowerCaseCL1); attributes.put("dbname", "someDB"); attributes.put("name", "somename"); + attributes.put(HIVE_TABLE_ATTR_SYNC_INFO, null); + attributes.put(HIVE_TABLE_ATTR_REPLICATED_FROM, "cl1"); + attributes.put(HIVE_TABLE_ATTR_REPLICATED_TO, "clx"); entity.setAttributes(attributes); return entity; @@ -141,7 +231,7 @@ public class ImportTransformsTest { AtlasEntity entity = new AtlasEntity("hive_column"); Map<String, Object> attributes = new HashMap<>(); - attributes.put(qualifiedName, String.format("col%s.TABLE1.default@cl1", index)); + attributes.put(ATTR_NAME_QUALIFIED_NAME, String.format("col%s.TABLE1.default@cl1", index)); attributes.put("name", "col" + index); entity.setAttributes(attributes); http://git-wip-us.apache.org/repos/asf/atlas/blob/03f2754d/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java index bbf3f63..d462491 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java @@ -24,15 +24,19 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.model.impexp.AtlasCluster; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasImportRequest; +import org.apache.atlas.model.impexp.AtlasImportResult; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.typedef.AtlasEntityDef; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.TestResourceFileUtils; import org.testng.SkipException; @@ -46,15 +50,12 @@ import java.io.IOException; import java.util.List; import static org.apache.atlas.model.impexp.AtlasExportRequest.OPTION_KEY_REPLICATED_TO; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createAtlasEntity; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadEntity; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel; import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runExportWithParameters; import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithParameters; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; @Guice(modules = TestModules.TestOnlyModule.class) public class ReplicationEntityAttributeTest extends ExportImportTestBase { @@ -92,23 +93,13 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase { @BeforeClass public void setup() throws IOException, AtlasBaseException { - loadBaseModel(typeDefStore, typeRegistry); - loadHiveModel(typeDefStore, typeRegistry); - createEntities(); - } - - private void createEntities() { + basicSetup(typeDefStore, typeRegistry); entityStore = new AtlasEntityStoreV2(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper); + createEntities(entityStore, ENTITIES_SUB_DIR, new String[]{"db", "table-columns"}); - createAtlasEntity(entityStore, loadEntity(ENTITIES_SUB_DIR,"db")); - createAtlasEntity(entityStore, loadEntity(ENTITIES_SUB_DIR, "table-columns")); - - try { - AtlasEntity.AtlasEntitiesWithExtInfo entities = entityStore.getByIds(ImmutableList.of(DB_GUID, TABLE_GUID)); - assertEquals(entities.getEntities().size(), 2); - } catch (AtlasBaseException e) { - throw new SkipException(String.format("getByIds: could not load '%s' & '%s'.", DB_GUID, TABLE_GUID)); - } + AtlasType refType = typeRegistry.getType("Referenceable"); + AtlasEntityDef entityDef = (AtlasEntityDef) typeDefStore.getByName(refType.getTypeName()); + assertNotNull(entityDef); } @BeforeMethod @@ -128,20 +119,21 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase { assertNotNull(zipSource.getCreationOrder()); assertEquals(zipSource.getCreationOrder().size(), expectedEntityCount); - assertClusterInfo(REPLICATED_TO_CLUSTER_NAME); + assertCluster(REPLICATED_TO_CLUSTER_NAME, null); assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_TO_CLUSTER); } @Test(dependsOnMethods = "exportWithReplicationToOption_AddsClusterObjectIdToReplicatedFromAttribute", enabled = false) public void importWithReplicationFromOption_AddsClusterObjectIdToReplicatedFromAttribute() throws AtlasBaseException, IOException { AtlasImportRequest request = getImportRequestWithReplicationOption(); - runImportWithParameters(importService, request, zipSource); + AtlasImportResult importResult = runImportWithParameters(importService, request, zipSource); - assertClusterInfo(REPLICATED_FROM_CLUSTER_NAME); + assertCluster(REPLICATED_FROM_CLUSTER_NAME, importResult); assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_FROM_CLUSTER); } private void assertReplicationAttribute(String attrNameReplication) throws AtlasBaseException { + pauseForIndexCreation(); AtlasEntity.AtlasEntitiesWithExtInfo entities = entityStore.getByIds(ImmutableList.of(DB_GUID, TABLE_GUID)); for (AtlasEntity e : entities.getEntities()) { Object ex = e.getAttribute(attrNameReplication); @@ -152,11 +144,25 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase { } } - private void assertClusterInfo(String name) { + private void assertCluster(String name, AtlasImportResult importResult) throws AtlasBaseException { AtlasCluster actual = clusterService.get(new AtlasCluster(name, name)); assertNotNull(actual); assertEquals(actual.getName(), name); + + if(importResult != null) { + assertClusterAdditionalInfo(actual, importResult); + } + } + + private void assertClusterAdditionalInfo(AtlasCluster cluster, AtlasImportResult importResult) throws AtlasBaseException { + AtlasExportRequest request = importResult.getExportResult().getRequest(); + AtlasEntityType type = (AtlasEntityType) typeRegistry.getType(request.getItemsToExport().get(0).getTypeName()); + AtlasEntity.AtlasEntityWithExtInfo entity = entityStore.getByUniqueAttributes(type, request.getItemsToExport().get(0).getUniqueAttributes()); + long actualLastModifiedTimestamp = (long) cluster.getAdditionalInfoRepl(entity.getEntity().getGuid()); + + assertTrue(cluster.getAdditionalInfo().size() > 0); + assertEquals(actualLastModifiedTimestamp, importResult.getExportResult().getLastModifiedTimestamp()); } private AtlasExportRequest getUpdateMetaInfoUpdateRequest() { http://git-wip-us.apache.org/repos/asf/atlas/blob/03f2754d/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json b/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json index 8282638..4441036 100644 --- a/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json +++ b/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json @@ -4,5 +4,8 @@ "cardinality": "SET", "isIndexable": false, "isOptional": true, - "isUnique": false + "isUnique": false, + "options": { + "isSoftReference": "true" + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/03f2754d/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index 8d7b40c..6f6921f 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -28,7 +28,6 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.discovery.SearchContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.clusterinfo.AtlasCluster; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasExportResult; @@ -450,7 +449,6 @@ public class AdminResource { /** * Fetch details of a cluster. * @param clusterName name of target cluster with which it is paired - * @param entityQualifiedName qualified name of top level entity * @return AtlasCluster * @throws AtlasBaseException */ @@ -458,8 +456,7 @@ public class AdminResource { @Path("/cluster/{clusterName}") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasCluster getCluster(@PathParam("clusterName") String clusterName, - @QueryParam("entity") String entityQualifiedName) throws AtlasBaseException { + public AtlasCluster getCluster(@PathParam("clusterName") String clusterName) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -478,22 +475,21 @@ public class AdminResource { @Path("/expimp/audit") @Consumes(Servlets.JSON_MEDIA_TYPE) @Produces(Servlets.JSON_MEDIA_TYPE) - public AtlasSearchResult getExportImportAudit(@QueryParam("sourceClusterName") String sourceCluster, - @QueryParam("targetCluster") String targetCluster, - @QueryParam("userName") String userName, - @QueryParam("operation") String operation, - @QueryParam("startTime") String startTime, - @QueryParam("endTime") String endTime, - @QueryParam("limit") int limit, - @QueryParam("offset") int offset) throws AtlasBaseException { + public List<ExportImportAuditEntry> getExportImportAudit(@QueryParam("clusterName") String cluster, + @QueryParam("userName") String userName, + @QueryParam("operation") String operation, + @QueryParam("startTime") String startTime, + @QueryParam("endTime") String endTime, + @QueryParam("limit") int limit, + @QueryParam("offset") int offset) throws AtlasBaseException { AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "getExportImportAudit(" + sourceCluster + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "getExportImportAudit(" + cluster + ")"); } - return exportImportAuditService.get(userName, operation, sourceCluster, targetCluster, startTime, endTime, limit, offset); + return exportImportAuditService.get(userName, operation, cluster, startTime, endTime, limit, offset); } finally { AtlasPerfTracer.log(perf); } http://git-wip-us.apache.org/repos/asf/atlas/blob/03f2754d/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java index fc804d2..7acd332 100644 --- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java @@ -21,13 +21,15 @@ package org.apache.atlas.web.resources; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.clusterinfo.AtlasCluster; +import org.apache.atlas.model.impexp.AtlasCluster; import org.apache.atlas.model.impexp.AtlasExportRequest; import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.model.impexp.AtlasImportResult; import org.apache.atlas.repository.impexp.ZipSource; import org.apache.atlas.utils.TestResourceFileUtils; import org.apache.atlas.web.integration.BaseResourceIT; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; import org.testng.annotations.Test; import java.io.ByteArrayInputStream; @@ -43,6 +45,13 @@ import static org.testng.Assert.assertTrue; public class AdminExportImportTestIT extends BaseResourceIT { private final String FILE_TO_IMPORT = "stocks-base.zip"; + private final String EXPORT_REQUEST_FILE = "export-incremental"; + private final String SOURCE_CLUSTER_NAME = "cl1"; + + static final String IMPORT_TRANSFORM_CLEAR_ATTRS = + "{ \"Asset\": { \"*\":[ \"clearAttrValue:replicatedToCluster,replicatedFromCluster\" ] } }"; + static final String IMPORT_TRANSFORM_SET_DELETED = + "{ \"Asset\": { \"*\":[ \"setDeleted\" ] } }"; @Test public void isActive() throws AtlasServiceException { @@ -50,25 +59,34 @@ public class AdminExportImportTestIT extends BaseResourceIT { } @Test(dependsOnMethods = "isActive") - public void importData() throws AtlasServiceException, IOException { + public void importData() throws AtlasServiceException { performImport(FILE_TO_IMPORT); + assertReplicationData("cl1"); } @Test(dependsOnMethods = "importData") public void exportData() throws AtlasServiceException, IOException, AtlasBaseException { - final int EXPECTED_CREATION_ORDER_SIZE = 13; + final int EXPECTED_CREATION_ORDER_SIZE = 10; - AtlasExportRequest request = TestResourceFileUtils.readObjectFromJson(".", "export-incremental", AtlasExportRequest.class); + AtlasExportRequest request = TestResourceFileUtils.readObjectFromJson(".", EXPORT_REQUEST_FILE, AtlasExportRequest.class); byte[] exportedBytes = atlasClientV2.exportData(request); assertNotNull(exportedBytes); ZipSource zs = new ZipSource(new ByteArrayInputStream(exportedBytes)); assertNotNull(zs.getExportResult()); - assertEquals(zs.getCreationOrder().size(), EXPECTED_CREATION_ORDER_SIZE); + assertTrue(zs.getCreationOrder().size() > EXPECTED_CREATION_ORDER_SIZE); } private void performImport(String fileToImport) throws AtlasServiceException { AtlasImportRequest request = new AtlasImportRequest(); + request.getOptions().put(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM, SOURCE_CLUSTER_NAME); + request.getOptions().put(AtlasImportRequest.TRANSFORMS_KEY, IMPORT_TRANSFORM_CLEAR_ATTRS); + + performImport(fileToImport, request); + } + + private void performImport(String fileToImport, AtlasImportRequest request) throws AtlasServiceException { + byte[] fileBytes = new byte[0]; try { fileBytes = Files.readAllBytes(Paths.get(TestResourceFileUtils.getTestFilePath(fileToImport))); @@ -82,4 +100,23 @@ public class AdminExportImportTestIT extends BaseResourceIT { assertNotNull(result.getMetrics()); assertEquals(result.getProcessedEntities().size(), 37); } + + private void assertReplicationData(String clusterName) throws AtlasServiceException { + AtlasCluster cluster = atlasClientV2.getCluster(clusterName); + assertNotNull(cluster); + assertNotNull(cluster.getAdditionalInfo()); + assertTrue(cluster.getAdditionalInfo().size() > 0); + } + + @AfterClass + protected void teardown() { + AtlasImportRequest request = new AtlasImportRequest(); + request.getOptions().put(AtlasImportRequest.TRANSFORMS_KEY, IMPORT_TRANSFORM_SET_DELETED); + + try { + performImport(FILE_TO_IMPORT, request); + } catch (AtlasServiceException e) { + throw new SkipException("performTeardown: failed! Subsequent tests results may be affected.", e); + } + } } http://git-wip-us.apache.org/repos/asf/atlas/blob/03f2754d/webapp/src/test/resources/json/export-incremental.json ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/json/export-incremental.json b/webapp/src/test/resources/json/export-incremental.json index 9710841..2e4bba9 100644 --- a/webapp/src/test/resources/json/export-incremental.json +++ b/webapp/src/test/resources/json/export-incremental.json @@ -2,10 +2,10 @@ "itemsToExport": [ { "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks@cl1" } - } ], "options": { - "fetchType": "full" + "fetchType": "incremental", + "replicatedTo": "cl2" } } http://git-wip-us.apache.org/repos/asf/atlas/blob/03f2754d/webapp/src/test/resources/stocks-base.zip ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/stocks-base.zip b/webapp/src/test/resources/stocks-base.zip index 40c7f37..8663805 100644 Binary files a/webapp/src/test/resources/stocks-base.zip and b/webapp/src/test/resources/stocks-base.zip differ
