This is an automated email from the ASF dual-hosted git repository.
mandarambawane pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new df293c4bd ATLAS-5034: Entities not created in Atlas when hive table
created on top a OFS/O3FS bucket/volume (#372)
df293c4bd is described below
commit df293c4bd1b8d25b9a9e7b32533364cfc9f7f0ce
Author: Aditya Gupta <aditya.gu...@freestoneinfotech.com>
AuthorDate: Mon Jun 16 16:14:47 2025 +0530
ATLAS-5034: Entities not created in Atlas when hive table created on top a
OFS/O3FS bucket/volume (#372)
---
.../apache/atlas/utils/AtlasPathExtractorUtil.java | 269 +++++++++++++++------
.../atlas/utils/AtlasPathExtractorUtilTest.java | 81 +++++--
2 files changed, 258 insertions(+), 92 deletions(-)
diff --git
a/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java
b/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java
index a60da1ec5..06558737f 100644
--- a/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java
+++ b/common/src/main/java/org/apache/atlas/utils/AtlasPathExtractorUtil.java
@@ -29,7 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URI;
-import java.util.Arrays;
+import java.util.Optional;
public class AtlasPathExtractorUtil {
// Common
@@ -90,6 +90,10 @@ public class AtlasPathExtractorUtil {
}
public static AtlasEntityWithExtInfo getPathEntity(Path path,
PathExtractorContext context) {
+ if (StringUtils.isEmpty(path.toString())) {
+ throw new IllegalArgumentException("Invalid Input: Path is Null");
+ }
+
AtlasEntityWithExtInfo entityWithExtInfo = new
AtlasEntityWithExtInfo();
String strPath = path.toString();
AtlasEntity ret;
@@ -344,111 +348,232 @@ public class AtlasPathExtractorUtil {
return ret;
}
- private static AtlasEntity addOzonePathEntity(Path path,
AtlasEntityExtInfo extInfo, PathExtractorContext context) {
- String strPath = path.toString();
+ /**
+ * Returns O3FS Authority i.e. "bucket-volume-Ozone_Service_Id"
+ * Example: query: 'o3fs://buck1.vol1.ozone1747118357/key1' -
O3FS_Authority: buck1.vol1.ozone1747118357
+ */
+ private static String o3fsAuthorityExtractor(String path) {
+ return Optional.ofNullable(path)
+ .filter(p -> p.startsWith(OZONE_3_SCHEME))
+ .map(p -> p.substring(OZONE_3_SCHEME.length()).split("/", 2))
+ .filter(parts -> parts.length > 0)
+ .map(parts -> parts[0])
+ .orElse("");
+ }
- LOG.debug("==> addOzonePathEntity(strPath={})", strPath);
+ /**
+ * Returns O3FS Keys defined in the path.
+ * Example: query: 'o3fs://buck1.vol1.ozone1747118357/key1' -> Keys: key1
+ * Example: query: 'o3fs://buck1.vol1.ozone1747118357/key1/key2' -> Keys:
[key1, key2]
+ */
+ private static String[] o3fsKeyExtractor(String path) {
+ return Optional.ofNullable(path)
+ .filter(p -> p.startsWith(OZONE_3_SCHEME))
+ .map(p -> p.substring(OZONE_3_SCHEME.length()).split("/", 2))
+ .filter(parts -> parts.length == 2)
+ .map(parts -> parts[1].split("/"))
+ .orElse(new String[] {""});
+ }
- String metadataNamespace = context.getMetadataNamespace();
- String ozoneScheme = path.toUri().getScheme();
- String pathQualifiedName = strPath + QNAME_SEP_METADATA_NAMESPACE
+ metadataNamespace;
- AtlasEntity ret = context.getEntity(pathQualifiedName);
+ /**
+ * Returns Count of O3FS paths - [bucket + volume + keys].
+ * Example: query: 'o3fs://buck1.vol1.ozone1747118357/key1': length = 1
(bucket) + 1 (volume) + 1 (key) = 3
+ * Example: query: 'o3fs://buck1.vol1.ozone1747118357/key1/key2': length =
1 (bucket) + 1 (volume) + 2 (key segments) = 4
+ *
+ * O3FS_AUTHORITY_WITH_KEYS: "buck1.vol1.ozone1747118357/key1/key2"
+ * Authority_And_Keys: [buck1.vol1.ozone1747118357, key1/key2]
+ * O3fs_Authority: "buck1.vol1.ozone1747118357"
+ * keyPath: "key1/key2"
+ */
+ private static int getO3fsPathLength(String path) {
+ if (!path.startsWith(OZONE_3_SCHEME)) {
+ return 0;
+ }
+ String o3fsAuthorityWithKeys = path.substring(OZONE_3_SCHEME.length());
+ String[] authorityAndKeys = o3fsAuthorityWithKeys.split("/", 2);
+ String o3fsAuthority = authorityAndKeys[0];
+ String keyPath = authorityAndKeys.length > 1 ?
authorityAndKeys[1].trim() : "";
+
+ // Count bucket and volume from authority
+ String[] bucketAndVolume = o3fsAuthority.split("\\.");
+ int length = 0;
+ if (bucketAndVolume.length >= 1) {
+ length++; // bucket
+ }
+ if (bucketAndVolume.length >= 2) {
+ length++; // volume
+ }
- if (ret == null) {
- //create ozone volume entity
- String volumeName = getOzoneVolumeName(path);
- String volumeQualifiedName = ozoneScheme + SCHEME_SEPARATOR +
volumeName + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
- AtlasEntity volumeEntity =
context.getEntity(volumeQualifiedName);
+ // Count key segments if present
+ if (!keyPath.isEmpty()) {
+ String[] keys = keyPath.split("/");
+ length += keys.length;
+ }
- if (volumeEntity == null) {
- volumeEntity = new AtlasEntity(OZONE_VOLUME);
+ return length;
+ }
- volumeEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME,
volumeQualifiedName);
- volumeEntity.setAttribute(ATTRIBUTE_NAME, volumeName);
+ /**
+ * Creates Ozone Entity for different Ozone Type Names: OZONE_BUCKET,
OZONE_VOLUME, OZONE_KEY and sets relationshipAttribute between them
+ */
+ private static AtlasEntity createOzoneEntity(PathExtractorContext context,
String typeName, String name, String qualifiedName, AtlasRelatedObjectId
relationship) {
+ AtlasEntity ozoneEntity = context.getEntity(qualifiedName);
- LOG.debug("adding entity: typeName={}, qualifiedName={}",
volumeEntity.getTypeName(),
volumeEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME));
+ if (ozoneEntity == null) {
+ ozoneEntity = new AtlasEntity(typeName);
+ ozoneEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, qualifiedName);
+ ozoneEntity.setAttribute(ATTRIBUTE_NAME, name);
- context.putEntity(volumeQualifiedName, volumeEntity);
+ if (relationship != null) {
+ String relationshipAttribute = typeName.equals(OZONE_BUCKET) ?
ATTRIBUTE_VOLUME : ATTRIBUTE_PARENT;
+ ozoneEntity.setRelationshipAttribute(relationshipAttribute,
relationship);
}
- extInfo.addReferredEntity(volumeEntity);
+ context.putEntity(qualifiedName, ozoneEntity);
+ LOG.info("Added entity: typeName={}, qualifiedName={}", typeName,
qualifiedName);
+ }
- //create ozone bucket entity
- String bucketName = getOzoneBucketName(path);
- String bucketQualifiedName = ozoneScheme + SCHEME_SEPARATOR +
volumeName + QNAME_SEP_ENTITY_NAME + bucketName + QNAME_SEP_METADATA_NAMESPACE
+ metadataNamespace;
- AtlasEntity bucketEntity =
context.getEntity(bucketQualifiedName);
+ return ozoneEntity;
+ }
- if (bucketEntity == null) {
- bucketEntity = new AtlasEntity(OZONE_BUCKET);
+ /**
+ * Adds Ozone Path Entity for Scheme: OZONE_SCHEME
+ */
+ private static AtlasEntity addOfsPathEntity(Path path, AtlasEntityExtInfo
extInfo, PathExtractorContext context) {
+ String metadataNamespace = context.getMetadataNamespace();
+ String ozoneScheme = path.toUri().getScheme();
+ String ofsVolumeName = getOzoneVolumeName(path);
+ String ofsVolumeQualifiedName = OZONE_SCHEME + ofsVolumeName +
QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
+
+ String ofsDirPath = path.toUri().getPath();
+ if (StringUtils.isEmpty(ofsDirPath)) {
+ ofsDirPath = Path.SEPARATOR;
+ }
- bucketEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME,
bucketQualifiedName);
- bucketEntity.setAttribute(ATTRIBUTE_NAME, bucketName);
- bucketEntity.setRelationshipAttribute(ATTRIBUTE_VOLUME,
AtlasTypeUtil.getAtlasRelatedObjectId(volumeEntity,
RELATIONSHIP_OZONE_VOLUME_BUCKET));
+ String[] ofsPath = ofsDirPath.split(Path.SEPARATOR);
+ if (ofsPath.length < 2) {
+ return null;
+ }
- LOG.debug("adding entity: typeName={}, qualifiedName={}",
bucketEntity.getTypeName(),
bucketEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME));
+ AtlasEntity volumeEntity = createOzoneEntity(context, OZONE_VOLUME,
ofsVolumeName, ofsVolumeQualifiedName, null);
+ extInfo.addReferredEntity(volumeEntity);
- context.putEntity(bucketQualifiedName, bucketEntity);
- }
+ if (ofsPath.length == 2) {
+ return volumeEntity;
+ }
- extInfo.addReferredEntity(bucketEntity);
+ String ofsBucketName = ofsPath[2];
+ String ofsBucketQualifiedName = OZONE_SCHEME + ofsVolumeName +
QNAME_SEP_ENTITY_NAME + ofsBucketName + QNAME_SEP_METADATA_NAMESPACE +
metadataNamespace;
+ AtlasEntity bucketEntity = createOzoneEntity(context, OZONE_BUCKET,
ofsBucketName, ofsBucketQualifiedName,
+ AtlasTypeUtil.getAtlasRelatedObjectId(volumeEntity,
RELATIONSHIP_OZONE_VOLUME_BUCKET));
+ extInfo.addReferredEntity(bucketEntity);
- AtlasRelatedObjectId parentObjId =
AtlasTypeUtil.getAtlasRelatedObjectId(bucketEntity,
RELATIONSHIP_OZONE_PARENT_CHILDREN);
- String parentPath = Path.SEPARATOR;
- String dirPath = path.toUri().getPath();
+ if (ofsPath.length == 3) {
+ return bucketEntity;
+ }
- if (StringUtils.isEmpty(dirPath)) {
- dirPath = Path.SEPARATOR;
- }
+ AtlasEntity currentOfsKeyEntity = null;
+ StringBuilder keyPathBuilder = new StringBuilder();
+ String keyQNamePrefix = ozoneScheme + SCHEME_SEPARATOR +
path.toUri().getAuthority();
- String keyQNamePrefix = ozoneScheme + SCHEME_SEPARATOR +
path.toUri().getAuthority();
- String[] subDirNames = dirPath.split(Path.SEPARATOR);
- String[] subDirNameArr = subDirNames;
+ AtlasEntity parentEntityForOfsKey = bucketEntity;
- if (ozoneScheme.equals(OZONE_SCHEME_NAME)) {
- subDirNames = Arrays.stream(subDirNameArr, 3,
subDirNameArr.length).toArray(String[]::new);
+ for (int i = 3; i < ofsPath.length; i++) {
+ String ofsKeyName = ofsPath[i];
+ if (StringUtils.isEmpty(ofsKeyName)) {
+ continue;
}
- boolean volumeBucketAdded = false;
+ keyPathBuilder.append(Path.SEPARATOR).append(ofsKeyName);
- for (String subDirName : subDirNames) {
- if (StringUtils.isEmpty(subDirName)) {
- continue;
- }
+ String ofsKeyQualifiedName = keyQNamePrefix
+ + Path.SEPARATOR + ofsVolumeName
+ + Path.SEPARATOR + ofsBucketName
+ + keyPathBuilder
+ + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
- String subDirPath;
+ currentOfsKeyEntity = createOzoneEntity(context, OZONE_KEY,
ofsKeyName, ofsKeyQualifiedName,
+
AtlasTypeUtil.getAtlasRelatedObjectId(parentEntityForOfsKey,
RELATIONSHIP_OZONE_PARENT_CHILDREN));
- if (ozoneScheme.equals(OZONE_SCHEME_NAME) &&
!volumeBucketAdded) {
- subDirPath = "%s%s" + Path.SEPARATOR + "%s" +
Path.SEPARATOR + "%s";
- subDirPath = String.format(subDirPath, parentPath,
subDirNameArr[1], subDirNameArr[2], subDirName);
- volumeBucketAdded = true;
- } else {
- subDirPath = parentPath + subDirName;
- }
+ parentEntityForOfsKey = currentOfsKeyEntity;
+ AtlasTypeUtil.getAtlasRelatedObjectId(parentEntityForOfsKey,
RELATIONSHIP_OZONE_PARENT_CHILDREN);
+ }
- String subDirQualifiedName = keyQNamePrefix + subDirPath +
QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== addOzonePathEntity(strPath={})", path.toString());
+ }
- ret = context.getEntity(subDirQualifiedName);
+ return currentOfsKeyEntity;
+ }
- if (ret == null) {
- ret = new AtlasEntity(OZONE_KEY);
+ /**
+ * Adds Ozone Path Entity for Scheme: OZONE_3_SCHEME
+ */
+ private static AtlasEntity addO3fsPathEntity(Path path, AtlasEntityExtInfo
extInfo, PathExtractorContext context) {
+ String[] o3fsKeys = o3fsKeyExtractor(path.toString());
+ int o3fsPathLength = getO3fsPathLength(path.toString());
+ String metadataNamespace = context.getMetadataNamespace();
+ String ozoneScheme = path.toUri().getScheme();
+ String o3fsBucketName = getOzoneBucketName(path);
+ String o3fsVolumeName = getOzoneVolumeName(path);
+ String o3fsVolumeQualifiedName = ozoneScheme + SCHEME_SEPARATOR +
o3fsVolumeName + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
+
+ AtlasEntity volumeEntity = createOzoneEntity(context, OZONE_VOLUME,
o3fsVolumeName, o3fsVolumeQualifiedName, null);
+ extInfo.addReferredEntity(volumeEntity);
+
+ String bucketQualifiedName = ozoneScheme + SCHEME_SEPARATOR +
o3fsVolumeName + QNAME_SEP_ENTITY_NAME + o3fsBucketName +
QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
+ AtlasEntity bucketEntity = createOzoneEntity(context, OZONE_BUCKET,
o3fsBucketName, bucketQualifiedName,
+ AtlasTypeUtil.getAtlasRelatedObjectId(volumeEntity,
RELATIONSHIP_OZONE_VOLUME_BUCKET));
+ extInfo.addReferredEntity(bucketEntity);
+
+ if (o3fsPathLength < 1) {
+ return null;
+ }
- ret.setRelationshipAttribute(ATTRIBUTE_PARENT,
parentObjId);
- ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME,
subDirQualifiedName);
- ret.setAttribute(ATTRIBUTE_NAME, subDirName);
+ if (o3fsPathLength == 1) {
+ return volumeEntity;
+ }
+ if (o3fsPathLength == 2) {
+ return bucketEntity;
+ }
- LOG.debug("adding entity: typeName={}, qualifiedName={}",
ret.getTypeName(), ret.getAttribute(ATTRIBUTE_QUALIFIED_NAME));
+ AtlasEntity currentO3fsKeyEntity = null;
+ AtlasEntity parentEntityForO3fsKey = bucketEntity;
+ StringBuilder keyPathBuilder = new StringBuilder();
+ String authority = o3fsAuthorityExtractor(path.toString());
- context.putEntity(subDirQualifiedName, ret);
- }
+ for (String o3fsKeyName : o3fsKeys) {
+ keyPathBuilder.append(Path.SEPARATOR).append(o3fsKeyName);
- parentObjId = AtlasTypeUtil.getAtlasRelatedObjectId(ret,
RELATIONSHIP_OZONE_PARENT_CHILDREN);
- parentPath = subDirPath + Path.SEPARATOR;
- }
+ String o3fsKeyQualifiedName = ozoneScheme + SCHEME_SEPARATOR +
authority + keyPathBuilder + QNAME_SEP_METADATA_NAMESPACE + metadataNamespace;
+
+ currentO3fsKeyEntity = createOzoneEntity(context, OZONE_KEY,
o3fsKeyName, o3fsKeyQualifiedName,
+
AtlasTypeUtil.getAtlasRelatedObjectId(parentEntityForO3fsKey,
RELATIONSHIP_OZONE_PARENT_CHILDREN));
+
+ parentEntityForO3fsKey = currentO3fsKeyEntity;
+ AtlasTypeUtil.getAtlasRelatedObjectId(parentEntityForO3fsKey,
RELATIONSHIP_OZONE_PARENT_CHILDREN);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== addOzonePathEntity(strPath={})", path.toString());
}
- LOG.debug("<== addOzonePathEntity(strPath={})", strPath);
+ return currentO3fsKeyEntity;
+ }
- return ret;
+ /**
+ * Adds Ozone Path Entity: OZONE_3_SCHEME | OZONE_3_SCHEME
+ */
+ private static AtlasEntity addOzonePathEntity(Path path,
AtlasEntityExtInfo extInfo, PathExtractorContext context) {
+ String strPath = path.toString();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> addOzonePathEntity(strPath={})", strPath);
+ }
+
+ return strPath.startsWith(OZONE_3_SCHEME)
+ ? addO3fsPathEntity(path, extInfo, context)
+ : addOfsPathEntity(path, extInfo, context);
}
private static AtlasEntity addGCSPathEntity(Path path, AtlasEntityExtInfo
extInfo, PathExtractorContext context) {
diff --git
a/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
b/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
index 6b95fd30c..83c84d785 100644
---
a/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
+++
b/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
@@ -28,6 +28,7 @@ import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@@ -88,22 +89,32 @@ public class AtlasPathExtractorUtilTest {
@Test(dataProvider = "ozonePathProvider")
public void testGetPathEntityOzone3Path(OzoneKeyValidator validator) {
- String scheme = validator.scheme;
+ String scheme = validator.scheme;
String ozonePath = scheme + validator.location;
PathExtractorContext extractorContext = new
PathExtractorContext(METADATA_NAMESPACE);
- Path path = new Path(ozonePath);
+ Path path = new Path(ozonePath);
AtlasEntityWithExtInfo entityWithExtInfo =
AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
- AtlasEntity entity =
entityWithExtInfo.getEntity();
+ AtlasEntity entity = entityWithExtInfo.getEntity();
assertNotNull(entity);
verifyOzoneKeyEntity(entity, validator);
- assertEquals(entityWithExtInfo.getReferredEntities().size(), 2);
+ if (entity.getTypeName() == OZONE_KEY) {
+ verifyReferredAndKnownEntities(entityWithExtInfo,
extractorContext, validator, validator.knownEntitiesCountTillKey, 2);
+ } else if (entity.getTypeName() == OZONE_BUCKET) {
+ verifyReferredAndKnownEntities(entityWithExtInfo,
extractorContext, validator, validator.knownEntitiesCountTillBucket, 2);
+ } else if (entity.getTypeName() == OZONE_VOLUME) {
+ verifyReferredAndKnownEntities(entityWithExtInfo,
extractorContext, validator, validator.knownEntitiesCountTillVolume, 1);
+ }
+ }
+
+ private void verifyReferredAndKnownEntities(AtlasEntityWithExtInfo
entityWithExtInfo, PathExtractorContext extractorContext, OzoneKeyValidator
validator, int knownEntitiesCountTillVolume, int referredEntitiesSize) {
+ assertEquals(entityWithExtInfo.getReferredEntities().size(),
referredEntitiesSize);
verifyOzoneEntities(entityWithExtInfo.getReferredEntities(),
validator);
- assertEquals(extractorContext.getKnownEntities().size(),
validator.knownEntitiesCount);
+ assertEquals(extractorContext.getKnownEntities().size(),
knownEntitiesCountTillVolume);
verifyOzoneEntities(extractorContext.getKnownEntities(), validator);
}
@@ -271,20 +282,40 @@ public class AtlasPathExtractorUtilTest {
"quarter_one", "ozone1/volume1/bucket1/quarter_one",
"sales", "ozone1/volume1/bucket1/quarter_one/sales")},
+ {new OzoneKeyValidator(OZONE_SCHEME,
"ozone1745922163/volume1/bucket1/mykey1/key.text",
+ "mykey1", "ozone1745922163/volume1/bucket1/mykey1",
+ "key.text",
"ozone1745922163/volume1/bucket1/mykey1/key.text")},
+
+ {new OzoneKeyValidator(OZONE_SCHEME, "ozone1/volume1/bucket1",
+ "bucket1", "volume1.bucket1")},
+
+ {new OzoneKeyValidator(OZONE_SCHEME, "ozone1/volume1",
+ "volume1", "volume1")},
+
{new OzoneKeyValidator(OZONE_3_SCHEME,
"bucket1.volume1.ozone1/files/file.txt",
"files", "bucket1.volume1.ozone1/files",
- "file.txt", "bucket1.volume1.ozone1/files/file.txt")},
+ "file.txt", "bucket1.volume1.ozone1/files/file.txt") },
{new OzoneKeyValidator(OZONE_3_SCHEME,
"bucket1.volume1.ozone1/file21.txt",
- "file21.txt", "bucket1.volume1.ozone1/file21.txt")},
+ "file21.txt", "bucket1.volume1.ozone1/file21.txt") },
{new OzoneKeyValidator(OZONE_3_SCHEME,
"bucket1.volume1.ozone1/quarter_one/sales",
"quarter_one", "bucket1.volume1.ozone1/quarter_one",
- "sales", "bucket1.volume1.ozone1/quarter_one/sales")},
+ "sales", "bucket1.volume1.ozone1/quarter_one/sales") },
{new OzoneKeyValidator(OZONE_3_SCHEME,
"bucket1.volume1.ozone1/quarter_one/sales/",
"quarter_one", "bucket1.volume1.ozone1/quarter_one",
- "sales", "bucket1.volume1.ozone1/quarter_one/sales")},
+ "sales", "bucket1.volume1.ozone1/quarter_one/sales") },
+
+ {new OzoneKeyValidator(OZONE_3_SCHEME,
"bucket1.volume1.ozone1/",
+ "bucket1", "volume1.bucket1")},
+
+ {new OzoneKeyValidator(OZONE_3_SCHEME,
"bucket1.volume1.ozone1/key1/key1.txt/",
+ "key1", "bucket1.volume1.ozone1/key1",
+ "key1.txt", "bucket1.volume1.ozone1/key1/key1.txt") },
+
+ {new OzoneKeyValidator(OZONE_3_SCHEME, "bucket1.volume1/key1/",
+ "key1", "bucket1.volume1/key1") },
};
}
@@ -309,8 +340,16 @@ public class AtlasPathExtractorUtilTest {
}
private void verifyOzoneKeyEntity(AtlasEntity entity, OzoneKeyValidator
validator) {
- assertEquals(entity.getTypeName(), OZONE_KEY);
- assertTrue(validator.validateNameQName(entity));
+ if (Objects.equals(entity.getTypeName(), OZONE_KEY)) {
+ assertEquals(entity.getTypeName(), OZONE_KEY);
+ assertTrue(validator.validateNameQName(entity));
+ } else if (Objects.equals(entity.getTypeName(), OZONE_BUCKET)) {
+ assertEquals(entity.getTypeName(), OZONE_BUCKET);
+ assertTrue(validator.validateNameQName(entity));
+ } else if (Objects.equals(entity.getTypeName(), OZONE_VOLUME)) {
+ assertEquals(entity.getTypeName(), OZONE_VOLUME);
+ assertTrue(validator.validateNameQName(entity));
+ }
}
private void verifyHDFSEntity(AtlasEntity entity, boolean toLowerCase) {
@@ -476,25 +515,27 @@ public class AtlasPathExtractorUtilTest {
private static class OzoneKeyValidator {
private final String scheme;
private final String location;
- private final int knownEntitiesCount;
+ private final int knownEntitiesCountTillKey;
+ private final int knownEntitiesCountTillBucket;
+ private final int knownEntitiesCountTillVolume;
private final Map<String, String> nameQNamePairs;
public OzoneKeyValidator(String scheme, String location, String...
pairs) {
this.scheme = scheme;
this.location = location;
this.nameQNamePairs = getPairMap(scheme, pairs);
- this.knownEntitiesCount = nameQNamePairs.size() + 2;
- }
+ this.knownEntitiesCountTillKey = nameQNamePairs.size() + 2;
+ this.knownEntitiesCountTillBucket = nameQNamePairs.size() + 1;
+ this.knownEntitiesCountTillVolume = nameQNamePairs.size(); }
public boolean validateNameQName(AtlasEntity entity) {
- String name = (String) entity.getAttribute(ATTRIBUTE_NAME);
-
- if (this.nameQNamePairs.containsKey(name)) {
- String qName = (String)
entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME);
+ String qName = (String)
entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME);
- return qName.equals(this.nameQNamePairs.get(name));
+ for (Map.Entry<String, String> nameQName :
nameQNamePairs.entrySet()) {
+ if (qName.equals(nameQName.getValue())) {
+ return true;
+ }
}
-
return false;
}
