This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b71d4086b1 Revert "HDDS-12589. Fix Incorrect FSO Key Listing for
Container-to-Key Mapping. (#8078)"
b71d4086b1 is described below
commit b71d4086b17a0fea5bd8fec5ba568c61faa74016
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Mar 18 10:39:42 2025 +0100
Revert "HDDS-12589. Fix Incorrect FSO Key Listing for Container-to-Key
Mapping. (#8078)"
This reverts commit 68e3842616ba98d6f1c9b76d8378ccb985141cbc.
Reason for revert: compile error
---
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 14 +--
.../ozone/recon/api/TestContainerEndpoint.java | 118 ---------------------
.../recon/tasks/TestContainerKeyMapperTask.java | 116 --------------------
3 files changed, 8 insertions(+), 240 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
index f5936e34d5..041bcc8e6b 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java
@@ -241,11 +241,6 @@ public Response getKeysForContainer(
for (ContainerKeyPrefix containerKeyPrefix : containerKeyPrefixMap
.keySet()) {
- // break the for loop if limit has been reached
- if (keyMetadataMap.size() == limit) {
- break;
- }
-
// Directly calling getSkipCache() on the Key/FileTable table
// instead of iterating since only full keys are supported now. We will
// try to get the OmKeyInfo object by searching the KEY_TABLE table
with
@@ -270,7 +265,10 @@ public Response getKeysForContainer(
List<ContainerBlockMetadata> blockIds =
getBlocks(matchedKeys, containerID);
- String ozoneKey = containerKeyPrefix.getKeyPrefix();
+ String ozoneKey = omMetadataManager.getOzoneKey(
+ omKeyInfo.getVolumeName(),
+ omKeyInfo.getBucketName(),
+ omKeyInfo.getKeyName());
lastKey = ozoneKey;
if (keyMetadataMap.containsKey(ozoneKey)) {
keyMetadataMap.get(ozoneKey).getVersions()
@@ -279,6 +277,10 @@ public Response getKeysForContainer(
keyMetadataMap.get(ozoneKey).getBlockIds()
.put(containerKeyPrefix.getKeyVersion(), blockIds);
} else {
+ // break the for loop if limit has been reached
+ if (keyMetadataMap.size() == limit) {
+ break;
+ }
KeyMetadata keyMetadata = new KeyMetadata();
keyMetadata.setBucket(omKeyInfo.getBucketName());
keyMetadata.setVolume(omKeyInfo.getVolumeName());
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index 0dfdf047c4..65967c6e24 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@ -1665,122 +1665,4 @@ public void
testGetOmContainersDeletedInSCMPrevContainerParam()
assertEquals(1, containerDiscrepancyInfoList.size());
assertEquals(2, containerDiscrepancyInfoList.get(0).getContainerID());
}
-
- /**
- * Helper method that creates duplicate FSO file keys – two keys having the
same file
- * name but under different directories. It creates the necessary volume,
bucket, and
- * directory entries, and then writes two keys using writeKeyToOm.
- */
- private void setUpDuplicateFSOFileKeys() throws IOException {
- // Ensure the volume exists.
- String volumeKey = reconOMMetadataManager.getVolumeKey(VOLUME_NAME);
- OmVolumeArgs volArgs = OmVolumeArgs.newBuilder()
- .setVolume(VOLUME_NAME)
- .setAdminName("TestUser")
- .setOwnerName("TestUser")
- .setObjectID(VOL_OBJECT_ID)
- .build();
- reconOMMetadataManager.getVolumeTable().put(volumeKey, volArgs);
-
- // Ensure the bucket exists.
- OmBucketInfo bucketInfo = OmBucketInfo.newBuilder()
- .setVolumeName(VOLUME_NAME)
- .setBucketName(BUCKET_NAME)
- .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED)
- .setObjectID(BUCKET_OBJECT_ID)
- .build();
- String bucketKey = reconOMMetadataManager.getBucketKey(VOLUME_NAME,
BUCKET_NAME);
- reconOMMetadataManager.getBucketTable().put(bucketKey, bucketInfo);
-
- // Create two directories: "dirA" and "dirB" with unique object IDs.
- // For a top-level directory in a bucket, the parent's object id is the
bucket's id.
- OmDirectoryInfo dirA = OmDirectoryInfo.newBuilder()
- .setName("dirA")
- .setParentObjectID(BUCKET_OBJECT_ID)
- .setUpdateID(1L)
- .setObjectID(5L) // Unique object id for dirA.
- .build();
- OmDirectoryInfo dirB = OmDirectoryInfo.newBuilder()
- .setName("dirB")
- .setParentObjectID(BUCKET_OBJECT_ID)
- .setUpdateID(1L)
- .setObjectID(6L) // Unique object id for dirB.
- .build();
- // Build DB directory keys. (The third parameter is used to form a unique
key.)
- String dirKeyA = reconOMMetadataManager.getOzonePathKey(VOL_OBJECT_ID,
BUCKET_OBJECT_ID, 5L, "dirA");
- String dirKeyB = reconOMMetadataManager.getOzonePathKey(VOL_OBJECT_ID,
BUCKET_OBJECT_ID, 6L, "dirB");
- reconOMMetadataManager.getDirectoryTable().put(dirKeyA, dirA);
- reconOMMetadataManager.getDirectoryTable().put(dirKeyB, dirB);
-
- // Use a common OmKeyLocationInfoGroup.
- OmKeyLocationInfoGroup locationInfoGroup = getLocationInfoGroup1();
-
- // Write two FSO keys with the same file name ("dupFile") but in different
directories.
- // The file name stored in OM is the full path (e.g., "dirA/dupFile" vs.
"dirB/dupFile").
- writeKeyToOm(reconOMMetadataManager,
- "dupFileKey1", // internal key name for the first key
- BUCKET_NAME,
- VOLUME_NAME,
- "dupFileKey1", // full file path for the first key
- 100L, // object id (example)
- 5L, // parent's object id for dirA (same as
dirA's object id)
- BUCKET_OBJECT_ID,
- VOL_OBJECT_ID,
- Collections.singletonList(locationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
-
- writeKeyToOm(reconOMMetadataManager,
- "dupFileKey1", // internal key name for the second key
- BUCKET_NAME,
- VOLUME_NAME,
- "dupFileKey1", // full file path for the second key
- 100L,
- 6L, // parent's object id for dirB
- BUCKET_OBJECT_ID,
- VOL_OBJECT_ID,
- Collections.singletonList(locationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
- }
-
- /**
- * Test method that sets up two duplicate FSO file keys (same file name but
in different directories)
- * and then verifies that the ContainerEndpoint returns two distinct key
records.
- */
- @Test
- public void testDuplicateFSOKeysForContainerEndpoint() throws IOException {
- // Set up duplicate FSO file keys.
- setUpDuplicateFSOFileKeys();
- NSSummaryTaskWithFSO nSSummaryTaskWithFso =
- new NSSummaryTaskWithFSO(reconNamespaceSummaryManager,
- reconOMMetadataManager, 10);
- nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager);
- // Reprocess the container key mappings so that the new keys are loaded.
- reprocessContainerKeyMapper();
-
- // Assume that FSO keys are mapped to container ID 20L (as in previous
tests for FSO).
- Response response = containerEndpoint.getKeysForContainer(20L, -1, "");
- KeysResponse keysResponse = (KeysResponse) response.getEntity();
- Collection<KeyMetadata> keyMetadataList = keysResponse.getKeys();
-
- // We expect two distinct keys.
- assertEquals(2, keysResponse.getTotalCount());
- assertEquals(2, keyMetadataList.size());
-
- for (KeyMetadata km : keyMetadataList) {
- String completePath = km.getCompletePath();
- assertNotNull(completePath);
- // Verify that the complete path reflects either directory "dirA" or
"dirB"
- if (completePath.contains("dirA")) {
- assertEquals(VOLUME_NAME + "/" + BUCKET_NAME + "/dirA/dupFileKey1",
completePath);
- } else if (completePath.contains("dirB")) {
- assertEquals(VOLUME_NAME + "/" + BUCKET_NAME + "/dirB/dupFileKey1",
completePath);
- } else {
- throw new AssertionError("Unexpected complete path: " + completePath);
- }
- }
- }
-
-
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
index 0cacbf209e..fb31537ec7 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
@@ -472,122 +472,6 @@ public void testFileTableProcess() throws Exception {
firstKeyPrefix.getKeyPrefix());
}
- @Test
- public void testDuplicateFSOKeysInDifferentDirectories() throws Exception {
- // Ensure container 1 is initially empty.
- Map<ContainerKeyPrefix, Integer> keyPrefixesForContainer =
- reconContainerMetadataManager.getKeyPrefixesForContainer(1L);
- assertThat(keyPrefixesForContainer).isEmpty();
-
- Pipeline pipeline = getRandomPipeline();
- // Create a common OmKeyLocationInfoGroup for all keys.
- List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
- BlockID blockID = new BlockID(1L, 1L);
- OmKeyLocationInfo omKeyLocationInfo = getOmKeyLocationInfo(blockID,
pipeline);
- omKeyLocationInfoList.add(omKeyLocationInfo);
- OmKeyLocationInfoGroup omKeyLocationInfoGroup =
- new OmKeyLocationInfoGroup(0L, omKeyLocationInfoList);
-
- // Define file names.
- String file1Key = "file1";
- String file2Key = "file2";
-
- // Define directory (parent) object IDs with shorter values.
- long dir1Id = -101L;
- long dir2Id = -102L;
- long dir3Id = -103L;
-
- // Write three FSO keys for "file1" with different parent object IDs.
- writeKeyToOm(reconOMMetadataManager,
- file1Key, // keyName
- BUCKET_NAME, // bucketName
- VOLUME_NAME, // volName
- file1Key, // fileName
- KEY_ONE_OBJECT_ID, // objectId
- dir1Id, // ObjectId for first directory
- BUCKET_ONE_OBJECT_ID, // bucketObjectId
- VOL_OBJECT_ID, // volumeObjectId
- Collections.singletonList(omKeyLocationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
-
- writeKeyToOm(reconOMMetadataManager,
- file1Key,
- BUCKET_NAME,
- VOLUME_NAME,
- file1Key,
- KEY_ONE_OBJECT_ID,
- dir2Id, // ObjectId for second directory
- BUCKET_ONE_OBJECT_ID,
- VOL_OBJECT_ID,
- Collections.singletonList(omKeyLocationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
-
- writeKeyToOm(reconOMMetadataManager,
- file1Key,
- BUCKET_NAME,
- VOLUME_NAME,
- file1Key,
- KEY_ONE_OBJECT_ID,
- dir3Id, // ObjectId for third directory
- BUCKET_ONE_OBJECT_ID,
- VOL_OBJECT_ID,
- Collections.singletonList(omKeyLocationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
-
- // Write three FSO keys for "file2" with different parent object IDs.
- writeKeyToOm(reconOMMetadataManager,
- "fso-file2",
- BUCKET_NAME,
- VOLUME_NAME,
- file2Key,
- KEY_ONE_OBJECT_ID,
- dir1Id,
- BUCKET_ONE_OBJECT_ID,
- VOL_OBJECT_ID,
- Collections.singletonList(omKeyLocationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
-
- writeKeyToOm(reconOMMetadataManager,
- "fso-file2",
- BUCKET_NAME,
- VOLUME_NAME,
- file2Key,
- KEY_ONE_OBJECT_ID,
- dir2Id,
- BUCKET_ONE_OBJECT_ID,
- VOL_OBJECT_ID,
- Collections.singletonList(omKeyLocationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
-
- writeKeyToOm(reconOMMetadataManager,
- "fso-file2",
- BUCKET_NAME,
- VOLUME_NAME,
- file2Key,
- KEY_ONE_OBJECT_ID,
- dir3Id,
- BUCKET_ONE_OBJECT_ID,
- VOL_OBJECT_ID,
- Collections.singletonList(omKeyLocationInfoGroup),
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- KEY_ONE_SIZE);
-
- // Reprocess container key mappings.
- ContainerKeyMapperTask containerKeyMapperTask =
- new ContainerKeyMapperTask(reconContainerMetadataManager,
omConfiguration);
- containerKeyMapperTask.reprocess(reconOMMetadataManager);
-
- // With our changes using the raw key prefix as the unique identifier,
- // we expect six distinct entries in container 1.
- keyPrefixesForContainer =
reconContainerMetadataManager.getKeyPrefixesForContainer(1L);
- assertEquals(6, keyPrefixesForContainer.size());
- }
-
private OmKeyInfo buildOmKeyInfo(String volume,
String bucket,
String key,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
