This is an automated email from the ASF dual-hosted git repository.
peterxcli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ae3358c0572 HDDS-13290. Correct the pagination semantic of
listMultipartUploads (#8975)
ae3358c0572 is described below
commit ae3358c0572fbbed5b5ced6a533833c3d048a568
Author: Peter Lee <[email protected]>
AuthorDate: Mon Sep 1 15:46:12 2025 +0800
HDDS-13290. Correct the pagination semantic of listMultipartUploads (#8975)
---
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 56 ++++++++++++++++++++++
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 12 ++---
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 12 +++++
3 files changed, 73 insertions(+), 7 deletions(-)
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index d3528eaaf7d..133bc031ab3 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -815,6 +815,17 @@ public void testListMultipartUploadsPagination() {
assertEquals(result.getKeyMarker(), keyMarker);
assertEquals(result.getMaxUploads(), 10);
+ // Verify next markers content
+ if (result.isTruncated()) {
+ MultipartUpload lastUploadOnPage = result.getMultipartUploads()
+ .get(result.getMultipartUploads().size() - 1);
+ assertEquals(lastUploadOnPage.getKey(), result.getNextKeyMarker());
+ assertEquals(lastUploadOnPage.getUploadId(),
result.getNextUploadIdMarker());
+ } else {
+ assertNull(result.getNextKeyMarker());
+ assertNull(result.getNextUploadIdMarker());
+ }
+
// Update markers for next page
keyMarker = result.getNextKeyMarker();
uploadIdMarker = result.getNextUploadIdMarker();
@@ -851,6 +862,51 @@ public void testListMultipartUploadsPagination() {
.collect(Collectors.toList()));
}
+ @Test
+ public void testListMultipartUploadsPaginationCornerCases() {
+ final String bucketName = getBucketName();
+ final String keyA = getKeyName("samekey");
+ final String keyB = getKeyName("after");
+
+ s3Client.createBucket(bucketName);
+
+ // Create multiple MPUs for the same key to verify upload-id-marker
semantics
+ List<String> keyAUploadIds = new ArrayList<>();
+ keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null,
null));
+ keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null,
null));
+ keyAUploadIds.add(initiateMultipartUpload(bucketName, keyA, null, null,
null));
+ // Also create another key to ensure listing proceeds past keyA
+ initiateMultipartUpload(bucketName, keyB, null, null, null);
+
+ // Sort upload IDs lexicographically to match listing order for the same
key
+ Collections.sort(keyAUploadIds);
+
+ // Case 1: key-marker=keyA and upload-id-marker set to the lowest uploadId
+ // Per spec, same-key uploads MAY be included if uploadId > marker
+ ListMultipartUploadsRequest request1 = new
ListMultipartUploadsRequest(bucketName)
+ .withKeyMarker(keyA)
+ .withUploadIdMarker(keyAUploadIds.get(0))
+ .withMaxUploads(100);
+ MultipartUploadListing result1 = s3Client.listMultipartUploads(request1);
+
+ List<MultipartUpload> uploads1 = result1.getMultipartUploads();
+ // Collect same-key uploads and verify none are <= marker
+ List<String> sameKeyIds1 = uploads1.stream()
+ .filter(u -> keyA.equals(u.getKey()))
+ .map(MultipartUpload::getUploadId)
+ .collect(Collectors.toList());
+ assertThat(sameKeyIds1).allSatisfy(id ->
assertTrue(id.compareTo(keyAUploadIds.get(0)) > 0));
+
+ // Case 2: key-marker=keyA and upload-id-marker set to the highest uploadId
+ // Expect no same-key (keyA) uploads to be returned
+ ListMultipartUploadsRequest request2 = new
ListMultipartUploadsRequest(bucketName)
+ .withKeyMarker(keyA)
+ .withUploadIdMarker(keyAUploadIds.get(2))
+ .withMaxUploads(100);
+ MultipartUploadListing result2 = s3Client.listMultipartUploads(request2);
+ assertTrue(result2.getMultipartUploads().stream().noneMatch(u ->
keyA.equals(u.getKey())));
+ }
+
@Test
public void testListParts(@TempDir Path tempDir) throws Exception {
final String bucketName = getBucketName();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 809974d9d7b..97af4c77af7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -986,14 +986,12 @@ public OmMultipartUploadList listMultipartUploads(String
volumeName,
OmMultipartUploadList.Builder resultBuilder =
OmMultipartUploadList.newBuilder();
if (withPagination && multipartUploadKeys.size() == maxUploads + 1) {
- int lastIndex = multipartUploadKeys.size() - 1;
- OmMultipartUpload lastUpload = multipartUploadKeys.get(lastIndex);
- resultBuilder.setNextKeyMarker(lastUpload.getKeyName())
- .setNextUploadIdMarker(lastUpload.getUploadId())
+ // Per spec, next markers should be the last element of the returned
list, not the lookahead.
+ multipartUploadKeys.remove(multipartUploadKeys.size() - 1);
+ OmMultipartUpload lastReturned =
multipartUploadKeys.get(multipartUploadKeys.size() - 1);
+ resultBuilder.setNextKeyMarker(lastReturned.getKeyName())
+ .setNextUploadIdMarker(lastReturned.getUploadId())
.setIsTruncated(true);
-
- // remove next upload from the list
- multipartUploadKeys.remove(lastIndex);
}
return resultBuilder
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 04e0998219f..48988192b2d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
+import static
org.apache.hadoop.hdds.StringUtils.getLexicographicallyHigherString;
import static org.apache.hadoop.ozone.OzoneConsts.DB_TRANSIENT_MARKER;
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
@@ -1506,6 +1507,10 @@ public <KEY, VALUE> long
countEstimatedRowsInTable(Table<KEY, VALUE> table)
return count;
}
+ /**
+ * Get the merged and paginated list of multipart upload keys from cache and
DB.
+ * Only return at most maxUploads + 1 keys (lexicographical order).
+ */
@Override
public List<OmMultipartUpload> getMultipartUploadKeys(
String volumeName, String bucketName, String prefix, String keyMarker,
@@ -1522,6 +1527,12 @@ public List<OmMultipartUpload> getMultipartUploadKeys(
if (StringUtils.isNotBlank(uploadIdMarker)) {
prefix = prefix + OM_KEY_PREFIX + uploadIdMarker;
}
+ // - If upload-id-marker is not specified,
+ // only the keys lexicographically greater than the specified key-marker
will be included in the list.
+ // - If upload-id-marker is specified,
+ // any multipart uploads for a key equal to the key-marker might also be
included,
+ // provided those multipart uploads have upload IDs lexicographically
greater than the specified upload-id-marker.
+ prefix = getLexicographicallyHigherString(prefix);
}
String seekKey = OmMultipartUpload.getDbKey(volumeName, bucketName,
prefix);
@@ -1551,6 +1562,7 @@ public List<OmMultipartUpload> getMultipartUploadKeys(
iterator = getMultipartInfoTable().iterator(prefixKey)) {
iterator.seek(seekKey);
+ // Try to get maxUploads + 1 keys to check if the list is truncated.
while (iterator.hasNext() && (noPagination || dbKeysCount < maxUploads +
1)) {
KeyValue<String, OmMultipartKeyInfo> entry = iterator.next();
// If it is marked for abort, skip it.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
